fix(mesh): close data-plane race that dropped early TcpData frames (#1086)

The tcp_open and tcp_open_reverse receivers registered their stream
entry only after awaiting resolveTarget / resolveContainerIp. The
peer, which is free to send TcpData immediately after the open frame,
hit the lookup-miss path in handleBinaryFrame and the bytes were
silently dropped. Probes passed because they only exercise the
handshake; real HTTP hung with 0 bytes received.

Reserve the stream entry synchronously, buffer early TcpData in a
per-stream pendingData queue capped at 1 MiB, and flush onto the
local socket inside the connect handler before sending tcp_open_ack.
The payload is copied because decodeBinaryFrame returns a subarray
view of the WS receive buffer; holding the view would pin the parent
buffer past its lifecycle.

Both sides of the data plane are patched:
- tcpStreamSwitchboard.onTcpOpen (forward, agent + proxy-mode peer)
- PilotTunnelBridge.handleTcpOpenReverse / acceptReverseLocal
  (reverse, primary acting as the local dial target)

Adds unit coverage for the race window and for the overflow path.
The cap constant lives in pilot/protocol.ts alongside the other
per-stream limits.
This commit is contained in:
Anso
2026-05-17 14:01:39 -04:00
committed by GitHub
parent 50b89db3b8
commit a318e6b3c1
5 changed files with 303 additions and 15 deletions
@@ -188,6 +188,72 @@ describe('TcpStreamSwitchboard.onTcpOpen (forward path)', () => {
expect(consumed).toBe(true);
});
it('buffers TcpData arriving during the resolveTarget await and flushes it on connect (F-9 race fix)', async () => {
const { port, server, firstConn } = await startEchoServer();
try {
// Resolver delayed by 50 ms simulates a real Dockerode inspect.
// Without the fix, the TcpData frame sent inside this window
// hits a missing this.tcpStreams entry and is silently dropped.
const { switchboard, sent } = makeSwitchboard({
resolve: () => new Promise((r) => setTimeout(() => r({ ok: true, host: '127.0.0.1', port }), 50)),
});
switchboard.handleJsonFrame({ t: 'tcp_open', s: 1, stack: 's', service: 'svc', port: 80 });
// Immediately push the "HTTP request" bytes before the resolver
// finishes. Reservation must catch them in pendingData.
const reqFrame = encodeBinaryFrame(BinaryFrameType.TcpData, 1, Buffer.from('GET / HTTP/1.1\r\n\r\n'));
switchboard.handleBinaryFrame(decodeBinaryFrame(reqFrame));
const targetSocket = await firstConn;
const echoed = new Promise<Buffer>((r) => {
let acc = Buffer.alloc(0);
targetSocket.on('data', (chunk: Buffer) => {
acc = Buffer.concat([acc, chunk]);
if (acc.length >= 'GET / HTTP/1.1\r\n\r\n'.length) r(acc);
});
});
const received = await echoed;
expect(received.toString()).toBe('GET / HTTP/1.1\r\n\r\n');
// tcp_open_ack should have followed (not preceded) the flush.
const ack = sent.map((s) => s.binary ? null : decodeJsonFrame(s.raw as string)).find((d) => d?.t === 'tcp_open_ack');
if (!ack || ack.t !== 'tcp_open_ack') throw new Error('no ack');
expect(ack.ok).toBe(true);
switchboard.cleanup('test');
} finally {
server.close();
}
});
it('caps pending-data buffer and sends tcp_close when a peer overflows it before resolve completes', async () => {
// Resolver that never resolves; reservation is held open and any
// queued bytes accumulate against the cap (1 MiB).
const { switchboard, sent } = makeSwitchboard({
resolve: () => new Promise(() => { /* never */ }),
});
try {
switchboard.handleJsonFrame({ t: 'tcp_open', s: 2, stack: 's', service: 'svc', port: 80 });
// 16 chunks of 96 KiB = 1.5 MiB total, crosses the 1 MiB cap on
// the chunk that pushes pendingBytes past the limit.
const chunkSize = 96 * 1024;
const payload = Buffer.alloc(chunkSize, 0x41);
for (let i = 0; i < 16; i++) {
const buf = encodeBinaryFrame(BinaryFrameType.TcpData, 2, payload);
switchboard.handleBinaryFrame(decodeBinaryFrame(buf));
}
await new Promise((r) => setImmediate(r));
const close = sent.map((s) => s.binary ? null : decodeJsonFrame(s.raw as string)).find((d) => d?.t === 'tcp_close' && d.s === 2);
expect(close).toBeDefined();
expect(switchboard.tcpStreamCount()).toBe(0);
} finally {
// Idle-timer + resolver closures both pin entry state across
// tests when the resolver never resolves; cleanup detaches them.
switchboard.cleanup('test');
}
});
it('tcp_close on a forward stream destroys the socket and drops the entry', async () => {
const { port, server, firstConn } = await startEchoServer();
try {
@@ -17,7 +17,9 @@ import { WebSocket } from 'ws';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import {
AGENT_REVERSE_ID_BASE,
BinaryFrameType,
decodeJsonFrame,
encodeBinaryFrame,
encodeJsonFrame,
} from '../pilot/protocol';
@@ -123,6 +125,58 @@ describe('PilotTunnelBridge handles tcp_open_reverse (Phase B)', () => {
vi.restoreAllMocks();
});
it('buffers TcpData arriving during the resolveContainerIp await and flushes it on connect (F-9 reverse race fix)', async () => {
const mockWs = makeMockTunnelWs();
const bridge = new PilotTunnelBridge(1, mockWs as unknown as WebSocket);
await bridge.start();
// Upstream that captures received bytes so we can assert the peer's
// request body was delivered (not silently dropped during the
// resolveContainerIp window).
let resolveReceived!: (buf: Buffer) => void;
const received = new Promise<Buffer>((r) => { resolveReceived = r; });
const upstream = net.createServer((socket) => {
let acc = Buffer.alloc(0);
socket.on('data', (chunk: Buffer) => {
acc = Buffer.concat([acc, chunk]);
if (acc.length >= 'POST /hook HTTP/1.1\r\n\r\n'.length) resolveReceived(acc);
});
});
await new Promise<void>((resolve) => upstream.listen(0, '127.0.0.1', () => resolve()));
const addr = upstream.address();
if (!addr || typeof addr === 'string') throw new Error('no address');
const upstreamPort = addr.port;
const localNodeId = (await import('../services/NodeRegistry')).NodeRegistry.getInstance().getDefaultNodeId();
// Delay the resolve to widen the race window; without the fix the
// TcpData frame below lands while this.streams has no entry for s
// and the bytes are dropped at the lookup-miss path in
// handleBinaryFrame, so the upstream never sees the request.
vi.spyOn(MeshService.getInstance(), 'resolveContainerIp')
.mockImplementation(() => new Promise((r) => setTimeout(() => r('127.0.0.1'), 50)));
const s = AGENT_REVERSE_ID_BASE + 3;
mockWs.emit('message', encodeJsonFrame({
t: 'tcp_open_reverse', s,
targetNodeId: localNodeId, stack: 'real', service: 'svc', port: upstreamPort,
}), false);
// Immediately push the "HTTP request" bytes while resolve is in flight.
mockWs.emit('message',
encodeBinaryFrame(BinaryFrameType.TcpData, s, Buffer.from('POST /hook HTTP/1.1\r\n\r\n')),
true);
const body = await received;
expect(body.toString()).toBe('POST /hook HTTP/1.1\r\n\r\n');
// ack:true should also have landed (after the flush).
const ack = await waitFor(() => findAck(mockWs, s));
expect(ack.ok).toBe(true);
upstream.close();
bridge.close();
vi.restoreAllMocks();
});
it('local target with a working dial sends ok:true and registers a reverse stream', async () => {
const mockWs = makeMockTunnelWs();
const bridge = new PilotTunnelBridge(1, mockWs as unknown as WebSocket);