fix(security): harden authentication and outbound targets (#1877)

* fix(security): harden auth and outbound targets

* fix(security): prevent login lockout and honor trusted schemes
This commit is contained in:
Anso
2026-09-01 20:52:12 +00:00
committed by GitHub
parent 82dca29314
commit 79b86ddcd4
76 changed files with 1457 additions and 212 deletions
+6 -5
View File
@@ -44,6 +44,12 @@ API_POLLING_RATE_LIMIT=300
# trailing slash. When unset, enrollment falls back to the request Host.
SENCHO_PUBLIC_URL=
# Comma-separated CIDRs of reverse proxies trusted to set forwarding headers.
# Set a single IPv4 proxy as a /32, a single IPv6 proxy as a /128, or use the
# proxy network CIDR when Sencho is behind one (for example, 192.168.1.50/32).
# Unset or invalid values make Sencho ignore forwarded client and scheme data.
SENCHO_TRUSTED_PROXY_CIDRS=
# ─── Pilot agent (remote host only) ──────────────────────────────
# These three vars are required ONLY on a remote host running as a
# pilot-agent reverse-tunnel container. The primary instance does not
@@ -53,11 +59,6 @@ SENCHO_PUBLIC_URL=
# this unset.
SENCHO_MODE=
# Comma-separated CIDRs of reverse proxies trusted to set X-Forwarded-Proto
# for Pilot Agent TLS termination. Unset or invalid: non-TLS Pilot upgrades are
# treated as non-confidential and hub registry credential delivery is skipped.
SENCHO_TRUSTED_PROXY_CIDRS=
# WebSocket-capable URL of the controlling Sencho instance. Use https://
# scheme; the agent rewrites it to wss:// for the tunnel upgrade.
SENCHO_PRIMARY_URL=
+1
View File
@@ -77,6 +77,7 @@ runs:
COMPOSE_DIR: ${{ inputs.compose-dir }}
PORT: ${{ inputs.port }}
NODE_ENV: test
SENCHO_E2E_ALLOW_LOOPBACK_OUTBOUND: 'true'
- name: Start frontend dev server
shell: bash
+10
View File
@@ -33,6 +33,7 @@
"semver": "^7.7.4",
"systeminformation": "^5.31.1",
"tar-stream": "^3.1.8",
"undici": "^8.10.0",
"ws": "^8.19.0",
"yaml": "^2.8.2",
"zod": "^4.3.6"
@@ -6098,6 +6099,15 @@
"dev": true,
"license": "MIT"
},
"node_modules/undici": {
"version": "8.10.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-8.10.0.tgz",
"integrity": "sha512-HvltHd7avK13QIw/oLe4qoOLyoVSoafqJ2jYOrtMRBkbYT31eiBQ8O0ehRKZiEZCMEyLFQNIADpgCWC5fALvYQ==",
"license": "MIT",
"engines": {
"node": ">=22.19.0"
}
},
"node_modules/undici-types": {
"version": "8.3.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
+1
View File
@@ -89,6 +89,7 @@
"semver": "^7.7.4",
"systeminformation": "^5.31.1",
"tar-stream": "^3.1.8",
"undici": "^8.10.0",
"ws": "^8.19.0",
"yaml": "^2.8.2",
"zod": "^4.3.6"
@@ -399,6 +399,7 @@ describe('AutoHealService.evaluate', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'http://remote:1852',
apiToken: 'tok',
trustedLoopback: false,
});
const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ ok: true } as Response);
@@ -434,6 +435,7 @@ describe('AutoHealService.evaluate', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'http://remote2:1852',
apiToken: 'tok2',
trustedLoopback: false,
});
const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValue({ ok: true } as Response);
@@ -63,6 +63,7 @@ beforeEach(() => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
const db = DatabaseService.getInstance().getDb();
db.prepare('DELETE FROM blueprint_deployments').run();
@@ -33,8 +33,16 @@ describe('fetchRemoteMeta Authorization header', () => {
await fetchRemoteMeta('https://remote.example.com:1852', 'real-token');
expect(getSpy).toHaveBeenCalledTimes(1);
const init = getSpy.mock.calls[0][1] as { headers: Record<string, string> };
const init = getSpy.mock.calls[0][1] as {
headers: Record<string, string>;
proxy?: boolean;
httpAgent?: unknown;
httpsAgent?: unknown;
};
expect(init.headers).toEqual({ Authorization: 'Bearer real-token' });
expect(init.proxy).toBe(false);
expect(init.httpAgent).toBeDefined();
expect(init.httpsAgent).toBeDefined();
});
it('omits Authorization entirely when token is empty (pilot-agent loopback)', async () => {
@@ -42,18 +50,21 @@ describe('fetchRemoteMeta Authorization header', () => {
data: { version: '0.76.7', capabilities: ['stacks'], startedAt: 1, updateError: null },
});
await fetchRemoteMeta('http://127.0.0.1:54321', '');
await fetchRemoteMeta('http://127.0.0.1:54321', '', true);
expect(getSpy).toHaveBeenCalledTimes(1);
const init = getSpy.mock.calls[0][1] as { headers: Record<string, string> };
const init = getSpy.mock.calls[0][1] as { headers: Record<string, string>; proxy?: boolean };
expect(init.headers).toEqual({});
expect(init.headers).not.toHaveProperty('Authorization');
expect(init.proxy).toBe(false);
expect(init).not.toHaveProperty('httpAgent');
expect(init).not.toHaveProperty('httpsAgent');
});
it('returns OFFLINE_META shape on transport failure', async () => {
vi.spyOn(axios, 'get').mockRejectedValue(new Error('connect ECONNREFUSED'));
const meta = await fetchRemoteMeta('http://127.0.0.1:54321', '');
const meta = await fetchRemoteMeta('http://127.0.0.1:54321', '', true);
expect(meta).toEqual({
version: null,
@@ -7,6 +7,7 @@ import request from 'supertest';
import jwt from 'jsonwebtoken';
import bcrypt from 'bcrypt';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
import { resetTrustedProxyBlockListCache } from '../helpers/trustedProxyCidrs';
let tmpDir: string;
let app: import('express').Express;
@@ -100,4 +101,35 @@ describe('GET /api/diagnostics/environment', () => {
});
}
});
it('ignores a forwarded HTTPS scheme from an untrusted peer', async () => {
const res = await request(app)
.get('/api/diagnostics/environment')
.set('Authorization', adminAuthHeader)
.set('Host', 'sencho.example.com')
.set('X-Forwarded-Proto', 'https');
expect(res.status).toBe(200);
const tls = (res.body.checks as Array<{ id: string; status: string }>).find(check => check.id === 'tls');
expect(tls?.status).toBe('warn');
});
it('honors a forwarded HTTPS scheme from an allowlisted proxy peer', async () => {
process.env.SENCHO_TRUSTED_PROXY_CIDRS = '127.0.0.0/8';
resetTrustedProxyBlockListCache();
try {
const res = await request(app)
.get('/api/diagnostics/environment')
.set('Authorization', adminAuthHeader)
.set('Host', 'sencho.example.com')
.set('X-Forwarded-Proto', 'https');
expect(res.status).toBe(200);
const tls = (res.body.checks as Array<{ id: string; status: string }>).find(check => check.id === 'tls');
expect(tls?.status).toBe('pass');
} finally {
delete process.env.SENCHO_TRUSTED_PROXY_CIDRS;
resetTrustedProxyBlockListCache();
}
});
});
+7 -7
View File
@@ -786,7 +786,7 @@ describe('bulk-assign orchestrator: remote fan-out', () => {
it('fans out to the remote local-assign receiver with Bearer auth and the template body', async () => {
const remoteId = addRemote('assign-remote-ok');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok', trustedLoopback: false });
const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(
JSON.stringify({ created: true, results: [{ stackName: 'r1', success: true }] }),
{ status: 200, headers: { 'content-type': 'application/json' } },
@@ -814,7 +814,7 @@ describe('bulk-assign orchestrator: remote fan-out', () => {
it('omits the Authorization header for a pilot-agent remote with an empty token', async () => {
const remoteId = addRemote('assign-remote-pilot', 'pilot_agent');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'http://127.0.0.1:9', apiToken: '' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'http://127.0.0.1:9', apiToken: '', trustedLoopback: true });
const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(
JSON.stringify({ created: false, results: [{ stackName: 'p1', success: true }] }),
{ status: 200, headers: { 'content-type': 'application/json' } },
@@ -850,7 +850,7 @@ describe('bulk-assign orchestrator: remote fan-out', () => {
it('treats a mixed-version remote (404 on local-assign) as a per-node failure', async () => {
const remoteId = addRemote('assign-remote-404');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok', trustedLoopback: false });
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('Not Found', { status: 404 }));
const res = await request(app)
@@ -867,7 +867,7 @@ describe('bulk-assign orchestrator: remote fan-out', () => {
it('reports a transport failure as a per-node failure', async () => {
const remoteId = addRemote('assign-remote-transport');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok', trustedLoopback: false });
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
const res = await request(app)
@@ -883,7 +883,7 @@ describe('bulk-assign orchestrator: remote fan-out', () => {
it('reports a malformed 200 body as a per-node failure', async () => {
const remoteId = addRemote('assign-remote-malformed');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok', trustedLoopback: false });
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(
JSON.stringify({ created: true, results: 'not-an-array' }),
{ status: 200, headers: { 'content-type': 'application/json' } },
@@ -903,7 +903,7 @@ describe('bulk-assign orchestrator: remote fan-out', () => {
it('fails a node whose 200 body returns an empty results array for a non-empty request', async () => {
const remoteId = addRemote('assign-remote-empty');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok', trustedLoopback: false });
// A well-shaped { created, results } body whose results are empty used to
// pass the bare Array.isArray check and read as a successful zero-stack
// assign. Membership validation must fail the node instead.
@@ -926,7 +926,7 @@ describe('bulk-assign orchestrator: remote fan-out', () => {
it('fails a node whose results omit one of the requested stacks', async () => {
const remoteId = addRemote('assign-remote-partial');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'https://remote.example.com:1852', apiToken: 'remote-tok', trustedLoopback: false });
// Two stacks requested, only one row returned: a partial body the control
// must not accept as a clean assign of the covered stack alone.
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(
@@ -66,7 +66,7 @@ afterEach(() => {
function mockTargetActive() {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => {
if (id === pilotNodeId) return { apiUrl: LOOPBACK, apiToken: '' };
if (id === pilotNodeId) return { apiUrl: LOOPBACK, apiToken: '', trustedLoopback: true };
return null;
});
}
@@ -90,8 +90,8 @@ afterEach(() => {
function mockTargets(opts: { pilotReachable: boolean; proxyReachable: boolean } = { pilotReachable: true, proxyReachable: true }) {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => {
if (id === pilotNodeId) return opts.pilotReachable ? { apiUrl: PILOT_LOOPBACK, apiToken: '' } : null;
if (id === proxyNodeId) return opts.proxyReachable ? { apiUrl: PROXY_URL, apiToken: PROXY_TOKEN } : null;
if (id === pilotNodeId) return opts.pilotReachable ? { apiUrl: PILOT_LOOPBACK, apiToken: '', trustedLoopback: true } : null;
if (id === proxyNodeId) return opts.proxyReachable ? { apiUrl: PROXY_URL, apiToken: PROXY_TOKEN, trustedLoopback: false } : null;
return null;
});
}
@@ -104,8 +104,8 @@ afterEach(() => {
function mockTargetForPilot() {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => {
if (id === pilotNodeId) return { apiUrl: LOOPBACK, apiToken: '' };
if (id === proxyNodeId) return { apiUrl: 'http://192.168.1.99:1852', apiToken: 'proxy-token' };
if (id === pilotNodeId) return { apiUrl: LOOPBACK, apiToken: '', trustedLoopback: true };
if (id === proxyNodeId) return { apiUrl: 'http://192.168.1.99:1852', apiToken: 'proxy-token', trustedLoopback: false };
return null;
});
}
@@ -627,7 +627,7 @@ describe('Snapshot restore: remote dossier notes (proxy PUT)', () => {
it('writes notes to a remote node via the proxy dossier PUT when opted in', async () => {
const { id, remoteId } = remoteDocSnapshot('rweb', 'documented');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tok' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tok', trustedLoopback: false });
const calls: Array<{ url: string; method?: string }> = [];
vi.stubGlobal('fetch', vi.fn(async (url: string, opts?: { method?: string }) => {
calls.push({ url, method: opts?.method });
@@ -647,7 +647,7 @@ describe('Snapshot restore: remote dossier notes (proxy PUT)', () => {
it('reports a non-fatal notesError when the remote dossier PUT fails but files restored', async () => {
const { id, remoteId } = remoteDocSnapshot('rweb2', 'documented');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tok' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tok', trustedLoopback: false });
vi.stubGlobal('fetch', vi.fn(async (url: string) => {
if (/\/dossier$/.test(url)) return { ok: false, status: 500, text: async () => 'boom' } as unknown as Response;
return { ok: true, status: 200, text: async () => '' } as unknown as Response;
@@ -668,7 +668,7 @@ describe('Snapshot restore: remote dossier notes (proxy PUT)', () => {
// restore-all is driven by snapshot id; the target node is resolved from
// the snapshot's stored files, so the returned remoteId is not needed here.
const { id } = remoteDocSnapshot('rweb3', 'documented');
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tok' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({ apiUrl: 'http://remote:1852', apiToken: 'tok', trustedLoopback: false });
vi.stubGlobal('fetch', vi.fn(async (url: string) => {
if (/\/dossier$/.test(url)) return { ok: false, status: 500, text: async () => 'boom' } as unknown as Response;
return { ok: true, status: 200, text: async () => '' } as unknown as Response;
@@ -928,6 +928,7 @@ describe('Snapshot restore: recovery generation contract', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'http://remote:1852',
apiToken: 'tok',
trustedLoopback: false,
});
}
@@ -51,7 +51,7 @@ function mockMeta(meta: RemoteMeta) {
function mockTarget() {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) =>
id === proxyNodeId ? { apiUrl: 'http://192.168.1.99:1852', apiToken: 'proxy-token' } : null,
id === proxyNodeId ? { apiUrl: 'http://192.168.1.99:1852', apiToken: 'proxy-token', trustedLoopback: false } : null,
);
}
@@ -29,6 +29,7 @@ import { GitOpsTransitions } from '../services/gitops/transitions';
import { insertHistory } from '../services/gitops/history';
import type { GitOpsApplicationRow } from '../services/gitops/types';
import { PROXY_DEPLOY_ACTOR_HEADER, PROXY_DEPLOY_SOURCE_HEADER } from '../services/license-headers';
import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets';
/** A minimal live Direct application row for GitOps read-path fixtures. */
function directApplicationFixture(id: string, stackName: string): GitOpsApplicationRow {
@@ -219,6 +220,20 @@ describe('POST /api/git-sources/browse: URL validation', () => {
expect(listRepoTree).not.toHaveBeenCalled();
listRepoTree.mockRestore();
});
it('rejects repository hosts that resolve to an unsafe address', async () => {
const res = await withLoopbackTargetProtection(() => request(app)
.post('/api/git-sources/browse')
.set('Authorization', `Bearer ${adminToken()}`)
.send({
repo_url: 'https://127.0.0.1:9999/repo.git',
branch: 'main',
auth_type: 'none',
}));
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/not allowed/i);
});
});
describe('PUT /api/stacks/:stackName/git-source — max-length caps', () => {
@@ -1757,6 +1772,15 @@ describe('POST /api/git-sources/ssh-host-key', () => {
expect(res.body.error).toMatch(/SSH repository URL/i);
});
it('rejects host-key probes to an unsafe address', async () => {
const res = await withLoopbackTargetProtection(() => request(app)
.post('/api/git-sources/ssh-host-key')
.set('Authorization', `Bearer ${adminToken()}`)
.send({ repo_url: 'ssh://git@127.0.0.1:22/example/repo.git' }));
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/not allowed/i);
});
it('returns scanned host keys for an SSH repository URL', async () => {
const scanHostKeys = vi.spyOn(
await import('../services/git/sshTrust'),
@@ -1771,12 +1795,13 @@ describe('POST /api/git-sources/ssh-host-key', () => {
const res = await request(app)
.post('/api/git-sources/ssh-host-key')
.set('Authorization', `Bearer ${adminToken()}`)
.send({ repo_url: 'git@github.com:example/repo.git' });
.send({ repo_url: 'git@pinned.example:example/repo.git' });
expect(res.status).toBe(200);
expect(res.body.host).toBe('github.com');
expect(res.body.host).toBe('pinned.example');
expect(res.body.port).toBe(22);
expect(res.body.keys).toHaveLength(1);
expect(res.body.keys[0].fingerprint).toBe('SHA256:fixtureFingerprint');
expect(scanHostKeys).toHaveBeenCalledWith('pinned.example', 22, '93.184.216.34');
scanHostKeys.mockRestore();
});
@@ -1822,7 +1847,7 @@ describe('POST /api/git-sources/ssh-host-key', () => {
const res = await request(app)
.post('/api/git-sources/ssh-host-key')
.set('Authorization', `Bearer ${adminToken()}`)
.send({ repo_url: 'ssh://git@git.example.com:2222/org/repo.git' });
.send({ repo_url: 'ssh://git@github.com:2222/org/repo.git' });
expect(res.status).toBe(500);
expect(res.body.error).toMatch(/Git source operation failed/i);
scanHostKeys.mockRestore();
@@ -165,7 +165,7 @@ async function startSshGitServer(bareDir: string, port: number): Promise<Omit<Ss
throw new Error(`sshd did not come up on port ${port}: ${stderr.trim() || '(no stderr)'}`, { cause: e });
}
const scanned = await scanHostKeys('127.0.0.1', port);
const scanned = await scanHostKeys('127.0.0.1', port, '127.0.0.1');
assertFixtureHostKey(port, hostKey.publicLine, scanned);
const knownHostsEntry = scanned.map((k) => k.line).join('\n');
+70 -7
View File
@@ -40,6 +40,7 @@ import {
} from '../services/git/credentialHelper';
import * as gitBinary from '../services/git/gitBinary';
import { nativeGitTransport, REF_MAX_LEN, startSizeWatchdog, verifyFastForward } from '../services/git/nativeGitTransport';
import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets';
import { GIT_ALLOWED_HOST_ENV_VAR } from '../services/git/credentialHelper';
const GIT_EXEC_PATH_STUB = 'C:/Program Files/Git/mingw64/libexec/git-core';
@@ -187,6 +188,7 @@ describe('classifyGitFailure (native git stderr corpus)', () => {
['size', { transportFailure: true as const, reason: 'size', maxBytes: 5 * 1024 * 1024, host: 'h', hasToken: false }, 'Repository exceeds the maximum clone size of 5 MB.'],
['tip-changed', { transportFailure: true as const, reason: 'tip-changed', host: 'h', hasToken: false }, 'Repository tip changed during fetch; retry the pull.'],
['ref-not-found', { transportFailure: true as const, reason: 'ref-not-found', host: 'h', hasToken: false }, 'The configured branch, tag, or commit was not found in the repository.'],
['ssh-auth-required', { transportFailure: true as const, reason: 'ssh-auth-required', host: 'h', hasToken: false }, 'SSH repository URLs require a deploy key.'],
['unsupported-ref', { transportFailure: true as const, reason: 'unsupported-ref', host: 'h', hasToken: false }, 'The configured commit is not reachable on this repository host. Use a branch or tag, or a commit the host advertises.'],
['timeout', { transportFailure: true as const, reason: 'timeout', host: 'github.com', hasToken: false }, 'Timed out reaching github.com.'],
] as const)('maps structured reason %s verbatim', (_label, failure, message) => {
@@ -560,6 +562,15 @@ describe('transport argv hardening', () => {
expect(mockSpawn).not.toHaveBeenCalled();
});
it('rejects SSH repository URLs without deploy-key authentication before spawning git', async () => {
await expect(nativeGitTransport.resolveRef({
repoUrl: 'ssh://git@ssh.example/org/repo.git',
ref: 'main',
workspaceRoot: os.tmpdir(),
})).rejects.toMatchObject({ transportFailure: true as const, reason: 'ssh-auth-required' });
expect(mockSpawn).not.toHaveBeenCalled();
});
it('rejects option-injecting ref names', async () => {
await expect(nativeGitTransport.resolveRef({
repoUrl: 'https://github.com/example/repo.git',
@@ -799,20 +810,53 @@ describe('resolve/fetch/verify flow', () => {
}
});
it('resolves an annotated tag through the peeled ^{} commit', async () => {
it('pins HTTPS to the validated address while resolving an annotated tag', async () => {
scriptSpawn([{
stdout: `${SHA_B}\trefs/tags/v1\n${SHA_A}\trefs/tags/v1^{}\n`,
}]);
const root = await makeWorkspace();
try {
await expect(nativeGitTransport.resolveRef({
repoUrl: 'https://github.com/example/repo.git',
repoUrl: 'https://pinned.example:8443/example/repo.git',
ref: 'v1',
timeoutMs: 5000,
workspaceRoot: root,
})).resolves.toMatchObject({ commitSha: SHA_A, kind: 'tag' });
const lsRemoteArgs = mockSpawn.mock.calls[0][1] as string[];
expect(lsRemoteArgs).toContain('refs/tags/v1^{}');
expect(lsRemoteArgs).toContain('http.followRedirects=false');
expect(lsRemoteArgs).toContain('http.proxy=');
expect(lsRemoteArgs).toContain('http.curloptResolve=pinned.example:8443:93.184.216.34');
const env = spawnEnv(0);
expect(env.HTTP_PROXY).toBe('');
expect(env.HTTPS_PROXY).toBe('');
expect(env.ALL_PROXY).toBe('');
expect(env.http_proxy).toBe('');
expect(env.https_proxy).toBe('');
expect(env.all_proxy).toBe('');
} finally {
await fs.rm(root, { recursive: true, force: true });
}
});
it('pins SSH to the validated address while retaining host identity and port', async () => {
scriptSpawn([{ stdout: `${SHA_A}\trefs/heads/main\n` }]);
const root = await makeWorkspace();
try {
await expect(nativeGitTransport.resolveRef({
repoUrl: 'ssh://git@pinned.example:2222/example/repo.git',
ref: 'main',
sshAuth: {
privateKey: '-----BEGIN OPENSSH PRIVATE KEY-----\nYWJj\n-----END OPENSSH PRIVATE KEY-----\n',
knownHostsEntry: 'pinned.example ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGb3JzL3Rlc3Q=\n',
},
timeoutMs: 5000,
workspaceRoot: root,
})).resolves.toMatchObject({ commitSha: SHA_A, kind: 'branch' });
const sshCommand = spawnEnv(0).GIT_SSH_COMMAND;
expect(sshCommand).toContain('Hostname=93.184.216.34');
expect(sshCommand).toContain('HostKeyAlias=[pinned.example]:2222');
} finally {
await fs.rm(root, { recursive: true, force: true });
}
@@ -836,12 +880,12 @@ describe('resolve/fetch/verify flow', () => {
it('self-resolves a full SHA without a network round trip', async () => {
const root = await makeWorkspace();
try {
await expect(nativeGitTransport.resolveRef({
repoUrl: 'https://github.com/example/repo.git',
await expect(withLoopbackTargetProtection(() => nativeGitTransport.resolveRef({
repoUrl: 'https://127.0.0.1/example/repo.git',
ref: SHA_A.toUpperCase(),
timeoutMs: 5000,
workspaceRoot: root,
})).resolves.toMatchObject({ commitSha: SHA_A, kind: 'sha' });
}))).resolves.toMatchObject({ commitSha: SHA_A, kind: 'sha' });
// The SHA needs no ls-remote: the identity IS the value.
expect(mockSpawn).not.toHaveBeenCalled();
} finally {
@@ -1061,6 +1105,26 @@ describe('clone failure classification and final size gate', () => {
}
});
it('rejects unsafe targets during fast-forward verification', async () => {
const root = await makeWorkspace();
try {
await expect(withLoopbackTargetProtection(() => verifyFastForward({
repoUrl: 'https://127.0.0.1/repo.git',
ancestorSha: SHA_B,
descendantSha: SHA_A,
timeoutMs: 5000,
workspaceRoot: root,
maxBytes: 100 * 1024 * 1024,
}))).rejects.toMatchObject({
transportFailure: true as const,
reason: 'unsafe-target',
});
expect(mockSpawn).not.toHaveBeenCalled();
} finally {
await fs.rm(root, { recursive: true, force: true });
}
});
it('deepens history exponentially with a bounded number of remote fetches', async () => {
scriptSpawn([
{ code: 0 },
@@ -1411,8 +1475,7 @@ describe('clone failure classification and final size gate', () => {
// simulated confirmation has not arrived yet: settling here
// would let a caller start cleaning up the workspace while the
// child tree is still alive.
await new Promise((r) => setTimeout(r, 55));
expect(killInvoked).toBe(true);
await vi.waitFor(() => expect(killInvoked).toBe(true), { timeout: 250 });
expect(settled).toBe(false);
await expect(promise).rejects.toMatchObject({ transportFailure: true as const, reason: 'timeout' });
@@ -0,0 +1,98 @@
import { vi } from 'vitest';
import http from 'http';
import https from 'https';
import type { LookupFunction } from 'net';
import { AsyncLocalStorage } from 'async_hooks';
const targetProtectionScope = new AsyncLocalStorage<boolean>();
export async function withLoopbackTargetProtection<T>(action: () => PromiseLike<T>): Promise<T> {
return targetProtectionScope.run(false, async () => action());
}
vi.mock('../../utils/outboundTarget', async (importOriginal) => {
const actual = await importOriginal<typeof import('../../utils/outboundTarget')>();
const fixtureAllows = (hostname: string): boolean => {
if (targetProtectionScope.getStore() === false) return false;
const normalized = hostname.replace(/^\[|\]$/g, '').toLowerCase();
return normalized === 'localhost'
|| normalized === '::1'
|| normalized.startsWith('127.')
|| normalized.startsWith('10.')
|| normalized.startsWith('192.168.')
|| /^172\.(1[6-9]|2\d|3[01])\./.test(normalized)
|| normalized.endsWith('.example.com')
|| normalized.endsWith('.example')
|| normalized.endsWith('.invalid')
|| normalized.endsWith('.local')
|| normalized === 'remote'
|| normalized === 'remote2'
|| normalized === 'good-host'
|| normalized === 'bad-host';
};
const fixtureAddress = (hostname: string): string => {
const normalized = hostname.replace(/^\[|\]$/g, '');
if (normalized === 'localhost') return '127.0.0.1';
if (normalized === 'pinned.example') return '93.184.216.34';
return normalized;
};
const fixtureLookup: LookupFunction = (hostname, options, callback): void => {
if (!fixtureAllows(hostname)) {
actual.safeOutboundLookup(hostname, options, callback);
return;
}
const normalized = hostname.replace(/^\[|\]$/g, '');
const address = fixtureAddress(hostname);
const family = normalized === '::1' ? 6 : 4;
if (options.all) callback(null, [{ address, family }]);
else callback(null, address, family);
};
const fixtureHttpAgent = new http.Agent({ lookup: fixtureLookup });
const fixtureHttpsAgent = new https.Agent({ lookup: fixtureLookup });
return {
...actual,
assertSafeOutboundHostname: async (
hostname: string,
): Promise<void> => {
if (fixtureAllows(hostname)) return;
await actual.assertSafeOutboundHostname(hostname);
},
assertSafeOutboundUrl: async (
raw: string,
): Promise<URL> => {
const url = new URL(raw);
if (fixtureAllows(url.hostname)) return url;
return actual.assertSafeOutboundUrl(raw);
},
resolveSafeOutboundHostname: async (hostname: string) => {
if (fixtureAllows(hostname)) {
const normalized = hostname.replace(/^\[|\]$/g, '');
return [{ address: fixtureAddress(hostname), family: normalized === '::1' ? 6 : 4 }];
}
return actual.resolveSafeOutboundHostname(hostname);
},
safeOutboundLookup: fixtureLookup,
safeHttpAgent: fixtureHttpAgent,
safeHttpsAgent: fixtureHttpsAgent,
safeAxiosTransport: (trustedLoopback = false) => ({
maxRedirects: 0,
proxy: false,
...(trustedLoopback ? {} : { httpAgent: fixtureHttpAgent, httpsAgent: fixtureHttpsAgent }),
}),
safeRemoteFetch: async (
input: Parameters<typeof actual.safeRemoteFetch>[0],
init?: Parameters<typeof actual.safeRemoteFetch>[1],
trustedLoopback?: boolean,
) => {
const url = input instanceof URL
? input
: new URL(typeof input === 'string' ? input : input.url);
if (fixtureAllows(url.hostname)) {
return globalThis.fetch(input, { ...init, redirect: 'error' });
}
return actual.safeRemoteFetch(input, init, trustedLoopback);
},
};
});
@@ -493,7 +493,7 @@ describe('GET /api/fleet/container-labels', () => {
]);
const db = DatabaseService.getInstance();
const remoteId = db.addNode({ name: 'remote-lbl', type: 'remote', compose_dir: '/app/compose', is_default: false, api_url: 'http://remote.invalid', api_token: 't' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't' } : null);
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't', trustedLoopback: false } : null);
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('nope', { status: 502 }));
try {
const res = await request(app).get('/api/fleet/container-labels').set('Cookie', authCookie);
@@ -511,7 +511,7 @@ describe('GET /api/fleet/container-labels', () => {
]);
const db = DatabaseService.getInstance();
const remoteId = db.addNode({ name: 'remote-bad', type: 'remote', compose_dir: '/app/compose', is_default: false, api_url: 'http://remote.invalid', api_token: 't' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't' } : null);
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't', trustedLoopback: false } : null);
// byLabel row missing valid key/value/source: must be rejected by the deep guard.
const malformed = JSON.stringify({ nodeId: remoteId, containers: [], byLabel: [{ key: 123, containers: [] }], partial: false, generatedAt: 0 });
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(malformed, { status: 200, headers: { 'content-type': 'application/json' } }));
@@ -528,7 +528,7 @@ describe('GET /api/fleet/container-labels', () => {
stubDockerList([]);
const db = DatabaseService.getInstance();
const remoteId = db.addNode({ name: 'remote-src', type: 'remote', compose_dir: '/app/compose', is_default: false, api_url: 'http://remote.invalid', api_token: 't' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't' } : null);
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't', trustedLoopback: false } : null);
const badSource = JSON.stringify({ nodeId: remoteId, containers: [], partial: false, generatedAt: 0, byLabel: [{ key: 'k', value: 'v', source: 'not-a-source', containers: [{ id: 'c', name: 'n' }] }] });
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(badSource, { status: 200, headers: { 'content-type': 'application/json' } }));
try {
@@ -544,7 +544,7 @@ describe('GET /api/fleet/container-labels', () => {
stubDockerList([]);
const db = DatabaseService.getInstance();
const remoteId = db.addNode({ name: 'remote-cont', type: 'remote', compose_dir: '/app/compose', is_default: false, api_url: 'http://remote.invalid', api_token: 't' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't' } : null);
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't', trustedLoopback: false } : null);
const badContainer = JSON.stringify({ nodeId: remoteId, byLabel: [], partial: false, generatedAt: 0, containers: [{}] });
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(badContainer, { status: 200, headers: { 'content-type': 'application/json' } }));
try {
@@ -560,7 +560,7 @@ describe('GET /api/fleet/container-labels', () => {
stubDockerList([]);
const db = DatabaseService.getInstance();
const remoteId = db.addNode({ name: 'remote-ref', type: 'remote', compose_dir: '/app/compose', is_default: false, api_url: 'http://remote.invalid', api_token: 't' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't' } : null);
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't', trustedLoopback: false } : null);
const badRef = JSON.stringify({ nodeId: remoteId, containers: [], partial: false, generatedAt: 0, byLabel: [{ key: 'k', value: 'v', source: 'runtime', containers: [{ id: 5 }] }] });
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(badRef, { status: 200, headers: { 'content-type': 'application/json' } }));
try {
@@ -0,0 +1,66 @@
import express from 'express';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import {
cleanupTestDb,
setupTestDb,
TEST_PASSWORD,
TEST_USERNAME,
} from './helpers/setupTestDb';
let authRouter: typeof import('../routes/auth').authRouter;
let tmpDir: string;
let app: import('express').Express;
beforeAll(async () => {
vi.stubEnv('NODE_ENV', 'production');
vi.resetModules();
tmpDir = await setupTestDb();
({ authRouter } = await import('../routes/auth'));
app = express();
app.set('trust proxy', 1);
app.use(express.json());
app.use('/api/auth', authRouter);
});
afterAll(() => {
cleanupTestDb(tmpDir);
vi.unstubAllEnvs();
});
describe('production login rate limiter', () => {
it('blocks repeated attempts from one source', async () => {
for (let attempt = 0; attempt < 5; attempt += 1) {
const failure = await request(app)
.post('/api/auth/login')
.set('X-Forwarded-For', '198.51.100.10')
.send({ username: TEST_USERNAME, password: 'wrong-password' });
expect(failure.status).toBe(401);
}
const blocked = await request(app)
.post('/api/auth/login')
.set('X-Forwarded-For', '198.51.100.10')
.send({ username: TEST_USERNAME, password: 'wrong-password' });
expect(blocked.status).toBe(429);
});
it('allows valid credentials after failures from other sources', async () => {
for (let attempt = 0; attempt < 20; attempt += 1) {
const failure = await request(app)
.post('/api/auth/login')
.set('X-Forwarded-For', `203.0.113.${attempt + 1}`)
.send({ username: TEST_USERNAME, password: 'wrong-password' });
expect(failure.status).toBe(401);
}
const validLogin = await request(app)
.post('/api/auth/login')
.set('X-Forwarded-For', '203.0.113.250')
.send({ username: TEST_USERNAME, password: TEST_PASSWORD });
expect(validLogin.status).toBe(200);
});
});
@@ -75,6 +75,7 @@ describe('MeshService.inspectStackServices dispatch (C-3 fix)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
const fetchMock = vi
@@ -114,6 +115,7 @@ describe('MeshService.inspectStackServices dispatch (C-3 fix)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('Internal Server Error', { status: 500 }));
@@ -73,6 +73,7 @@ describe('MeshService.listStacksOnNode dispatch (F8)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
const fetchMock = vi
@@ -112,6 +113,7 @@ describe('MeshService.listStacksOnNode dispatch (F8)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('Internal Server Error', { status: 500 }));
@@ -160,6 +162,7 @@ describe('MeshService.listStacksOnNode dispatch (F8)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(
JSON.stringify({ stacks: ['ok-string', 42, null, { not: 'a string' }] }),
@@ -223,6 +226,7 @@ describe('MeshService.listStacksOnNode dispatch (F8)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
vi.spyOn(globalThis, 'fetch').mockImplementation(() => {
throw new MeshError('push_failed', 'simulated transport failure');
@@ -9,6 +9,7 @@
*/
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
@@ -74,6 +75,28 @@ describe('MeshProxyTunnelDialer', () => {
expect(dialer.hasBridge(nodeId)).toBe(false);
});
it('rejects an unsafe proxy target before opening a WebSocket', async () => {
const dialer = MeshProxyTunnelDialer.resetForTest(0);
const db = DatabaseService.getInstance();
const nodeId = db.addNode({
name: 'proxy-test-unsafe-target',
type: 'remote',
compose_dir: '',
is_default: false,
mode: 'proxy',
api_url: 'http://127.0.0.1:1852',
api_token: 'test-token',
});
const result = await withLoopbackTargetProtection(() => dialer.ensureBridge(nodeId));
expect(result).toBeNull();
expect(dialer.getRecentFailure(nodeId)).toMatchObject({
code: 'network_error',
message: 'The target address is not allowed.',
});
});
it('expires the recent-failure cache entry after the cache TTL window', async () => {
const dialer = MeshProxyTunnelDialer.resetForTest(0);
const result = await dialer.ensureBridge(8888);
@@ -47,6 +47,7 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
const fetchMock = vi
@@ -82,6 +83,7 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
const fetchMock = vi
@@ -112,6 +114,7 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('ECONNREFUSED'));
@@ -139,6 +142,7 @@ describe('MeshService.removeOverrideFromNode (remote dispatch)', () => {
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'remote-tok',
trustedLoopback: false,
});
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response('another operation is already in progress', { status: 500 }),
@@ -196,7 +196,7 @@ describe('networking summary', () => {
it('degrades a remote that errors to a node-error while keeping the hub', async () => {
const db = DatabaseService.getInstance();
const remoteId = db.addNode({ name: 'remote-degrade', type: 'remote', compose_dir: '/app/compose', is_default: false, api_url: 'http://remote.invalid', api_token: 't' });
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't' } : null);
vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockImplementation((id: number) => id === remoteId ? { apiUrl: 'http://remote.invalid', apiToken: 't', trustedLoopback: false } : null);
vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('not found', { status: 404 }));
try {
const res = await request(app).get('/api/fleet/networking-summary').set('Authorization', authHeader);
@@ -19,6 +19,7 @@ import { WebSocket } from 'ws';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import { MeshProxyTunnelDialer } from '../services/MeshProxyTunnelDialer';
import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets';
let tmpDir: string;
let app: import('express').Express;
@@ -146,6 +147,45 @@ describe('node-management write routes require node:manage', () => {
expect(res.status).toBe(200);
expect(db.getNode(id)).toBeUndefined();
});
it('rejects proxy nodes whose API URL resolves to an unsafe address', async () => {
const res = await request(app)
.post('/api/nodes')
.set('Authorization', `Bearer ${tokenForRole('admin')}`)
.send({
name: `nm-unsafe-${Date.now()}`,
type: 'remote',
mode: 'proxy',
api_url: 'http://[::ffff:127.0.0.1]:1852',
api_token: 'test-token',
});
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/not allowed/i);
});
it('rejects updating a proxy node to an unsafe API URL', async () => {
const db = DatabaseService.getInstance();
const id = db.addNode({
name: `nm-update-unsafe-${Date.now()}`,
type: 'remote',
mode: 'proxy',
compose_dir: '/tmp/x',
is_default: false,
api_url: 'https://remote.example.com:1852',
api_token: 'test-token',
});
const res = await withLoopbackTargetProtection(() => request(app)
.put(`/api/nodes/${id}`)
.set('Authorization', `Bearer ${tokenForRole('admin')}`)
.send({ api_url: 'http://127.0.0.1:1852' }));
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/not allowed/i);
expect(db.getNode(id)?.api_url).toBe('https://remote.example.com:1852');
db.deleteNode(id);
});
});
describe('deleting a node tears down its tunnel or mesh bridge', () => {
@@ -77,6 +77,7 @@ describe('NodeRegistry.fetchMetaForNode', () => {
vi.spyOn(reg, 'getProxyTarget').mockReturnValue({
apiUrl: 'http://127.0.0.1:54321',
apiToken: '',
trustedLoopback: true,
});
const axiosSpy = vi.spyOn(axios, 'get').mockResolvedValue({
data: {
@@ -96,8 +97,10 @@ describe('NodeRegistry.fetchMetaForNode', () => {
expect(axiosSpy).toHaveBeenCalledTimes(1);
const url = axiosSpy.mock.calls[0][0];
expect(url).toBe('http://127.0.0.1:54321/api/meta');
const init = axiosSpy.mock.calls[0][1] as { headers: Record<string, string> };
const init = axiosSpy.mock.calls[0][1] as { headers: Record<string, string>; httpAgent?: unknown; httpsAgent?: unknown };
expect(init.headers).toEqual({});
expect(init.httpAgent).toBeUndefined();
expect(init.httpsAgent).toBeUndefined();
db.deleteNode(nodeId);
});
@@ -118,6 +121,7 @@ describe('NodeRegistry.fetchMetaForNode', () => {
vi.spyOn(reg, 'getProxyTarget').mockReturnValue({
apiUrl: 'https://remote.example.com:1852',
apiToken: 'real-token',
trustedLoopback: false,
});
const axiosSpy = vi.spyOn(axios, 'get').mockResolvedValue({
data: { version: '0.76.7', capabilities: [], startedAt: 1, updateError: null },
@@ -125,8 +129,10 @@ describe('NodeRegistry.fetchMetaForNode', () => {
await reg.fetchMetaForNode(nodeId);
const init = axiosSpy.mock.calls[0][1] as { headers: Record<string, string> };
const init = axiosSpy.mock.calls[0][1] as { headers: Record<string, string>; httpAgent?: unknown; httpsAgent?: unknown };
expect(init.headers).toEqual({ Authorization: 'Bearer real-token' });
expect(init.httpAgent).toBeDefined();
expect(init.httpsAgent).toBeDefined();
db.deleteNode(nodeId);
});
+1 -1
View File
@@ -59,7 +59,7 @@ describe('POST /api/nodes - api_url SSRF validation (C2 fix)', () => {
.set('Authorization', authHeader)
.send({ name: 'bad-node', type: 'remote', api_url: 'http://localhost:6379' });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/loopback/i);
expect(res.body.error).toMatch(/not allowed/i);
});
it('rejects 127.0.0.1 api_url', async () => {
@@ -0,0 +1,183 @@
import http from 'http';
import { afterEach, describe, expect, it, vi } from 'vitest';
import {
assertSafeOutboundUrl,
isBlockedOutboundAddress,
safeOutboundLookup,
safeAxiosTransport,
safeRemoteFetch,
UnsafeOutboundTargetError,
} from '../utils/outboundTarget';
import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets';
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllEnvs();
});
describe('outbound target validation', () => {
it.each([
'127.0.0.1',
'169.254.169.254',
'100.100.100.200',
'192.0.2.10',
'198.18.0.10',
'198.51.100.10',
'203.0.113.10',
'0.0.0.0',
'224.0.0.1',
'::1',
'fe80::1',
'ff02::1',
'100::1',
'2001:db8::1',
'fd00:ec2::254',
'::ffff:7f00:1',
])('blocks unsafe address %s', (address) => {
expect(isBlockedOutboundAddress(address)).toBe(true);
});
it.each([
'10.0.0.10',
'172.16.0.10',
'192.168.1.10',
'fd12:3456:789a::10',
'8.8.8.8',
'::ffff:808:808',
])('allows private and ordinary unicast address %s', (address) => {
expect(isBlockedOutboundAddress(address)).toBe(false);
});
it('rejects a URL whose literal host is unsafe', async () => {
await expect(withLoopbackTargetProtection(() =>
assertSafeOutboundUrl('https://[::ffff:127.0.0.1]/repo.git')))
.rejects.toBeInstanceOf(UnsafeOutboundTargetError);
});
it('accepts private LAN targets', async () => {
await expect(assertSafeOutboundUrl('http://192.168.1.50:1852'))
.resolves.toMatchObject({ hostname: '192.168.1.50' });
});
it('allows only loopback targets when the E2E test override is active', async () => {
const actual = await vi.importActual<typeof import('../utils/outboundTarget')>('../utils/outboundTarget');
vi.stubEnv('NODE_ENV', 'test');
vi.stubEnv('SENCHO_E2E_ALLOW_LOOPBACK_OUTBOUND', 'true');
await expect(actual.resolveSafeOutboundHostname('127.0.0.1'))
.resolves.toEqual([{ address: '127.0.0.1', family: 4 }]);
await expect(actual.resolveSafeOutboundHostname('169.254.169.254'))
.rejects.toMatchObject({ reason: 'blocked' });
});
it('ignores the E2E loopback override outside the test environment', async () => {
const actual = await vi.importActual<typeof import('../utils/outboundTarget')>('../utils/outboundTarget');
vi.stubEnv('NODE_ENV', 'production');
vi.stubEnv('SENCHO_E2E_ALLOW_LOOPBACK_OUTBOUND', 'true');
await expect(actual.resolveSafeOutboundHostname('127.0.0.1'))
.rejects.toMatchObject({ reason: 'blocked' });
});
it('rejects an unsafe address inside the connection lookup', async () => {
const error = await withLoopbackTargetProtection(() =>
new Promise<NodeJS.ErrnoException | null>((resolve) => {
safeOutboundLookup('127.0.0.1', {}, (lookupError) => resolve(lookupError));
}));
expect(error).toBeInstanceOf(UnsafeOutboundTargetError);
});
it('rejects a hostname when any resolved address is unsafe', async () => {
const actual = await vi.importActual<typeof import('../utils/outboundTarget')>('../utils/outboundTarget');
const resolveMixed = async () => [
{ address: '93.184.216.34', family: 4 as const },
{ address: '169.254.169.254', family: 4 as const },
];
await expect(actual.resolveSafeOutboundHostname('mixed.example', resolveMixed))
.rejects.toMatchObject({ reason: 'blocked' });
});
it('rejects an exact metadata address returned by DNS', async () => {
const actual = await vi.importActual<typeof import('../utils/outboundTarget')>('../utils/outboundTarget');
const resolveMetadata = async () => [
{ address: '100.100.100.200', family: 4 as const },
];
await expect(actual.resolveSafeOutboundHostname('metadata.example', resolveMetadata))
.rejects.toMatchObject({ reason: 'blocked' });
});
it('rechecks DNS at connection time after safe validation', async () => {
const actual = await vi.importActual<typeof import('../utils/outboundTarget')>('../utils/outboundTarget');
const resolveSafe = async () => [{ address: '93.184.216.34', family: 4 as const }];
await expect(actual.resolveSafeOutboundHostname('rebinding.example', resolveSafe)).resolves.toHaveLength(1);
const rebindingLookup = actual.createSafeOutboundLookup((_hostname, _options, callback) => {
callback(null, [{ address: '169.254.169.254', family: 4 }]);
});
const connectionError = await new Promise<NodeJS.ErrnoException | null>((resolve) => {
rebindingLookup('rebinding.example', {}, (lookupError) => resolve(lookupError));
});
expect(connectionError).toMatchObject({ reason: 'blocked' });
});
it('rejects unsafe addresses at fetch connection time', async () => {
await expect(withLoopbackTargetProtection(() =>
safeRemoteFetch('http://127.0.0.1:1852/api/meta')))
.rejects.toBeInstanceOf(UnsafeOutboundTargetError);
});
it('rejects redirects without contacting the redirect destination', async () => {
const actual = await vi.importActual<typeof import('../utils/outboundTarget')>('../utils/outboundTarget');
let destinationRequests = 0;
const destination = http.createServer((_req, res) => {
destinationRequests += 1;
res.end('unexpected');
});
await new Promise<void>((resolve) => destination.listen(0, '127.0.0.1', resolve));
const destinationAddress = destination.address();
if (!destinationAddress || typeof destinationAddress === 'string') throw new Error('Missing destination address');
const redirect = http.createServer((_req, res) => {
res.writeHead(302, { Location: `http://127.0.0.1:${destinationAddress.port}/target` });
res.end();
});
await new Promise<void>((resolve) => redirect.listen(0, '127.0.0.1', resolve));
const redirectAddress = redirect.address();
if (!redirectAddress || typeof redirectAddress === 'string') throw new Error('Missing redirect address');
try {
await expect(actual.safeRemoteFetch(
`http://127.0.0.1:${redirectAddress.port}/start`,
{},
true,
)).rejects.toThrow();
expect(destinationRequests).toBe(0);
} finally {
await Promise.all([
new Promise<void>((resolve, reject) => destination.close((error) => error ? reject(error) : resolve())),
new Promise<void>((resolve, reject) => redirect.close((error) => error ? reject(error) : resolve())),
]);
}
});
it('isolates protected loopback checks from concurrent fixture requests', async () => {
const [ordinary, protectedResult] = await Promise.all([
assertSafeOutboundUrl('http://127.0.0.1:1852').then(() => 'allowed'),
withLoopbackTargetProtection(() => assertSafeOutboundUrl('http://127.0.0.1:1852'))
.then(() => 'allowed', () => 'blocked'),
]);
expect(ordinary).toBe('allowed');
expect(protectedResult).toBe('blocked');
});
it('disables environment proxy routing for guarded Axios requests', () => {
expect(safeAxiosTransport(false)).toMatchObject({
maxRedirects: 0,
proxy: false,
});
});
});
@@ -18,6 +18,7 @@ import request from 'supertest';
import crypto from 'crypto';
import { parse as parseYaml } from 'yaml';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
import { resetTrustedProxyBlockListCache } from '../helpers/trustedProxyCidrs';
interface ComposeService {
image: string;
@@ -223,6 +224,39 @@ describe('SENCHO_PUBLIC_URL override in mintPilotEnrollment', () => {
expect(parsed.services.agent.environment.SENCHO_PRIMARY_URL).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
});
it('ignores a forwarded HTTPS scheme from an untrusted peer', async () => {
delete process.env.SENCHO_PUBLIC_URL;
const res = await request(app)
.post('/api/nodes')
.set('Cookie', adminCookie)
.set('Host', 'sencho.example.com')
.set('X-Forwarded-Proto', 'https')
.send({ name: 'pilot-untrusted-forwarded-scheme', type: 'remote', mode: 'pilot_agent' });
const parsed = parseYaml(res.body.enrollment.composeYaml) as ComposeFile;
expect(parsed.services.agent.environment.SENCHO_PRIMARY_URL).toBe('http://sencho.example.com');
});
it('honors a forwarded HTTPS scheme from an allowlisted proxy peer', async () => {
delete process.env.SENCHO_PUBLIC_URL;
process.env.SENCHO_TRUSTED_PROXY_CIDRS = '127.0.0.0/8';
resetTrustedProxyBlockListCache();
try {
const res = await request(app)
.post('/api/nodes')
.set('Cookie', adminCookie)
.set('Host', 'sencho.example.com')
.set('X-Forwarded-Proto', 'https')
.send({ name: 'pilot-trusted-forwarded-scheme', type: 'remote', mode: 'pilot_agent' });
const parsed = parseYaml(res.body.enrollment.composeYaml) as ComposeFile;
expect(parsed.services.agent.environment.SENCHO_PRIMARY_URL).toBe('https://sencho.example.com');
} finally {
delete process.env.SENCHO_TRUSTED_PROXY_CIDRS;
resetTrustedProxyBlockListCache();
}
});
it('falls back to request host when env var is malformed', async () => {
process.env.SENCHO_PUBLIC_URL = 'not a url';
const res = await request(app)
@@ -17,6 +17,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets';
describe('remote proxy mount order', () => {
let tmpDir: string;
@@ -65,6 +66,16 @@ describe('remote proxy mount order', () => {
expect(res.body?.error).toMatch(/unreachable/i);
});
it('refuses an unsafe target from an existing node row', async () => {
const res = await withLoopbackTargetProtection(() => request(app)
.get('/api/stacks')
.set('Authorization', authHeader)
.set('x-node-id', String(remoteNodeId)));
expect(res.status).toBe(502);
expect(res.body?.error).toMatch(/not allowed/i);
});
it('routes local requests (no x-node-id) to the local handler', async () => {
const res = await request(app)
.get('/api/stacks')
@@ -57,7 +57,7 @@ describe('pilot-agent-mode proxy role header parity', () => {
const registry = NodeRegistry.getInstance();
const orig = registry.getProxyTarget.bind(registry);
vi.spyOn(registry, 'getProxyTarget').mockImplementation((nid: number) => {
if (nid === pilotNodeId) return { apiUrl: `http://127.0.0.1:${port}`, apiToken: '' };
if (nid === pilotNodeId) return { apiUrl: `http://127.0.0.1:${port}`, apiToken: '', trustedLoopback: true };
return orig(nid);
});
});
@@ -66,7 +66,7 @@ describe('registryDeliveryOutbound', () => {
apiPath: '/api/stacks/demo/deploy',
nodeId,
node,
target: { apiUrl: 'http://remote:1852', apiToken: 'token' },
target: { apiUrl: 'http://remote:1852', apiToken: 'token', trustedLoopback: false },
body,
});
@@ -86,7 +86,7 @@ describe('registryDeliveryOutbound', () => {
apiPath: '/api/stacks/demo/deploy',
nodeId,
node,
target: { apiUrl: 'http://remote:1852', apiToken: 'token' },
target: { apiUrl: 'http://remote:1852', apiToken: 'token', trustedLoopback: false },
body,
});
@@ -122,7 +122,7 @@ describe('registryDeliveryOutbound', () => {
apiPath: '/api/stacks/demo/deploy',
nodeId,
node,
target: { apiUrl: 'http://remote:1852', apiToken: 'token' },
target: { apiUrl: 'http://remote:1852', apiToken: 'token', trustedLoopback: false },
body: {},
abortSignal: controller.signal,
});
@@ -160,7 +160,7 @@ describe('registryDeliveryOutbound', () => {
apiPath: '/api/stacks/demo/deploy',
nodeId,
node,
target: { apiUrl: 'http://remote:1852', apiToken: 'token' },
target: { apiUrl: 'http://remote:1852', apiToken: 'token', trustedLoopback: false },
body: {},
});
@@ -210,7 +210,7 @@ describe('registryDeliveryOutbound', () => {
apiPath: '/api/stacks/demo/deploy',
nodeId,
node,
target: { apiUrl: 'http://remote:1852', apiToken: 'token' },
target: { apiUrl: 'http://remote:1852', apiToken: 'token', trustedLoopback: false },
body: {},
abortSignal: controller.signal,
});
@@ -0,0 +1,37 @@
import { IncomingMessage } from 'http';
import { Socket } from 'net';
import { PassThrough } from 'stream';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { wsProxyServer } from '../proxy/websocketProxy';
import { handleRemoteForwarder } from '../websocket/remoteForwarder';
import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets';
afterEach(() => vi.restoreAllMocks());
describe('remote WebSocket target validation', () => {
it('rejects an unsafe target before invoking the WebSocket proxy', async () => {
const req = new IncomingMessage(new Socket());
req.url = '/api/containers/demo/logs?nodeId=2';
req.headers.host = 'sencho.example';
const socket = new PassThrough();
socket.resume();
const proxySpy = vi.spyOn(wsProxyServer, 'ws').mockImplementation(() => {});
vi.spyOn(console, 'error').mockImplementation(() => {});
await withLoopbackTargetProtection(() => handleRemoteForwarder(
req,
socket,
Buffer.alloc(0),
{
pathname: '/api/containers/demo/logs',
target: {
apiUrl: 'http://127.0.0.1:1852',
apiToken: 'test-token',
trustedLoopback: false,
},
},
));
expect(proxySpy).not.toHaveBeenCalled();
});
});
+19 -1
View File
@@ -31,6 +31,12 @@ describe('sshTrust URL parsing', () => {
expect(parsed?.port).toBe(2222);
});
it('preserves absolute paths in default-port ssh:// URLs', () => {
const parsed = parseSshUrl('ssh://git@host.example/abs/path/repo.git');
expect(parsed?.href).toBe('git@host.example:/abs/path/repo.git');
expect(parsed?.pathname).toBe('/abs/path/repo.git');
});
it('classifies transport kind from mixed inputs', () => {
expect(parseRepoTransportUrl('https://github.com/org/repo.git')?.kind).toBe('https');
expect(parseRepoTransportUrl('git@host:org/repo.git')?.kind).toBe('ssh');
@@ -72,11 +78,23 @@ describe('ssh credential canonicalization', () => {
describe('ssh command builder', () => {
it('enforces strict host key checking', () => {
const cmd = buildSshCommand('/tmp/key', '/tmp/known_hosts');
const cmd = buildSshCommand('/tmp/key', '/tmp/known_hosts', {
address: '10.0.0.8',
hostKeyAlias: 'git.internal.example',
});
expect(cmd).toContain('StrictHostKeyChecking=yes');
expect(cmd).toContain('UserKnownHostsFile=/tmp/known_hosts');
expect(cmd).toContain('IdentitiesOnly=yes');
});
it('pins the address while retaining the repository host identity', () => {
const cmd = buildSshCommand('/tmp/key', '/tmp/known_hosts', {
address: '10.0.0.8',
hostKeyAlias: '[git.internal.example]:2222',
});
expect(cmd).toContain('Hostname=10.0.0.8');
expect(cmd).toContain('HostKeyAlias=[git.internal.example]:2222');
});
});
describe('SSH stderr classification', () => {
@@ -0,0 +1,68 @@
import request from 'supertest';
import { beforeEach, describe, expect, it } from 'vitest';
import { createApp } from '../app';
import { isSecureRequest } from '../helpers/cookies';
import { resetTrustedProxyBlockListCache } from '../helpers/trustedProxyCidrs';
describe('Express trusted proxy configuration', () => {
beforeEach(() => {
delete process.env.SENCHO_TRUSTED_PROXY_CIDRS;
resetTrustedProxyBlockListCache();
});
it('ignores forwarded client addresses from an untrusted direct peer', async () => {
const app = createApp();
app.get('/peer-ip', (req, res) => res.json({ ip: req.ip }));
const res = await request(app)
.get('/peer-ip')
.set('X-Forwarded-For', '203.0.113.50');
expect(res.status).toBe(200);
expect(res.body.ip).not.toBe('203.0.113.50');
});
it('honors forwarded client addresses from an allowlisted proxy peer', async () => {
process.env.SENCHO_TRUSTED_PROXY_CIDRS = '127.0.0.0/8';
resetTrustedProxyBlockListCache();
const app = createApp();
app.get('/peer-ip', (req, res) => res.json({ ip: req.ip }));
const res = await request(app)
.get('/peer-ip')
.set('X-Forwarded-For', '203.0.113.50');
expect(res.status).toBe(200);
expect(res.body.ip).toBe('203.0.113.50');
});
it('ignores a forwarded HTTPS scheme from an untrusted direct peer', async () => {
const app = createApp();
app.get('/request-scheme', (req, res) => {
res.json({ protocol: req.protocol, secure: isSecureRequest(req) });
});
const res = await request(app)
.get('/request-scheme')
.set('X-Forwarded-Proto', 'https');
expect(res.status).toBe(200);
expect(res.body).toEqual({ protocol: 'http', secure: false });
});
it('honors a forwarded HTTPS scheme from an allowlisted proxy peer', async () => {
process.env.SENCHO_TRUSTED_PROXY_CIDRS = '127.0.0.0/8';
resetTrustedProxyBlockListCache();
const app = createApp();
app.get('/request-scheme', (req, res) => {
res.json({ protocol: req.protocol, secure: isSecureRequest(req) });
});
const res = await request(app)
.get('/request-scheme')
.set('X-Forwarded-Proto', 'https');
expect(res.status).toBe(200);
expect(res.body).toEqual({ protocol: 'https', secure: true });
});
});
@@ -23,6 +23,13 @@ describe('trustedProxyCidrs', () => {
expect(isTrustedProxyPeer('192.168.1.1')).toBe(false);
});
it('matches IPv4-mapped IPv6 peers against IPv4 CIDRs', () => {
process.env.SENCHO_TRUSTED_PROXY_CIDRS = '10.0.0.0/8';
resetTrustedProxyBlockListCache();
expect(isTrustedProxyPeer('::ffff:10.1.2.3')).toBe(true);
expect(isTrustedProxyPeer('::ffff:192.168.1.1')).toBe(false);
});
it('fails closed on invalid entries', () => {
process.env.SENCHO_TRUSTED_PROXY_CIDRS = 'not-a-cidr';
resetTrustedProxyBlockListCache();
+3 -3
View File
@@ -6,6 +6,7 @@ import helmet from 'helmet';
import { globalApiLimiter, pollingLimiter } from './middleware/rateLimiters';
import { conditionalJsonParser } from './middleware/jsonParser';
import { nodeContextMiddleware } from './middleware/nodeContext';
import { isTrustedProxyPeer } from './helpers/trustedProxyCidrs';
import { normalizeAcceptEncoding } from './middleware/normalizeAcceptEncoding';
import './types/express';
@@ -43,9 +44,8 @@ import './types/express';
export function createApp(): express.Express {
const app = express();
// 1. Trust the first reverse proxy (nginx, Traefik, etc.) for correct
// req.protocol, req.ip, and secure cookie detection behind a proxy.
app.set('trust proxy', 1);
// 1. Trust forwarding headers only from explicitly configured proxy peers.
app.set('trust proxy', (address: string) => isTrustedProxyPeer(address));
// 2. Security headers.
// crossOriginEmbedderPolicy: disabled because Monaco editor workers lack COEP headers.
@@ -6,6 +6,7 @@ import { PROXY_TIER_HEADER } from '../services/license-headers';
import { LicenseService } from '../services/LicenseService';
import { isValidStackName } from '../utils/validation';
import { getErrorMessage } from '../utils/errors';
import { safeAxiosTransport } from '../utils/outboundTarget';
const REMOTE_STACKS_TIMEOUT_MS = 30_000;
@@ -59,6 +60,7 @@ export async function assertStackExistsOnNode(
try {
const res = await axios.get(`${baseUrl}/api/stacks`, {
...safeAxiosTransport(target.trustedLoopback),
headers,
timeout: REMOTE_STACKS_TIMEOUT_MS,
validateStatus: () => true,
+1 -1
View File
@@ -2,7 +2,7 @@ import type { Request } from 'express';
/** True when the request arrived over HTTPS, either directly or via a trusted TLS-terminating proxy. */
export const isSecureRequest = (req: Request): boolean => {
return req.secure || req.headers['x-forwarded-proto'] === 'https';
return req.secure;
};
/**
+11 -2
View File
@@ -1,6 +1,7 @@
import { DatabaseService, type Node } from '../services/DatabaseService';
import { FileSystemService } from '../services/FileSystemService';
import { NodeRegistry } from '../services/NodeRegistry';
import { safeRemoteFetch } from '../utils/outboundTarget';
import { formatNoTargetError } from '../utils/remoteTarget';
import { getErrorMessage } from '../utils/errors';
@@ -89,8 +90,16 @@ async function summarizeRemoteNode(node: Node): Promise<NodeLabelSummary> {
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
try {
const [labelsRes, assignmentsRes] = await Promise.all([
fetch(`${base}/api/labels`, { headers, signal: AbortSignal.timeout(SUMMARY_FETCH_TIMEOUT_MS) }),
fetch(`${base}/api/labels/assignments`, { headers, signal: AbortSignal.timeout(SUMMARY_FETCH_TIMEOUT_MS) }),
safeRemoteFetch(
`${base}/api/labels`,
{ headers, signal: AbortSignal.timeout(SUMMARY_FETCH_TIMEOUT_MS) },
target.trustedLoopback,
),
safeRemoteFetch(
`${base}/api/labels/assignments`,
{ headers, signal: AbortSignal.timeout(SUMMARY_FETCH_TIMEOUT_MS) },
target.trustedLoopback,
),
]);
if (!labelsRes.ok || !assignmentsRes.ok) {
// Surface the remote's own error body (e.g. a token/tier message) the same
+5 -4
View File
@@ -2,6 +2,7 @@ import type { Node } from '../services/DatabaseService';
import DockerController from '../services/DockerController';
import { FileSystemService } from '../services/FileSystemService';
import { NodeRegistry } from '../services/NodeRegistry';
import { safeRemoteFetch } from '../utils/outboundTarget';
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
import {
PrunePlanStaleError,
@@ -343,12 +344,12 @@ async function fetchRemotePlan(node: Node, targets: FleetPruneTarget[], scope: P
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;
try {
const response = await fetch(`${proxyTarget.apiUrl.replace(/\/$/, '')}/api/system/prune/plan`, {
const response = await safeRemoteFetch(`${proxyTarget.apiUrl.replace(/\/$/, '')}/api/system/prune/plan`, {
method: 'POST',
headers,
body: JSON.stringify({ targets, scope }),
signal: AbortSignal.timeout(REMOTE_PLAN_TIMEOUT_MS),
});
}, proxyTarget.trustedLoopback);
const data: unknown = await response.json().catch(() => null);
if (!response.ok) {
const message = data && typeof data === 'object' && typeof (data as { error?: unknown }).error === 'string'
@@ -471,12 +472,12 @@ async function executeRemote(entry: Preflight, targets: FleetPruneTarget[], scop
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;
try {
const response = await fetch(`${proxyTarget.apiUrl.replace(/\/$/, '')}/api/system/prune/system`, {
const response = await safeRemoteFetch(`${proxyTarget.apiUrl.replace(/\/$/, '')}/api/system/prune/system`, {
method: 'POST',
headers,
body: JSON.stringify({ targets, scope, planFingerprint: plan.fingerprint }),
signal: AbortSignal.timeout(REMOTE_PLAN_TIMEOUT_MS),
});
}, proxyTarget.trustedLoopback);
const data: unknown = await response.json().catch(() => null);
const record = data && typeof data === 'object' ? data as Record<string, unknown> : null;
if (!response.ok) {
@@ -8,6 +8,7 @@ import {
} from '../services/CapabilityRegistry';
import { remoteAdvertisesCapability } from './remoteCapabilities';
import { getErrorMessage } from '../utils/errors';
import { safeRemoteFetch } from '../utils/outboundTarget';
const SYNC_TIMEOUT_MS = 15_000;
@@ -91,10 +92,14 @@ function clearPending(nodeId: number, ruleId: number): void {
DatabaseService.getInstance().deleteNotificationSuppressionPendingRetraction(ruleId, nodeId);
}
function resolveRemoteApi(node: Node): { baseUrl: string; apiToken: string } | null {
function resolveRemoteApi(node: Node): { baseUrl: string; apiToken: string; trustedLoopback: boolean } | null {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target?.apiUrl) return null;
return { baseUrl: target.apiUrl.replace(/\/$/, ''), apiToken: target.apiToken };
return {
baseUrl: target.apiUrl.replace(/\/$/, ''),
apiToken: target.apiToken,
trustedLoopback: target.trustedLoopback,
};
}
/** Non-2xx (including opaque 404) is always failure; never treat missing routes as applied. */
@@ -119,12 +124,12 @@ async function pushRuleToNode(node: Node, rule: NotificationSuppressionRule): Pr
console.warn(`[SuppressionSync] Skipping node "${node.name}": no proxy target`);
return;
}
const res = await fetch(`${remote.baseUrl}/api/notification-suppression-rules/replica`, {
const res = await safeRemoteFetch(`${remote.baseUrl}/api/notification-suppression-rules/replica`, {
method: 'POST',
headers: buildRemoteHeaders(remote.apiToken),
body: JSON.stringify({ rule: replicaPayload(rule) }),
signal: AbortSignal.timeout(SYNC_TIMEOUT_MS),
});
}, remote.trustedLoopback);
if (!res.ok) await throwHttpFailure(res);
const outcome = await readOutcome(res);
if (outcome !== 'applied') {
@@ -148,12 +153,12 @@ export async function deleteRuleOnNode(
throw new Error(err);
}
try {
const res = await fetch(`${remote.baseUrl}/api/notification-suppression-rules/replica/${ruleId}`, {
const res = await safeRemoteFetch(`${remote.baseUrl}/api/notification-suppression-rules/replica/${ruleId}`, {
method: 'DELETE',
headers: buildRemoteHeaders(remote.apiToken),
body: JSON.stringify(retraction),
signal: AbortSignal.timeout(SYNC_TIMEOUT_MS),
});
}, remote.trustedLoopback);
if (!res.ok) await throwHttpFailure(res);
const outcome = await readOutcome(res);
if (outcome !== 'applied') {
@@ -1,7 +1,8 @@
import axios from 'axios';
import type { Node } from '../services/DatabaseService';
import { DatabaseService } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry';
import { NodeRegistry, type ProxyTarget } from '../services/NodeRegistry';
import { safeAxiosTransport } from '../utils/outboundTarget';
import { PilotTunnelManager } from '../services/PilotTunnelManager';
import { RegistryDeliveryService } from '../services/RegistryDeliveryService';
import type { RegistryDeliveryDiscoverResponse } from '../services/RegistryDeliveryService';
@@ -26,7 +27,7 @@ export interface AugmentRegistryDeliveryInput {
apiPath: string;
nodeId: number;
node: Node;
target: { apiUrl: string; apiToken: string };
target: ProxyTarget;
body: Record<string, unknown>;
sourceKind?: string;
prepId?: string;
@@ -61,7 +62,7 @@ export async function wouldAttemptRegistryDelivery(
}
async function callTargetDiscover(
target: { apiUrl: string; apiToken: string },
target: ProxyTarget,
body: Record<string, unknown>,
abortSignal?: AbortSignal,
): Promise<RegistryDeliveryDiscoverResponse> {
@@ -70,6 +71,7 @@ async function callTargetDiscover(
}
const base = target.apiUrl.replace(/\/$/, '');
const res = await axios.post(`${base}/api/registry-delivery/discover`, body, {
...safeAxiosTransport(target.trustedLoopback),
headers: { Authorization: `Bearer ${target.apiToken}` },
timeout: 30_000,
maxBodyLength: REGISTRY_DELIVERY_FIELD_LIMIT_BYTES,
+2 -1
View File
@@ -1,5 +1,6 @@
import type { Request, Response } from 'express';
import type { Node } from '../services/DatabaseService';
import type { ProxyTarget } from '../services/NodeRegistry';
import { augmentJsonBodyForRegistryDelivery, wouldAttemptRegistryDelivery } from './registryDeliveryOutbound';
export interface RegistryDeliveryProxyResult {
@@ -18,7 +19,7 @@ export async function augmentRemoteProxyWithRegistryDelivery(
req: Request,
nodeId: number,
node: Node,
target: { apiUrl: string; apiToken: string },
target: ProxyTarget,
rawBody: Buffer,
): Promise<RegistryDeliveryProxyResult> {
const apiPath = `/api${req.path}`;
+6 -4
View File
@@ -30,7 +30,7 @@ function parseCidrEntry(raw: string): { family: 4 | 6; address: string; prefix:
/**
* Parse SENCHO_TRUSTED_PROXY_CIDRS once at process start. Invalid or duplicate
* entries fail closed by returning null (treat all upgrades as non-confidential).
* entries fail closed by returning null, so forwarding headers are ignored.
*/
export function getTrustedProxyBlockList(): net.BlockList | null {
if (cachedBlockList !== undefined) {
@@ -91,12 +91,14 @@ export function isTrustedProxyPeer(peerAddress: string | undefined): boolean {
const blockList = getTrustedProxyBlockList();
if (!blockList) return false;
const family = net.isIP(peerAddress);
const mappedIpv4 = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/i.exec(peerAddress)?.[1];
const normalizedPeer = mappedIpv4 ?? peerAddress;
const family = net.isIP(normalizedPeer);
if (family === 4) {
return blockList.check(peerAddress, 'ipv4');
return blockList.check(normalizedPeer, 'ipv4');
}
if (family === 6) {
return blockList.check(peerAddress, 'ipv6');
return blockList.check(normalizedPeer, 'ipv6');
}
return false;
}
+1 -1
View File
@@ -14,7 +14,7 @@ import { DatabaseService } from '../services/DatabaseService';
// with a 300/min safety net to prevent resource exhaustion.
// Tier W (Webhooks): CI/CD webhook triggers at 500/min (shared datacenter IPs).
// Tier 2 (Standard): All other endpoints at 200/min.
// Tier 3 (Auth): Strict brute-force protection (5-10 attempts / 15min).
// Tier 3 (Auth): Login protection at 5 attempts/IP per 15 minutes.
//
// Enterprise adaptations:
// - Internal node-to-node traffic (node_proxy JWTs) bypasses all rate limiters.
+39 -4
View File
@@ -52,6 +52,12 @@ import type { PermissionAction } from '../middleware/permissions';
import { SETTING_WRITE_PERMISSIONS } from '../routes/settings';
import { rejectApiTokenScope } from '../middleware/apiTokenScope';
import { RegistryDeliveryService } from '../services/RegistryDeliveryService';
import {
assertSafeOutboundUrl,
safeHttpAgent,
safeHttpsAgent,
UnsafeOutboundTargetError,
} from '../utils/outboundTarget';
import {
classifyRegistryDeliveryRouteClass,
getRegistryDeliveryTotalBodyLimit,
@@ -318,7 +324,11 @@ export function createRemoteProxyMiddleware(): RequestHandler {
pathRewrite: (path: string) => '/api' + path,
};
const proxy = createProxyMiddleware<Request, Response>({ ...baseOptions, on: sharedOn });
const createStreamingProxy = (agent?: typeof safeHttpAgent | typeof safeHttpsAgent) =>
createProxyMiddleware<Request, Response>({ ...baseOptions, ...(agent ? { agent } : {}), on: sharedOn });
const proxy = createStreamingProxy();
const safeHttpProxy = createStreamingProxy(safeHttpAgent);
const safeHttpsProxy = createStreamingProxy(safeHttpsAgent);
/**
* The identity hop: buffers the response so node ids inside it can be
@@ -328,8 +338,10 @@ export function createRemoteProxyMiddleware(): RequestHandler {
* buffering is exactly what the streaming hop must never do. Logs, event
* streams, and downloads keep flowing through that one untouched.
*/
const identityProxy = createProxyMiddleware<Request, Response>({
const createIdentityProxy = (agent?: typeof safeHttpAgent | typeof safeHttpsAgent) =>
createProxyMiddleware<Request, Response>({
...baseOptions,
...(agent ? { agent } : {}),
selfHandleResponse: true,
// Bounded so a remote that sends headers and then stalls cannot pin the
// buffered body and both sockets indefinitely. No pathFilter: the
@@ -376,6 +388,9 @@ export function createRemoteProxyMiddleware(): RequestHandler {
},
},
});
const identityProxy = createIdentityProxy();
const safeHttpIdentityProxy = createIdentityProxy(safeHttpAgent);
const safeHttpsIdentityProxy = createIdentityProxy(safeHttpsAgent);
return (req: Request, res: Response, next: NextFunction): void => {
// The `/api/` mount strips the `/api` prefix, so req.path is now `/auth/…`,
@@ -418,6 +433,20 @@ export function createRemoteProxyMiddleware(): RequestHandler {
}
const runGatedProxy = async (): Promise<void> => {
if (node.mode === 'proxy') {
try {
await assertSafeOutboundUrl(target.apiUrl);
} catch (error: unknown) {
if (!(error instanceof UnsafeOutboundTargetError)) throw error;
const message = error.reason === 'blocked'
? 'Remote node target is not allowed.'
: 'Remote node target host could not be resolved.';
console.warn(`[Proxy] Refused remote target for node ${node.id}: ${error.reason}`);
res.status(502).json({ error: message });
return;
}
}
if (isStackDownWithRemoveVolumes(req)) {
const supported = await remoteAdvertisesCapability(req.nodeId, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY);
if (!supported) {
@@ -779,7 +808,10 @@ export function createRemoteProxyMiddleware(): RequestHandler {
}
req.gitopsIdentity = { query: prepared.search.toString(), preRewritePath: req.path };
beginProxyTiming(req, res);
identityProxy(req, res, next);
const selectedIdentityProxy = node.mode === 'pilot_agent'
? identityProxy
: target.apiUrl.startsWith('https:') ? safeHttpsIdentityProxy : safeHttpIdentityProxy;
selectedIdentityProxy(req, res, next);
return;
}
@@ -787,7 +819,10 @@ export function createRemoteProxyMiddleware(): RequestHandler {
if (req.registryDeliveryAbortController?.signal.aborted) {
return;
}
proxy(req, res, next);
const selectedProxy = node.mode === 'pilot_agent'
? proxy
: target.apiUrl.startsWith('https:') ? safeHttpsProxy : safeHttpProxy;
selectedProxy(req, res, next);
};
runGatedProxy().catch(next);
+2 -2
View File
@@ -43,12 +43,12 @@ diagnosticsRouter.get('/', async (req: Request, res: Response): Promise<void> =>
// compose directory and its host path mapping, TLS, disk headroom). Same admin
// session gate as the recovery report. proto / host come from the request so
// the TLS verdict reflects how this browser reached the dashboard; behind a
// reverse proxy that terminates TLS, x-forwarded-proto carries the real scheme.
// trusted reverse proxy that terminates TLS, req.protocol carries the real scheme.
diagnosticsRouter.get('/environment', async (req: Request, res: Response): Promise<void> => {
if (!requireUserSession(req, res)) return;
if (!requireAdmin(req, res)) return;
try {
const proto = (req.get('x-forwarded-proto')?.split(',')[0].trim()) || req.protocol;
const proto = req.protocol;
const host = req.get('host') || '';
const report = await collectEnvironmentReport(buildRealProbes({ proto, host }));
try {
+45 -31
View File
@@ -7,7 +7,7 @@ import { DatabaseService, type Node, type StackDossierFields } from '../services
import { ControlIdentityMismatchError, FleetSyncService, StaleSyncPushError } from '../services/FleetSyncService';
import { MAX_SYNC_ROWS, SYNC_ERROR_CODES } from '../services/fleetSyncConstants';
import { FleetUpdateTrackerService, type UpdateTracker, type TerminalStatus, UPDATE_TIMEOUT_MS, UPDATE_TIMEOUT_MSG, TERMINAL_TTL_MS } from '../services/FleetUpdateTrackerService';
import { NodeRegistry } from '../services/NodeRegistry';
import { NodeRegistry, type ProxyTarget } from '../services/NodeRegistry';
import { computeNodeNetworkingSummary, type NodeNetworkingSummary } from '../services/network/networkingSummary';
import DockerController from '../services/DockerController';
import { getHostMemory, memoryToWire, type MemoryWire } from '../helpers/hostMemory';
@@ -79,6 +79,7 @@ import { buildNodeLabelInventory, VALID_LABEL_SOURCES, type NodeLabelInventory }
import { labelInventoryOptionsFromRequest, requireRevealAdmin } from '../helpers/labelInventoryRequest';
import { PROXY_TIER_HEADER, deployProvenanceHeaders } from '../services/license-headers';
import { LicenseService } from '../services/LicenseService';
import { safeRemoteFetch } from '../utils/outboundTarget';
const updateTracker = FleetUpdateTrackerService.getInstance();
/** Sync lock for remote reapply while meta is fetched (before the pollable tracker exists). */
@@ -391,9 +392,9 @@ async function fetchRemoteNodeOverview(node: Node, db: DatabaseService): Promise
try {
const [statsRes, systemStatsRes, stacksRes] = await Promise.allSettled([
fetch(`${baseUrl}/api/stats`, { headers, signal: AbortSignal.timeout(10000) }),
fetch(`${baseUrl}/api/system/stats`, { headers, signal: AbortSignal.timeout(10000) }),
fetch(`${baseUrl}/api/stacks`, { headers, signal: AbortSignal.timeout(10000) }),
safeRemoteFetch(`${baseUrl}/api/stats`, { headers, signal: AbortSignal.timeout(10000) }, target.trustedLoopback),
safeRemoteFetch(`${baseUrl}/api/system/stats`, { headers, signal: AbortSignal.timeout(10000) }, target.trustedLoopback),
safeRemoteFetch(`${baseUrl}/api/stacks`, { headers, signal: AbortSignal.timeout(10000) }, target.trustedLoopback),
]);
interface RemoteSystemStats {
@@ -684,7 +685,7 @@ fleetRouter.get('/configuration', authMiddleware, async (req: Request, res: Resp
}
try {
const resp = await fetch(
const resp = await safeRemoteFetch(
`${target.apiUrl.replace(/\/$/, '')}/api/dashboard/configuration`,
{
headers: {
@@ -693,6 +694,7 @@ fleetRouter.get('/configuration', authMiddleware, async (req: Request, res: Resp
},
signal: AbortSignal.timeout(10000),
},
target.trustedLoopback,
);
const raw = resp.ok ? (await resp.json() as ConfigurationStatus) : null;
const configuration = raw ? normalizeRemoteConfigurationStatus(raw) : null;
@@ -703,7 +705,11 @@ fleetRouter.get('/configuration', authMiddleware, async (req: Request, res: Resp
status: configuration ? 'online' : 'offline',
configuration,
};
} catch {
} catch (error: unknown) {
console.warn(
`[Fleet] Configuration fetch failed for node "${sanitizeForLog(node.name)}":`,
getErrorMessage(error, 'unknown'),
);
return { id: node.id, name: node.name, type: 'remote', status: 'offline', configuration: null };
}
}),
@@ -746,12 +752,13 @@ fleetRouter.get('/dependency-map', authMiddleware, async (req: Request, res: Res
return { nodeId: node.id, nodeName: node.name, status: 'error', graph: null, error: formatNoTargetError(node) };
}
const resp = await fetch(
const resp = await safeRemoteFetch(
`${target.apiUrl.replace(/\/$/, '')}/api/dependency-map/node-graph`,
{
headers: { ...(target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {}) },
signal: AbortSignal.timeout(15000),
},
target.trustedLoopback,
);
if (!resp.ok) {
const errBody = await resp.json().catch(() => null) as { error?: string } | null;
@@ -877,12 +884,13 @@ fleetRouter.get('/container-labels', authMiddleware, async (req: Request, res: R
}
const revealQs = options.revealSecrets ? '?reveal=1' : '';
const resp = await fetch(
const resp = await safeRemoteFetch(
`${target.apiUrl.replace(/\/$/, '')}/api/system/container-labels${revealQs}`,
{
headers: { ...(target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {}) },
signal: AbortSignal.timeout(30000),
},
target.trustedLoopback,
);
if (!resp.ok) {
const errBody = await resp.json().catch(() => null) as { error?: string } | null;
@@ -986,9 +994,10 @@ fleetRouter.get('/networking-summary', authMiddleware, async (req: Request, res:
if (!target) {
return { nodeId: node.id, nodeName: node.name, status: 'error', summary: null, error: formatNoTargetError(node) };
}
const resp = await fetch(
const resp = await safeRemoteFetch(
`${target.apiUrl.replace(/\/$/, '')}/api/networking/summary`,
{ headers: { ...(target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {}) }, signal: AbortSignal.timeout(15000) },
target.trustedLoopback,
);
if (!resp.ok) {
return { nodeId: node.id, nodeName: node.name, status: 'error', summary: null, error: `Remote returned ${resp.status}` };
@@ -1036,10 +1045,10 @@ fleetRouter.get('/node/:nodeId/stacks', authMiddleware, async (req: Request, res
res.status(503).json({ error: formatNoTargetError(node) });
return;
}
const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks`, {
const response = await safeRemoteFetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks`, {
headers: target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {},
signal: AbortSignal.timeout(10000),
});
}, target.trustedLoopback);
if (!response.ok) {
res.status(502).json({ error: 'Failed to fetch stacks from remote node' });
return;
@@ -1083,10 +1092,10 @@ fleetRouter.get('/node/:nodeId/stacks/:stackName/containers', authMiddleware, as
res.status(503).json({ error: formatNoTargetError(node) });
return;
}
const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, {
const response = await safeRemoteFetch(`${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(stackName)}/containers`, {
headers: target.apiToken ? { Authorization: `Bearer ${target.apiToken}` } : {},
signal: AbortSignal.timeout(10000),
});
}, target.trustedLoopback);
if (!response.ok) {
res.status(502).json({ error: 'Failed to fetch containers from remote node' });
return;
@@ -1402,25 +1411,25 @@ fleetRouter.get('/update-status/release-notes', authMiddleware, async (req: Requ
// sent as null/invalid), and an older remote that predates this field simply
// ignores the extra body key and behaves as before.
function postSystemEndpoint(
target: { apiUrl: string; apiToken: string },
target: ProxyTarget,
endpoint: '/api/system/update' | '/api/system/reapply-compose',
body: Record<string, unknown> = {},
) {
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
return fetch(`${target.apiUrl.replace(/\/$/, '')}${endpoint}`, {
return safeRemoteFetch(`${target.apiUrl.replace(/\/$/, '')}${endpoint}`, {
method: 'POST',
headers,
body: JSON.stringify(body),
signal: AbortSignal.timeout(10000),
});
}, target.trustedLoopback);
}
function postSystemUpdate(target: { apiUrl: string; apiToken: string }, targetVersion?: string) {
function postSystemUpdate(target: ProxyTarget, targetVersion?: string) {
return postSystemEndpoint(target, '/api/system/update', targetVersion ? { targetVersion } : {});
}
function postSystemReapplyCompose(target: { apiUrl: string; apiToken: string }) {
function postSystemReapplyCompose(target: ProxyTarget) {
return postSystemEndpoint(target, '/api/system/reapply-compose');
}
@@ -2039,7 +2048,7 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
try {
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
const response = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/fleet-actions/labels/local-stop`, {
const response = await safeRemoteFetch(`${target.apiUrl.replace(/\/$/, '')}/api/fleet-actions/labels/local-stop`, {
method: 'POST',
headers,
body: JSON.stringify({
@@ -2048,7 +2057,7 @@ fleetRouter.post('/labels/fleet-stop', authMiddleware, async (req: Request, res:
...(allowedStacks ? { stackNames: [...allowedStacks] } : {}),
}),
signal: AbortSignal.timeout(60000),
});
}, target.trustedLoopback);
if (!response.ok) {
const err = (await response.json().catch(() => ({}))) as { error?: string };
return { nodeId: node.id, nodeName: node.name, reachable: false, matched: false, stackResults: [], error: err.error || `Remote returned ${response.status}` };
@@ -2218,12 +2227,12 @@ fleetRouter.post('/labels/bulk-assign', authMiddleware, async (req: Request, res
try {
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;
const response = await fetch(`${proxyTarget.apiUrl.replace(/\/$/, '')}/api/fleet-actions/labels/local-assign`, {
const response = await safeRemoteFetch(`${proxyTarget.apiUrl.replace(/\/$/, '')}/api/fleet-actions/labels/local-assign`, {
method: 'POST',
headers,
body: JSON.stringify({ label: template, stackNames: target.stackNames }),
signal: AbortSignal.timeout(60000),
});
}, proxyTarget.trustedLoopback);
if (!response.ok) {
const err = (await response.json().catch(() => ({}))) as { error?: string };
const message = err.error || `Remote returned ${response.status}`;
@@ -2452,12 +2461,12 @@ fleetRouter.post('/prune/estimate', authMiddleware, async (req: Request, res: Re
// serialized and one failure should short-circuit later targets there.)
const perTarget = await Promise.all(targets.map(async (target): Promise<FleetEstimateTargetResult> => {
try {
const response = await fetch(`${baseUrl}/api/system/prune/estimate`, {
const response = await safeRemoteFetch(`${baseUrl}/api/system/prune/estimate`, {
method: 'POST',
headers: estimateHeaders,
body: JSON.stringify({ target, scope }),
signal: AbortSignal.timeout(15000),
});
}, proxyTarget.trustedLoopback);
if (!response.ok) {
const errBody = (await response.json().catch(() => ({}))) as { error?: string };
return { bytes: 0, error: errBody.error || `Remote returned ${response.status}` };
@@ -2750,6 +2759,7 @@ class SnapshotProxyTargetError extends Error {
interface RemoteProxyContext {
baseUrl: string;
headers: Record<string, string>;
trustedLoopback: boolean;
}
// Builds an error from a failed remote response so the thrown message names the
@@ -2787,7 +2797,11 @@ function buildRemoteProxyContext(node: Node): RemoteProxyContext | null {
[PROXY_TIER_HEADER]: proxyHeaders.tier,
};
if (proxyTarget.apiToken) headers.Authorization = `Bearer ${proxyTarget.apiToken}`;
return { baseUrl: proxyTarget.apiUrl.replace(/\/$/, ''), headers };
return {
baseUrl: proxyTarget.apiUrl.replace(/\/$/, ''),
headers,
trustedLoopback: proxyTarget.trustedLoopback,
};
}
// Writes a snapshot stack's files back to its node. Existing stacks capture a
@@ -2814,12 +2828,12 @@ async function applySnapshotStackFiles(
const ctx = buildRemoteProxyContext(node);
if (!ctx) throw new SnapshotProxyTargetError(formatNoTargetError(node));
const applyRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/fleet-snapshot-apply`, {
const applyRes = await safeRemoteFetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/fleet-snapshot-apply`, {
method: 'POST',
headers: ctx.headers,
body: JSON.stringify({ files: applyFiles }),
signal: AbortSignal.timeout(FLEET_SNAPSHOT_APPLY_TIMEOUT_MS),
});
}, ctx.trustedLoopback);
if (!applyRes.ok) throw await remoteStackError('Failed to restore stack files', applyRes);
}
@@ -2854,7 +2868,7 @@ async function redeploySnapshotStack(node: Node, stackName: string): Promise<voi
if (!augmented.ok) {
throw new Error(augmented.error);
}
const deployRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/deploy`, {
const deployRes = await safeRemoteFetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/deploy`, {
method: 'POST',
headers: {
...ctx.headers,
@@ -2862,7 +2876,7 @@ async function redeploySnapshotStack(node: Node, stackName: string): Promise<voi
},
body: JSON.stringify(augmented.body),
signal: AbortSignal.timeout(30000),
});
}, ctx.trustedLoopback);
if (!deployRes.ok) throw await remoteStackError('Failed to redeploy stack', deployRes);
}
@@ -2899,12 +2913,12 @@ async function restoreSnapshotStackDossier(node: Node, stackName: string, fields
}
const ctx = buildRemoteProxyContext(node);
if (!ctx) throw new SnapshotProxyTargetError(formatNoTargetError(node));
const putRes = await fetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/dossier`, {
const putRes = await safeRemoteFetch(`${ctx.baseUrl}/api/stacks/${encodeURIComponent(stackName)}/dossier`, {
method: 'PUT',
headers: ctx.headers,
body: JSON.stringify(fields),
signal: AbortSignal.timeout(15000),
});
}, ctx.trustedLoopback);
if (!putRes.ok) throw await remoteStackError('Failed to restore dossier notes', putRes);
}
+31 -2
View File
@@ -15,10 +15,11 @@ import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelec
import { isValidGitSourcePath, isValidStackName } from '../utils/validation';
import { sendGitSourceError, webhookPullStatus } from '../utils/gitSourceHttp';
import { sanitizeForLog } from '../utils/safeLog';
import { repoUrlRejectionMessage } from '../services/gitops/repoIdentity';
import { parseStorableRepoUrl, repoUrlRejectionMessage } from '../services/gitops/repoIdentity';
import { REF_MAX_LEN } from '../services/git/nativeGitTransport';
import { validateCaBundlePem } from '../services/git/caBundle';
import { auditActorUsername } from '../helpers/auditActor';
import { assertSafeOutboundHostname, resolveSafeOutboundHostname, UnsafeOutboundTargetError } from '../utils/outboundTarget';
// Reasonable upper bounds so a caller cannot flood the service with huge
// payloads. Generous compared to anything a real Git provider emits.
@@ -59,6 +60,25 @@ async function handleBrowse(
res.status(400).json({ error: repoUrlError });
return;
}
const parsedRepo = parseStorableRepoUrl(repo_url);
if (!parsedRepo.ok) {
res.status(400).json({ error: 'Repository URL is invalid' });
return;
}
const repoHostname = parsedRepo.kind === 'https' ? parsedRepo.url.hostname : parsedRepo.ssh.host;
try {
await assertSafeOutboundHostname(repoHostname);
} catch (error: unknown) {
if (error instanceof UnsafeOutboundTargetError) {
res.status(400).json({
error: error.reason === 'blocked'
? 'Repository host is not allowed'
: 'Repository host could not be resolved',
});
return;
}
throw error;
}
if (branch.length > MAX_BRANCH_LENGTH) {
res.status(400).json({ error: 'The branch, tag, or commit SHA is too long.' });
return;
@@ -150,13 +170,22 @@ gitSourcesRouter.post('/ssh-host-key', async (req: Request, res: Response): Prom
res.status(400).json({ error: 'Host key probe requires an SSH repository URL' });
return;
}
const keys = await scanHostKeys(parsed.host, parsed.port);
const [{ address }] = await resolveSafeOutboundHostname(parsed.host);
const keys = await scanHostKeys(parsed.host, parsed.port, address);
res.json({
host: parsed.host,
port: parsed.port,
keys,
});
} catch (error) {
if (error instanceof UnsafeOutboundTargetError) {
res.status(400).json({
error: error.reason === 'blocked'
? 'Repository host is not allowed'
: 'Repository host could not be resolved',
});
return;
}
sendGitSourceError(res, error);
}
});
+14 -5
View File
@@ -4,6 +4,7 @@ import { CronExpressionParser } from 'cron-parser';
import DockerController from '../services/DockerController';
import { DatabaseService } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry';
import { safeRemoteFetch } from '../utils/outboundTarget';
import { CacheService } from '../services/CacheService';
import {
createAutoUpdateDigestGateState,
@@ -274,16 +275,20 @@ imageUpdatesRouter.get('/fleet', authMiddleware, async (req: Request, res: Respo
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS);
try {
const resp = await fetch(`${baseUrl}/api/image-updates`, {
const resp = await safeRemoteFetch(`${baseUrl}/api/image-updates`, {
headers: proxyTarget.apiToken
? { Authorization: `Bearer ${proxyTarget.apiToken}` }
: {},
signal: controller.signal,
});
}, proxyTarget.trustedLoopback);
clearTimeout(timeout);
if (resp.ok) return { nodeId: node.id, data: await resp.json() as Record<string, boolean> };
} catch {
} catch (error: unknown) {
clearTimeout(timeout);
console.warn(
`[Image updates] Status fetch failed for node "${sanitizeForLog(node.name)}":`,
getErrorMessage(error, 'unknown'),
);
}
return null;
}),
@@ -348,17 +353,21 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request,
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS);
try {
const resp = await fetch(`${baseUrl}/api/image-updates/refresh`, {
const resp = await safeRemoteFetch(`${baseUrl}/api/image-updates/refresh`, {
method: 'POST',
headers: proxyTarget.apiToken
? { Authorization: `Bearer ${proxyTarget.apiToken}` }
: {},
signal: controller.signal,
});
}, proxyTarget.trustedLoopback);
clearTimeout(timeout);
return { nodeId: node.id, status: resp.status };
} catch (e) {
clearTimeout(timeout);
console.warn(
`[Image updates] Refresh failed for node "${sanitizeForLog(node.name)}":`,
getErrorMessage(e, 'unknown'),
);
return { nodeId: node.id, status: 0, error: e };
}
}),
+25 -4
View File
@@ -27,6 +27,7 @@ import { logDebugTiming } from '../utils/requestTiming';
import { BlueprintReconciler } from '../services/BlueprintReconciler';
import { recordPlacementShift, snapshotPlacementWith } from '../services/gitops/nodePlacementProducers';
import { projectCommittedRevisions } from '../helpers/gitopsResponse';
import { assertSafeOutboundUrl, safeRemoteFetch, UnsafeOutboundTargetError } from '../utils/outboundTarget';
const NODE_SCOPE_MESSAGE = 'API tokens cannot manage nodes.';
const REMOTE_META_CACHE_TTL = 3 * 60 * 1000;
@@ -59,9 +60,7 @@ function resolvePrimaryUrl(req: Request): string {
if (check.valid) return override.replace(/\/$/, '');
console.warn(`[Enrollment] SENCHO_PUBLIC_URL is set but invalid (${check.reason}); falling back to request host.`);
}
const forwardedProto = req.headers['x-forwarded-proto'];
const protoHeader = Array.isArray(forwardedProto) ? forwardedProto[0] : forwardedProto;
const protocol = protoHeader || req.protocol || 'http';
const protocol = req.protocol;
const host = req.get('host') || 'localhost:1852';
return `${protocol}://${host}`;
}
@@ -251,6 +250,17 @@ nodesRouter.post('/', enrollmentLimiter, async (req: Request, res: Response) =>
if (!urlCheck.valid) {
return res.status(400).json({ error: urlCheck.reason });
}
try {
await assertSafeOutboundUrl(api_url);
} catch (error: unknown) {
if (error instanceof UnsafeOutboundTargetError) {
const message = error.reason === 'blocked'
? 'API URL target is not allowed'
: 'API URL host could not be resolved';
return res.status(400).json({ error: message });
}
throw error;
}
}
const id = DatabaseService.getInstance().addNode({
@@ -381,6 +391,17 @@ nodesRouter.put('/:id', async (req: Request, res: Response) => {
if (!urlCheck.valid) {
return res.status(400).json({ error: urlCheck.reason });
}
try {
await assertSafeOutboundUrl(updates.api_url);
} catch (error: unknown) {
if (error instanceof UnsafeOutboundTargetError) {
const message = error.reason === 'blocked'
? 'API URL target is not allowed'
: 'API URL host could not be resolved';
return res.status(400).json({ error: message });
}
throw error;
}
}
DatabaseService.getInstance().updateNode(id, updates);
@@ -590,7 +611,7 @@ nodesRouter.post('/:id/fleet-sync/reset-anchor', async (req: Request, res: Respo
const baseUrl = node.api_url.replace(/\/$/, '');
let peerResponse: globalThis.Response;
try {
peerResponse = await fetch(`${baseUrl}/api/fleet/role/reanchor`, {
peerResponse = await safeRemoteFetch(`${baseUrl}/api/fleet/role/reanchor`, {
method: 'POST',
headers: {
Authorization: `Bearer ${node.api_token}`,
+3 -2
View File
@@ -9,6 +9,7 @@ import { NotificationService } from './NotificationService';
import { PROXY_TIER_HEADER } from './license-headers';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { safeRemoteFetch } from '../utils/outboundTarget';
// Dockerode listContainers shape (subset used here)
type ContainerInfo = {
@@ -177,14 +178,14 @@ export class AutoHealService {
}
const baseUrl = target.apiUrl.replace(/\/$/, '');
try {
const res = await fetch(`${baseUrl}/api/auto-heal/policies`, {
const res = await safeRemoteFetch(`${baseUrl}/api/auto-heal/policies`, {
method: 'GET',
headers: {
'Authorization': `Bearer ${target.apiToken}`,
[PROXY_TIER_HEADER]: proxyHeaders.tier,
},
signal: AbortSignal.timeout(LEASE_REFRESH_TIMEOUT_MS),
});
}, target.trustedLoopback);
if (res.ok) {
this.leaseRefreshFailures.delete(nodeId);
} else {
+16 -2
View File
@@ -13,6 +13,7 @@ import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './St
import { DeployedStackDeletionService } from './DeployedStackDeletionService';
import { FileSystemService } from './FileSystemService';
import { NodeRegistry } from './NodeRegistry';
import { safeAxiosTransport } from '../utils/outboundTarget';
import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers';
import { LicenseService } from './LicenseService';
import { assertPolicyGateAllows, buildSystemPolicyGateOptions, describePolicyBlock, triggerPostDeployScan } from '../helpers/policyGate';
@@ -180,6 +181,7 @@ export class BlueprintService {
if (!target) return null;
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(BLUEPRINT_MARKER_FILENAME)}`;
const res = await axios.get(url, {
...safeAxiosTransport(target.trustedLoopback),
headers: this.remoteHeaders(target.apiToken),
timeout: REMOTE_HTTP_TIMEOUT_MS,
validateStatus: () => true,
@@ -229,6 +231,7 @@ export class BlueprintService {
let listRes;
try {
listRes = await axios.get(`${baseUrl}/api/stacks`, {
...safeAxiosTransport(target.trustedLoopback),
headers: this.remoteHeaders(target.apiToken),
timeout: REMOTE_HTTP_TIMEOUT_MS,
validateStatus: () => true,
@@ -433,6 +436,7 @@ export class BlueprintService {
if (!target) return { allRunning: false, detail: 'remote node not reachable (no proxy target)' };
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/containers`;
const res = await axios.get(url, {
...safeAxiosTransport(target.trustedLoopback),
headers: this.remoteHeaders(target.apiToken),
timeout: REMOTE_HTTP_TIMEOUT_MS,
validateStatus: () => true,
@@ -659,7 +663,12 @@ export class BlueprintService {
const res = await axios.post(
`${baseUrl}/api/blueprints/apply-local`,
augmented.body,
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
{
...safeAxiosTransport(target.trustedLoopback),
headers,
timeout: REMOTE_HTTP_TIMEOUT_MS,
validateStatus: () => true,
},
);
if (res.status === 404) {
throw new BlueprintRemoteUpgradeRequiredError(
@@ -691,7 +700,12 @@ export class BlueprintService {
res = await axios.post(
`${baseUrl}/api/blueprints/withdraw-local`,
{ stackName: blueprint.name, blueprintId: blueprint.id },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
{
...safeAxiosTransport(target.trustedLoopback),
headers,
timeout: REMOTE_HTTP_TIMEOUT_MS,
validateStatus: () => true,
},
);
} catch (err) {
const message = BlueprintService.formatError(err);
+8 -1
View File
@@ -5,6 +5,7 @@ import semver from 'semver';
import { SENCHO_VERSION } from '../generated/version';
import { isDebugEnabled } from '../utils/debug';
import type { ImagePinKind } from '../helpers/selfUpdateCompose';
import { assertSafeOutboundUrl, safeAxiosTransport } from '../utils/outboundTarget';
const IMAGE_PIN_KINDS: readonly ImagePinKind[] = ['floating', 'semver', 'digest', 'unknown'];
@@ -239,10 +240,16 @@ function redactUrlCredentials(url: string): string {
}
/** Fetch /api/meta from a remote Sencho instance. Returns empty data on failure. */
export async function fetchRemoteMeta(baseUrl: string, apiToken: string): Promise<RemoteMeta> {
export async function fetchRemoteMeta(
baseUrl: string,
apiToken: string,
trustedLoopback = false,
): Promise<RemoteMeta> {
const safeUrl = redactUrlCredentials(baseUrl);
try {
if (!trustedLoopback) await assertSafeOutboundUrl(baseUrl);
const res = await axios.get(`${baseUrl.replace(/\/$/, '')}/api/meta`, {
...safeAxiosTransport(trustedLoopback),
headers: apiToken ? { Authorization: `Bearer ${apiToken}` } : {},
timeout: 5000,
});
+2
View File
@@ -5,6 +5,7 @@ import { NodeRegistry } from './NodeRegistry';
import { NotificationService } from './NotificationService';
import { isDebugEnabled } from '../utils/debug';
import { sanitizeForLog } from '../utils/safeLog';
import { safeAxiosTransport } from '../utils/outboundTarget';
import {
FleetResource,
MAX_SYNC_ROWS,
@@ -500,6 +501,7 @@ export class FleetSyncService {
`${baseUrl}/api/fleet/sync/${resource}`,
payload,
{
...safeAxiosTransport(false),
headers: { Authorization: `Bearer ${node.api_token}` },
timeout: 15_000,
},
@@ -11,6 +11,7 @@ import { PilotMetrics } from './PilotMetrics';
import type { MeshActivityType } from './MeshService';
import { LicenseService } from './LicenseService';
import { PROXY_TIER_HEADER } from './license-headers';
import { assertSafeOutboundUrl, safeOutboundLookup } from '../utils/outboundTarget';
/**
* Central-side dialer for proxy-mode mesh tunnels.
@@ -247,7 +248,9 @@ export class MeshProxyTunnelDialer extends EventEmitter {
const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
let ws: WebSocket;
try {
await assertSafeOutboundUrl(target.apiUrl);
ws = new WebSocket(wsUrl, {
lookup: safeOutboundLookup,
headers: {
Authorization: `Bearer ${target.apiToken}`,
[PROXY_TIER_HEADER]: proxyHeaders.tier,
+3 -2
View File
@@ -20,6 +20,7 @@ import { lookupContainerIp } from '../mesh/containerLookup';
import { STREAM_PENDING_DATA_MAX_BYTES } from '../pilot/protocol';
import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
import { isDebugEnabled } from '../utils/debug';
import { safeRemoteFetch } from '../utils/outboundTarget';
import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation';
import { getErrorMessage } from '../utils/errors';
import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants';
@@ -2518,12 +2519,12 @@ export class MeshService extends EventEmitter implements MeshForwarderHost {
}
}
return await fetch(url, {
return await safeRemoteFetch(url, {
method,
headers,
body: bodyToSend === undefined ? undefined : JSON.stringify(bodyToSend),
signal: AbortSignal.timeout(timeoutMs),
});
}, target.trustedLoopback);
}
/**
+19 -8
View File
@@ -4,6 +4,11 @@ import { EventEmitter } from 'events';
import { DatabaseService, Node } from './DatabaseService';
import { fetchRemoteMeta, OFFLINE_META, RemoteMeta } from './CapabilityRegistry';
import { PilotTunnelManager } from './PilotTunnelManager';
import { assertSafeOutboundUrl, safeAxiosTransport } from '../utils/outboundTarget';
export type ProxyTarget =
| { apiUrl: string; apiToken: ''; trustedLoopback: true }
| { apiUrl: string; apiToken: string; trustedLoopback: false };
/**
* NodeRegistry: Manages connections for multiple nodes.
@@ -105,18 +110,18 @@ export class NodeRegistry extends EventEmitter {
* bridge; the bridge strips the bearer token and re-authenticates
* implicitly via the pre-verified tunnel socket.
*/
public getProxyTarget(nodeId: number): { apiUrl: string; apiToken: string } | null {
public getProxyTarget(nodeId: number): ProxyTarget | null {
const node = DatabaseService.getInstance().getNode(nodeId);
if (!node || node.type !== 'remote') return null;
if (node.mode === 'pilot_agent') {
const loopbackUrl = PilotTunnelManager.getInstance().getLoopbackUrl(nodeId);
if (!loopbackUrl) return null;
return { apiUrl: loopbackUrl, apiToken: '' };
return { apiUrl: loopbackUrl, apiToken: '', trustedLoopback: true };
}
if (!node.api_url || !node.api_token) return null;
return { apiUrl: node.api_url, apiToken: node.api_token };
return { apiUrl: node.api_url, apiToken: node.api_token, trustedLoopback: false };
}
/**
@@ -127,7 +132,7 @@ export class NodeRegistry extends EventEmitter {
public async fetchMetaForNode(nodeId: number): Promise<RemoteMeta> {
const target = this.getProxyTarget(nodeId);
if (!target) return { ...OFFLINE_META };
return fetchRemoteMeta(target.apiUrl, target.apiToken);
return fetchRemoteMeta(target.apiUrl, target.apiToken, target.trustedLoopback);
}
/**
@@ -226,10 +231,16 @@ export class NodeRegistry extends EventEmitter {
const baseUrl = node.api_url.replace(/\/$/, '');
const headers = { Authorization: `Bearer ${node.api_token}` };
const requestConfig = {
...safeAxiosTransport(false),
headers,
timeout: 8000,
};
try {
await assertSafeOutboundUrl(baseUrl);
// Step 1: Verify auth. A 401 here means wrong token - surface that clearly.
const authRes = await axios.get(`${baseUrl}/api/auth/check`, { headers, timeout: 8000 });
const authRes = await axios.get(`${baseUrl}/api/auth/check`, requestConfig);
if (authRes.status !== 200) throw new Error(`Unexpected status ${authRes.status}`);
db.updateNodeStatus(node.id, 'online');
@@ -237,9 +248,9 @@ export class NodeRegistry extends EventEmitter {
// Step 2: Fetch Docker stats in parallel. Use allSettled so a slow or missing
// endpoint doesn't fail the whole test - each field falls back to '-' gracefully.
const [statsResult, sysResult, imagesResult, metaResult] = await Promise.allSettled([
axios.get(`${baseUrl}/api/stats`, { headers, timeout: 8000 }),
axios.get(`${baseUrl}/api/system/stats`, { headers, timeout: 8000 }),
axios.get(`${baseUrl}/api/system/images`, { headers, timeout: 8000 }),
axios.get(`${baseUrl}/api/stats`, requestConfig),
axios.get(`${baseUrl}/api/system/stats`, requestConfig),
axios.get(`${baseUrl}/api/system/images`, requestConfig),
fetchRemoteMeta(baseUrl, node.api_token!),
]);
@@ -7,8 +7,9 @@ import type { RegistryDeliveryEvidencePage } from '../types/registryDeliveryEvid
import { getErrorMessage } from '../utils/errors';
import { isDebugEnabled } from '../utils/debug';
import { DatabaseService } from './DatabaseService';
import { NodeRegistry } from './NodeRegistry';
import { NodeRegistry, type ProxyTarget } from './NodeRegistry';
import { PilotTunnelManager } from './PilotTunnelManager';
import { safeAxiosTransport } from '../utils/outboundTarget';
const RECONCILE_INTERVAL_MS = 5 * 60 * 1000;
const RECONCILE_INITIAL_DELAY_MS = 30_000;
@@ -140,7 +141,7 @@ export class RegistryDeliveryReconciler {
}
private async fetchEvidencePage(
target: { apiUrl: string; apiToken: string },
target: ProxyTarget,
cursor: number,
limit: number,
): Promise<RegistryDeliveryEvidencePage> {
@@ -151,6 +152,7 @@ export class RegistryDeliveryReconciler {
}
const res = await axios.get(`${base}/api/registry-delivery/evidence`, {
...safeAxiosTransport(target.trustedLoopback),
headers,
params: { cursor, limit },
timeout: 30_000,
+13 -11
View File
@@ -22,10 +22,11 @@ import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { safeRemoteFetch } from '../utils/outboundTarget';
import { formatNoTargetError } from '../utils/remoteTarget';
import { sanitizeForLog } from '../utils/safeLog';
import { captureLocalNodeFiles, captureRemoteNodeFiles, buildSnapshotDocumentation, type SnapshotNodeData } from '../utils/snapshot-capture';
import { NodeRegistry } from './NodeRegistry';
import { NodeRegistry, type ProxyTarget } from './NodeRegistry';
import { NotificationService } from './NotificationService';
import TrivyService from './TrivyService';
import type { ScanAllNodeImagesResult } from './TrivyService';
@@ -992,7 +993,7 @@ export class SchedulerService {
}
const startTime = Date.now();
try {
const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
const response = await safeRemoteFetch(`${baseUrl}/api/auto-update/execute`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -1001,7 +1002,7 @@ export class SchedulerService {
},
body: JSON.stringify({ target }),
signal: AbortSignal.timeout(300_000), // 5 minute timeout for long updates
});
}, proxyTarget.trustedLoopback);
if (!response.ok) {
throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
@@ -1027,7 +1028,7 @@ export class SchedulerService {
}
const startTime = Date.now();
try {
const response = await fetch(`${baseUrl}/api/auto-update/execute`, {
const response = await safeRemoteFetch(`${baseUrl}/api/auto-update/execute`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -1036,7 +1037,7 @@ export class SchedulerService {
},
body: JSON.stringify({ targets }),
signal: AbortSignal.timeout(300_000),
});
}, proxyTarget.trustedLoopback);
// Older remotes only accept { target }. Fall back to one call per
// stack so mixed-version fleets still complete the label schedule.
@@ -1109,7 +1110,7 @@ export class SchedulerService {
}
}
private requireRemoteProxyTarget(nodeId: number): { apiUrl: string; apiToken: string } {
private requireRemoteProxyTarget(nodeId: number): ProxyTarget {
const proxyTarget = NodeRegistry.getInstance().getProxyTarget(nodeId);
if (!proxyTarget) {
throw new Error(this.remoteProxyFailureMessage(nodeId, this.noProxyTargetDetail(nodeId)));
@@ -1187,13 +1188,13 @@ export class SchedulerService {
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
try {
const response = await fetch(`${baseUrl}/api/containers?all=true`, {
const response = await safeRemoteFetch(`${baseUrl}/api/containers?all=true`, {
headers: {
'Authorization': `Bearer ${proxyTarget.apiToken}`,
[PROXY_TIER_HEADER]: proxyHeaders.tier,
},
signal: AbortSignal.timeout(60_000),
});
}, proxyTarget.trustedLoopback);
if (!response.ok) {
throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
}
@@ -1218,7 +1219,7 @@ export class SchedulerService {
const baseUrl = proxyTarget.apiUrl.replace(/\/$/, '');
const proxyHeaders = LicenseService.getInstance().getProxyHeaders();
try {
const response = await fetch(
const response = await safeRemoteFetch(
`${baseUrl}/api/containers/${encodeURIComponent(containerId)}/${action}`,
{
method: 'POST',
@@ -1229,6 +1230,7 @@ export class SchedulerService {
},
signal: AbortSignal.timeout(300_000),
},
proxyTarget.trustedLoopback,
);
if (!response.ok) {
throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
@@ -1260,7 +1262,7 @@ export class SchedulerService {
if (!augmented.ok) {
throw new Error(augmented.error);
}
const response = await fetch(`${baseUrl}/api/stacks/${routeSuffix}`, {
const response = await safeRemoteFetch(`${baseUrl}/api/stacks/${routeSuffix}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -1270,7 +1272,7 @@ export class SchedulerService {
},
body: JSON.stringify(augmented.body),
signal: AbortSignal.timeout(300_000),
});
}, proxyTarget.trustedLoopback);
if (!response.ok) {
throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response)));
}
+7 -6
View File
@@ -9,6 +9,7 @@ import { resolveAllEnvFilePaths } from '../routes/stacks';
import { getErrorMessage } from '../utils/errors';
import { formatNoTargetError } from '../utils/remoteTarget';
import { isDebugEnabled } from '../utils/debug';
import { safeRemoteFetch } from '../utils/outboundTarget';
export type SecretKv = Record<string, string>;
export type DiffStatus = 'added' | 'changed' | 'removed' | 'unchanged';
@@ -225,10 +226,10 @@ async function resolveEnvFileRemote(node: Node, stackName: string, basename: str
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers: Record<string, string> = {};
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
const res = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/envs`, {
const res = await safeRemoteFetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/envs`, {
headers,
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
}, target.trustedLoopback);
if (!res.ok) {
if (res.status === 404 && basename === '.env') {
return { absolutePath: '.env' };
@@ -268,10 +269,10 @@ async function readEnvRemote(node: Node, stackName: string, absolutePath: string
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
const url = new URL(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`);
if (absolutePath !== '.env') url.searchParams.set('file', absolutePath);
const res = await fetch(url.toString(), {
const res = await safeRemoteFetch(url.toString(), {
headers,
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
}, target.trustedLoopback);
if (res.status === 404) return '';
if (!res.ok) throw new Error(`failed to read env (HTTP ${res.status})`);
return await res.text();
@@ -287,12 +288,12 @@ async function writeEnvRemote(node: Node, stackName: string, absolutePath: strin
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
const url = new URL(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`);
if (absolutePath !== '.env') url.searchParams.set('file', absolutePath);
const res = await fetch(url.toString(), {
const res = await safeRemoteFetch(url.toString(), {
method: 'PUT',
headers,
body: JSON.stringify({ content }),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
}, target.trustedLoopback);
if (!res.ok) {
const body = await res.text().catch(() => '');
throw new Error(`failed to write env (HTTP ${res.status}${body ? ': ' + body.slice(0, 200) : ''})`);
+3 -2
View File
@@ -9,6 +9,7 @@ import { HealthGateService } from './HealthGateService';
import { LicenseService } from './LicenseService';
import { PROXY_TIER_HEADER } from './license-headers';
import { NodeRegistry } from './NodeRegistry';
import { safeRemoteFetch } from '../utils/outboundTarget';
import { getErrorMessage } from '../utils/errors';
import { redactSensitiveText } from '../utils/safeLog';
import { isValidStackName } from '../utils/validation';
@@ -368,12 +369,12 @@ export class WebhookService {
}
bodyToSend = augmented.body;
}
return await fetch(url, {
return await safeRemoteFetch(url, {
method,
headers,
body: method === 'GET' || bodyToSend === undefined ? undefined : JSON.stringify(bodyToSend),
signal: controller.signal,
});
}, target.trustedLoopback);
} catch (err) {
if (controller.signal.aborted) {
throw new Error('Remote node request timed out', { cause: err });
+12
View File
@@ -27,6 +27,9 @@ export type TransportFacingCode =
/** Structured failure raised by the native transport; classified below. */
export type TransportFailureReason =
| 'invalid-url'
| 'unsafe-target'
| 'target-unresolved'
| 'ssh-auth-required'
| 'invalid-ref'
| 'git-missing'
| 'git-old'
@@ -52,6 +55,9 @@ interface TransportFailureBase {
*/
export type TransportFailure = TransportFailureBase & (
| { reason: 'invalid-url' }
| { reason: 'unsafe-target' }
| { reason: 'target-unresolved' }
| { reason: 'ssh-auth-required' }
| { reason: 'invalid-ref' }
| { reason: 'git-missing'; stderr?: string }
| { reason: 'git-old'; stderr?: string }
@@ -108,6 +114,12 @@ export function classifyGitFailure(
switch (failure.reason) {
case 'invalid-url':
return { code: 'GIT_ERROR', message: 'Unsupported repository URL. Use https:// or SSH (git@host:org/repo.git or ssh://) without embedded credentials.' };
case 'unsafe-target':
return { code: 'GIT_ERROR', message: 'Repository host is not allowed.' };
case 'target-unresolved':
return { code: 'NETWORK_TIMEOUT', message: `Could not resolve${dest}. Check the repository URL and your network or DNS.` };
case 'ssh-auth-required':
return { code: 'GIT_ERROR', message: 'SSH repository URLs require a deploy key.' };
case 'invalid-ref':
return { code: 'GIT_ERROR', message: 'Unsupported ref name. Use a branch name, a tag name, or a full commit SHA as the remote reports it.' };
case 'git-missing':
+68 -9
View File
@@ -22,6 +22,7 @@ import {
type ParsedRepoUrl,
} from './sshTrust';
import { writeDeployKey, writeKnownHosts } from './sshCredentialFiles';
import { resolveSafeOutboundHostname, UnsafeOutboundTargetError } from '../../utils/outboundTarget';
/**
* Native git transport: every Git operation is an `execFile`-style spawn of
@@ -32,8 +33,8 @@ import { writeDeployKey, writeKnownHosts } from './sshCredentialFiles';
* - `GIT_CONFIG_NOSYSTEM=1` plus an isolated empty HOME/USERPROFILE so the
* operator's ~/.gitconfig (credential helpers, insteadOf rewrites, hooks)
* cannot influence fetches.
* - `protocol.allow=never` with only https re-enabled: no file://, git://,
* ext::, or ssh:// this early in the program.
* - `protocol.allow=never` with only the validated target protocol (HTTPS or
* SSH) re-enabled: file://, git://, and ext:: remain blocked.
* - `core.hooksPath` pointed at an empty directory we own, so repository
* scripts can never run. (A literal /dev/null works on Linux but not
* Windows; an empty dir is portable.)
@@ -293,6 +294,12 @@ function buildEnv(
// An inherited trace flag would widen the log surface with packet
// dumps that can carry URL material.
GIT_TRACE: '',
HTTP_PROXY: '',
HTTPS_PROXY: '',
ALL_PROXY: '',
http_proxy: '',
https_proxy: '',
all_proxy: '',
HOME: homeDir,
};
if (process.platform === 'win32') {
@@ -449,6 +456,7 @@ async function commonArgs(
*/
async function prepareInvocation(
workspaceRoot: string,
target: ResolvedRepoTarget,
repoUrl: string,
token?: string | null,
sshAuth?: ResolveRequest['sshAuth'],
@@ -456,10 +464,18 @@ async function prepareInvocation(
): Promise<{ layout: WorkspaceLayout; env: NodeJS.ProcessEnv; baseArgs: string[]; allowedHost: string | null; caPath: string | null }> {
const layout = await prepareWorkspace(workspaceRoot);
let sshCommand: string | null = null;
if (sshAuth) {
if (target.kind === 'ssh') {
if (!sshAuth) {
throw {
transportFailure: true,
reason: 'ssh-auth-required',
host: target.sshTarget.hostKeyAlias,
hasToken: Boolean(token),
} satisfies TransportFailure;
}
const keyPath = await writeDeployKey(layout.metaDir, sshAuth.privateKey);
const knownPath = await writeKnownHosts(layout.metaDir, sshAuth.knownHostsEntry);
sshCommand = buildSshCommand(keyPath, knownPath);
sshCommand = buildSshCommand(keyPath, knownPath, target.sshTarget);
}
const helperPath = token ? await writeCredentialHelper(layout.metaDir) : null;
const parsed = parseRepoTransportUrl(repoUrl);
@@ -467,7 +483,8 @@ async function prepareInvocation(
? credentialScopeHost(parsed.host)
: null;
const env = buildEnv(layout.homeDir, token, helperPath, sshCommand, allowedHost);
const { args: baseArgs, caPath } = await commonArgs(layout, helperPath, Boolean(sshAuth), caBundlePem);
const { args: commonBaseArgs, caPath } = await commonArgs(layout, helperPath, target.kind === 'ssh', caBundlePem);
const baseArgs = [...commonBaseArgs, ...target.gitArgs];
return { layout, env, baseArgs, allowedHost, caPath };
}
@@ -765,6 +782,7 @@ export async function verifyFastForward(req: {
const hasToken = Boolean(req.token) || Boolean(req.sshAuth);
await ensureBinaryReady(hasToken);
const repo = assertValidRepoUrl(req.repoUrl, hasToken);
const target = await resolveSafeRepoTarget(repo, hasToken);
const host = repoHostLabel(repo);
const timeoutMs = req.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const deadline = Date.now() + timeoutMs;
@@ -775,7 +793,7 @@ export async function verifyFastForward(req: {
}
};
const { env, baseArgs, caPath } = await prepareInvocation(
req.workspaceRoot, req.repoUrl, req.token, req.sshAuth, req.caBundlePem,
req.workspaceRoot, target, req.repoUrl, req.token, req.sshAuth, req.caBundlePem,
);
// Resolved once if the host refuses a redirect, then reused by the deepen
// rounds so they do not each re-walk the same chain.
@@ -988,6 +1006,46 @@ export async function verifyFastForward(req: {
}
}
type ResolvedRepoTarget =
| { kind: 'https'; gitArgs: string[] }
| { kind: 'ssh'; gitArgs: []; sshTarget: { address: string; hostKeyAlias: string } };
async function resolveSafeRepoTarget(repo: ParsedRepoUrl, hasToken: boolean): Promise<ResolvedRepoTarget> {
try {
if (repo.kind === 'https') {
const url = new URL(repo.href);
const [{ address, family }] = await resolveSafeOutboundHostname(url.hostname);
const port = url.port || '443';
const curlAddress = family === 6 ? `[${address}]` : address;
return {
kind: 'https',
gitArgs: [
'-c', 'http.followRedirects=false',
'-c', 'http.proxy=',
'-c', `http.curloptResolve=${url.hostname}:${port}:${curlAddress}`,
],
};
}
const [{ address }] = await resolveSafeOutboundHostname(repo.host);
const hostKeyAlias = repo.port && repo.port !== 22
? `[${repo.host}]:${repo.port}`
: repo.host;
return {
kind: 'ssh',
gitArgs: [],
sshTarget: { address, hostKeyAlias },
};
} catch (error: unknown) {
if (!(error instanceof UnsafeOutboundTargetError)) throw error;
throw {
transportFailure: true,
reason: error.reason === 'blocked' ? 'unsafe-target' : 'target-unresolved',
host: repoHostLabel(repo),
hasToken,
} satisfies TransportFailure;
}
}
export const nativeGitTransport: GitTransport = {
async resolveRef(req: ResolveRequest): Promise<ResolveResult> {
const hasToken = Boolean(req.token) || Boolean(req.sshAuth);
@@ -1003,9 +1061,9 @@ export const nativeGitTransport: GitTransport = {
}
assertValidRef(req.ref, repoHostLabel(repo), hasToken);
const target = await resolveSafeRepoTarget(repo, hasToken);
const { env, baseArgs, caPath } = await prepareInvocation(
req.workspaceRoot, req.repoUrl, req.token, req.sshAuth, req.caBundlePem,
req.workspaceRoot, target, req.repoUrl, req.token, req.sshAuth, req.caBundlePem,
);
const found = await lsRemoteRefs(
repo, req.ref, env, baseArgs,
@@ -1020,6 +1078,7 @@ export const nativeGitTransport: GitTransport = {
const hasToken = Boolean(req.token) || Boolean(req.sshAuth);
await ensureBinaryReady(hasToken);
const repo = assertValidRepoUrl(req.repoUrl, hasToken);
const target = await resolveSafeRepoTarget(repo, hasToken);
assertValidRef(req.ref, repoHostLabel(repo), hasToken);
// The credential scope host is set inside prepareInvocation via
@@ -1027,7 +1086,7 @@ export const nativeGitTransport: GitTransport = {
// credentials for any other host. A refused redirect is resolved and
// approved by the preflight below before any retry.
const { layout, env, baseArgs, caPath } = await prepareInvocation(
req.workspaceRoot, req.repoUrl, req.token, req.sshAuth, req.caBundlePem,
req.workspaceRoot, target, req.repoUrl, req.token, req.sshAuth, req.caBundlePem,
);
const checkout = path.join(req.workspaceRoot, 'repo');
const timeoutMs = req.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+33 -20
View File
@@ -68,7 +68,7 @@ export function parseSshUrl(raw: string): ParsedSshRepoUrl | null {
if (pathname === '/' || pathname.includes('..')) return null;
const user = url.username;
const href = port === DEFAULT_SSH_PORT
? `${user}@${url.hostname}:${pathname.slice(1)}`
? `${user}@${url.hostname}:${pathname}`
: `ssh://${user}@${url.hostname}:${port}${pathname}`;
return { href, host: url.hostname, port, pathname };
}
@@ -206,11 +206,11 @@ export interface ScannedHostKey {
line: string;
}
function runSshKeyscan(host: string, port: number): Promise<{ stdout: string; stderr: string; exitCode: number }> {
function runSshKeyscan(address: string, port: number): Promise<{ stdout: string; stderr: string; exitCode: number }> {
return new Promise((resolve, reject) => {
const args = port === DEFAULT_SSH_PORT
? ['-H', host]
: ['-p', String(port), '-H', host];
? [address]
: ['-p', String(port), address];
const child = spawn('ssh-keyscan', args, { windowsHide: true });
let stdout = '';
let stderr = '';
@@ -231,8 +231,8 @@ function runSshKeyscan(host: string, port: number): Promise<{ stdout: string; st
}
/** Fetch host keys from the server without trusting them (probe step only). */
export async function scanHostKeys(host: string, port: number): Promise<ScannedHostKey[]> {
const result = await runSshKeyscan(host, port);
export async function scanHostKeys(host: string, port: number, address: string): Promise<ScannedHostKey[]> {
const result = await runSshKeyscan(address, port);
if (result.exitCode !== 0 && !result.stdout.trim()) {
throw new Error(result.stderr.trim() || 'ssh-keyscan failed');
}
@@ -240,11 +240,13 @@ export async function scanHostKeys(host: string, port: number): Promise<ScannedH
for (const line of result.stdout.split(/\r?\n/)) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const fingerprint = fingerprintFromKnownHostsLine(trimmed);
if (!fingerprint) continue;
const material = keyMaterialFromKnownHostsLine(trimmed);
const keyType = material?.keyType ?? 'unknown';
keys.push({ keyType, fingerprint, line: trimmed });
if (!material) continue;
const knownHost = port === DEFAULT_SSH_PORT ? host : `[${host}]:${port}`;
const knownHostsLine = `${knownHost} ${material.keyType} ${material.keyBase64}`;
const fingerprint = fingerprintFromKnownHostsLine(knownHostsLine);
if (!fingerprint) continue;
keys.push({ keyType: material.keyType, fingerprint, line: knownHostsLine });
}
if (keys.length === 0) {
throw new Error('No host keys returned from ssh-keyscan');
@@ -256,17 +258,28 @@ export async function scanHostKeys(host: string, port: number): Promise<ScannedH
* Build GIT_SSH_COMMAND / core.sshCommand value enforcing strict host-key
* checking against our per-fetch known_hosts file and a single deploy key.
*/
export function buildSshCommand(keyPath: string, knownHostsPath: string): string {
function shellQuote(value: string): string {
return `'${value.replace(/'/g, `'"'"'`)}'`;
}
export function buildSshCommand(
keyPath: string,
knownHostsPath: string,
target: { address: string; hostKeyAlias: string },
): string {
const key = keyPath.split(path.sep).join('/');
const known = knownHostsPath.split(path.sep).join('/');
return [
const args = [
'ssh',
'-o', 'BatchMode=yes',
'-o', 'StrictHostKeyChecking=yes',
'-o', 'UserKnownHostsFile=' + known,
'-o', 'IdentitiesOnly=yes',
'-o', 'IdentityAgent=none',
'-F', '/dev/null',
'-i', key,
].join(' ');
'-o BatchMode=yes',
'-o StrictHostKeyChecking=yes',
`-o ${shellQuote(`UserKnownHostsFile=${known}`)}`,
'-o IdentitiesOnly=yes',
'-o IdentityAgent=none',
'-F /dev/null',
`-i ${shellQuote(key)}`,
];
args.push(`-o ${shellQuote(`Hostname=${target.address}`)}`);
args.push(`-o ${shellQuote(`HostKeyAlias=${target.hostKeyAlias}`)}`);
return args.join(' ');
}
+211
View File
@@ -0,0 +1,211 @@
import dns, { promises as dnsPromises, type LookupAddress, type LookupAllOptions } from 'dns';
import http from 'http';
import https from 'https';
import net, { type LookupFunction } from 'net';
import { Agent as UndiciAgent, fetch as undiciFetch, type RequestInfo, type RequestInit, type Response } from 'undici';
const blockedIpv4 = new net.BlockList();
blockedIpv4.addSubnet('0.0.0.0', 8, 'ipv4');
blockedIpv4.addSubnet('127.0.0.0', 8, 'ipv4');
blockedIpv4.addSubnet('169.254.0.0', 16, 'ipv4');
blockedIpv4.addSubnet('192.0.0.0', 24, 'ipv4');
blockedIpv4.addSubnet('192.0.2.0', 24, 'ipv4');
blockedIpv4.addSubnet('192.88.99.0', 24, 'ipv4');
blockedIpv4.addSubnet('198.18.0.0', 15, 'ipv4');
blockedIpv4.addSubnet('198.51.100.0', 24, 'ipv4');
blockedIpv4.addSubnet('203.0.113.0', 24, 'ipv4');
blockedIpv4.addSubnet('224.0.0.0', 4, 'ipv4');
blockedIpv4.addSubnet('240.0.0.0', 4, 'ipv4');
blockedIpv4.addAddress('100.100.100.200', 'ipv4');
const blockedIpv6 = new net.BlockList();
blockedIpv6.addAddress('::', 'ipv6');
blockedIpv6.addAddress('::1', 'ipv6');
blockedIpv6.addSubnet('100::', 64, 'ipv6');
blockedIpv6.addSubnet('2001:db8::', 32, 'ipv6');
blockedIpv6.addSubnet('fe80::', 10, 'ipv6');
blockedIpv6.addSubnet('ff00::', 8, 'ipv6');
blockedIpv6.addAddress('fd00:ec2::254', 'ipv6');
const loopbackIpv4 = new net.BlockList();
loopbackIpv4.addSubnet('127.0.0.0', 8, 'ipv4');
export class UnsafeOutboundTargetError extends Error {
public readonly reason: 'blocked' | 'unresolved';
public readonly code = 'EACCES';
public constructor(reason: 'blocked' | 'unresolved') {
super(reason === 'blocked'
? 'The target address is not allowed.'
: 'The target host could not be resolved.');
this.name = 'UnsafeOutboundTargetError';
this.reason = reason;
}
}
export function isBlockedOutboundAddress(address: string): boolean {
const family = net.isIP(address);
if (family === 4) return blockedIpv4.check(address, 'ipv4');
if (family === 6) {
const mappedIpv4 = ipv4FromMappedIpv6(address);
return mappedIpv4
? blockedIpv4.check(mappedIpv4, 'ipv4')
: blockedIpv6.check(address, 'ipv6');
}
return true;
}
function isE2eLoopbackAllowed(address: string): boolean {
if (process.env.NODE_ENV !== 'test' || process.env.SENCHO_E2E_ALLOW_LOOPBACK_OUTBOUND !== 'true') {
return false;
}
if (net.isIPv4(address)) return loopbackIpv4.check(address, 'ipv4');
if (!net.isIPv6(address)) return false;
const mappedIpv4 = ipv4FromMappedIpv6(address);
return mappedIpv4
? loopbackIpv4.check(mappedIpv4, 'ipv4')
: address === '::1';
}
function isDisallowedOutboundAddress(address: string): boolean {
return isBlockedOutboundAddress(address) && !isE2eLoopbackAllowed(address);
}
function ipv4FromMappedIpv6(address: string): string | null {
const mapped = address.match(/^(?:::ffff:|0:0:0:0:0:ffff:)(.+)$/i)?.[1];
if (!mapped) return null;
if (net.isIPv4(mapped)) return mapped;
const words = mapped.split(':');
if (words.length !== 2 || words.some((word) => !/^[0-9a-f]{1,4}$/i.test(word))) return null;
const high = Number.parseInt(words[0], 16);
const low = Number.parseInt(words[1], 16);
return `${high >> 8}.${high & 0xff}.${low >> 8}.${low & 0xff}`;
}
function lookupHostname(url: URL): string {
return url.hostname.startsWith('[') && url.hostname.endsWith(']')
? url.hostname.slice(1, -1)
: url.hostname;
}
export async function assertSafeOutboundHostname(hostname: string): Promise<void> {
await resolveSafeOutboundHostname(hostname);
}
type ResolveAllAddresses = (hostname: string) => Promise<LookupAddress[]>;
type ResolvedOutboundAddresses = [LookupAddress, ...LookupAddress[]];
const systemResolveAllAddresses: ResolveAllAddresses = (hostname) =>
dnsPromises.lookup(hostname, { all: true, verbatim: true });
export async function resolveSafeOutboundHostname(
hostname: string,
resolveAllAddresses: ResolveAllAddresses = systemResolveAllAddresses,
): Promise<ResolvedOutboundAddresses> {
const normalizedHostname = hostname.startsWith('[') && hostname.endsWith(']')
? hostname.slice(1, -1)
: hostname;
if (net.isIP(normalizedHostname) !== 0) {
if (isDisallowedOutboundAddress(normalizedHostname)) throw new UnsafeOutboundTargetError('blocked');
return [{ address: normalizedHostname, family: net.isIPv4(normalizedHostname) ? 4 : 6 }];
}
let addresses: LookupAddress[];
try {
addresses = await resolveAllAddresses(normalizedHostname);
} catch {
throw new UnsafeOutboundTargetError('unresolved');
}
const [first, ...rest] = addresses;
if (!first) throw new UnsafeOutboundTargetError('unresolved');
if (addresses.some(({ address }) => isDisallowedOutboundAddress(address))) {
throw new UnsafeOutboundTargetError('blocked');
}
return [first, ...rest];
}
type LookupAllAddresses = (
hostname: string,
options: LookupAllOptions,
callback: (error: NodeJS.ErrnoException | null, addresses: LookupAddress[]) => void,
) => void;
const systemLookupAllAddresses: LookupAllAddresses = (hostname, options, callback) => {
dns.lookup(hostname, options, callback);
};
export function createSafeOutboundLookup(lookupAllAddresses: LookupAllAddresses): LookupFunction {
return (hostname, options, callback): void => lookupAllAddresses(hostname, { ...options, all: true }, (error, addresses) => {
if (error) {
callback(error, '', 0);
return;
}
if (!Array.isArray(addresses) || addresses.length === 0) {
callback(new UnsafeOutboundTargetError('unresolved'), '', 0);
return;
}
if (addresses.some(({ address }) => isDisallowedOutboundAddress(address))) {
callback(new UnsafeOutboundTargetError('blocked'), '', 0);
return;
}
if (options.all) {
callback(null, addresses);
return;
}
callback(null, addresses[0].address, addresses[0].family);
});
}
export const safeOutboundLookup = createSafeOutboundLookup(systemLookupAllAddresses);
export const safeHttpAgent = new http.Agent({ lookup: safeOutboundLookup });
export const safeHttpsAgent = new https.Agent({ lookup: safeOutboundLookup });
const safeFetchDispatcher = new UndiciAgent({ connect: { lookup: safeOutboundLookup } });
export function safeAxiosTransport(trustedLoopback = false): {
maxRedirects: number;
proxy: false;
httpAgent?: http.Agent;
httpsAgent?: https.Agent;
} {
return {
maxRedirects: 0,
proxy: false,
...(trustedLoopback ? {} : { httpAgent: safeHttpAgent, httpsAgent: safeHttpsAgent }),
};
}
export async function safeRemoteFetch(
input: RequestInfo,
init: RequestInit = {},
trustedLoopback = false,
): Promise<Response> {
if (!trustedLoopback) {
const raw = input instanceof URL
? input.toString()
: typeof input === 'string' ? input : input.url;
const host = lookupHostname(new URL(raw));
if (net.isIP(host) !== 0 && isDisallowedOutboundAddress(host)) {
throw new UnsafeOutboundTargetError('blocked');
}
}
try {
return await undiciFetch(input, {
...init,
...(trustedLoopback ? {} : { dispatcher: safeFetchDispatcher }),
redirect: 'error',
});
} catch (error: unknown) {
const cause = error instanceof Error ? error.cause : undefined;
if (cause instanceof UnsafeOutboundTargetError) throw cause;
throw error;
}
}
export async function assertSafeOutboundUrl(
raw: string,
): Promise<URL> {
const url = new URL(raw);
await assertSafeOutboundHostname(lookupHostname(url));
return url;
}
+9 -8
View File
@@ -9,6 +9,7 @@ import { FileSystemService } from '../services/FileSystemService';
import { NodeRegistry } from '../services/NodeRegistry';
import { formatNoTargetError } from './remoteTarget';
import { isDebugEnabled } from './debug';
import { safeRemoteFetch } from './outboundTarget';
// Presence map over every operator-authored dossier field. Typing it as
// Record<keyof StackDossierFields, true> makes the build fail if a field is
@@ -215,10 +216,10 @@ export async function captureRemoteNodeFiles(node: CaptureNode, captureDocs = fa
const headers: Record<string, string> = {};
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
const stacksRes = await fetch(`${baseUrl}/api/stacks`, {
const stacksRes = await safeRemoteFetch(`${baseUrl}/api/stacks`, {
headers,
signal: AbortSignal.timeout(15000),
});
}, target.trustedLoopback);
if (!stacksRes.ok) throw new Error('Failed to fetch stacks from remote node');
const stackNames = await stacksRes.json() as string[];
@@ -231,10 +232,10 @@ export async function captureRemoteNodeFiles(node: CaptureNode, captureDocs = fa
let composeContent: string;
try {
const composeRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
const composeRes = await safeRemoteFetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}`, {
headers,
signal: AbortSignal.timeout(15000),
});
}, target.trustedLoopback);
if (!composeRes.ok) {
const reason = `compose.yaml fetch failed (HTTP ${composeRes.status}); stack skipped`;
console.warn(`[Fleet Snapshot] ${reason} ("${stackName}" on "${node.name}")`);
@@ -255,10 +256,10 @@ export async function captureRemoteNodeFiles(node: CaptureNode, captureDocs = fa
files.push({ filename: 'compose.yaml', content: composeContent });
try {
const envRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
const envRes = await safeRemoteFetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/env`, {
headers,
signal: AbortSignal.timeout(15000),
});
}, target.trustedLoopback);
// The remote replies 200 with an empty body and X-Env-Exists: false when a
// stack has no .env. Treat that as absent (matching the local ENOENT path)
// so restore does not write a spurious empty .env. An older remote that
@@ -280,10 +281,10 @@ export async function captureRemoteNodeFiles(node: CaptureNode, captureDocs = fa
let dossier: StackDossierFields | undefined;
if (captureDocs) {
try {
const dossierRes = await fetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/dossier`, {
const dossierRes = await safeRemoteFetch(`${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/dossier`, {
headers,
signal: AbortSignal.timeout(15000),
});
}, target.trustedLoopback);
if (dossierRes.ok) {
const fields = pickDossierFields(await dossierRes.json() as Record<string, unknown>);
if (dossierHasContent(fields)) dossier = fields;
+7 -4
View File
@@ -1,5 +1,7 @@
import path from 'path';
import net from 'net';
import { sanitizeForLog } from './safeLog';
import { isBlockedOutboundAddress } from './outboundTarget';
/**
* Stack name must only contain URL-safe characters with no path separators.
@@ -33,12 +35,13 @@ export function isValidRemoteUrl(
if (!['http:', 'https:'].includes(url.protocol)) {
return { valid: false, reason: 'API URL must use http:// or https://' };
}
// Node.js URL API preserves brackets for IPv6: new URL('http://[::1]').hostname === '[::1]'
const loopback = /^(localhost|127(\.\d+){3}|\[::1\]|0\.0\.0\.0)$/i;
if (loopback.test(url.hostname)) {
const hostname = url.hostname.startsWith('[') && url.hostname.endsWith(']')
? url.hostname.slice(1, -1)
: url.hostname;
if (hostname.toLowerCase() === 'localhost' || (net.isIP(hostname) !== 0 && isBlockedOutboundAddress(hostname))) {
return {
valid: false,
reason: 'API URL cannot point to localhost or loopback - use the actual host address',
reason: 'API URL target is not allowed',
};
}
return { valid: true, url };
+22 -4
View File
@@ -6,6 +6,14 @@ import { wsProxyServer } from '../proxy/websocketProxy';
import { getErrorMessage } from '../utils/errors';
import { rejectUpgrade as reject } from './reject';
import { consoleSessionPathForPathname } from '../helpers/consoleSession';
import type { ProxyTarget } from '../services/NodeRegistry';
import {
assertSafeOutboundUrl,
safeHttpAgent,
safeHttpsAgent,
safeRemoteFetch,
UnsafeOutboundTargetError,
} from '../utils/outboundTarget';
/**
* Forward a WebSocket upgrade to a remote Sencho instance. Handles the
@@ -26,7 +34,7 @@ export async function handleRemoteForwarder(
head: Buffer,
opts: {
pathname: string;
target: { apiUrl: string; apiToken: string };
target: ProxyTarget;
/** Hub browser operator; recorded as acting_as on the remote audit trail. */
actingAs?: string;
},
@@ -35,7 +43,16 @@ export async function handleRemoteForwarder(
if (!target.apiUrl) return reject(socket, 503, 'Service Unavailable');
const wsTarget = target.apiUrl.replace(/\/$/, '').replace(/^https?/, (m) => m === 'https' ? 'wss' : 'ws');
const isPilotLoopback = target.apiToken === '';
const isPilotLoopback = target.trustedLoopback;
if (!isPilotLoopback) {
try {
await assertSafeOutboundUrl(target.apiUrl);
} catch (error: unknown) {
const reason = error instanceof UnsafeOutboundTargetError ? error.reason : 'validation-error';
console.error(`[WS Proxy] Refused remote target (${reason}):`, getErrorMessage(error, 'unknown'));
return reject(socket, 502, 'Bad Gateway');
}
}
// Interactive console paths (host console / container exec) are guarded on
// the remote by an isProxyToken check that rejects the long-lived api_token.
@@ -49,7 +66,7 @@ export async function handleRemoteForwarder(
if (sessionPath && !isPilotLoopback) {
try {
const consoleHeaders = LicenseService.getInstance().getProxyHeaders();
const tokenRes = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/system/console-token`, {
const tokenRes = await safeRemoteFetch(`${target.apiUrl.replace(/\/$/, '')}/api/system/console-token`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${target.apiToken}`,
@@ -98,5 +115,6 @@ export async function handleRemoteForwarder(
const fwdUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
fwdUrl.searchParams.delete('nodeId');
req.url = fwdUrl.pathname + (fwdUrl.searchParams.toString() ? `?${fwdUrl.searchParams.toString()}` : '');
wsProxyServer.ws(req, socket, head, { target: wsTarget });
const agent = target.apiUrl.startsWith('https:') ? safeHttpsAgent : safeHttpAgent;
wsProxyServer.ws(req, socket, head, isPilotLoopback ? { target: wsTarget } : { target: wsTarget, agent });
}
+1
View File
@@ -9,6 +9,7 @@ export default defineConfig({
// Build the baseline DB (schema + migrations + admin seed) once; each
// test file's setupTestDb copies it instead of re-running migrations.
globalSetup: ['./src/__tests__/helpers/vitestGlobalSetup.ts'],
setupFiles: ['./src/__tests__/helpers/allowLoopbackTargets.ts'],
// Each test file gets its own worker so singletons are fresh between files.
pool: 'forks',
// Cap concurrency: each worker dynamic-imports the full Express stack
+2
View File
@@ -97,6 +97,8 @@ Tick **Deploy after create** to run `docker compose up -d` immediately after the
| **Authentication** | **Public (no auth)** for public repos, **Personal Access Token** for private HTTPS repos, or **SSH deploy key** for private SSH repos |
| **Apply behavior** | See the three modes below |
Private repository hosts on your LAN or VPN are supported. Sencho refuses repository targets that resolve to loopback, link-local, multicast, or selected special-use addresses.
Saving runs a reachability check against the repository. If the URL is wrong, the token is invalid, the ref does not exist, or a file is missing, Sencho surfaces the error inline and nothing is persisted.
### Multiple compose files
+2
View File
@@ -271,6 +271,8 @@ http://100.64.0.2:1852 ← Tailscale IP, encrypted by the VPN tunnel
All traffic between nodes is encrypted by the VPN. Sencho does not need to do anything additional.
Remote node URLs may use private LAN, VPC, or VPN addresses. Sencho refuses targets that resolve to loopback, link-local, multicast, or selected special-use addresses.
#### Reverse proxy (Caddy, Nginx, Traefik)
If you prefer TLS termination at each node, place a reverse proxy in front of each Sencho instance. [Caddy](https://caddyserver.com/) is the simplest option; it auto-provisions HTTPS certificates from Let's Encrypt with zero configuration:
+4 -5
View File
@@ -55,6 +55,7 @@ These tune optional subsystems. Most deployments never set them; the defaults ar
| `GITSOURCE_MAX_PATH_DEPTH` | `64` | Maximum directory depth for a materialized repository path. Deeper paths are refused. |
| `GITSOURCE_MAX_FILE_BYTES` | `10485760` | Maximum size of a single materialized file (10 MB). Oversized files are refused. |
| `SENCHO_PUBLIC_URL` | *(request host)* | Set on the primary instance. Its externally reachable `http(s)://` URL, no trailing slash, baked into pilot enrollment so remote agents dial the public hostname rather than the address the admin used at setup. |
| `SENCHO_TRUSTED_PROXY_CIDRS` | *(unset)* | Comma-separated CIDRs of reverse proxy peers trusted to supply forwarded client addresses and schemes. Use `/32` for one IPv4 proxy, `/128` for one IPv6 proxy, or the proxy network CIDR. Unset or invalid values make Sencho ignore forwarding headers. |
| `SENCHO_COMPOSE_COMMAND_TIMEOUT_MS` | `1800000` | Hard timeout for a single Compose command (pull, up, down) during deploy and update, in milliseconds (30 minutes). Sencho kills the command and reports failure if it runs longer than this, regardless of whether it is still producing output. Raise it only for very large images or slow storage. |
| `SENCHO_COMPOSE_STALL_TIMEOUT_MS` | `600000` | Idle-output backstop for deploy and update Compose steps (pull and recreate), separate from the hard timeout above. If a step produces no output for this long while still running, Sencho stops it so a hung image pull surfaces a clear failure and the in-app recovery actions instead of spinning. Raise it on slow links or for heavy local image builds. |
| `SENCHO_ZFS_ARCSTATS_PATH` | *(auto)* | Path **inside the container** to the OpenZFS ARC kstat file, for [ZFS ARC-aware host memory](#zfs-arc-aware-host-memory). Sencho checks this path first, then `/host/proc/spl/kstat/zfs/arcstats`, then `/proc/spl/kstat/zfs/arcstats`. Set it only when your ARC stats live at a non-standard path. |
@@ -62,10 +63,6 @@ These tune optional subsystems. Most deployments never set them; the defaults ar
Running a remote host as a pilot agent uses four more variables (`SENCHO_MODE`, `SENCHO_PRIMARY_URL`, `SENCHO_ENROLL_TOKEN`, and `SENCHO_PILOT_CA_FILE`), set only on the remote agent container. Sencho bakes them into the enrollment Compose file it generates, so you rarely write them by hand. See [Pilot Agent](/features/pilot-agent) for the full enrollment walkthrough.
| Variable | Default | Description |
|----------|---------|-------------|
| `SENCHO_TRUSTED_PROXY_CIDRS` | *(unset)* | Comma-separated CIDRs of reverse proxies that may set `X-Forwarded-Proto` for Pilot Agent TLS termination. When unset or invalid, non-TLS Pilot upgrades are treated as non-confidential and hub registry credential delivery is skipped for that hop. Set this when a TLS-terminating proxy sits in front of the primary and pilots connect through it. |
## ZFS ARC-aware host memory
On OpenZFS hosts (TrueNAS SCALE, Proxmox, ZFS on Ubuntu or Debian) the ZFS ARC cache can hold a large share of RAM. ARC is reclaimable on demand, but the Linux kernel reports it as unavailable, so a naive reading counts ARC as used memory and can raise false host-memory alerts.
@@ -249,7 +246,7 @@ services:
## Reverse proxy setup
Sencho works behind any reverse proxy. The only requirement is that WebSocket connections are forwarded correctly (used for live logs, container terminals, and the host console).
Sencho works behind any reverse proxy. Forward WebSocket upgrades, the original client address, and the original request scheme. Set `SENCHO_TRUSTED_PROXY_CIDRS` to the direct proxy as a CIDR (for example, `192.168.1.50/32` for one IPv4 address or `fd12:3456:789a::50/128` for one IPv6 address), or to the proxy network CIDR, so Sencho accepts those forwarding headers only from that peer.
### Nginx
@@ -268,6 +265,8 @@ server {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 3600s;
}
}
+1 -1
View File
@@ -49,7 +49,7 @@ test.describe('Node management', () => {
// Scope to the dialog so we target the submit button, not the trigger
await page.getByRole('dialog').getByRole('button', { name: /add node/i }).click();
await expect(page.getByText(/loopback|localhost/i)).toBeVisible({ timeout: 5_000 });
await expect(page.getByText(/target is not allowed/i)).toBeVisible({ timeout: 5_000 });
});
test('adding a node with an invalid URL shows an error', async ({ page }) => {