diff --git a/.env.example b/.env.example index 14df7b50..a088e1f4 100644 --- a/.env.example +++ b/.env.example @@ -53,6 +53,11 @@ SENCHO_PUBLIC_URL= # this unset. SENCHO_MODE= +# Comma-separated CIDRs of reverse proxies trusted to set X-Forwarded-Proto +# for Pilot Agent TLS termination. Unset or invalid: non-TLS Pilot upgrades are +# treated as non-confidential and hub registry credential delivery is skipped. +SENCHO_TRUSTED_PROXY_CIDRS= + # WebSocket-capable URL of the controlling Sencho instance. Use https:// # scheme; the agent rewrites it to wss:// for the tunnel upgrade. SENCHO_PRIMARY_URL= diff --git a/backend/src/__tests__/blueprints-compose-apply.test.ts b/backend/src/__tests__/blueprints-compose-apply.test.ts index 4c764ff9..27e49da7 100644 --- a/backend/src/__tests__/blueprints-compose-apply.test.ts +++ b/backend/src/__tests__/blueprints-compose-apply.test.ts @@ -186,6 +186,8 @@ describe('Blueprint compose apply (real filesystem)', () => { ).rejects.toThrow(/deploy blew up/); expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(priorMarker); + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')) + .toBe('services:\n old:\n image: nginx\n'); expect(await fsPromises.access(stackDir).then(() => true, () => false)).toBe(true); }); @@ -214,6 +216,47 @@ describe('Blueprint compose apply (real filesystem)', () => { await expectMissing(path.join(stackDir, '.blueprint.json')); expect(deploySpy).not.toHaveBeenCalled(); }); + + it('does not mutate an existing stack when compose snapshot read fails', async () => { + const nodeId = seedLocalNode(); + const stackName = `bp-snapshot-fail-${counter}`; + const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); + const original = 'services:\n old:\n image: nginx\n'; + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original); + await fsPromises.writeFile( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: 10, revision: 1, lastApplied: 1 }, null, 2), + ); + + vi.spyOn(FileSystemService.prototype, 'readStackFile').mockResolvedValue({ + content: undefined, + binary: false, + oversized: true, + size: 3 * 1024 * 1024, + mime: 'text/yaml', + mtimeMs: Date.now(), + }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ + recoveryId: null, + deployedGenerationId: null, + }); + const writeSpy = vi.spyOn(FileSystemService.prototype, 'writeStackFile'); + + await expect( + BlueprintService.getInstance().applyLocalUnderLock( + nodeId, + stackName, + 'services:\n new:\n image: redis:7\n', + JSON.stringify({ blueprintId: 10, revision: 2, lastApplied: Date.now() }, null, 2), + '/api/blueprints/test/apply', + ), + ).rejects.toThrow(/Cannot snapshot existing compose/i); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(original); + expect(deploySpy).not.toHaveBeenCalled(); + expect(writeSpy).not.toHaveBeenCalled(); + }); }); describe('FileSystemService.removeAlternateRootComposeFiles', () => { diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts index e9748212..e2d6b1ef 100644 --- a/backend/src/__tests__/compose-service.test.ts +++ b/backend/src/__tests__/compose-service.test.ts @@ -36,7 +36,13 @@ const { mockBuildUnifiedHeldImagePredicate, mockGetRecovery, mockFsStat, -} = vi.hoisted(() => ({ + mockCleanupDockerAuthTempDir, + mockCreateDockerAuthTempDir, + mockRecordRegistryDeliveryEvent, + mockPreparedSourceFinalize, +} = vi.hoisted(() => { + const mockCleanupDockerAuthTempDir = vi.fn(); + return { mockSpawn: vi.fn(), mockGetContainersByStack: vi.fn().mockResolvedValue([]), mockGetLegacyOrphanContainersByStack: vi.fn().mockResolvedValue([]), @@ -89,6 +95,35 @@ const { mockBuildUnifiedHeldImagePredicate: vi.fn().mockReturnValue(() => false), mockGetRecovery: vi.fn().mockReturnValue(undefined), mockFsStat: vi.fn().mockResolvedValue({ isDirectory: () => true }), + mockCleanupDockerAuthTempDir, + mockCreateDockerAuthTempDir: vi.fn(() => ({ + dirPath: '/tmp/sencho-docker-test', + kind: 'local' as const, + cleanup: mockCleanupDockerAuthTempDir, + })), + mockRecordRegistryDeliveryEvent: vi.fn(), + mockPreparedSourceFinalize: vi.fn(), + }; +}); + +vi.mock('../helpers/dockerAuthTempDir', () => ({ + createDockerAuthTempDir: mockCreateDockerAuthTempDir, +})); + +vi.mock('../helpers/registryDeliveryEvidence', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + recordRegistryDeliveryEvent: (...args: unknown[]) => mockRecordRegistryDeliveryEvent(...args), + }; +}); + +vi.mock('../services/preparedSourceStore', () => ({ + PreparedSourceStore: { + getInstance: () => ({ + finalize: (...args: unknown[]) => mockPreparedSourceFinalize(...args), + }), + }, })); vi.mock('child_process', () => ({ spawn: mockSpawn, execFile: vi.fn() })); @@ -1301,6 +1336,7 @@ describe('ComposeService - withRegistryAuth', () => { it('creates temp config dir when registries exist', async () => { mockGetRegistries.mockReturnValue([{ url: 'https://registry.example.com', username: 'user', password: 'pass' }]); mockResolveDockerConfig.mockResolvedValue({ config: { auths: { 'registry.example.com': { auth: 'dXNlcjpwYXNz' } } }, warnings: [] }); + mockGetGlobalSettings.mockReturnValue({ delivery_source_id: 'test-delivery-source' }); setupAutoCloseSpawn(); mockListContainers.mockResolvedValue([]); @@ -1310,10 +1346,8 @@ describe('ComposeService - withRegistryAuth', () => { await vi.advanceTimersByTimeAsync(3100); await promise; - expect(mockMkdtempSync).toHaveBeenCalled(); - expect(mockWriteFileSync).toHaveBeenCalled(); - expect(mockUnlinkSync).toHaveBeenCalled(); - expect(mockRmdirSync).toHaveBeenCalled(); + expect(mockCreateDockerAuthTempDir).toHaveBeenCalled(); + expect(mockCleanupDockerAuthTempDir).toHaveBeenCalled(); }); it('surfaces resolveDockerConfig warnings to the WebSocket output', async () => { @@ -1338,7 +1372,8 @@ describe('ComposeService - withRegistryAuth', () => { it('cleans up temp dir even on command failure', async () => { mockGetRegistries.mockReturnValue([{ url: 'https://registry.example.com' }]); - mockResolveDockerConfig.mockResolvedValue({ config: { auths: {} }, warnings: [] }); + mockResolveDockerConfig.mockResolvedValue({ config: { auths: { 'registry.example.com': { auth: 'dGVzdA==' } } }, warnings: [] }); + mockGetGlobalSettings.mockReturnValue({ delivery_source_id: 'test-delivery-source' }); // Make spawn fail mockSpawn.mockImplementation(() => { @@ -1356,7 +1391,100 @@ describe('ComposeService - withRegistryAuth', () => { await vi.runAllTimersAsync(); const error = await result; expect(error).not.toBeNull(); - expect(mockUnlinkSync).toHaveBeenCalled(); + expect(mockCleanupDockerAuthTempDir).toHaveBeenCalled(); + }); +}); + +describe('ComposeService - registry delivery cleanup logging', () => { + const deliveryEnvelope = { + attestation: 'test-token', + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: 'delivery-src-1', + }; + + async function deployWithDeliveryContext(): Promise { + const { runWithRegistryDeliveryContext } = await import('../helpers/registryDeliveryContext'); + const svc = ComposeService.getInstance(1); + await runWithRegistryDeliveryContext({ + envelope: deliveryEnvelope, + nodeId: 1, + stack: 'my-stack', + stage: 'stack-deploy', + seamResult: { auths: { 'registry.example.com': { auth: 'dGVzdA==' } }, prepId: 'prep-1' }, + seamSettled: true, + }, async () => { + const promise = svc.deployStack('my-stack'); + await vi.advanceTimersByTimeAsync(3100); + await promise; + }); + } + + beforeEach(() => { + mockGetRegistries.mockReturnValue([]); + mockGetGlobalSettings.mockReturnValue({ delivery_source_id: 'delivery-src-1' }); + mockCleanupDockerAuthTempDir.mockReset(); + mockCleanupDockerAuthTempDir.mockImplementation(() => undefined); + mockRecordRegistryDeliveryEvent.mockReset(); + mockPreparedSourceFinalize.mockReset(); + mockPreparedSourceFinalize.mockImplementation(() => undefined); + setupAutoCloseSpawn(); + mockListContainers.mockResolvedValue([]); + }); + + it('logs cleanup failure when evidence records successfully', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + mockCleanupDockerAuthTempDir.mockImplementation(() => { + throw new Error('cleanup failed'); + }); + + await deployWithDeliveryContext(); + + expect(mockRecordRegistryDeliveryEvent).toHaveBeenCalledWith(expect.objectContaining({ + eventType: 'cleanup_failed', + })); + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining('Registry delivery temp dir cleanup failed'), + expect.any(String), + 'cleanup failed', + ); + errorSpy.mockRestore(); + }); + + it('logs both cleanup and evidence failures without changing deploy success', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + mockCleanupDockerAuthTempDir.mockImplementation(() => { + throw new Error('cleanup failed'); + }); + mockRecordRegistryDeliveryEvent.mockImplementation(() => { + throw new Error('evidence failed'); + }); + + await deployWithDeliveryContext(); + + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining('Registry delivery cleanup and evidence both failed'), + expect.any(String), + 'evidence failed', + 'cleanup failed', + ); + errorSpy.mockRestore(); + }); + + it('logs prepared-source finalization failure without changing deploy success', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + mockPreparedSourceFinalize.mockImplementation(() => { + throw new Error('finalize failed'); + }); + + await deployWithDeliveryContext(); + + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining('Registry delivery prepared-source finalize failed'), + 'prep-1', + 'finalize failed', + ); + errorSpy.mockRestore(); }); }); @@ -1543,6 +1671,39 @@ describe('ComposeService - idle-output stall backstop', () => { await promise; }); + it('terminates compose when the registry delivery abort signal fires', async () => { + const { runWithRegistryDeliveryContext } = await import('../helpers/registryDeliveryContext'); + const proc = createMockProcess(); + mockSpawn.mockReturnValue(proc); + const abortController = new AbortController(); + + const svc = ComposeService.getInstance(1); + const result = runWithRegistryDeliveryContext( + { + envelope: { + attestation: 'unused', + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: 'test', + }, + nodeId: 1, + stack: 'my-stack', + stage: 'stack-deploy', + abortSignal: abortController.signal, + }, + () => svc.runCommand('my-stack', 'restart'), + ).then(() => null, (e: Error) => e); + + await waitForSpawn(); + abortController.abort(); + proc.emit('close', null); + + const error = await result; + expect(proc.kill).toHaveBeenCalledWith('SIGTERM'); + expect(error).not.toBeNull(); + expect((error as Error).message).toContain('OPERATION_ABORTED'); + }); + it('falls back to the default stall window when the env value is invalid', async () => { process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '0'; // invalid → default (10min) mockListContainers.mockResolvedValue([]); diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index 5c166247..1236005f 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -20,6 +20,7 @@ import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; import type { TransportFailure } from '../services/git/errors'; import { GitOpsStore } from '../services/gitops/store'; import { GitOpsTransitions } from '../services/gitops/transitions'; +import { StackOpLockService } from '../services/StackOpLockService'; import { buildGenerationRow, directSourceIdentity, @@ -124,6 +125,8 @@ beforeEach(() => { mockRecoveryLinkGateOrRetain.mockReset(); mockRecoveryGet.mockReturnValue({ id: 'rec-test-1', is_current: 1 }); + StackOpLockService.resetForTests(); + // Wipe persisted git sources between tests const db = DatabaseService.getInstance(); for (const s of db.getGitSources()) db.deleteGitSource(s.stack_name); @@ -1416,6 +1419,7 @@ describe('GitSourceService.handleWebhookPull debounce', () => { 'git_apply', 'system:webhook', expect.any(Function), + undefined, ); runExclusive.mockRestore(); diff --git a/backend/src/__tests__/git-transport.test.ts b/backend/src/__tests__/git-transport.test.ts index dc95e293..d2f31f0e 100644 --- a/backend/src/__tests__/git-transport.test.ts +++ b/backend/src/__tests__/git-transport.test.ts @@ -1320,7 +1320,7 @@ describe('clone failure classification and final size gate', () => { // Simulate a real OS termination confirmation arriving after // a short, non-zero delay rather than synchronously with the // kill call. - setTimeout(() => child.emit('close', null), 40); + setTimeout(() => child.emit('close', null), 150); }; return child; }); @@ -1341,7 +1341,7 @@ describe('clone failure classification and final size gate', () => { // simulated confirmation has not arrived yet: settling here // would let a caller start cleaning up the workspace while the // child tree is still alive. - await new Promise((r) => setTimeout(r, 60)); + await new Promise((r) => setTimeout(r, 55)); expect(killInvoked).toBe(true); expect(settled).toBe(false); diff --git a/backend/src/__tests__/pilot-http-cancel.test.ts b/backend/src/__tests__/pilot-http-cancel.test.ts new file mode 100644 index 00000000..278a29ab --- /dev/null +++ b/backend/src/__tests__/pilot-http-cancel.test.ts @@ -0,0 +1,165 @@ +/** + * Tests for Pilot http_cancel: bridge notifies agent on client disconnect, + * agent destroys the in-flight loopback HTTP request. + */ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import http from 'http'; +import { EventEmitter } from 'events'; +import { WebSocket } from 'ws'; +import { decodeJsonFrame } from '../pilot/protocol'; +import { PilotTunnelBridge } from '../services/PilotTunnelBridge'; +import { PilotAgent } from '../pilot/agent'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +function makeMockTunnelWs(): EventEmitter & { + sent: unknown[]; + readyState: number; + bufferedAmount: number; + send: (data: unknown) => void; + ping: () => void; + close: () => void; +} { + const ws = new EventEmitter() as EventEmitter & { + sent: unknown[]; + readyState: number; + bufferedAmount: number; + send: (data: unknown) => void; + ping: () => void; + close: () => void; + }; + ws.sent = []; + ws.readyState = WebSocket.OPEN; + ws.bufferedAmount = 0; + ws.send = (data: unknown) => { ws.sent.push(data); }; + ws.ping = () => { /* no-op */ }; + ws.close = () => { ws.readyState = WebSocket.CLOSED; ws.emit('close'); }; + return ws; +} + +function jsonFramesSent(mockWs: ReturnType) { + return mockWs.sent + .filter((item): item is string => typeof item === 'string') + .map((raw) => decodeJsonFrame(raw)); +} + +describe('PilotTunnelBridge http_cancel on client disconnect', () => { + let bridge: PilotTunnelBridge; + let mockWs: ReturnType; + let loopbackUrl: string; + + beforeAll(async () => { + mockWs = makeMockTunnelWs(); + bridge = new PilotTunnelBridge(1, mockWs as unknown as WebSocket); + await bridge.start(); + loopbackUrl = bridge.getLoopbackUrl(); + }); + + afterAll(() => { + bridge.close(); + }); + + it('sends http_cancel and http_err when the loopback client disconnects mid-flight', async () => { + const url = new URL(loopbackUrl); + const req = http.request({ + host: url.hostname, + port: Number(url.port), + method: 'POST', + path: '/api/stacks/my-stack/deploy', + headers: { 'content-type': 'application/json' }, + }); + req.on('error', () => { /* expected when the client disconnects mid-flight */ }); + req.end(JSON.stringify({})); + + await new Promise((resolve) => { + const check = () => { + const frames = jsonFramesSent(mockWs); + if (frames.some((f) => f.t === 'http_req')) { + resolve(); + return; + } + setTimeout(check, 10); + }; + check(); + }); + + const httpReq = jsonFramesSent(mockWs).find((f) => f.t === 'http_req'); + expect(httpReq?.t).toBe('http_req'); + if (httpReq?.t !== 'http_req') throw new Error('expected http_req'); + + req.destroy(); + + await new Promise((resolve) => { + const check = () => { + const frames = jsonFramesSent(mockWs); + const cancel = frames.find((f) => f.t === 'http_cancel' && f.s === httpReq.s); + const err = frames.find((f) => f.t === 'http_err' && f.s === httpReq.s); + if (cancel && err) { + resolve(); + return; + } + setTimeout(check, 10); + }; + check(); + }); + + const frames = jsonFramesSent(mockWs); + expect(frames).toContainEqual({ t: 'http_cancel', s: httpReq.s }); + expect(frames).toContainEqual({ + t: 'http_err', + s: httpReq.s, + code: 'tunnel_down', + message: 'client aborted', + }); + }, 10_000); +}); + +describe('PilotAgent http_cancel handling', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + await import('../index'); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('destroys the loopback ClientRequest on http_cancel', () => { + const destroySpy = vi.spyOn(http.ClientRequest.prototype, 'destroy'); + + const agent = new PilotAgent({ + primaryUrl: 'http://primary.invalid', + loopbackPort: 9, + initialToken: 'test-token', + enrollToken: null, + enrolling: false, + }); + + const agentInternals = agent as unknown as { + ws: { send: (data: string) => void; readyState: number } | null; + handleJsonFrame: (frame: ReturnType) => void; + httpStreams: Map; + }; + agentInternals.ws = { send: () => { /* no-op */ }, readyState: WebSocket.OPEN }; + + agentInternals.handleJsonFrame({ + t: 'http_req', + s: 99, + method: 'GET', + path: '/api/health', + headers: {}, + }); + expect(agentInternals.httpStreams.has(99)).toBe(true); + + agentInternals.handleJsonFrame({ t: 'http_cancel', s: 99 }); + + expect(destroySpy).toHaveBeenCalled(); + expect(agentInternals.httpStreams.has(99)).toBe(false); + + // Idempotent: second cancel is ignored without throwing. + agentInternals.handleJsonFrame({ t: 'http_cancel', s: 99 }); + + destroySpy.mockRestore(); + }); +}); diff --git a/backend/src/__tests__/pilot-protocol.test.ts b/backend/src/__tests__/pilot-protocol.test.ts index df35f9d1..14330a76 100644 --- a/backend/src/__tests__/pilot-protocol.test.ts +++ b/backend/src/__tests__/pilot-protocol.test.ts @@ -44,6 +44,11 @@ describe('JSON frame roundtrip', () => { expect(close.t).toBe('ws_close'); }); + it('roundtrips an http_cancel frame', () => { + const decoded = decodeJsonFrame(encodeJsonFrame({ t: 'http_cancel', s: 12 })); + expect(decoded).toEqual({ t: 'http_cancel', s: 12 }); + }); + it('rejects malformed JSON', () => { expect(() => decodeJsonFrame('not json')).toThrow(); }); diff --git a/backend/src/__tests__/registry-delivery-compose-env.test.ts b/backend/src/__tests__/registry-delivery-compose-env.test.ts new file mode 100644 index 00000000..ec9dd292 --- /dev/null +++ b/backend/src/__tests__/registry-delivery-compose-env.test.ts @@ -0,0 +1,70 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import { + discoverRegistryReferences, +} from '../services/registryReferenceDiscovery'; +import { + mergeComposeEnvVars, + resolveComposeEnvForDiscovery, +} from '../helpers/registryDeliveryComposeEnv'; + +describe('registryDeliveryComposeEnv', () => { + const envKey = 'SENCHO_REGDELIVERY_TEST_REGISTRY'; + + afterEach(() => { + delete process.env[envKey]; + }); + + it('resolves registry host from project .env variables', () => { + const dir = path.join(process.env.TMPDIR || '/tmp', `sencho-compose-env-${Date.now()}`); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'compose.yaml'), + 'services:\n app:\n image: ${REGISTRY}/org/private:latest\n', + ); + fs.writeFileSync(path.join(dir, '.env'), 'REGISTRY=ghcr.io\n'); + + const env = resolveComposeEnvForDiscovery(dir); + const result = discoverRegistryReferences(dir, env); + expect(result.referencedHosts).toEqual(['ghcr.io']); + }); + + it('lets process environment override .env for compose variables', () => { + const dir = path.join(process.env.TMPDIR || '/tmp', `sencho-compose-env-override-${Date.now()}`); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'compose.yaml'), + `services:\n app:\n image: \${${envKey}}/org/private:latest\n`, + ); + fs.writeFileSync(path.join(dir, '.env'), `${envKey}=ghcr.io\n`); + + process.env[envKey] = 'quay.io'; + const env = resolveComposeEnvForDiscovery(dir); + const result = discoverRegistryReferences(dir, env); + expect(result.referencedHosts).toEqual(['quay.io']); + }); + + it('merges request env over .env before process overrides', () => { + const merged = mergeComposeEnvVars({ FOO: 'from-dotenv' }, { FOO: 'from-request' }); + expect(merged.FOO).toBe('from-request'); + + const previous = process.env.BAR; + process.env.BAR = 'from-process'; + try { + const withProcess = mergeComposeEnvVars({ BAR: 'from-dotenv' }); + expect(withProcess.BAR).toBe('from-process'); + } finally { + if (previous === undefined) delete process.env.BAR; + else process.env.BAR = previous; + } + }); + + it('ignores unsafe request env keys', () => { + const unsafe = JSON.parse('{"__proto__": "evil", "REGISTRY": "ghcr.io"}') as Record; + const merged = mergeComposeEnvVars({}, unsafe); + expect(merged.REGISTRY).toBe('ghcr.io'); + expect(Object.keys(merged)).not.toContain('__proto__'); + expect(Object.prototype.hasOwnProperty.call(merged, '__proto__')).toBe(false); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-evidence-reconciliation.test.ts b/backend/src/__tests__/registry-delivery-evidence-reconciliation.test.ts new file mode 100644 index 00000000..4cf7fbbc --- /dev/null +++ b/backend/src/__tests__/registry-delivery-evidence-reconciliation.test.ts @@ -0,0 +1,148 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import axios from 'axios'; +import { setupTestDb } from './helpers/setupTestDb'; +import { DatabaseService } from '../services/DatabaseService'; +import { + importRegistryDeliveryEvidencePage, + listRegistryDeliveryEvidencePage, + recordRegistryDeliveryEvent, +} from '../helpers/registryDeliveryEvidence'; +import { RegistryDeliveryReconciler } from '../services/RegistryDeliveryReconciler'; + +describe('registry delivery evidence persistence', () => { + beforeEach(async () => { + await setupTestDb(); + RegistryDeliveryReconciler.resetForTests(); + vi.restoreAllMocks(); + }); + + it('assigns monotonic seq values and pages by cursor', () => { + const db = DatabaseService.getInstance(); + const deliverySourceId = db.getGlobalSettings().delivery_source_id!; + + recordRegistryDeliveryEvent({ + deliverySourceId, + eventType: 'swept_local', + tempDirId: 'local-abc', + }); + recordRegistryDeliveryEvent({ + deliverySourceId, + eventType: 'swept_delivered', + tempDirId: 'delivered-def', + }); + + const page1 = listRegistryDeliveryEvidencePage(deliverySourceId, 0, 1); + expect(page1.events).toHaveLength(1); + expect(page1.events[0]?.event_type).toBe('swept_local'); + expect(page1.nextCursor).toBe(page1.events[0]?.seq); + + const page2 = listRegistryDeliveryEvidencePage(deliverySourceId, page1.nextCursor, 10); + expect(page2.events).toHaveLength(1); + expect(page2.events[0]?.event_type).toBe('swept_delivered'); + }); + + it('imports a page in one transaction and advances the cursor', () => { + const db = DatabaseService.getInstance(); + const deliverySourceId = db.getGlobalSettings().delivery_source_id!; + const hubNodeId = 1; + const beforeSeq = listRegistryDeliveryEvidencePage(deliverySourceId, 0, 1000).nextCursor; + + recordRegistryDeliveryEvent({ + deliverySourceId, + eventType: 'operation_completed', + stack: 'demo', + op: 'stack-deploy', + }); + const page = listRegistryDeliveryEvidencePage(deliverySourceId, beforeSeq, 10); + expect(page.events).toHaveLength(1); + const result = importRegistryDeliveryEvidencePage(hubNodeId, deliverySourceId, page.events); + expect(result.imported).toBe(1); + expect(result.lastSeq).toBe(page.events[0]?.seq); + expect(db.getRegistryDeliveryImportCursor(deliverySourceId)).toBe(page.events[0]?.seq); + + const duplicate = importRegistryDeliveryEvidencePage(hubNodeId, deliverySourceId, page.events); + expect(duplicate.imported).toBe(0); + }); + + it('keeps the original hub_node_id_snapshot when the same source is re-imported', () => { + const db = DatabaseService.getInstance(); + const deliverySourceId = db.getGlobalSettings().delivery_source_id!; + const beforeSeq = listRegistryDeliveryEvidencePage(deliverySourceId, 0, 1000).nextCursor; + + recordRegistryDeliveryEvent({ + deliverySourceId, + eventType: 'operation_completed', + stack: 'demo', + op: 'stack-deploy', + }); + const page = listRegistryDeliveryEvidencePage(deliverySourceId, beforeSeq, 10); + expect(page.events).toHaveLength(1); + importRegistryDeliveryEvidencePage(10, deliverySourceId, page.events); + importRegistryDeliveryEvidencePage(20, deliverySourceId, page.events); + + const imported = db.getDb().prepare( + 'SELECT hub_node_id_snapshot FROM registry_delivery_imported_events WHERE event_id = ?', + ).all(page.events[0]!.event_id) as Array<{ hub_node_id_snapshot: number }>; + expect(imported).toHaveLength(1); + expect(imported[0]?.hub_node_id_snapshot).toBe(10); + }); + + it('writes a retention_gap event with pruned_through_seq when old rows are pruned', () => { + const db = DatabaseService.getInstance(); + const deliverySourceId = db.getGlobalSettings().delivery_source_id!; + + const seq = recordRegistryDeliveryEvent({ + deliverySourceId, + eventType: 'swept_local', + tempDirId: 'local-old', + }); + db.getDb().prepare( + 'UPDATE registry_delivery_events SET created_at = ? WHERE seq = ?', + ).run(Date.now() - (120 * 24 * 60 * 60 * 1000), seq); + + const pruned = db.cleanupOldDeliveryEvents(90); + expect(pruned).toBe(1); + + const gap = listRegistryDeliveryEvidencePage(deliverySourceId, seq - 1, 10); + const retention = gap.events.find((event) => event.event_type === 'retention_gap'); + expect(retention?.pruned_through_seq).toBe(seq); + }); + + it('attributes imported evidence to the remote node id', async () => { + const db = DatabaseService.getInstance(); + const remoteNodeId = 42; + const deliverySourceId = 'remote-source-id'; + db.getDb().prepare( + `INSERT INTO nodes (id, name, type, api_url, api_token, mode, compose_dir, is_default, status, created_at) + VALUES (?, 'remote-test', 'remote', 'http://remote:1852', 'token', 'direct', ?, 0, 'online', ?) + ON CONFLICT(id) DO UPDATE SET + type = excluded.type, + api_url = excluded.api_url, + api_token = excluded.api_token, + mode = excluded.mode`, + ).run(remoteNodeId, process.env.COMPOSE_DIR!, Date.now()); + + vi.spyOn(axios, 'get').mockResolvedValue({ + status: 200, + data: { + deliverySourceId, + events: [{ + event_id: 'evt-remote-1', + seq: 99, + event_type: 'operation_completed', + created_at: Date.now(), + stack: 'demo', + op: 'stack-deploy', + }], + nextCursor: 99, + }, + }); + + await RegistryDeliveryReconciler.getInstance().reconcileNode(remoteNodeId); + + const row = db.getDb().prepare( + 'SELECT hub_node_id_snapshot FROM registry_delivery_imported_events WHERE event_id = ?', + ).get('evt-remote-1') as { hub_node_id_snapshot: number } | undefined; + expect(row?.hub_node_id_snapshot).toBe(remoteNodeId); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-from-git-prep.test.ts b/backend/src/__tests__/registry-delivery-from-git-prep.test.ts new file mode 100644 index 00000000..c8495274 --- /dev/null +++ b/backend/src/__tests__/registry-delivery-from-git-prep.test.ts @@ -0,0 +1,149 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import { setupTestDb } from './helpers/setupTestDb'; +import { GitSourceService } from '../services/GitSourceService'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { PreparedSourceStore } from '../services/preparedSourceStore'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { runWithRegistryDeliveryContext } from '../helpers/registryDeliveryContext'; +import { + writeGitCandidatePreparedMeta, +} from '../helpers/registryDeliveryGitCandidate'; +import { hashDeliverySourceDir, hashActionSet } from '../helpers/registryDeliveryHashes'; +import { candidateRelPathForSha } from '../services/gitops/createStagingMarker'; +import type { FetchResult, MaterializationResult } from '../services/GitSourceService'; + +describe('createStackFromGit prepared git candidate consumption', () => { + beforeEach(async () => { + await setupTestDb(); + RegistryDeliveryService.resetForTests(); + const deliverySourceId = RegistryDeliveryService.getInstance().getDeliverySourceId(); + PreparedSourceStore.getInstance().configure(deliverySourceId); + }); + + it('uses the prepared git candidate instead of fetchFromGit when prepId is set', async () => { + const svc = GitSourceService.getInstance(); + const fetchSpy = vi.spyOn( + svc as unknown as { fetchFromGit: () => Promise }, + 'fetchFromGit', + ).mockRejectedValue(new Error('fetchFromGit must not run when prepId is present')); + + const stagingDir = path.join(process.env.TMPDIR || '/tmp', `sencho-git-prep-${Date.now()}`); + fs.mkdirSync(stagingDir, { recursive: true }); + fs.writeFileSync( + path.join(stagingDir, 'compose.yaml'), + 'services:\n app:\n image: nginx:latest\n', + ); + + const commitSha = 'a'.repeat(40); + const candidateRelPath = candidateRelPathForSha(commitSha); + const materialization: MaterializationResult = { + inventory: { + inputs: [], + refusals: [], + buildContexts: [], + dynamic: [], + counts: { managed: 0, unmanaged: 0, refused: 0 }, + }, + contextCopyPlans: [], + candidateRelPath, + validation: { ok: true }, + }; + + await writeGitCandidatePreparedMeta(stagingDir, { + version: 1, + commitSha, + resolvedRefKind: 'branch', + candidateRelPath, + composeFiles: [{ path: 'compose.yaml', content: 'services:\n app:\n image: nginx:latest\n' }], + envContent: null, + materialization, + warnings: [], + }); + + const sourceHash = hashDeliverySourceDir(stagingDir); + const entry = await PreparedSourceStore.getInstance().prepareFromDirectory( + 'git-candidate', + sourceHash, + stagingDir, + ); + + const stackName = `from-git-prep-${Date.now()}`; + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const delivery = RegistryDeliveryService.getInstance(); + + const restoreSpy = vi.spyOn( + svc as unknown as { + restoreCreateFromPreparedGitCandidate: ( + prepId: string, + managedRoot: string, + rootPreexisted: boolean, + gitopsOperationId: string, + staged: { candidateRelPath: string | null }, + ) => Promise<{ fetched: FetchResult; materialization: MaterializationResult }>; + }, + 'restoreCreateFromPreparedGitCandidate', + ).mockResolvedValue({ + fetched: { + composeFiles: [{ path: 'compose.yaml', content: 'services:\n app:\n image: nginx:latest\n' }], + envContent: null, + commitSha, + resolvedRefKind: 'branch', + warnings: [], + }, + materialization, + }); + + try { + await runWithRegistryDeliveryContext({ + envelope: { + attestation: delivery.signAttestation({ + nodeIdClaim: nodeId, + stack: stackName, + op: 'from-git-deploy-now', + sourceHash, + referencedHostsHash: delivery.hashHostList([]), + coveredHostsHash: delivery.hashHostList([]), + actionSetHash: hashActionSet(['stack:create']), + prepId: entry.prepId, + }), + prepId: entry.prepId, + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: delivery.getDeliverySourceId(), + }, + nodeId, + stack: stackName, + stage: 'from-git-deploy-now', + }, () => svc.createStackFromGit({ + stackName, + repoUrl: 'https://github.com/example/demo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + })); + } catch { + // Downstream create steps may fail in this isolated test; the contract + // under test is the prepared-source branch before fetchFromGit. + } + + expect(fetchSpy).not.toHaveBeenCalled(); + expect(restoreSpy).toHaveBeenCalledWith( + entry.prepId, + expect.any(String), + false, + expect.any(String), + expect.objectContaining({ candidateRelPath: null }), + ); + + fetchSpy.mockRestore(); + restoreSpy.mockRestore(); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-git-apply-classifier.test.ts b/backend/src/__tests__/registry-delivery-git-apply-classifier.test.ts new file mode 100644 index 00000000..3fc7c1c0 --- /dev/null +++ b/backend/src/__tests__/registry-delivery-git-apply-classifier.test.ts @@ -0,0 +1,24 @@ +import { describe, it, expect } from 'vitest'; +import { classifyRegistryDeliveryOp } from '../helpers/registryOpClassifier'; +import { classifyRegistryDeliveryRouteClass } from '../helpers/registryDeliveryBodyLimits'; + +describe('git-apply-auto-deploy route classification', () => { + it('classifies git-source apply as git-apply-auto-deploy', () => { + const result = classifyRegistryDeliveryOp( + 'POST', + '/api/stacks/my-stack/git-source/apply', + ); + expect(result).toEqual({ + eligible: true, + stage: 'git-apply-auto-deploy', + stack: 'my-stack', + }); + }); + + it('assigns bulk-label-git body budget to git-source apply', () => { + expect(classifyRegistryDeliveryRouteClass( + 'POST', + '/api/stacks/my-stack/git-source/apply', + )).toBe('bulk-label-git'); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-git-apply-prep.test.ts b/backend/src/__tests__/registry-delivery-git-apply-prep.test.ts new file mode 100644 index 00000000..0f9c1770 --- /dev/null +++ b/backend/src/__tests__/registry-delivery-git-apply-prep.test.ts @@ -0,0 +1,144 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import { setupTestDb } from './helpers/setupTestDb'; +import { GitSourceService } from '../services/GitSourceService'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { PreparedSourceStore } from '../services/preparedSourceStore'; +import { + writeGitCandidatePreparedMeta, +} from '../helpers/registryDeliveryGitCandidate'; +import { hashDeliverySourceDir } from '../helpers/registryDeliveryHashes'; +import { candidateRelPathForSha } from '../services/gitops/createStagingMarker'; +import { stackManagedRoot } from '../services/gitops/directApplication'; +import type { MaterializationResult } from '../services/GitSourceService'; + +describe('restoreApplyFromPreparedGitCandidate', () => { + beforeEach(async () => { + await setupTestDb(); + RegistryDeliveryService.resetForTests(); + const deliverySourceId = RegistryDeliveryService.getInstance().getDeliverySourceId(); + PreparedSourceStore.getInstance().configure(deliverySourceId); + }); + + it('installs prepared bytes at the pending candidate path', async () => { + const stagingDir = path.join(process.env.TMPDIR || '/tmp', `sencho-git-apply-prep-${Date.now()}`); + fs.mkdirSync(stagingDir, { recursive: true }); + fs.writeFileSync( + path.join(stagingDir, 'compose.yaml'), + 'services:\n app:\n image: nginx:latest\n', + ); + + const commitSha = 'd'.repeat(40); + const candidateRelPath = candidateRelPathForSha(commitSha); + const materialization: MaterializationResult = { + inventory: { + inputs: [], + refusals: [], + buildContexts: [], + dynamic: [], + counts: { managed: 0, unmanaged: 0, refused: 0 }, + }, + contextCopyPlans: [], + candidateRelPath, + validation: { ok: true }, + }; + + await writeGitCandidatePreparedMeta(stagingDir, { + version: 1, + commitSha, + resolvedRefKind: 'branch', + candidateRelPath, + composeFiles: [{ path: 'compose.yaml', content: 'services:\n app:\n image: nginx:latest\n' }], + envContent: null, + materialization, + warnings: [], + }); + + const sourceHash = hashDeliverySourceDir(stagingDir); + const entry = await PreparedSourceStore.getInstance().prepareFromDirectory( + 'git-candidate', + sourceHash, + stagingDir, + ); + + const stackName = `apply-restore-${Date.now()}`; + const svc = GitSourceService.getInstance(); + await ( + svc as unknown as { + restoreApplyFromPreparedGitCandidate: ( + prepId: string, + stackName: string, + commitSha: string, + candidateRelPath: string, + ) => Promise; + } + ).restoreApplyFromPreparedGitCandidate( + entry.prepId, + stackName, + commitSha, + candidateRelPath, + ); + + const installed = path.join(stackManagedRoot(stackName), candidateRelPath, 'compose.yaml'); + expect(fs.existsSync(installed)).toBe(true); + expect(PreparedSourceStore.getInstance().getEntry(entry.prepId)?.state).toBe('prepared'); + }); + + it('rejects commit or candidate path mismatches', async () => { + const stagingDir = path.join(process.env.TMPDIR || '/tmp', `sencho-git-apply-mismatch-${Date.now()}`); + fs.mkdirSync(stagingDir, { recursive: true }); + fs.writeFileSync(path.join(stagingDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n'); + + const commitSha = 'e'.repeat(40); + const candidateRelPath = candidateRelPathForSha(commitSha); + const materialization: MaterializationResult = { + inventory: { + inputs: [], + refusals: [], + buildContexts: [], + dynamic: [], + counts: { managed: 0, unmanaged: 0, refused: 0 }, + }, + contextCopyPlans: [], + candidateRelPath, + validation: { ok: true }, + }; + + await writeGitCandidatePreparedMeta(stagingDir, { + version: 1, + commitSha, + resolvedRefKind: 'branch', + candidateRelPath, + composeFiles: [{ path: 'compose.yaml', content: 'services:\n app:\n image: nginx\n' }], + envContent: null, + materialization, + warnings: [], + }); + + const entry = await PreparedSourceStore.getInstance().prepareFromDirectory( + 'git-candidate', + hashDeliverySourceDir(stagingDir), + stagingDir, + ); + + const svc = GitSourceService.getInstance(); + const restore = ( + svc as unknown as { + restoreApplyFromPreparedGitCandidate: ( + prepId: string, + stackName: string, + commitSha: string, + candidateRelPath: string, + ) => Promise; + } + ).restoreApplyFromPreparedGitCandidate.bind(svc); + + await expect(restore( + entry.prepId, + `mismatch-${Date.now()}`, + 'f'.repeat(40), + candidateRelPath, + )).rejects.toThrow(/commit mismatch/i); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-git-candidate-meta.test.ts b/backend/src/__tests__/registry-delivery-git-candidate-meta.test.ts new file mode 100644 index 00000000..f4d36b1a --- /dev/null +++ b/backend/src/__tests__/registry-delivery-git-candidate-meta.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import { + fetchResultFromPreparedMeta, + installGitCandidatePayloadToManagedRoot, + readGitCandidatePreparedMeta, + writeGitCandidatePreparedMeta, + GIT_CANDIDATE_PREPARED_META_FILE, +} from '../helpers/registryDeliveryGitCandidate'; +import { candidateRelPathForSha } from '../services/gitops/createStagingMarker'; +import type { MaterializationResult } from '../services/GitSourceService'; + +describe('registryDeliveryGitCandidate helpers', () => { + it('round-trips metadata and installs candidate bytes', async () => { + const payloadDir = path.join(process.env.TMPDIR || '/tmp', `sencho-git-meta-${Date.now()}`); + const managedRoot = path.join(process.env.TMPDIR || '/tmp', `sencho-git-managed-${Date.now()}`); + fs.mkdirSync(payloadDir, { recursive: true }); + fs.mkdirSync(managedRoot, { recursive: true }); + fs.writeFileSync(path.join(payloadDir, 'compose.yaml'), 'services:\n web:\n image: nginx\n'); + + const commitSha = 'b'.repeat(40); + const candidateRelPath = candidateRelPathForSha(commitSha); + const materialization: MaterializationResult = { + inventory: { + inputs: [], + refusals: [], + buildContexts: [], + dynamic: [], + counts: { managed: 0, unmanaged: 0, refused: 0 }, + }, + contextCopyPlans: [], + candidateRelPath, + validation: { ok: true }, + }; + + await writeGitCandidatePreparedMeta(payloadDir, { + version: 1, + commitSha, + resolvedRefKind: 'branch', + candidateRelPath, + composeFiles: [{ path: 'compose.yaml', content: 'services:\n web:\n image: nginx\n' }], + envContent: null, + materialization, + warnings: ['submodule skipped'], + }); + + const meta = await readGitCandidatePreparedMeta(payloadDir); + expect(meta.commitSha).toBe(commitSha); + expect(fetchResultFromPreparedMeta(meta).warnings).toEqual(['submodule skipped']); + + await installGitCandidatePayloadToManagedRoot(payloadDir, managedRoot, candidateRelPath); + const installedCompose = path.join(managedRoot, candidateRelPath, 'compose.yaml'); + expect(fs.existsSync(installedCompose)).toBe(true); + expect(fs.existsSync(path.join(managedRoot, candidateRelPath, GIT_CANDIDATE_PREPARED_META_FILE))).toBe(false); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-hashes.test.ts b/backend/src/__tests__/registry-delivery-hashes.test.ts new file mode 100644 index 00000000..54e34309 --- /dev/null +++ b/backend/src/__tests__/registry-delivery-hashes.test.ts @@ -0,0 +1,52 @@ +import crypto from 'crypto'; +import { describe, it, expect } from 'vitest'; +import { hashActionSet, hashBlueprintPostApplySource, hashProjectSource } from '../helpers/registryDeliveryHashes'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +describe('registryDeliveryHashes', () => { + it('hashActionSet is order-independent', () => { + const a = hashActionSet(['stack:deploy', 'stack:edit']); + const b = hashActionSet(['stack:edit', 'stack:deploy']); + expect(a).toBe(b); + }); + + it('hashProjectSource is not raw compose bytes', () => { + const content = 'services:\n web:\n image: nginx\n'; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-hash-test-')); + try { + fs.writeFileSync(path.join(dir, 'compose.yaml'), content); + const projectHash = hashProjectSource(dir); + const rawHash = crypto.createHash('sha256').update(content).digest('hex'); + expect(projectHash).not.toBe(rawHash); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('hashProjectSource changes when compose content changes', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-hash-test-')); + try { + fs.writeFileSync(path.join(dir, 'compose.yaml'), 'services:\n web:\n image: nginx\n'); + const before = hashProjectSource(dir); + fs.writeFileSync(path.join(dir, 'compose.yaml'), 'services:\n web:\n image: nginx:2\n'); + const after = hashProjectSource(dir); + expect(before).not.toBe(after); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it('hashBlueprintPostApplySource matches hashProjectSource for an empty .env file', () => { + const compose = 'services:\n web:\n image: nginx\n'; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-hash-test-')); + try { + fs.writeFileSync(path.join(dir, 'compose.yaml'), compose); + fs.writeFileSync(path.join(dir, '.env'), ''); + expect(hashBlueprintPostApplySource(compose, '')).toBe(hashProjectSource(dir)); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/backend/src/__tests__/registry-delivery-lock-context.test.ts b/backend/src/__tests__/registry-delivery-lock-context.test.ts new file mode 100644 index 00000000..186d2afe --- /dev/null +++ b/backend/src/__tests__/registry-delivery-lock-context.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { setupTestDb } from './helpers/setupTestDb'; +import { StackOpLockService } from '../services/StackOpLockService'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { resolveRegistryAuthAtSeam } from '../helpers/registryDeliverySeam'; +import { hashActionSet, hashProjectSource } from '../helpers/registryDeliveryHashes'; +import fs from 'fs'; +import path from 'path'; + +describe('registry delivery stack lock context', () => { + beforeEach(async () => { + await setupTestDb(); + StackOpLockService.resetForTests(); + RegistryDeliveryService.resetForTests(); + }); + + it('stores lock context on tryAcquire and clears it on release', () => { + const locks = StackOpLockService.getInstance(); + const acquired = locks.tryAcquire(1, 'ctx-stack', 'deploy', 'admin', { + opId: 'op-123', + kind: 'stack-deploy', + }); + expect(acquired.acquired).toBe(true); + expect(locks.get(1, 'ctx-stack')?.context).toEqual({ + opId: 'op-123', + kind: 'stack-deploy', + }); + locks.release(1, 'ctx-stack'); + expect(locks.get(1, 'ctx-stack')).toBeUndefined(); + }); + + it('rejects seam when held lock context does not match attestation', async () => { + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const stackName = 'ctx-seam-stack'; + const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n web:\n image: nginx\n'); + + const delivery = RegistryDeliveryService.getInstance(); + const sourceHash = hashProjectSource(stackDir); + const attestation = delivery.signAttestation({ + nodeIdClaim: nodeId, + stack: stackName, + op: 'stack-deploy', + sourceHash, + referencedHostsHash: delivery.hashHostList([]), + coveredHostsHash: delivery.hashHostList([]), + actionSetHash: hashActionSet(['stack:deploy']), + }); + + StackOpLockService.getInstance().tryAcquire(nodeId, stackName, 'deploy', 'admin', { + opId: 'wrong-op', + kind: 'stack-deploy', + }); + + await expect(resolveRegistryAuthAtSeam({ + envelope: { + attestation, + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: delivery.getDeliverySourceId(), + }, + nodeId, + stack: stackName, + stage: 'stack-deploy', + })).rejects.toThrow(/lock context mismatch/i); + + StackOpLockService.getInstance().release(nodeId, stackName); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-middleware.test.ts b/backend/src/__tests__/registry-delivery-middleware.test.ts new file mode 100644 index 00000000..19a881a0 --- /dev/null +++ b/backend/src/__tests__/registry-delivery-middleware.test.ts @@ -0,0 +1,84 @@ +import express from 'express'; +import request from 'supertest'; +import { describe, it, expect, beforeEach } from 'vitest'; +import { REGISTRY_DELIVERY_BODY_FIELD } from '../helpers/registryDeliveryBodyLimits'; +import { + registryDeliveryApiPath, + registryDeliveryMiddleware, +} from '../middleware/registryDelivery'; +import { setupTestDb } from './helpers/setupTestDb'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; + +describe('registryDeliveryMiddleware', () => { + beforeEach(async () => { + await setupTestDb(); + RegistryDeliveryService.resetForTests(); + }); + + it('normalizes Express-mounted paths to /api/... for classification', () => { + expect(registryDeliveryApiPath({ path: '/stacks/jackett/deploy' })).toBe( + '/api/stacks/jackett/deploy', + ); + }); + + it('engages on stack deploy when mounted at /api (rejects invalid attestation)', async () => { + const app = express(); + app.use(express.json()); + app.use('/api', registryDeliveryMiddleware); + app.post('/api/stacks/:name/deploy', (_req, res) => { + res.status(200).json({ ok: true }); + }); + + const res = await request(app) + .post('/api/stacks/jackett/deploy') + .send({ + [REGISTRY_DELIVERY_BODY_FIELD]: { + attestation: 'not-a-jwt', + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: 'test-source', + }, + }); + + expect(res.status).toBe(400); + expect(res.body).toEqual({ error: 'Invalid registry delivery envelope' }); + }); + + it('passes through eligible routes without a delivery field', async () => { + const app = express(); + app.use(express.json()); + app.use('/api', registryDeliveryMiddleware); + app.post('/api/stacks/:name/deploy', (_req, res) => { + res.status(200).json({ ok: true }); + }); + + const res = await request(app) + .post('/api/stacks/jackett/deploy') + .send({}); + + expect(res.status).toBe(200); + expect(res.body).toEqual({ ok: true }); + }); + + it('does not engage on non-delivery routes', async () => { + const app = express(); + app.use(express.json()); + app.use('/api', registryDeliveryMiddleware); + app.get('/api/stacks', (_req, res) => { + res.status(200).json({ ok: true }); + }); + + const res = await request(app) + .get('/api/stacks') + .send({ + [REGISTRY_DELIVERY_BODY_FIELD]: { + attestation: 'not-a-jwt', + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: 'test-source', + }, + }); + + expect(res.status).toBe(200); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-outbound.test.ts b/backend/src/__tests__/registry-delivery-outbound.test.ts new file mode 100644 index 00000000..bbcefeaf --- /dev/null +++ b/backend/src/__tests__/registry-delivery-outbound.test.ts @@ -0,0 +1,224 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { setupTestDb } from './helpers/setupTestDb'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { DatabaseService } from '../services/DatabaseService'; +import { REGISTRY_DELIVERY_BODY_FIELD } from '../helpers/registryDeliveryBodyLimits'; +import { classifyRegistryDeliveryOp } from '../helpers/registryOpClassifier'; + +const mockRemoteAdvertises = vi.fn(); +const mockAxiosPost = vi.fn(); +const mockIsProxyConfidential = vi.fn(); +const mockIsTunnelConfidential = vi.fn(); + +vi.mock('../helpers/remoteCapabilities', () => ({ + remoteAdvertisesCapability: (...args: unknown[]) => mockRemoteAdvertises(...args), +})); + +vi.mock('axios', () => ({ + default: { + post: (...args: unknown[]) => mockAxiosPost(...args), + }, +})); + +vi.mock('../services/PilotTunnelManager', () => ({ + PilotTunnelManager: { + getInstance: () => ({ + isTunnelConfidential: (...args: unknown[]) => mockIsTunnelConfidential(...args), + }), + }, +})); + +let augmentJsonBodyForRegistryDelivery: typeof import('../helpers/registryDeliveryOutbound').augmentJsonBodyForRegistryDelivery; +let wouldAttemptRegistryDelivery: typeof import('../helpers/registryDeliveryOutbound').wouldAttemptRegistryDelivery; + +describe('registryDeliveryOutbound', () => { + beforeEach(async () => { + await setupTestDb(); + RegistryDeliveryService.resetForTests(); + vi.clearAllMocks(); + mockIsProxyConfidential.mockReturnValue(true); + mockIsTunnelConfidential.mockReturnValue(true); + }); + + beforeEach(async () => { + ({ augmentJsonBodyForRegistryDelivery, wouldAttemptRegistryDelivery } = await import('../helpers/registryDeliveryOutbound')); + const delivery = RegistryDeliveryService.getInstance(); + vi.spyOn(delivery, 'isProxyTransportConfidential').mockImplementation( + () => mockIsProxyConfidential(), + ); + }); + + it('classifies blueprint apply-local as eligible', () => { + const result = classifyRegistryDeliveryOp('POST', '/api/blueprints/apply-local'); + expect(result.eligible).toBe(true); + expect(result.stage).toBe('blueprint-apply'); + }); + + it('passes through unchanged when remote lacks delivery capability', async () => { + mockRemoteAdvertises.mockResolvedValue(false); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const node = DatabaseService.getInstance().getNode(nodeId)!; + const body = { foo: 'bar' }; + + const result = await augmentJsonBodyForRegistryDelivery({ + method: 'POST', + apiPath: '/api/stacks/demo/deploy', + nodeId, + node, + target: { apiUrl: 'http://remote:1852', apiToken: 'token' }, + body, + }); + + expect(result).toEqual({ ok: true, body, augmented: false }); + expect(mockAxiosPost).not.toHaveBeenCalled(); + }); + + it('passes through unchanged when transport is not confidential', async () => { + mockRemoteAdvertises.mockResolvedValue(true); + mockIsProxyConfidential.mockReturnValue(false); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const node = DatabaseService.getInstance().getNode(nodeId)!; + const body = { foo: 'bar' }; + + const result = await augmentJsonBodyForRegistryDelivery({ + method: 'POST', + apiPath: '/api/stacks/demo/deploy', + nodeId, + node, + target: { apiUrl: 'http://remote:1852', apiToken: 'token' }, + body, + }); + + expect(result).toEqual({ ok: true, body, augmented: false }); + expect(mockAxiosPost).not.toHaveBeenCalled(); + }); + + it('returns false from wouldAttemptRegistryDelivery when remote lacks capability', async () => { + mockRemoteAdvertises.mockResolvedValue(false); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const node = DatabaseService.getInstance().getNode(nodeId)!; + + const result = await wouldAttemptRegistryDelivery( + nodeId, + node, + 'POST', + '/api/stacks/demo/deploy', + ); + + expect(result).toBe(false); + }); + + it('returns aborted when discover is cancelled', async () => { + mockRemoteAdvertises.mockResolvedValue(true); + mockAxiosPost.mockImplementation(() => new Promise(() => {})); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const node = DatabaseService.getInstance().getNode(nodeId)!; + const controller = new AbortController(); + controller.abort(); + + const result = await augmentJsonBodyForRegistryDelivery({ + method: 'POST', + apiPath: '/api/stacks/demo/deploy', + nodeId, + node, + target: { apiUrl: 'http://remote:1852', apiToken: 'token' }, + body: {}, + abortSignal: controller.signal, + }); + + expect(result).toEqual({ ok: false, status: 499, error: 'Request aborted' }); + expect(mockAxiosPost).not.toHaveBeenCalled(); + }); + + it('augments deploy body when capability and transport are present', async () => { + mockRemoteAdvertises.mockResolvedValue(true); + const delivery = RegistryDeliveryService.getInstance(); + const discover = { + referencedHosts: ['ghcr.io'], + coveredHosts: [], + sourceHash: 'abc', + actionSetHash: 'def', + deliverySourceId: delivery.getDeliverySourceId(), + attestation: delivery.signAttestation({ + nodeIdClaim: 1, + stack: 'demo', + op: 'stack-deploy', + sourceHash: 'abc', + referencedHostsHash: delivery.hashHostList(['ghcr.io']), + coveredHostsHash: delivery.hashHostList([]), + actionSetHash: 'def', + }), + }; + mockAxiosPost.mockResolvedValue({ status: 200, data: discover }); + + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const node = DatabaseService.getInstance().getNode(nodeId)!; + + const result = await augmentJsonBodyForRegistryDelivery({ + method: 'POST', + apiPath: '/api/stacks/demo/deploy', + nodeId, + node, + target: { apiUrl: 'http://remote:1852', apiToken: 'token' }, + body: {}, + }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.augmented).toBe(true); + expect(result.body[REGISTRY_DELIVERY_BODY_FIELD]).toBeDefined(); + expect(mockAxiosPost).toHaveBeenCalledOnce(); + }); + + it('returns aborted when hub envelope build is cancelled after discover', async () => { + mockRemoteAdvertises.mockResolvedValue(true); + const delivery = RegistryDeliveryService.getInstance(); + const discover = { + referencedHosts: ['ghcr.io'], + coveredHosts: [], + sourceHash: 'abc', + actionSetHash: 'def', + deliverySourceId: delivery.getDeliverySourceId(), + attestation: delivery.signAttestation({ + nodeIdClaim: 1, + stack: 'demo', + op: 'stack-deploy', + sourceHash: 'abc', + referencedHostsHash: delivery.hashHostList(['ghcr.io']), + coveredHostsHash: delivery.hashHostList([]), + actionSetHash: 'def', + }), + }; + mockAxiosPost.mockResolvedValue({ status: 200, data: discover }); + + let releaseEnvelope: (() => void) | undefined; + vi.spyOn(delivery, 'buildHubEnvelope').mockImplementation(() => new Promise((resolve) => { + releaseEnvelope = () => resolve({ + attestation: discover.attestation, + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: discover.deliverySourceId, + }); + })); + + const controller = new AbortController(); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const node = DatabaseService.getInstance().getNode(nodeId)!; + const pending = augmentJsonBodyForRegistryDelivery({ + method: 'POST', + apiPath: '/api/stacks/demo/deploy', + nodeId, + node, + target: { apiUrl: 'http://remote:1852', apiToken: 'token' }, + body: {}, + abortSignal: controller.signal, + }); + + setTimeout(() => controller.abort(), 10); + setTimeout(() => releaseEnvelope?.(), 50); + + const result = await pending; + expect(result).toEqual({ ok: false, status: 499, error: 'Request aborted' }); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-prep-claim.test.ts b/backend/src/__tests__/registry-delivery-prep-claim.test.ts new file mode 100644 index 00000000..381588f6 --- /dev/null +++ b/backend/src/__tests__/registry-delivery-prep-claim.test.ts @@ -0,0 +1,123 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import jwt from 'jsonwebtoken'; +import { setupTestDb } from './helpers/setupTestDb'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { StackOpLockService } from '../services/StackOpLockService'; +import { PreparedSourceStore } from '../services/preparedSourceStore'; +import { resolveRegistryAuthAtSeam } from '../helpers/registryDeliverySeam'; +import { hashActionSet, hashDeliverySourceDir } from '../helpers/registryDeliveryHashes'; +import { discoverRegistryReferences } from '../services/registryReferenceDiscovery'; +import { normalizeImageHost } from '../services/RegistryService'; + +function acquireLockForAttestation( + nodeId: number, + stack: string, + attestation: string, + stage: string, +): void { + const payload = jwt.decode(attestation) as jwt.JwtPayload; + StackOpLockService.getInstance().tryAcquire(nodeId, stack, 'deploy', 'admin', { + opId: String(payload.jti_t), + kind: stage, + }); +} + +describe('registryDeliverySeam prepared source claim', () => { + beforeEach(async () => { + await setupTestDb(); + RegistryDeliveryService.resetForTests(); + StackOpLockService.resetForTests(); + const deliverySourceId = RegistryDeliveryService.getInstance().getDeliverySourceId(); + PreparedSourceStore.getInstance().configure(deliverySourceId); + }); + + it('claims prepId and merges delivered credentials from prepared payload', async () => { + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const stackName = 'prep-claim-stack'; + const stagingDir = path.join(process.env.TMPDIR || '/tmp', `sencho-prep-${Date.now()}`); + fs.mkdirSync(stagingDir, { recursive: true }); + fs.writeFileSync( + path.join(stagingDir, 'compose.yaml'), + 'services:\n app:\n image: ghcr.io/example/private/app:latest\n', + ); + + const sourceHash = hashDeliverySourceDir(stagingDir); + const entry = await PreparedSourceStore.getInstance().prepareFromDirectory( + 'request-generated', + sourceHash, + stagingDir, + ); + const payloadPath = PreparedSourceStore.getInstance().peekPayloadPath(entry.prepId); + const referencedHosts = discoverRegistryReferences(payloadPath).referencedHosts; + const delivery = RegistryDeliveryService.getInstance(); + const attestation = delivery.signAttestation({ + nodeIdClaim: nodeId, + stack: stackName, + op: 'template-deploy', + sourceHash, + referencedHostsHash: delivery.hashHostList(referencedHosts), + coveredHostsHash: delivery.hashHostList([]), + actionSetHash: hashActionSet(['stack:create', 'stack:deploy']), + prepId: entry.prepId, + }); + + const envelope = { + attestation, + prepId: entry.prepId, + auths: [{ + host: 'ghcr.io', + username: 'hub-user', + password: 'hub-pass', + }], + notAfter: Date.now() + 60_000, + deliverySourceId: delivery.getDeliverySourceId(), + }; + + acquireLockForAttestation(nodeId, stackName, attestation, 'template-deploy'); + + const result = await resolveRegistryAuthAtSeam({ + envelope, + nodeId, + stack: stackName, + stage: 'template-deploy', + }); + + const ghcrKey = referencedHosts.map(normalizeImageHost).find(h => h.includes('ghcr')) ?? 'ghcr.io'; + expect(result.auths[ghcrKey]).toBeDefined(); + expect(result.prepId).toBe(entry.prepId); + expect(PreparedSourceStore.getInstance().getEntry(entry.prepId)?.state).toBe('claimed'); + }); + + it('rejects a substituted prepId at the seam', async () => { + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const delivery = RegistryDeliveryService.getInstance(); + const attestation = delivery.signAttestation({ + nodeIdClaim: nodeId, + stack: 'mismatch', + op: 'template-deploy', + sourceHash: 'abc', + referencedHostsHash: delivery.hashHostList([]), + coveredHostsHash: delivery.hashHostList([]), + actionSetHash: hashActionSet(['stack:create', 'stack:deploy']), + prepId: 'deadbeefdeadbeefdeadbeefdeadbeef', + }); + + acquireLockForAttestation(nodeId, 'mismatch', attestation, 'template-deploy'); + + await expect(resolveRegistryAuthAtSeam({ + envelope: { + attestation, + prepId: 'cafebabe', + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: delivery.getDeliverySourceId(), + }, + nodeId, + stack: 'mismatch', + stage: 'template-deploy', + })).rejects.toThrow(/prepId/i); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-proxy-abort.test.ts b/backend/src/__tests__/registry-delivery-proxy-abort.test.ts new file mode 100644 index 00000000..3530bd7d --- /dev/null +++ b/backend/src/__tests__/registry-delivery-proxy-abort.test.ts @@ -0,0 +1,116 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { EventEmitter } from 'events'; +import type { Request, Response } from 'express'; +import { setupTestDb } from './helpers/setupTestDb'; +import { DatabaseService } from '../services/DatabaseService'; +import { NodeRegistry } from '../services/NodeRegistry'; + +const mockWouldAttempt = vi.fn(); + +vi.mock('../helpers/registryDeliveryOutbound', () => ({ + wouldAttemptRegistryDelivery: (...args: unknown[]) => mockWouldAttempt(...args), +})); + +describe('registry delivery proxy hop abort', () => { + beforeEach(async () => { + await setupTestDb(); + vi.clearAllMocks(); + }); + + function mockReqRes(): { req: Request; res: Response } { + const req = new EventEmitter() as Request; + const res = new EventEmitter() as Response; + Object.defineProperty(res, 'writableEnded', { value: false, writable: true }); + return { req, res }; + } + + it('aborts the hop signal when the client disconnects during capability probing', async () => { + const { ensureRegistryDeliveryHopAbortController } = await import('../helpers/registryDeliveryProxy'); + const { req, res } = mockReqRes(); + + let resolveProbe: ((value: boolean) => void) | undefined; + mockWouldAttempt.mockImplementation(() => new Promise((resolve) => { + resolveProbe = resolve; + })); + + ensureRegistryDeliveryHopAbortController(req, res); + const probe = mockWouldAttempt( + NodeRegistry.getInstance().getDefaultNodeId(), + DatabaseService.getInstance().getNode(NodeRegistry.getInstance().getDefaultNodeId())!, + 'POST', + '/api/blueprints/apply-local', + ); + + req.emit('aborted'); + resolveProbe?.(true); + + await probe; + expect(req.registryDeliveryAbortController?.signal.aborted).toBe(true); + }); + + it('returns aborted from decideRegistryDeliveryProxyHop when disconnected during the probe', async () => { + const { decideRegistryDeliveryProxyHop } = await import('../helpers/registryDeliveryProxy'); + const { req, res } = mockReqRes(); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const node = DatabaseService.getInstance().getNode(nodeId)!; + + let resolveProbe: ((value: boolean) => void) | undefined; + mockWouldAttempt.mockImplementation(() => new Promise((resolve) => { + resolveProbe = resolve; + })); + + const decision = decideRegistryDeliveryProxyHop( + req, + res, + nodeId, + node, + 'POST', + '/api/blueprints/apply-local', + ); + + req.emit('aborted'); + resolveProbe?.(true); + + await expect(decision).resolves.toEqual({ action: 'aborted' }); + }); + + it('maps aborted decisions to stop at the proxy gate', async () => { + const { evaluateRegistryDeliveryProxyGate } = await import('../helpers/registryDeliveryProxy'); + const { req, res } = mockReqRes(); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const node = DatabaseService.getInstance().getNode(nodeId)!; + + Object.defineProperty(req, 'aborted', { value: true, configurable: true }); + + await expect( + evaluateRegistryDeliveryProxyGate( + req, + res, + nodeId, + node, + 'POST', + '/api/blueprints/apply-local', + ), + ).resolves.toEqual({ outcome: 'stop' }); + }); + + it('maps capability miss to continue at the proxy gate', async () => { + const { evaluateRegistryDeliveryProxyGate } = await import('../helpers/registryDeliveryProxy'); + const { req, res } = mockReqRes(); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const node = DatabaseService.getInstance().getNode(nodeId)!; + + mockWouldAttempt.mockResolvedValue(false); + + await expect( + evaluateRegistryDeliveryProxyGate( + req, + res, + nodeId, + node, + 'POST', + '/api/blueprints/apply-local', + ), + ).resolves.toEqual({ outcome: 'continue' }); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-proxy-gate.test.ts b/backend/src/__tests__/registry-delivery-proxy-gate.test.ts new file mode 100644 index 00000000..e8358091 --- /dev/null +++ b/backend/src/__tests__/registry-delivery-proxy-gate.test.ts @@ -0,0 +1,172 @@ +/** + * Production proxy orchestration for registry credential delivery: abort must + * stop forwarding, compressed bodies pass through when delivery is unavailable, + * and return 415 only when delivery would run. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import http from 'http'; +import zlib from 'zlib'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; +import { REMOTE_REGISTRY_CREDENTIALS_CAPABILITY } from '../services/CapabilityRegistry'; + +let tmpDir: string; +let app: import('express').Express; +let authHeader: string; +let remoteWithCapabilityId: number; +let remoteWithoutCapabilityId: number; + +const capturedHops: Array<{ method: string; url: string }> = []; +let metaDelayMs = 0; + +function createRemoteServer(capabilities: string[]): http.Server { + return http.createServer((req, res) => { + if (req.url?.startsWith('/api/meta')) { + const respond = () => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ version: '0.97.1', capabilities })); + }; + if (metaDelayMs > 0) { + setTimeout(respond, metaDelayMs); + return; + } + respond(); + return; + } + capturedHops.push({ method: req.method ?? '', url: req.url ?? '' }); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true })); + }); +} + +async function listen(server: http.Server): Promise { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return (server.address() as import('net').AddressInfo).port; +} + +let capableServer: http.Server; +let incapableServer: http.Server; +let capablePort: number; +let incapablePort: number; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + + const { DatabaseService } = await import('../services/DatabaseService'); + const { RegistryDeliveryService } = await import('../services/RegistryDeliveryService'); + vi.spyOn(RegistryDeliveryService.getInstance(), 'isProxyTransportConfidential').mockReturnValue(true); + + capableServer = createRemoteServer([REMOTE_REGISTRY_CREDENTIALS_CAPABILITY]); + incapableServer = createRemoteServer([]); + capablePort = await listen(capableServer); + incapablePort = await listen(incapableServer); + + remoteWithCapabilityId = DatabaseService.getInstance().addNode({ + name: 'regdelivery-capable', + type: 'remote', + mode: 'proxy', + compose_dir: '/tmp', + is_default: false, + api_url: `http://127.0.0.1:${capablePort}`, + api_token: 'capable-token', + }); + + remoteWithoutCapabilityId = DatabaseService.getInstance().addNode({ + name: 'regdelivery-incapable', + type: 'remote', + mode: 'proxy', + compose_dir: '/tmp', + is_default: false, + api_url: `http://127.0.0.1:${incapablePort}`, + api_token: 'incapable-token', + }); + + const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1h' }); + authHeader = `Bearer ${token}`; +}); + +afterAll(async () => { + await Promise.all([ + new Promise((resolve) => capableServer.close(() => resolve())), + new Promise((resolve) => incapableServer.close(() => resolve())), + ]); + cleanupTestDb(tmpDir); + vi.restoreAllMocks(); +}); + +beforeEach(() => { + capturedHops.length = 0; + metaDelayMs = 0; +}); + +describe('remoteNodeProxy registry delivery gate', () => { + const deployPath = '/api/stacks/reg-proxy-gate/deploy'; + const gzipBody = zlib.gzipSync(Buffer.from('{}', 'utf-8')); + + it('rejects gzip-encoded deploy when delivery would run', async () => { + const res = await request(app) + .post(deployPath) + .set('Authorization', authHeader) + .set('x-node-id', String(remoteWithCapabilityId)) + .set('Content-Encoding', 'gzip') + .send(gzipBody); + + expect(res.status).toBe(415); + expect(res.body.code).toBe('encoding_unsupported'); + expect(capturedHops.some((h) => h.url.includes('/deploy'))).toBe(false); + }); + + it('forwards gzip-encoded deploy unchanged when delivery is unavailable', async () => { + const res = await request(app) + .post(deployPath) + .set('Authorization', authHeader) + .set('x-node-id', String(remoteWithoutCapabilityId)) + .set('Content-Encoding', 'gzip') + .send(gzipBody); + + expect(res.status).toBe(200); + expect(capturedHops.some((h) => h.url.includes('/deploy'))).toBe(true); + }); + + it('does not forward deploy after client abort during capability probing', async () => { + metaDelayMs = 800; + const server = await new Promise((resolve) => { + const s = app.listen(0, '127.0.0.1', () => resolve(s)); + }); + const port = (server.address() as import('net').AddressInfo).port; + + const outcome = await new Promise<{ aborted: boolean }>((resolve) => { + const req = http.request( + { + hostname: '127.0.0.1', + port, + path: deployPath, + method: 'POST', + headers: { + Authorization: authHeader, + 'x-node-id': String(remoteWithCapabilityId), + 'Content-Type': 'application/json', + 'Content-Length': '2', + }, + }, + () => resolve({ aborted: false }), + ); + req.on('error', (err: NodeJS.ErrnoException) => { + if (err.code === 'ECONNRESET' || err.message === 'aborted') { + resolve({ aborted: true }); + return; + } + resolve({ aborted: false }); + }); + req.write('{}'); + req.end(); + setTimeout(() => req.destroy(), 50); + }); + + await new Promise((resolve) => server.close(() => resolve())); + expect(outcome.aborted).toBe(true); + expect(capturedHops.some((h) => h.url.includes('/deploy'))).toBe(false); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-seam.test.ts b/backend/src/__tests__/registry-delivery-seam.test.ts new file mode 100644 index 00000000..867a6d9e --- /dev/null +++ b/backend/src/__tests__/registry-delivery-seam.test.ts @@ -0,0 +1,160 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import jwt from 'jsonwebtoken'; +import { setupTestDb } from './helpers/setupTestDb'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { StackOpLockService } from '../services/StackOpLockService'; +import { resolveRegistryAuthAtSeam } from '../helpers/registryDeliverySeam'; +import { hashActionSet, hashProjectSource } from '../helpers/registryDeliveryHashes'; +import { discoverRegistryReferences } from '../services/registryReferenceDiscovery'; +import { normalizeImageHost } from '../services/RegistryService'; + +function acquireLockForAttestation( + nodeId: number, + stack: string, + attestation: string, + stage: string, +): void { + const payload = jwt.decode(attestation) as jwt.JwtPayload; + StackOpLockService.getInstance().tryAcquire(nodeId, stack, 'deploy', 'admin', { + opId: String(payload.jti_t), + kind: stage, + }); +} + +describe('registryDeliverySeam', () => { + beforeEach(async () => { + await setupTestDb(); + RegistryDeliveryService.resetForTests(); + StackOpLockService.resetForTests(); + }); + + it('merges delivered credentials for target-missing hosts at the seam', async () => { + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const stackName = 'regcred-test'; + const composeDir = NodeRegistry.getInstance().getComposeDir(nodeId); + const stackDir = path.join(composeDir, stackName); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync( + path.join(stackDir, 'compose.yaml'), + 'services:\n app:\n image: ghcr.io/example/private/app:latest\n', + ); + + const sourceHash = hashProjectSource(stackDir); + const referencedHosts = discoverRegistryReferences(stackDir).referencedHosts; + const delivery = RegistryDeliveryService.getInstance(); + const attestation = delivery.signAttestation({ + nodeIdClaim: nodeId, + stack: stackName, + op: 'stack-deploy', + sourceHash, + referencedHostsHash: delivery.hashHostList(referencedHosts), + coveredHostsHash: delivery.hashHostList([]), + actionSetHash: hashActionSet(['stack:deploy']), + }); + + const envelope = { + attestation, + auths: [{ + host: 'ghcr.io', + username: 'hub-user', + password: 'hub-pass', + }], + notAfter: Date.now() + 60_000, + deliverySourceId: delivery.getDeliverySourceId(), + }; + + acquireLockForAttestation(nodeId, stackName, attestation, 'stack-deploy'); + + const result = await resolveRegistryAuthAtSeam({ + envelope, + nodeId, + stack: stackName, + stage: 'stack-deploy', + }); + + const ghcrKey = referencedHosts.map(normalizeImageHost).find(h => h.includes('ghcr')) ?? 'ghcr.io'; + expect(result.auths[ghcrKey]).toBeDefined(); + }); + + it('rejects replayed jti at the seam', async () => { + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const stackName = 'regcred-replay'; + const composeDir = NodeRegistry.getInstance().getComposeDir(nodeId); + const stackDir = path.join(composeDir, stackName); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n'); + + const sourceHash = hashProjectSource(stackDir); + const referencedHosts = discoverRegistryReferences(stackDir).referencedHosts; + const delivery = RegistryDeliveryService.getInstance(); + const attestation = delivery.signAttestation({ + nodeIdClaim: nodeId, + stack: stackName, + op: 'stack-deploy', + sourceHash, + referencedHostsHash: delivery.hashHostList(referencedHosts), + coveredHostsHash: delivery.hashHostList([]), + actionSetHash: hashActionSet(['stack:deploy']), + }); + + const envelope = { + attestation, + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: delivery.getDeliverySourceId(), + }; + + acquireLockForAttestation(nodeId, stackName, attestation, 'stack-deploy'); + + await resolveRegistryAuthAtSeam({ + envelope, + nodeId, + stack: stackName, + stage: 'stack-deploy', + }); + + await expect(resolveRegistryAuthAtSeam({ + envelope, + nodeId, + stack: stackName, + stage: 'stack-deploy', + })).rejects.toThrow(/consumed/i); + }); + + it('rejects seam when stack lock is not held', async () => { + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const stackName = 'regcred-no-lock'; + const composeDir = NodeRegistry.getInstance().getComposeDir(nodeId); + const stackDir = path.join(composeDir, stackName); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n'); + + const sourceHash = hashProjectSource(stackDir); + const referencedHosts = discoverRegistryReferences(stackDir).referencedHosts; + const delivery = RegistryDeliveryService.getInstance(); + const attestation = delivery.signAttestation({ + nodeIdClaim: nodeId, + stack: stackName, + op: 'stack-deploy', + sourceHash, + referencedHostsHash: delivery.hashHostList(referencedHosts), + coveredHostsHash: delivery.hashHostList([]), + actionSetHash: hashActionSet(['stack:deploy']), + }); + + await expect(resolveRegistryAuthAtSeam({ + envelope: { + attestation, + auths: [], + notAfter: Date.now() + 60_000, + deliverySourceId: delivery.getDeliverySourceId(), + }, + nodeId, + stack: stackName, + stage: 'stack-deploy', + })).rejects.toThrow(/stack lock required/i); + }); +}); diff --git a/backend/src/__tests__/registry-delivery-service.test.ts b/backend/src/__tests__/registry-delivery-service.test.ts new file mode 100644 index 00000000..cf4d7ce9 --- /dev/null +++ b/backend/src/__tests__/registry-delivery-service.test.ts @@ -0,0 +1,114 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import { setupTestDb } from './helpers/setupTestDb'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { hashActionSet, hashProjectSource } from '../helpers/registryDeliveryHashes'; +import { discoverRegistryReferences } from '../services/registryReferenceDiscovery'; +import { resolveComposeEnvForDiscovery } from '../helpers/registryDeliveryComposeEnv'; + +describe('RegistryDeliveryService', () => { + beforeEach(async () => { + await setupTestDb(); + RegistryDeliveryService.resetForTests(); + }); + + it('evicts expired consumed jtis before accepting new ones', () => { + const delivery = RegistryDeliveryService.getInstance(); + const now = Date.now(); + + delivery.consumeAttestationJti('expired-a', now - 1_000); + expect(() => delivery.consumeAttestationJti('fresh', now + 60_000)).not.toThrow(); + }); + + it('reclaims replay-store capacity after expired jtis are evicted', () => { + const delivery = RegistryDeliveryService.getInstance(); + delivery.setReplayStoreCapacityForTests(2); + const now = Date.now(); + + delivery.consumeAttestationJti('expired-slot', now - 1); + delivery.consumeAttestationJti('active-slot', now + 60_000); + expect(() => delivery.consumeAttestationJti('fresh-after-evict', now + 60_000)).not.toThrow(); + expect(() => delivery.consumeAttestationJti('overflow', now + 60_000)).toThrow(/capacity/i); + }); + + it('rejects a jti that is still within its replay window', () => { + const delivery = RegistryDeliveryService.getInstance(); + delivery.consumeAttestationJti('active', Date.now() + 60_000); + expect(() => delivery.consumeAttestationJti('active', Date.now() + 60_000)).toThrow(/consumed/i); + }); + + it('rejects restore-candidate discover when stack name is invalid', async () => { + const delivery = RegistryDeliveryService.getInstance(); + + await expect(delivery.discoverOnTarget({ + op: 'stack-deploy', + sourceKind: 'restore-candidate', + stack: '../escape', + actionSetHash: hashActionSet(['stack:deploy']), + })).rejects.toThrow(/invalid stack name/i); + }); + + it('blueprint body-content discover matches post-apply live-project hash and hosts', async () => { + const delivery = RegistryDeliveryService.getInstance(); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const stackName = 'bp-regdisc-parity'; + const composeDir = NodeRegistry.getInstance().getComposeDir(nodeId); + const stackDir = path.join(composeDir, stackName); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync( + path.join(stackDir, 'compose.yaml'), + 'services:\n old:\n image: nginx:latest\n', + ); + fs.writeFileSync(path.join(stackDir, '.env'), 'REGISTRY=ghcr.io\n'); + + const incomingCompose = 'services:\n app:\n image: ${REGISTRY}/org/private:latest\n'; + const discover = await delivery.discoverOnTarget({ + op: 'blueprint-apply', + sourceKind: 'body-content', + stack: stackName, + composeContent: incomingCompose, + actionSetHash: hashActionSet(['stack:deploy']), + }); + + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), incomingCompose); + const seamHash = hashProjectSource(stackDir); + const seamHosts = discoverRegistryReferences( + stackDir, + resolveComposeEnvForDiscovery(stackDir), + ).referencedHosts; + + expect(discover.sourceHash).toBe(seamHash); + expect(discover.referencedHosts).toEqual(seamHosts); + expect(discover.referencedHosts).toEqual(['ghcr.io']); + }); + + it('blueprint body-content discover matches seam hash when .env exists but is empty', async () => { + const delivery = RegistryDeliveryService.getInstance(); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const stackName = 'bp-regdisc-empty-env'; + const composeDir = NodeRegistry.getInstance().getComposeDir(nodeId); + const stackDir = path.join(composeDir, stackName); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync( + path.join(stackDir, 'compose.yaml'), + 'services:\n old:\n image: nginx:latest\n', + ); + fs.writeFileSync(path.join(stackDir, '.env'), ''); + + const incomingCompose = 'services:\n app:\n image: nginx:alpine\n'; + const discover = await delivery.discoverOnTarget({ + op: 'blueprint-apply', + sourceKind: 'body-content', + stack: stackName, + composeContent: incomingCompose, + actionSetHash: hashActionSet(['stack:deploy']), + }); + + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), incomingCompose); + const seamHash = hashProjectSource(stackDir); + + expect(discover.sourceHash).toBe(seamHash); + }); +}); diff --git a/backend/src/__tests__/registry-reference-discovery.test.ts b/backend/src/__tests__/registry-reference-discovery.test.ts new file mode 100644 index 00000000..59c213b3 --- /dev/null +++ b/backend/src/__tests__/registry-reference-discovery.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import { + discoverRegistryReferences, + discoverRegistryReferencesFromComposeContent, + parseDockerfileReferences, +} from '../services/registryReferenceDiscovery'; + +describe('registryReferenceDiscovery', () => { + it('discovers hosts from compose files and Dockerfiles', () => { + const dir = path.join(process.env.TMPDIR || '/tmp', `sencho-refdisc-${Date.now()}`); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'compose.yaml'), + 'services:\n app:\n image: ghcr.io/org/private-app:latest\n', + ); + fs.writeFileSync( + path.join(dir, 'Dockerfile'), + 'FROM docker.io/library/node:20\nCOPY --from=ghcr.io/org/cache:1 /app /app\n', + ); + + const result = discoverRegistryReferences(dir); + expect(result.referencedHosts).toContain('ghcr.io'); + expect(result.referencedHosts).toContain('index.docker.io'); + }); + + it('discovers hosts from inline compose content', () => { + const result = discoverRegistryReferencesFromComposeContent( + 'services:\n app:\n image: ghcr.io/org/private-app:latest\n', + ); + expect(result.referencedHosts).toEqual(['ghcr.io']); + }); + + it('ignores numeric COPY --from stages in isolation', () => { + const hosts = parseDockerfileReferences('COPY --from=0 /src /dest\n'); + expect(hosts).toEqual([]); + }); + + it('parses FROM lines in Dockerfiles', () => { + const hosts = parseDockerfileReferences('FROM alpine:3\n'); + expect(hosts).toEqual(['index.docker.io']); + }); + + it('rejects oversized Dockerfiles', () => { + const dir = path.join(process.env.TMPDIR || '/tmp', `sencho-refdisc-big-${Date.now()}`); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'compose.yaml'), 'services:\n web:\n image: nginx\n'); + fs.writeFileSync(path.join(dir, 'Dockerfile'), 'x'.repeat(1_048_577)); + + expect(() => discoverRegistryReferences(dir)).toThrow(/size limit/i); + }); + + it('resolves registry variables from .env when env map is supplied', () => { + const dir = path.join(process.env.TMPDIR || '/tmp', `sencho-refdisc-env-${Date.now()}`); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, 'compose.yaml'), + 'services:\n app:\n image: ${REGISTRY}/org/private:latest\n', + ); + + const result = discoverRegistryReferences(dir, { REGISTRY: 'ghcr.io' }); + expect(result.referencedHosts).toEqual(['ghcr.io']); + }); +}); diff --git a/backend/src/__tests__/trusted-proxy-cidrs.test.ts b/backend/src/__tests__/trusted-proxy-cidrs.test.ts new file mode 100644 index 00000000..aad68fda --- /dev/null +++ b/backend/src/__tests__/trusted-proxy-cidrs.test.ts @@ -0,0 +1,37 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { + getTrustedProxyBlockList, + isTrustedProxyPeer, + resetTrustedProxyBlockListCache, +} from '../helpers/trustedProxyCidrs'; + +describe('trustedProxyCidrs', () => { + beforeEach(() => { + delete process.env.SENCHO_TRUSTED_PROXY_CIDRS; + resetTrustedProxyBlockListCache(); + }); + + it('returns null when unset', () => { + expect(getTrustedProxyBlockList()).toBeNull(); + expect(isTrustedProxyPeer('10.0.0.1')).toBe(false); + }); + + it('matches IPv4 CIDR members', () => { + process.env.SENCHO_TRUSTED_PROXY_CIDRS = '10.0.0.0/8'; + resetTrustedProxyBlockListCache(); + expect(isTrustedProxyPeer('10.1.2.3')).toBe(true); + expect(isTrustedProxyPeer('192.168.1.1')).toBe(false); + }); + + it('fails closed on invalid entries', () => { + process.env.SENCHO_TRUSTED_PROXY_CIDRS = 'not-a-cidr'; + resetTrustedProxyBlockListCache(); + expect(getTrustedProxyBlockList()).toBeNull(); + }); + + it('fails closed on duplicate entries', () => { + process.env.SENCHO_TRUSTED_PROXY_CIDRS = '10.0.0.0/8,10.0.0.0/8'; + resetTrustedProxyBlockListCache(); + expect(getTrustedProxyBlockList()).toBeNull(); + }); +}); diff --git a/backend/src/app.ts b/backend/src/app.ts index d0ca05e5..4755a954 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -12,7 +12,7 @@ import './types/express'; /** * Build an Express app with the full middleware pipeline installed. * - * Canonical middleware order (18 steps). Do not reorder without re-running the + * Canonical middleware order (19 steps). Do not reorder without re-running the * regression checklist in `docs/internal/architecture/middleware-order.md`. * * 1. trust proxy @@ -29,10 +29,11 @@ import './types/express'; * 12. auditLog (at /api) -- registered in index.ts * 13. enforceApiTokenScope (at /api) -- registered in index.ts * 14. hubOnlyGuard (at /api) -- middleware/hubOnlyGuard.ts, registered in index.ts - * 15. createRemoteProxyMiddleware -- proxy/remoteNodeProxy.ts, registered in index.ts - * 16. routes -- registered in index.ts from routes/* - * 17. static serving + SPA fallback -- registered in index.ts - * 18. errorHandler -- registered in index.ts + * 15. registryDeliveryMiddleware (at /api) -- middleware/registryDelivery.ts, registered in index.ts + * 16. createRemoteProxyMiddleware -- proxy/remoteNodeProxy.ts, registered in index.ts + * 17. routes -- registered in index.ts from routes/* + * 18. static serving + SPA fallback -- registered in index.ts + * 19. errorHandler -- registered in index.ts * * Steps 11 to 14 and 16 must run after the public auth routers (meta, auth, * mfa, sso) are registered so those routes stay reachable without a session diff --git a/backend/src/bootstrap/shutdown.ts b/backend/src/bootstrap/shutdown.ts index fe2ba8af..82c62a03 100644 --- a/backend/src/bootstrap/shutdown.ts +++ b/backend/src/bootstrap/shutdown.ts @@ -15,6 +15,8 @@ import { MeshService } from '../services/MeshService'; import { BlueprintReconciler } from '../services/BlueprintReconciler'; import { CveIntelService } from '../services/CveIntelService'; import { PilotMetrics } from '../services/PilotMetrics'; +import { PreparedSourceStore } from '../services/preparedSourceStore'; +import { RegistryDeliveryReconciler } from '../services/RegistryDeliveryReconciler'; /** * Wire graceful shutdown handlers. Docker sends SIGTERM when the container @@ -65,6 +67,12 @@ export function installShutdownHandlers(server: Server): void { try { PilotMetrics.flush(); } catch (e) { console.warn('[Shutdown] PilotMetrics flush failed:', (e as Error).message); } + try { PreparedSourceStore.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] PreparedSourceStore cleanup failed:', (e as Error).message); + } + try { RegistryDeliveryReconciler.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] RegistryDeliveryReconciler cleanup failed:', (e as Error).message); + } try { DatabaseService.getInstance().flushAuditLogBuffer(); } catch (e) { console.warn('[Shutdown] Audit log flush failed:', (e as Error).message); } diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index 5fa15cc9..e9453541 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -35,6 +35,11 @@ import { setGitOpsEventSink } from '../services/gitops/publish'; import { NotificationService } from '../services/NotificationService'; import { sanitizeForLog } from '../utils/safeLog'; import { PORT } from '../helpers/constants'; +import { sweepDockerAuthTempDirs, classifyDockerAuthChildName } from '../helpers/dockerAuthTempDir'; +import { recordRegistryDeliveryEvent } from '../helpers/registryDeliveryEvidence'; +import { PreparedSourceStore } from '../services/preparedSourceStore'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { RegistryDeliveryReconciler } from '../services/RegistryDeliveryReconciler'; import { LOW_MEMORY_FLOOR_BYTES } from '../utils/spawnErrors'; function isPilotMode(): boolean { @@ -237,6 +242,41 @@ export async function startServer(server: Server): Promise { console.warn('[GitManifest] Managed-area sweep failed:', err instanceof Error ? err.message : String(err)); } + // Registry delivery recovery sweeps must settle before any mutation-capable + // producer starts (AUD-36). + try { + const deliverySourceId = RegistryDeliveryService.getInstance().getDeliverySourceId(); + PreparedSourceStore.getInstance().configure(deliverySourceId); + const dockerSweep = await sweepDockerAuthTempDirs(deliverySourceId); + for (const basename of dockerSweep.swept) { + const kind = classifyDockerAuthChildName(basename); + recordRegistryDeliveryEvent({ + deliverySourceId, + eventType: kind === 'delivered' ? 'swept_delivered' : 'swept_local', + tempDirId: basename, + }); + } + for (const basename of dockerSweep.legacyOrphansObserved) { + recordRegistryDeliveryEvent({ + deliverySourceId, + eventType: 'legacy_orphan_observed', + tempDirId: basename, + }); + } + const preparedSwept = await PreparedSourceStore.getInstance().sweepOrphans(deliverySourceId); + for (const basename of preparedSwept) { + recordRegistryDeliveryEvent({ + deliverySourceId, + eventType: 'swept_prepared', + tempDirId: basename, + }); + } + PreparedSourceStore.getInstance().start(); + RegistryDeliveryReconciler.getInstance().start(); + } catch (err) { + console.warn('[RegistryDelivery] Startup sweep failed:', err instanceof Error ? err.message : String(err)); + } + // Synchronous starts: schedule background timers and continue. None of // these fire their first tick for at least a few seconds, so they // safely run alongside the async initializers below. @@ -262,6 +302,7 @@ export async function startServer(server: Server): Promise { PilotTunnelManager.getInstance().on('tunnel-up', (nodeId: number) => { invalidateRemoteMetaCache(nodeId); void SuppressionRetractionRetryService.getInstance().flushNode(nodeId); + void RegistryDeliveryReconciler.getInstance().reconcileNode(nodeId); }); // Most async initializers still run in parallel. Docker event monitoring diff --git a/backend/src/helpers/composeInputParse.ts b/backend/src/helpers/composeInputParse.ts index 5558aa60..9e287288 100644 --- a/backend/src/helpers/composeInputParse.ts +++ b/backend/src/helpers/composeInputParse.ts @@ -88,7 +88,8 @@ interface FileContext { /** True when the path contains a Compose `$VAR` / `${VAR}` interpolation form. */ export function isDynamicPath(p: string): boolean { - return /\$\{[^}]*\}|\$[A-Za-z_][A-Za-z0-9_]*/.test(p); + if (p.includes('${')) return true; + return /\$[A-Za-z_][A-Za-z0-9_]*/.test(p); } /** Absolute (POSIX, Windows drive/UNC, drive-relative, root-relative) or home-relative host path. */ diff --git a/backend/src/helpers/dockerAuthTempDir.ts b/backend/src/helpers/dockerAuthTempDir.ts new file mode 100644 index 00000000..dfece688 --- /dev/null +++ b/backend/src/helpers/dockerAuthTempDir.ts @@ -0,0 +1,172 @@ +import crypto from 'crypto'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { ensureTrustedRoot, validateTrustedRoot } from './privateRootValidator'; + +export const DOCKER_AUTH_MARKER_FILE = '.sencho-docker-auth'; +export const DOCKER_AUTH_PARENT_PREFIX = 'sencho-docker-auth-'; +export const DOCKER_AUTH_LOCAL_CHILD_PREFIX = 'local-'; +export const DOCKER_AUTH_DELIVERED_CHILD_PREFIX = 'delivered-'; + +export type DockerAuthChildKind = 'local' | 'delivered'; + +export interface DockerAuthTempDirHandle { + dirPath: string; + kind: DockerAuthChildKind; + cleanup: () => void; +} + +function deliverySourceHash(deliverySourceId: string): string { + return crypto.createHash('sha256').update(`docker-auth:${deliverySourceId}`).digest('hex'); +} + +export function getDockerAuthParentPath(deliverySourceId: string): string { + return path.join(os.tmpdir(), `${DOCKER_AUTH_PARENT_PREFIX}${deliverySourceHash(deliverySourceId)}`); +} + +function childPrefixForKind(kind: DockerAuthChildKind): string { + return kind === 'local' ? DOCKER_AUTH_LOCAL_CHILD_PREFIX : DOCKER_AUTH_DELIVERED_CHILD_PREFIX; +} + +function isTypedChildName(name: string): DockerAuthChildKind | null { + if (name.startsWith(DOCKER_AUTH_LOCAL_CHILD_PREFIX)) return 'local'; + if (name.startsWith(DOCKER_AUTH_DELIVERED_CHILD_PREFIX)) return 'delivered'; + return null; +} + +function publishMarker(childDir: string): void { + const markerPath = path.join(childDir, DOCKER_AUTH_MARKER_FILE); + const fd = fs.openSync(markerPath, 'wx', 0o600); + try { + fs.writeSync(fd, 'docker-auth\n'); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +function removeChildDir(childDir: string): void { + try { + const configPath = path.join(childDir, 'config.json'); + try { fs.unlinkSync(configPath); } catch { /* may not exist */ } + try { fs.unlinkSync(path.join(childDir, DOCKER_AUTH_MARKER_FILE)); } catch { /* may not exist */ } + fs.rmdirSync(childDir); + } catch { + /* best effort */ + } +} + +/** + * Create a typed Docker-auth child directory with marker-before-payload publication. + */ +export function createDockerAuthTempDir( + deliverySourceId: string, + kind: DockerAuthChildKind, + config: { auths: Record }, +): DockerAuthTempDirHandle { + const parentPath = getDockerAuthParentPath(deliverySourceId); + const parentValidation = ensureTrustedRoot({ rootPath: parentPath, kind: 'docker-auth' }); + if (!parentValidation.ok) { + throw new Error(parentValidation.reason); + } + + const childName = `${childPrefixForKind(kind)}${crypto.randomBytes(8).toString('hex')}`; + const childDir = path.join(parentPath, childName); + + if (fs.existsSync(childDir)) { + throw new Error('docker-auth child collision'); + } + + fs.mkdirSync(childDir, { mode: 0o700 }); + try { + publishMarker(childDir); + const configPath = path.join(childDir, 'config.json'); + const fd = fs.openSync(configPath, 'wx', 0o600); + try { + fs.writeSync(fd, JSON.stringify(config)); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + } catch (error) { + removeChildDir(childDir); + throw error; + } + + return { + dirPath: childDir, + kind, + cleanup: () => removeChildDir(childDir), + }; +} + +export interface DockerAuthSweepResult { + swept: string[]; + preservedMarkerless: string[]; + legacyOrphansObserved: string[]; +} + +/** + * Sweep marked crash remnants under the owned Docker-auth parent. Markerless + * children are preserved because they contain no published payload. + */ +export async function sweepDockerAuthTempDirs(deliverySourceId: string): Promise { + const parentPath = getDockerAuthParentPath(deliverySourceId); + const result: DockerAuthSweepResult = { + swept: [], + preservedMarkerless: [], + legacyOrphansObserved: [], + }; + + const parentValidation = validateTrustedRoot({ rootPath: parentPath, kind: 'docker-auth' }); + if (!parentValidation.ok) { + return result; + } + + let entries: string[]; + try { + entries = await fs.promises.readdir(parentPath); + } catch { + return result; + } + + for (const entry of entries) { + const childPath = path.join(parentPath, entry); + let stat: fs.Stats; + try { + stat = await fs.promises.lstat(childPath); + } catch { + continue; + } + if (!stat.isDirectory() || stat.isSymbolicLink()) continue; + if (!isTypedChildName(entry)) continue; + + const markerPath = path.join(childPath, DOCKER_AUTH_MARKER_FILE); + if (!fs.existsSync(markerPath)) { + result.preservedMarkerless.push(entry); + continue; + } + + removeChildDir(childPath); + result.swept.push(entry); + } + + // Observe but do not delete legacy top-level sencho-docker-* dirs. + try { + const tmpEntries = await fs.promises.readdir(os.tmpdir()); + for (const entry of tmpEntries) { + if (entry.startsWith('sencho-docker-') && !entry.startsWith(DOCKER_AUTH_PARENT_PREFIX)) { + result.legacyOrphansObserved.push(entry); + } + } + } catch { + /* ignore */ + } + + return result; +} + +export function classifyDockerAuthChildName(name: string): DockerAuthChildKind | null { + return isTypedChildName(name); +} diff --git a/backend/src/helpers/dockerComposeRunner.ts b/backend/src/helpers/dockerComposeRunner.ts new file mode 100644 index 00000000..596e00e2 --- /dev/null +++ b/backend/src/helpers/dockerComposeRunner.ts @@ -0,0 +1,43 @@ +import { spawn } from 'child_process'; +import os from 'os'; +import path from 'path'; +import { managedAreaBase } from '../services/gitops/managedPaths'; + +export function runDockerCompose( + args: string[], + cwd: string, + timeoutMs: number, +): Promise<{ code: number; stdout: string; stderr: string }> { + const resolvedCwd = path.resolve(cwd); + const managedBase = path.resolve(managedAreaBase()); + const tmpBase = path.resolve(os.tmpdir()); + // Canonical inline js/path-injection barrier, kept in the same scope as the + // spawn cwd sink below. CodeQL does not credit a barrier separated from the + // sink by the Promise-executor closure, so spawn is hoisted out of it. + // Two sequential single-condition guards, not one compound negated-AND, + // since CodeQL's barrier-guard recognizer only credits the simple shape. + if (!resolvedCwd.startsWith(managedBase + path.sep)) { + if (!resolvedCwd.startsWith(tmpBase + path.sep)) { + return Promise.resolve({ code: -1, stdout: '', stderr: 'Invalid working directory' }); + } + } + const child = spawn('docker', args, { cwd: resolvedCwd }); + return new Promise((resolve) => { + let stdout = ''; + let stderr = ''; + const timer = setTimeout(() => { + try { child.kill('SIGKILL'); } catch { /* best effort */ } + resolve({ code: -1, stdout, stderr: stderr + '\nValidation timed out.' }); + }, timeoutMs); + child.stdout.on('data', (d) => { stdout += d.toString(); }); + child.stderr.on('data', (d) => { stderr += d.toString(); }); + child.on('close', (code) => { + clearTimeout(timer); + resolve({ code: code ?? -1, stdout, stderr }); + }); + child.on('error', (err) => { + clearTimeout(timer); + resolve({ code: -1, stdout, stderr: stderr + '\n' + err.message }); + }); + }); +} diff --git a/backend/src/helpers/privateRootValidator.ts b/backend/src/helpers/privateRootValidator.ts new file mode 100644 index 00000000..e65eef68 --- /dev/null +++ b/backend/src/helpers/privateRootValidator.ts @@ -0,0 +1,91 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +export interface TrustedRootValidationOptions { + /** Absolute path to the application-owned root directory. */ + rootPath: string; + /** Human-readable kind for error messages (e.g. docker-auth, prepared-source). */ + kind: string; +} + +export type TrustedRootValidationResult = + | { + ok: true; + resolvedRoot: string; + resolvedTempRoot: string; + } + | { + ok: false; + reason: string; + }; + +function getExpectedUid(): number | null { + if (typeof process.getuid === 'function') { + return process.getuid(); + } + return null; +} + +function hasPrivateDirectoryMode(mode: number): boolean { + return (mode & 0o777) === 0o700; +} + +/** + * Validate that an application-owned root directory is safe to use for + * sensitive payload writes and startup sweeps. Fails closed when the path is + * missing, not a directory, a symlink, owned by another user, or has loose + * permissions, or when resolved containment escapes the system temp root. + */ +export function validateTrustedRoot(options: TrustedRootValidationOptions): TrustedRootValidationResult { + const { rootPath, kind } = options; + + let rootStat: fs.Stats; + try { + rootStat = fs.lstatSync(rootPath); + } catch { + return { ok: false, reason: `${kind} root does not exist` }; + } + + if (!rootStat.isDirectory()) { + return { ok: false, reason: `${kind} root is not a directory` }; + } + if (rootStat.isSymbolicLink()) { + return { ok: false, reason: `${kind} root is a symlink` }; + } + if (!hasPrivateDirectoryMode(rootStat.mode)) { + return { ok: false, reason: `${kind} root permissions are not private` }; + } + + const expectedUid = getExpectedUid(); + if (expectedUid !== null && rootStat.uid !== expectedUid) { + return { ok: false, reason: `${kind} root is not owned by this process` }; + } + + let resolvedRoot: string; + let resolvedTempRoot: string; + try { + resolvedRoot = fs.realpathSync(rootPath); + resolvedTempRoot = fs.realpathSync(os.tmpdir()); + } catch { + return { ok: false, reason: `${kind} root could not be resolved` }; + } + + const relative = path.relative(resolvedTempRoot, resolvedRoot); + if (relative.startsWith('..') || path.isAbsolute(relative)) { + return { ok: false, reason: `${kind} root escapes the temporary root` }; + } + + return { ok: true, resolvedRoot, resolvedTempRoot }; +} + +/** + * Create a trusted root directory with mode 0700 when absent, then validate it. + */ +export function ensureTrustedRoot(options: TrustedRootValidationOptions): TrustedRootValidationResult { + const { rootPath } = options; + if (!fs.existsSync(rootPath)) { + fs.mkdirSync(rootPath, { recursive: true, mode: 0o700 }); + } + return validateTrustedRoot(options); +} diff --git a/backend/src/helpers/proxyExemptPaths.ts b/backend/src/helpers/proxyExemptPaths.ts index c1c23e1d..094bf073 100644 --- a/backend/src/helpers/proxyExemptPaths.ts +++ b/backend/src/helpers/proxyExemptPaths.ts @@ -63,6 +63,7 @@ export const HUB_ONLY_PREFIXES: readonly string[] = [ '/api/secrets/', '/api/blueprints/', '/api/node-labels/', + '/api/registry-delivery/', ]; /** Returns true when the path is hub-only and must not be proxied to a remote node. */ diff --git a/backend/src/helpers/registryDeliveryBodyLimits.ts b/backend/src/helpers/registryDeliveryBodyLimits.ts new file mode 100644 index 00000000..f8a3d22f --- /dev/null +++ b/backend/src/helpers/registryDeliveryBodyLimits.ts @@ -0,0 +1,74 @@ +/** Reserved JSON field name for hub-to-target delivery envelopes. */ +export const REGISTRY_DELIVERY_BODY_FIELD = '__sencho_registry_delivery'; + +/** Maximum UTF-8 size of the delivery field alone. */ +export const REGISTRY_DELIVERY_FIELD_LIMIT_BYTES = 64 * 1024; + +const KIB = 1024; + +export type RegistryDeliveryRouteClass = + | 'stack-deploy-update' + | 'bulk-label-git' + | 'template-deploy' + | 'scheduler-selector'; + +const ROUTE_CLASS_LIMITS: Record = { + 'stack-deploy-update': 512 * KIB, + 'bulk-label-git': 256 * KIB, + 'template-deploy': 128 * KIB, + 'scheduler-selector': 64 * KIB, +}; + +function matchesAny(path: string, patterns: RegExp[]): boolean { + return patterns.some(pattern => pattern.test(path)); +} + +/** + * Classify an API path into a delivery body budget group. Returns null when the + * path is not eligible for delivery body augmentation. + */ +export function classifyRegistryDeliveryRouteClass(method: string, apiPath: string): RegistryDeliveryRouteClass | null { + const upper = method.toUpperCase(); + if (upper !== 'POST' && upper !== 'PUT' && upper !== 'PATCH') return null; + + if (matchesAny(apiPath, [ + /^\/api\/stacks\/[^/]+\/(deploy|update|pull-update|rollback)(\/|$)/, + /^\/api\/stacks\/[^/]+\/services\/[^/]+\/(update|pull-update)(\/|$)/, + /^\/api\/blueprints\/apply-local$/, + ])) { + return 'stack-deploy-update'; + } + + if (matchesAny(apiPath, [ + /^\/api\/stacks\/bulk-update/, + /^\/api\/labels\/[^/]+\/action/, + /^\/api\/stacks\/from-git/, + /^\/api\/stacks\/[^/]+\/git-source\/apply/, + /^\/api\/fleet\/[^/]+\/snapshot/, + ])) { + return 'bulk-label-git'; + } + + if (apiPath === '/api/templates/deploy') { + return 'template-deploy'; + } + + if (matchesAny(apiPath, [ + /^\/api\/scheduled-tasks\/[^/]+\/execute/, + /^\/api\/image-updates\/selector/, + ])) { + return 'scheduler-selector'; + } + + return null; +} + +/** Total HTTP JSON body limit for a classified route (original body + delivery field). */ +export function getRegistryDeliveryTotalBodyLimit(routeClass: RegistryDeliveryRouteClass): number { + return ROUTE_CLASS_LIMITS[routeClass]; +} + +/** Whether this path may receive a delivery envelope in its JSON body. */ +export function isRegistryDeliveryAugmentedRoute(method: string, apiPath: string): boolean { + return classifyRegistryDeliveryRouteClass(method, apiPath) !== null; +} diff --git a/backend/src/helpers/registryDeliveryComposeEnv.ts b/backend/src/helpers/registryDeliveryComposeEnv.ts new file mode 100644 index 00000000..75499185 --- /dev/null +++ b/backend/src/helpers/registryDeliveryComposeEnv.ts @@ -0,0 +1,60 @@ +import fs from 'fs'; +import path from 'path'; + +import { loadDotEnv } from '../services/ImageUpdateService'; + +function isSafeComposeEnvKey(key: string): boolean { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(key); +} + +/** + * Merge compose variable maps with Docker Compose precedence: + * request overrides, then .env, then process.env overrides both. + */ +export function mergeComposeEnvVars( + dotEnv: Record = {}, + requestEnv?: Record, +): Record { + const merged: Record = { ...dotEnv }; + if (requestEnv) { + for (const [key, value] of Object.entries(requestEnv)) { + if (typeof value === 'string' && isSafeComposeEnvKey(key)) { + merged[key] = value; + } + } + } + for (const [key, value] of Object.entries(process.env)) { + if (value !== undefined) merged[key] = value; + } + return merged; +} + +/** Best-effort read of a stack project `.env` file from disk. */ +export function loadDotEnvFromProjectDir(projectDir: string): Record { + const baseResolved = path.resolve(projectDir); + const envPath = path.resolve(baseResolved, '.env'); + if (!envPath.startsWith(baseResolved + path.sep)) { + return {}; + } + try { + if (!fs.existsSync(envPath)) return {}; + return loadDotEnv(fs.readFileSync(envPath, 'utf8')); + } catch { + return {}; + } +} + +/** Resolve compose env for registry discovery on a project directory. */ +export function resolveComposeEnvForDiscovery( + projectDir: string, + requestEnv?: Record, +): Record { + return mergeComposeEnvVars(loadDotEnvFromProjectDir(projectDir), requestEnv); +} + +/** Resolve compose env for inline compose content discovery. */ +export function resolveComposeEnvForContent( + requestEnv?: Record, +): Record { + return mergeComposeEnvVars({}, requestEnv); +} diff --git a/backend/src/helpers/registryDeliveryContext.ts b/backend/src/helpers/registryDeliveryContext.ts new file mode 100644 index 00000000..5f94e0aa --- /dev/null +++ b/backend/src/helpers/registryDeliveryContext.ts @@ -0,0 +1,56 @@ +import { AsyncLocalStorage } from 'async_hooks'; + +import { attestationJtiFromToken } from './registryDeliveryEvidence'; +import type { RegistryDeliveryStage } from './registryOpClassifier'; + +export interface RegistryDeliveryAuthEntry { + host: string; + username: string; + password: string; + expiresAt?: number; +} + +export interface RegistryDeliveryEnvelope { + attestation: string; + prepId?: string; + auths: RegistryDeliveryAuthEntry[]; + notAfter: number; + deliverySourceId: string; +} + +export interface RegistryDeliveryContext { + envelope: RegistryDeliveryEnvelope; + nodeId: number; + stack: string; + stage: RegistryDeliveryStage; + service?: string; + abortSignal?: AbortSignal; + onFinalize?: () => void; + seamSettled?: boolean; + seamResult?: { auths: Record; prepId?: string }; +} + +const storage = new AsyncLocalStorage(); + +export function runWithRegistryDeliveryContext( + context: RegistryDeliveryContext, + fn: () => T, +): T { + return storage.run(context, fn); +} + +export function getRegistryDeliveryContext(): RegistryDeliveryContext | undefined { + return storage.getStore(); +} + +export function getRegistryDeliveryLockContext(): { opId: string; kind: string } | undefined { + const ctx = getRegistryDeliveryContext(); + if (!ctx) return undefined; + const jti = attestationJtiFromToken(ctx.envelope.attestation); + if (!jti) return undefined; + return { opId: jti, kind: ctx.stage }; +} + +export function clearRegistryDeliveryContext(): void { + storage.disable(); +} diff --git a/backend/src/helpers/registryDeliveryDiscoverPayload.ts b/backend/src/helpers/registryDeliveryDiscoverPayload.ts new file mode 100644 index 00000000..3fd6d8f9 --- /dev/null +++ b/backend/src/helpers/registryDeliveryDiscoverPayload.ts @@ -0,0 +1,108 @@ +import { hashActionSet } from './registryDeliveryHashes'; +import { classifyRegistryDeliveryOp, type RegistryDeliveryStage } from './registryOpClassifier'; + +function resolveStackName(body: Record, classificationStack?: string): string | undefined { + if (classificationStack) return classificationStack; + const stackName = body.stackName ?? body.stack_name; + return typeof stackName === 'string' && stackName.length > 0 ? stackName : undefined; +} + +function sourceKindForStage(stage: RegistryDeliveryStage): string { + switch (stage) { + case 'template-deploy': + return 'request-generated'; + case 'from-git-deploy-now': + case 'git-apply-auto-deploy': + return 'git-candidate'; + case 'stack-deploy': + return 'live-project'; + default: + return 'live-project'; + } +} + +function requiredActionsForStage(stage: RegistryDeliveryStage): string[] { + switch (stage) { + case 'git-apply-auto-deploy': + return ['stack:edit', 'stack:deploy']; + case 'template-deploy': + case 'from-git-deploy-now': + return ['stack:deploy', 'stack:create']; + default: + return ['stack:deploy']; + } +} + +export function buildRegistryDiscoverPayload(options: { + method: string; + apiPath: string; + body: Record; +}): Record | null { + const classification = classifyRegistryDeliveryOp(options.method, options.apiPath); + if (!classification.eligible || !classification.stage) return null; + + const stack = resolveStackName(options.body, classification.stack); + const stage = classification.stage; + const isRollback = Boolean(options.apiPath.match(/^\/api\/stacks\/[^/]+\/rollback$/)); + const sourceKind = isRollback ? 'restore-candidate' : sourceKindForStage(stage); + + const payload: Record = { + stack, + op: isRollback ? 'stack-deploy' : stage, + service: classification.service, + sourceKind, + actionSetHash: hashActionSet( + isRollback ? ['stack:deploy'] : requiredActionsForStage(stage), + ), + }; + + if (stage === 'template-deploy') { + payload.stackName = options.body.stackName; + payload.template = options.body.template; + payload.envVars = options.body.envVars; + } + + if (stage === 'from-git-deploy-now') { + payload.stack = options.body.stack_name; + payload.git = { + stackName: options.body.stack_name, + repo_url: options.body.repo_url, + branch: options.body.branch, + compose_path: options.body.compose_path, + compose_paths: options.body.compose_paths, + context_dir: options.body.context_dir, + sync_env: options.body.sync_env, + env_path: options.body.env_path, + auth_type: options.body.auth_type, + token: options.body.token, + auto_apply_on_webhook: options.body.auto_apply_on_webhook, + auto_deploy_on_apply: options.body.auto_deploy_on_apply, + }; + } + + if (stage === 'git-apply-auto-deploy') { + payload.gitApply = true; + } + + if (isRollback) { + payload.restoreVariant = 'backup'; + } + + if (stage === 'blueprint-apply') { + payload.sourceKind = 'body-content'; + payload.composeContent = options.body.composeContent; + payload.stackName = options.body.stackName; + } + + return payload; +} + +export function resolveDeliveryStack( + method: string, + apiPath: string, + body: Record | undefined, +): string | undefined { + const classification = classifyRegistryDeliveryOp(method, apiPath); + if (!classification.eligible) return undefined; + return resolveStackName(body ?? {}, classification.stack); +} diff --git a/backend/src/helpers/registryDeliveryEvidence.ts b/backend/src/helpers/registryDeliveryEvidence.ts new file mode 100644 index 00000000..f124697f --- /dev/null +++ b/backend/src/helpers/registryDeliveryEvidence.ts @@ -0,0 +1,71 @@ +import crypto from 'crypto'; +import jwt from 'jsonwebtoken'; + +import { DatabaseService } from '../services/DatabaseService'; +import type { + RegistryDeliveryEventInput, + RegistryDeliveryEventRow, + RegistryDeliveryEvidencePage, +} from '../types/registryDeliveryEvidence'; + +export function hashPrepId(prepId: string): string { + return crypto.createHash('sha256').update(prepId).digest('hex'); +} + +export function attestationJtiFromToken(attestation: string | undefined): string | null { + if (!attestation) return null; + try { + const decoded = jwt.decode(attestation); + if (!decoded || typeof decoded === 'string') return null; + return typeof decoded.jti_t === 'string' ? decoded.jti_t : null; + } catch { + return null; + } +} + +export function recordRegistryDeliveryEvent(input: RegistryDeliveryEventInput): number { + return DatabaseService.getInstance().insertRegistryDeliveryEvent({ + deliverySourceId: input.deliverySourceId, + eventType: input.eventType, + stack: input.stack ?? null, + op: input.op ?? null, + attestationJti: input.attestationJti ?? null, + prepIdSha256: input.prepIdSha256 ?? null, + tempDirId: input.tempDirId ?? null, + sourceHash: input.sourceHash ?? null, + prunedThroughSeq: input.prunedThroughSeq ?? null, + }); +} + +export function listRegistryDeliveryEvidencePage( + deliverySourceId: string, + cursor: number, + limit: number, +): RegistryDeliveryEvidencePage { + const events = DatabaseService.getInstance().listRegistryDeliveryEvents( + deliverySourceId, + cursor, + limit, + ); + const nextCursor = events.length > 0 + ? events[events.length - 1]!.seq + : cursor; + return { + deliverySourceId, + events, + nextCursor, + limit, + }; +} + +export function importRegistryDeliveryEvidencePage( + hubNodeIdSnapshot: number, + deliverySourceId: string, + events: RegistryDeliveryEventRow[], +): { imported: number; lastSeq: number } { + return DatabaseService.getInstance().importRegistryDeliveryEventPage( + hubNodeIdSnapshot, + deliverySourceId, + events, + ); +} diff --git a/backend/src/helpers/registryDeliveryGitCandidate.ts b/backend/src/helpers/registryDeliveryGitCandidate.ts new file mode 100644 index 00000000..276a1ded --- /dev/null +++ b/backend/src/helpers/registryDeliveryGitCandidate.ts @@ -0,0 +1,124 @@ +import { promises as fsPromises } from 'fs'; +import path from 'path'; + +import type { ComposeFile, FetchResult, MaterializationResult } from '../services/GitSourceService'; +import type { RefKind } from '../services/git/types'; +import { validateCandidateRelPath } from '../services/gitops/createStagingMarker'; + +export const GIT_CANDIDATE_PREPARED_META_FILE = '.sencho-git-candidate-meta.json'; + +export interface GitCandidatePreparedMeta { + version: 1; + commitSha: string; + resolvedRefKind: RefKind; + candidateRelPath: string; + composeFiles: ComposeFile[]; + envContent: string | null; + materialization: MaterializationResult; + warnings: string[]; +} + +export async function writeGitCandidatePreparedMeta( + stagingDir: string, + meta: GitCandidatePreparedMeta, +): Promise { + const metaPath = path.join(stagingDir, GIT_CANDIDATE_PREPARED_META_FILE); + await fsPromises.writeFile(metaPath, JSON.stringify(meta), { encoding: 'utf8', mode: 0o600 }); +} + +export async function readGitCandidatePreparedMeta(payloadPath: string): Promise { + const metaPath = path.join(payloadPath, GIT_CANDIDATE_PREPARED_META_FILE); + let raw: string; + try { + raw = await fsPromises.readFile(metaPath, 'utf8'); + } catch { + throw new Error('Prepared git candidate is missing metadata'); + } + const parsed = JSON.parse(raw) as GitCandidatePreparedMeta; + if ( + parsed.version !== 1 + || typeof parsed.commitSha !== 'string' + || typeof parsed.resolvedRefKind !== 'string' + || typeof parsed.candidateRelPath !== 'string' + || !parsed.materialization + || !Array.isArray(parsed.composeFiles) + || !Array.isArray(parsed.warnings) + ) { + throw new Error('Invalid git candidate prepared metadata'); + } + return parsed; +} + +export function fetchResultFromPreparedMeta(meta: GitCandidatePreparedMeta): FetchResult { + return { + composeFiles: meta.composeFiles, + envContent: meta.envContent, + commitSha: meta.commitSha, + resolvedRefKind: meta.resolvedRefKind, + warnings: meta.warnings, + }; +} + +/** + * Restore prepared candidate bytes into the git-managed area for promotion. + * Payload layout matches a flat copy of the candidate directory (meta file excluded). + */ +export async function installGitCandidatePayloadToManagedRoot( + payloadPath: string, + managedRoot: string, + candidateRelPath: string, +): Promise { + const managedResolved = path.resolve(managedRoot); + const pathReason = validateCandidateRelPath(candidateRelPath, managedResolved); + if (pathReason) { + throw new Error(pathReason); + } + const candidateDest = path.resolve(managedResolved, candidateRelPath); + if (!candidateDest.startsWith(managedResolved + path.sep)) { + throw new Error('Invalid candidate path'); + } + // Canonical js/path-injection barrier inline with the mkdir sink. + await fsPromises.mkdir(candidateDest, { recursive: true, mode: 0o700 }); + const entries = await fsPromises.readdir(payloadPath, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name === GIT_CANDIDATE_PREPARED_META_FILE) continue; + if (entry.isSymbolicLink()) continue; + const src = path.join(payloadPath, entry.name); + const dest = path.resolve(candidateDest, entry.name); + if (!dest.startsWith(managedResolved + path.sep)) continue; + if (entry.isDirectory()) { + await copyTree(src, dest, managedResolved); + continue; + } + if (entry.isFile()) { + // Canonical js/path-injection barrier inline with the copy sink. + await fsPromises.copyFile(src, dest); + await fsPromises.chmod(dest, 0o600); + } + } +} + +async function copyTree(srcDir: string, destDir: string, managedResolved: string): Promise { + const resolvedDestDir = path.resolve(destDir); + if (!resolvedDestDir.startsWith(managedResolved + path.sep)) { + return; + } + // Canonical js/path-injection barrier inline with the mkdir sink. + await fsPromises.mkdir(resolvedDestDir, { recursive: true, mode: 0o700 }); + const entries = await fsPromises.readdir(srcDir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isSymbolicLink()) continue; + const src = path.join(srcDir, entry.name); + const dest = path.resolve(resolvedDestDir, entry.name); + if (!dest.startsWith(managedResolved + path.sep)) continue; + if (entry.isDirectory()) { + await copyTree(src, dest, managedResolved); + continue; + } + if (entry.isFile()) { + // Canonical js/path-injection barrier inline with the copy sink. + await fsPromises.copyFile(src, dest); + await fsPromises.chmod(dest, 0o600); + } + } +} diff --git a/backend/src/helpers/registryDeliveryHashes.ts b/backend/src/helpers/registryDeliveryHashes.ts new file mode 100644 index 00000000..9855b867 --- /dev/null +++ b/backend/src/helpers/registryDeliveryHashes.ts @@ -0,0 +1,120 @@ +import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +const COMPOSE_FILENAMES = [ + 'compose.yaml', + 'compose.yml', + 'docker-compose.yaml', + 'docker-compose.yml', + '.env', +]; + +/** Stable hash of inline compose body content (body-content discovery). */ +export function hashComposeBodyContent(composeContent: string): string { + const hash = crypto.createHash('sha256'); + hash.update('compose.yaml'); + hash.update('\0'); + hash.update(composeContent); + hash.update('\n'); + return hash.digest('hex'); +} + +/** + * Stable hash of the post-apply blueprint project bundle: incoming compose.yaml + * plus the existing stack .env bytes when present. + */ +export function hashBlueprintPostApplySource( + composeContent: string, + envFileContent?: string | null, +): string { + const hash = crypto.createHash('sha256'); + hash.update('compose.yaml'); + hash.update('\0'); + hash.update(composeContent); + hash.update('\n'); + if (envFileContent !== null && envFileContent !== undefined) { + hash.update('.env'); + hash.update('\0'); + hash.update(envFileContent); + hash.update('\n'); + } + return hash.digest('hex'); +} + +/** Stable hash of the live project file bundle used for live-project delivery. */ +export function hashProjectSource(projectDir: string): string { + const hash = crypto.createHash('sha256'); + const baseResolved = path.resolve(projectDir); + for (const name of COMPOSE_FILENAMES) { + const filePath = path.resolve(baseResolved, name); + if (!filePath.startsWith(baseResolved + path.sep)) continue; + const content = readRegularFileSync(filePath, baseResolved); + if (!content) continue; + hash.update(name); + hash.update('\0'); + hash.update(content); + hash.update('\n'); + } + return hash.digest('hex'); +} + +function readRegularFileSync(filePath: string, baseResolved: string): Buffer | null { + const resolved = path.resolve(filePath); + if (!resolved.startsWith(baseResolved + path.sep)) return null; + try { + const fd = fs.openSync(resolved, 'r'); + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile()) return null; + const buf = Buffer.alloc(stat.size); + fs.readSync(fd, buf, 0, stat.size, 0); + return buf; + } finally { + fs.closeSync(fd); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} + +function hashDirectoryTree(rootDir: string, hash: crypto.Hash, current = ''): void { + const baseResolved = path.resolve(rootDir); + const abs = current ? path.resolve(baseResolved, current) : baseResolved; + if (current && !abs.startsWith(baseResolved + path.sep)) { + return; + } + const entries = fs.readdirSync(abs, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isSymbolicLink()) continue; + const rel = current ? path.join(current, entry.name) : entry.name; + const filePath = path.resolve(baseResolved, rel); + if (!filePath.startsWith(baseResolved + path.sep)) continue; + if (entry.isDirectory()) { + hashDirectoryTree(rootDir, hash, rel); + continue; + } + if (!entry.isFile()) continue; + const content = readRegularFileSync(filePath, baseResolved); + if (!content) continue; + hash.update(rel.split(path.sep).join('/')); + hash.update('\0'); + hash.update(content); + hash.update('\n'); + } +} + +/** Stable hash of a prepared source bundle directory (all regular files). */ +export function hashDeliverySourceDir(projectDir: string): string { + const hash = crypto.createHash('sha256'); + hashDirectoryTree(projectDir, hash); + return hash.digest('hex'); +} + +export function hashActionSet(actions: readonly string[]): string { + return crypto + .createHash('sha256') + .update(actions.slice().sort().join('\n')) + .digest('hex'); +} diff --git a/backend/src/helpers/registryDeliveryMaterialize.ts b/backend/src/helpers/registryDeliveryMaterialize.ts new file mode 100644 index 00000000..76870ea6 --- /dev/null +++ b/backend/src/helpers/registryDeliveryMaterialize.ts @@ -0,0 +1,69 @@ +import { promises as fsPromises } from 'fs'; +import path from 'path'; +import { FileSystemService } from '../services/FileSystemService'; +import { PreparedSourceStore } from '../services/preparedSourceStore'; +import { isValidStackName } from '../utils/validation'; + +export async function copyPreparedPayloadDirectory(srcDir: string, destDir: string): Promise { + const destRoot = path.resolve(destDir); + await fsPromises.mkdir(destRoot, { recursive: true, mode: 0o700 }); + const entries = await fsPromises.readdir(srcDir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isSymbolicLink()) continue; + const src = path.join(srcDir, entry.name); + const dest = path.resolve(destRoot, entry.name); + if (!dest.startsWith(destRoot + path.sep)) continue; + if (entry.isDirectory()) { + await copyTree(src, dest, destRoot); + continue; + } + if (entry.isFile()) { + await fsPromises.copyFile(src, dest); + await fsPromises.chmod(dest, 0o600); + } + } +} + +/** + * Copy a prepared payload bundle into a stack directory before compose runs. + * Used when hop-1 discovery stored the exact bytes the operation will execute. + */ +export async function materializePreparedSourceToStack( + prepId: string, + nodeId: number, + stackName: string, +): Promise { + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack name'); + } + const store = PreparedSourceStore.getInstance(); + const payloadPath = store.peekPayloadPath(prepId); + const fsSvc = FileSystemService.getInstance(nodeId); + const baseResolved = path.resolve(fsSvc.getBaseDir()); + const stackDir = path.resolve(baseResolved, stackName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + await fsPromises.mkdir(stackDir, { recursive: true, mode: 0o700 }); + + await copyPreparedPayloadDirectory(payloadPath, stackDir); +} + +async function copyTree(srcDir: string, destDir: string, stackRoot: string): Promise { + await fsPromises.mkdir(destDir, { recursive: true, mode: 0o700 }); + const entries = await fsPromises.readdir(srcDir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isSymbolicLink()) continue; + const src = path.join(srcDir, entry.name); + const dest = path.resolve(destDir, entry.name); + if (!dest.startsWith(stackRoot + path.sep)) continue; + if (entry.isDirectory()) { + await copyTree(src, dest, stackRoot); + continue; + } + if (entry.isFile()) { + await fsPromises.copyFile(src, dest); + await fsPromises.chmod(dest, 0o600); + } + } +} diff --git a/backend/src/helpers/registryDeliveryOutbound.ts b/backend/src/helpers/registryDeliveryOutbound.ts new file mode 100644 index 00000000..e2191470 --- /dev/null +++ b/backend/src/helpers/registryDeliveryOutbound.ts @@ -0,0 +1,217 @@ +import axios from 'axios'; +import type { Node } from '../services/DatabaseService'; +import { DatabaseService } from '../services/DatabaseService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { PilotTunnelManager } from '../services/PilotTunnelManager'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import type { RegistryDeliveryDiscoverResponse } from '../services/RegistryDeliveryService'; +import { REMOTE_REGISTRY_CREDENTIALS_CAPABILITY } from '../services/CapabilityRegistry'; +import { remoteAdvertisesCapability } from './remoteCapabilities'; +import { + classifyRegistryDeliveryRouteClass, + getRegistryDeliveryTotalBodyLimit, + REGISTRY_DELIVERY_BODY_FIELD, + REGISTRY_DELIVERY_FIELD_LIMIT_BYTES, +} from './registryDeliveryBodyLimits'; +import { classifyRegistryDeliveryOp } from './registryOpClassifier'; +import { buildRegistryDiscoverPayload } from './registryDeliveryDiscoverPayload'; +import { getErrorMessage } from '../utils/errors'; + +export type RegistryDeliveryAugmentResult = + | { ok: true; body: Record; augmented: boolean } + | { ok: false; status: number; error: string }; + +export interface AugmentRegistryDeliveryInput { + method: string; + apiPath: string; + nodeId: number; + node: Node; + target: { apiUrl: string; apiToken: string }; + body: Record; + sourceKind?: string; + prepId?: string; + abortSignal?: AbortSignal; +} + +function isTransportConfidential(nodeId: number, node: Node): boolean { + const delivery = RegistryDeliveryService.getInstance(); + if (node.mode === 'pilot_agent') { + return PilotTunnelManager.getInstance().isTunnelConfidential(nodeId); + } + return delivery.isProxyTransportConfidential(nodeId); +} + +export async function wouldAttemptRegistryDelivery( + nodeId: number, + node: Node, + method: string, + apiPath: string, +): Promise { + const classification = classifyRegistryDeliveryOp(method, apiPath); + if (!classification.eligible || !classification.stage) { + return false; + } + if (!isTransportConfidential(nodeId, node)) { + return false; + } + if (!await remoteAdvertisesCapability(nodeId, REMOTE_REGISTRY_CREDENTIALS_CAPABILITY)) { + return false; + } + return classifyRegistryDeliveryRouteClass(method, apiPath) != null; +} + +async function callTargetDiscover( + target: { apiUrl: string; apiToken: string }, + body: Record, + abortSignal?: AbortSignal, +): Promise { + if (abortSignal?.aborted) { + throw Object.assign(new Error('Registry delivery aborted'), { status: 499, code: 'ABORTED' }); + } + const base = target.apiUrl.replace(/\/$/, ''); + const res = await axios.post(`${base}/api/registry-delivery/discover`, body, { + headers: { Authorization: `Bearer ${target.apiToken}` }, + timeout: 30_000, + maxBodyLength: REGISTRY_DELIVERY_FIELD_LIMIT_BYTES, + signal: abortSignal, + validateStatus: () => true, + }); + if (abortSignal?.aborted) { + throw Object.assign(new Error('Registry delivery aborted'), { status: 499, code: 'ABORTED' }); + } + if (res.status < 200 || res.status >= 300) { + const message = typeof res.data?.error === 'string' + ? res.data.error + : 'Registry delivery discovery failed on target'; + throw Object.assign(new Error(message), { status: res.status }); + } + return res.data as RegistryDeliveryDiscoverResponse; +} + +/** + * Run hop-1 discover, assemble the delivery envelope, and merge it into a JSON + * body for direct hub-to-remote fetch callers. When capability or confidentiality + * is absent, returns the input body unchanged (AUD-30). + */ +export async function augmentJsonBodyForRegistryDelivery( + input: AugmentRegistryDeliveryInput, +): Promise { + const classification = classifyRegistryDeliveryOp(input.method, input.apiPath); + if (!classification.eligible || !classification.stage) { + return { ok: true, body: input.body, augmented: false }; + } + + if (input.abortSignal?.aborted) { + return { ok: false, status: 499, error: 'Request aborted' }; + } + + const confidential = isTransportConfidential(input.nodeId, input.node); + const capable = await remoteAdvertisesCapability(input.nodeId, REMOTE_REGISTRY_CREDENTIALS_CAPABILITY); + if (!confidential || !capable) { + return { ok: true, body: input.body, augmented: false }; + } + + const routeClass = classifyRegistryDeliveryRouteClass(input.method, input.apiPath); + if (!routeClass) { + return { ok: true, body: input.body, augmented: false }; + } + + try { + const discoverBody = buildRegistryDiscoverPayload({ + method: input.method, + apiPath: input.apiPath, + body: input.body, + }); + if (!discoverBody) { + return { ok: true, body: input.body, augmented: false }; + } + + const discover = await callTargetDiscover(input.target, discoverBody, input.abortSignal); + if (input.abortSignal?.aborted) { + return { ok: false, status: 499, error: 'Request aborted' }; + } + const envelope = await RegistryDeliveryService.getInstance().buildHubEnvelope(input.nodeId, discover); + if (input.abortSignal?.aborted) { + return { ok: false, status: 499, error: 'Request aborted' }; + } + if (!envelope) { + return { ok: true, body: input.body, augmented: false }; + } + + const envelopeJson = JSON.stringify(envelope); + if (Buffer.byteLength(envelopeJson, 'utf8') > REGISTRY_DELIVERY_FIELD_LIMIT_BYTES) { + return { + ok: false, + status: 413, + error: 'Registry delivery envelope too large', + }; + } + + const parsed = { ...input.body }; + parsed[REGISTRY_DELIVERY_BODY_FIELD] = envelope; + const augmented = Buffer.from(JSON.stringify(parsed), 'utf-8'); + const totalLimit = getRegistryDeliveryTotalBodyLimit(routeClass); + if (augmented.length > totalLimit) { + return { + ok: false, + status: 413, + error: 'Request body exceeds registry delivery limit', + }; + } + + if (input.abortSignal?.aborted) { + return { ok: false, status: 499, error: 'Request aborted' }; + } + + return { ok: true, body: parsed, augmented: true }; + } catch (error) { + if (input.abortSignal?.aborted || (error as { code?: string }).code === 'ABORTED') { + return { ok: false, status: 499, error: 'Request aborted' }; + } + if (axios.isCancel(error)) { + return { ok: false, status: 499, error: 'Request aborted' }; + } + const status = Number((error as { status?: number }).status) || 500; + console.error( + '[registryDeliveryOutbound] hop-1 failed:', + getErrorMessage(error, 'unknown'), + ); + return { + ok: false, + status, + error: status >= 500 ? 'Registry delivery failed' : getErrorMessage(error, 'Registry delivery failed'), + }; + } +} + +/** + * Convenience wrapper for services that already validated the remote target. + * Loads node + proxy target from the registry. + */ +export async function prepareOutboundRegistryDeliveryBody(options: { + method: string; + apiPath: string; + nodeId: number; + body?: Record | null; + sourceKind?: string; + prepId?: string; + abortSignal?: AbortSignal; +}): Promise { + const body = options.body ?? {}; + const node = DatabaseService.getInstance().getNode(options.nodeId); + const target = NodeRegistry.getInstance().getProxyTarget(options.nodeId); + if (!node || !target) { + return { ok: true, body, augmented: false }; + } + return augmentJsonBodyForRegistryDelivery({ + method: options.method, + apiPath: options.apiPath, + nodeId: options.nodeId, + node, + target, + body, + sourceKind: options.sourceKind, + prepId: options.prepId, + abortSignal: options.abortSignal, + }); +} diff --git a/backend/src/helpers/registryDeliveryPrepare.ts b/backend/src/helpers/registryDeliveryPrepare.ts new file mode 100644 index 00000000..8608dec0 --- /dev/null +++ b/backend/src/helpers/registryDeliveryPrepare.ts @@ -0,0 +1,215 @@ +import os from 'os'; +import path from 'path'; +import { promises as fsPromises } from 'fs'; +import type { Template } from '../services/TemplateService'; +import { templateService } from '../services/TemplateService'; +import { FileSystemService } from '../services/FileSystemService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { PreparedSourceStore } from '../services/preparedSourceStore'; +import { hashDeliverySourceDir, hashBlueprintPostApplySource, hashProjectSource } from './registryDeliveryHashes'; +import type { RegistryDeliveryDiscoverRequest } from '../services/RegistryDeliveryService'; +import type { CreateStackFromGitInput } from '../services/GitSourceService'; +import { isValidStackName } from '../utils/validation'; +import { loadDotEnv } from '../services/ImageUpdateService'; +import { discoverRegistryReferencesFromComposeContent } from '../services/registryReferenceDiscovery'; +import { mergeComposeEnvVars } from './registryDeliveryComposeEnv'; + +export interface PreparedSourceResult { + prepId: string; + sourceHash: string; +} + +export interface BlueprintPostApplyDiscovery { + sourceHash: string; + referencedHosts: string[]; +} + +/** + * Discover registry references for blueprint body-content using the post-apply + * project shape without writing sensitive material to a temp directory. + */ +export async function resolveBlueprintPostApplyDiscovery( + stackName: string, + composeContent: string, + nodeId: number, +): Promise { + const fs = FileSystemService.getInstance(nodeId); + let envFileContent: string | null = null; + if (await fs.envExists(stackName)) { + envFileContent = await fs.getEnvContent(stackName); + } + const dotEnv = envFileContent ? loadDotEnv(envFileContent) : {}; + const sourceHash = hashBlueprintPostApplySource(composeContent, envFileContent); + const discovery = discoverRegistryReferencesFromComposeContent( + composeContent, + mergeComposeEnvVars(dotEnv), + ); + return { sourceHash, referencedHosts: discovery.referencedHosts }; +} + +export async function prepareRequestGeneratedSource(input: { + stackName: string; + template: Template; + envVars?: Record; +}): Promise { + const composeYaml = templateService.generateComposeFromTemplate(input.template, input.stackName); + const stagingDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-regprep-')); + try { + await fsPromises.writeFile(path.join(stagingDir, 'compose.yaml'), composeYaml, { mode: 0o600 }); + if (input.envVars && Object.keys(input.envVars).length > 0) { + const envString = templateService.generateEnvString(input.envVars); + await fsPromises.writeFile(path.join(stagingDir, '.env'), envString, { mode: 0o600 }); + } + const sourceHash = hashProjectSource(stagingDir); + const entry = await PreparedSourceStore.getInstance().prepareFromDirectory( + 'request-generated', + sourceHash, + stagingDir, + ); + return { prepId: entry.prepId, sourceHash }; + } catch (error) { + await fsPromises.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); + throw error; + } +} + +export async function prepareRestoreCandidateFromBackup( + stackName: string, + nodeId: number, +): Promise { + const fsSvc = FileSystemService.getInstance(nodeId); + const backupInfo = await fsSvc.getBackupInfo(stackName); + if (!backupInfo.exists) { + throw new Error('No backup available for restore preparation'); + } + const stagingDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-regprep-')); + try { + await fsSvc.copyBackupSlotToDir(stackName, stagingDir); + const sourceHash = hashDeliverySourceDir(stagingDir); + const entry = await PreparedSourceStore.getInstance().prepareFromDirectory( + 'restore-candidate', + sourceHash, + stagingDir, + ); + return { prepId: entry.prepId, sourceHash }; + } catch (error) { + await fsPromises.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); + throw error; + } +} + +export async function prepareRestoreCandidateFromRecoveryGeneration( + stackName: string, + nodeId: number, + generationId: string, +): Promise { + const { RollbackGenerationStore } = await import('../services/RollbackGenerationStore'); + const stagingDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-regprep-')); + try { + await RollbackGenerationStore.copyPresentFilesToDir(nodeId, stackName, generationId, stagingDir); + const sourceHash = hashDeliverySourceDir(stagingDir); + const entry = await PreparedSourceStore.getInstance().prepareFromDirectory( + 'restore-candidate', + sourceHash, + stagingDir, + ); + return { prepId: entry.prepId, sourceHash }; + } catch (error) { + await fsPromises.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); + throw error; + } +} + +async function prepareRestoreCandidateForStack( + stackName: string, + nodeId: number, +): Promise { + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const currentGen = StackUpdateRecoveryService.getInstance().getCurrent(nodeId, stackName); + if (currentGen?.id) { + return prepareRestoreCandidateFromRecoveryGeneration(stackName, nodeId, currentGen.id); + } + return prepareRestoreCandidateFromBackup(stackName, nodeId); +} + +function gitInputFromDiscover(request: RegistryDeliveryDiscoverRequest): CreateStackFromGitInput | null { + const git = request.git; + if (!git || typeof git !== 'object') return null; + const record = git as Record; + const stackName = typeof request.stack === 'string' ? request.stack : typeof record.stackName === 'string' ? record.stackName : ''; + const repoUrl = typeof record.repo_url === 'string' ? record.repo_url : typeof record.repoUrl === 'string' ? record.repoUrl : ''; + const branch = typeof record.branch === 'string' ? record.branch : ''; + const composePaths = Array.isArray(record.compose_paths) + ? record.compose_paths.filter((p): p is string => typeof p === 'string') + : typeof record.compose_path === 'string' + ? [record.compose_path] + : []; + if (!stackName || !repoUrl || !branch || composePaths.length === 0) { + return null; + } + return { + stackName, + repoUrl, + branch, + composePaths, + contextDir: typeof record.context_dir === 'string' ? record.context_dir : null, + syncEnv: record.sync_env === true, + envPath: typeof record.env_path === 'string' ? record.env_path : null, + authType: record.auth_type === 'token' ? 'token' : 'none', + token: typeof record.token === 'string' ? record.token : null, + autoApplyOnWebhook: record.auto_apply_on_webhook === true, + autoDeployOnApply: record.auto_deploy_on_apply === true, + }; +} + +export async function prepareGitCandidateSource( + request: RegistryDeliveryDiscoverRequest, +): Promise { + if (request.gitApply === true && request.stack) { + const { GitSourceService } = await import('../services/GitSourceService'); + return GitSourceService.getInstance().prepareRegistryDeliveryFromPending(request.stack); + } + const input = gitInputFromDiscover(request); + if (!input) { + throw new Error('Git candidate discovery is missing required fields'); + } + const { GitSourceService } = await import('../services/GitSourceService'); + return GitSourceService.getInstance().prepareRegistryDeliveryFromGit(input); +} + +export async function prepareSourceForDiscover( + request: RegistryDeliveryDiscoverRequest, +): Promise { + switch (request.sourceKind) { + case 'request-generated': { + const stackName = typeof request.stackName === 'string' + ? request.stackName + : typeof request.stack === 'string' + ? request.stack + : ''; + if (!stackName || !request.template || typeof request.template !== 'object') { + throw new Error('Template discovery is missing stackName or template'); + } + return prepareRequestGeneratedSource({ + stackName, + template: request.template as Template, + envVars: request.envVars as Record | undefined, + }); + } + case 'restore-candidate': { + const stack = request.stack; + if (!stack || !isValidStackName(stack)) { + throw new Error('Invalid stack name'); + } + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + return prepareRestoreCandidateForStack(stack, nodeId); + } + case 'git-candidate': + return prepareGitCandidateSource(request); + case 'live-project': + case 'body-content': + return null; + default: + throw new Error(`Unsupported registry delivery source kind: ${request.sourceKind}`); + } +} diff --git a/backend/src/helpers/registryDeliveryProxy.ts b/backend/src/helpers/registryDeliveryProxy.ts new file mode 100644 index 00000000..3f3b85a3 --- /dev/null +++ b/backend/src/helpers/registryDeliveryProxy.ts @@ -0,0 +1,158 @@ +import type { Request, Response } from 'express'; +import type { Node } from '../services/DatabaseService'; +import { augmentJsonBodyForRegistryDelivery, wouldAttemptRegistryDelivery } from './registryDeliveryOutbound'; + +export interface RegistryDeliveryProxyResult { + /** When false, respond to the client with status/error instead of forwarding. */ + forward: boolean; + status?: number; + error?: string; +} + +/** + * When delivery can be negotiated, run hop-1 discover, assemble the envelope, + * and buffer an augmented JSON body on req.rawBody. When capability or + * confidentiality is absent, leaves the request unchanged (AUD-30). + */ +export async function augmentRemoteProxyWithRegistryDelivery( + req: Request, + nodeId: number, + node: Node, + target: { apiUrl: string; apiToken: string }, + rawBody: Buffer, +): Promise { + const apiPath = `/api${req.path}`; + + let parsed: Record = {}; + if (rawBody.length > 0) { + try { + parsed = JSON.parse(rawBody.toString('utf-8')) as Record; + } catch { + return { forward: false, status: 400, error: 'Request body is not valid JSON' }; + } + } + + const result = await augmentJsonBodyForRegistryDelivery({ + method: req.method, + apiPath, + nodeId, + node, + target, + body: parsed, + abortSignal: req.registryDeliveryAbortController?.signal, + }); + + if (!result.ok) { + return { forward: false, status: result.status, error: result.error }; + } + + if (req.registryDeliveryAbortController?.signal.aborted) { + return { forward: false, status: 499, error: 'Request aborted' }; + } + + if (result.augmented || rawBody.length === 0) { + req.rawBody = Buffer.from(JSON.stringify(result.body), 'utf-8'); + } else if (rawBody.length > 0) { + req.rawBody = rawBody; + } + + return { forward: true }; +} + +/** Bind hop-1 abort to client disconnect before any async capability work. */ +export function ensureRegistryDeliveryHopAbortController(req: Request, res: Response): void { + if (req.registryDeliveryAbortController) return; + const abortController = new AbortController(); + req.registryDeliveryAbortController = abortController; + const onReqAborted = () => { + if (!abortController.signal.aborted) { + abortController.abort(); + } + }; + const onResClose = () => { + if (!res.writableEnded && !abortController.signal.aborted) { + abortController.abort(); + } + }; + let detached = false; + const detach = () => { + if (detached) return; + detached = true; + req.off('aborted', onReqAborted); + res.off('close', onResClose); + }; + req.on('aborted', onReqAborted); + res.on('close', onResClose); + res.once('finish', detach); + res.once('close', detach); +} + +export type RegistryDeliveryProxyHopDecision = + | { action: 'attempt' } + | { action: 'skip' } + | { action: 'aborted' }; + +export type RegistryDeliveryProxyGateResult = + | { outcome: 'continue' } + | { outcome: 'stop' } + | { outcome: 'run-delivery' }; + +/** + * Register abort listeners, then decide whether hop-1 registry delivery runs. + * Abort is wired before the capability probe so a client disconnect during the + * probe still cancels the hop. Aborted is distinct from skip so callers do not + * forward consequential requests after cancellation. + */ +export async function decideRegistryDeliveryProxyHop( + req: Request, + res: Response, + nodeId: number, + node: Node, + method: string, + deliveryApiPath: string, +): Promise { + if (req.destroyed || req.aborted) { + return { action: 'aborted' }; + } + ensureRegistryDeliveryHopAbortController(req, res); + if (req.registryDeliveryAbortController?.signal.aborted) { + return { action: 'aborted' }; + } + const wouldAttempt = await wouldAttemptRegistryDelivery(nodeId, node, method, deliveryApiPath); + if ( + req.destroyed + || req.aborted + || req.registryDeliveryAbortController?.signal.aborted + ) { + return { action: 'aborted' }; + } + return wouldAttempt ? { action: 'attempt' } : { action: 'skip' }; +} + +/** + * Map an eligible-route registry delivery decision to proxy gate behavior. + */ +export async function evaluateRegistryDeliveryProxyGate( + req: Request, + res: Response, + nodeId: number, + node: Node, + method: string, + deliveryApiPath: string, +): Promise { + const decision = await decideRegistryDeliveryProxyHop( + req, + res, + nodeId, + node, + method, + deliveryApiPath, + ); + if (decision.action === 'aborted') { + return { outcome: 'stop' }; + } + if (decision.action === 'attempt') { + return { outcome: 'run-delivery' }; + } + return { outcome: 'continue' }; +} diff --git a/backend/src/helpers/registryDeliverySeam.ts b/backend/src/helpers/registryDeliverySeam.ts new file mode 100644 index 00000000..cc5da13b --- /dev/null +++ b/backend/src/helpers/registryDeliverySeam.ts @@ -0,0 +1,229 @@ +import path from 'path'; +import type { JwtPayload } from 'jsonwebtoken'; +import { FileSystemService } from '../services/FileSystemService'; +import { + RegistryService, + normalizeImageHost, +} from '../services/RegistryService'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { PreparedSourceStore } from '../services/preparedSourceStore'; +import { StackOpLockService } from '../services/StackOpLockService'; +import { discoverRegistryReferences } from '../services/registryReferenceDiscovery'; +import type { RegistryDeliveryEnvelope } from './registryDeliveryContext'; +import { hashActionSet, hashProjectSource } from './registryDeliveryHashes'; +import type { RegistryDeliveryStage } from './registryOpClassifier'; +import { isValidStackName } from '../utils/validation'; +import { resolveComposeEnvForDiscovery } from './registryDeliveryComposeEnv'; + +export interface RegistryDeliverySeamInput { + envelope: RegistryDeliveryEnvelope; + nodeId: number; + stack: string; + stage: RegistryDeliveryStage; + service?: string; +} + +export interface RegistryDeliverySeamResult { + auths: Record; + prepId?: string; +} + +function requiredActionsForStage(stage: RegistryDeliveryStage): string[] { + switch (stage) { + case 'git-apply-auto-deploy': + return ['stack:edit', 'stack:deploy']; + case 'template-deploy': + case 'from-git-deploy-now': + return ['stack:create', 'stack:deploy']; + case 'fleet-label': + case 'stack-deploy': + case 'stack-update': + case 'stack-pull-update': + case 'service-update': + case 'service-pull-update': + case 'webhook-deploy': + case 'scheduler-auto-update': + case 'scheduler-auto-start': + case 'mesh-redeploy': + case 'blueprint-apply': + case 'fleet-snapshot': + return ['stack:deploy']; + default: + return ['stack:deploy']; + } +} + +function assertClaim( + condition: boolean, + message: string, +): asserts condition { + if (!condition) { + throw new Error(message); + } +} + +function encodeAuth(username: string, password: string): { auth: string } { + return { + auth: Buffer.from(`${username}:${password}`).toString('base64'), + }; +} + +/** + * Execute the registry delivery seam: claim prepared sources when present, + * re-hash source inputs, re-verify the attestation, burn jti_t, check notAfter, + * and merge target-local with delivered credentials (target available always wins). + */ +export async function resolveRegistryAuthAtSeam( + input: RegistryDeliverySeamInput, +): Promise { + const delivery = RegistryDeliveryService.getInstance(); + const payload = delivery.parseAttestation(input.envelope.attestation); + + assertClaim( + typeof payload.nodeIdClaim === 'number' && payload.nodeIdClaim === input.nodeId, + 'Attestation node mismatch', + ); + if (input.stack && payload.stack && payload.stack !== input.stack) { + throw new Error('Attestation stack mismatch'); + } + if (payload.op && payload.op !== input.stage) { + throw new Error('Attestation operation mismatch'); + } + if (input.service && payload.service && payload.service !== input.service) { + throw new Error('Attestation service mismatch'); + } + + const heldLock = StackOpLockService.getInstance().get(input.nodeId, input.stack); + const jtiForLock = payload.jti_t; + assertClaim( + !!heldLock, + 'Stack lock required for registry delivery', + ); + assertClaim( + typeof jtiForLock === 'string' + && heldLock.context?.opId === jtiForLock + && heldLock.context?.kind === input.stage, + 'Stack lock context mismatch', + ); + + const prepId = input.envelope.prepId ?? (typeof payload.prepId === 'string' ? payload.prepId : undefined); + let sourceHash: string; + let referencedHosts: string[]; + + if (prepId) { + assertClaim( + payload.prepId === prepId, + 'Attestation prepId mismatch', + ); + const store = PreparedSourceStore.getInstance(); + const entry = store.claim(prepId); + const payloadPath = store.getPayloadPath(prepId); + sourceHash = entry.sourceHash; + assertClaim( + payload.sourceHash === sourceHash, + 'Prepared source hash mismatch', + ); + const discovery = discoverRegistryReferences( + payloadPath, + resolveComposeEnvForDiscovery(payloadPath), + ); + referencedHosts = discovery.referencedHosts; + } else { + if (!isValidStackName(input.stack)) { + throw new Error('Invalid stack name'); + } + const fs = FileSystemService.getInstance(input.nodeId); + const baseResolved = path.resolve(fs.getBaseDir()); + const projectDir = path.resolve(baseResolved, input.stack); + if (!projectDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + sourceHash = hashProjectSource(projectDir); + assertClaim( + payload.sourceHash === sourceHash, + 'Project source hash mismatch', + ); + const discovery = discoverRegistryReferences( + projectDir, + resolveComposeEnvForDiscovery(projectDir), + ); + referencedHosts = discovery.referencedHosts; + } + + const referencedHostsHash = delivery.hashHostList(referencedHosts); + assertClaim( + payload.referencedHostsHash === referencedHostsHash, + 'Referenced hosts hash mismatch', + ); + + const registry = RegistryService.getInstance(); + const coveredHosts: string[] = []; + for (const host of referencedHosts) { + const resolution = await registry.resolveDockerConfigForHostDetailed(host); + if (resolution.state === 'unavailable') { + throw new Error(`Registry credentials unavailable for ${host}`); + } + if (resolution.state === 'available') { + coveredHosts.push(host); + } + } + + const coveredHostsHash = delivery.hashHostList(coveredHosts); + assertClaim( + payload.coveredHostsHash === coveredHostsHash, + 'Covered hosts hash mismatch', + ); + + const actionSetHash = hashActionSet(requiredActionsForStage(input.stage)); + assertClaim( + payload.actionSetHash === actionSetHash, + 'Action set hash mismatch', + ); + + const jti = payload.jti_t; + assertClaim(typeof jti === 'string' && jti.length > 0, 'Attestation missing jti'); + const expiresAtMs = typeof payload.exp === 'number' ? payload.exp * 1000 : Date.now() + 900_000; + delivery.consumeAttestationJti(jti, expiresAtMs); + + if (Date.now() >= input.envelope.notAfter) { + throw new Error('Registry delivery envelope expired'); + } + + const referencedSet = new Set(referencedHosts.map(normalizeImageHost)); + const coveredSet = new Set(coveredHosts.map(normalizeImageHost)); + const merged: Record = {}; + + for (const host of referencedHosts) { + const normalized = normalizeImageHost(host); + const resolution = await registry.resolveDockerConfigForHostDetailed(host); + if (resolution.state === 'available' && resolution.auth) { + merged[normalized] = encodeAuth(resolution.auth.username, resolution.auth.password); + } + } + + for (const entry of input.envelope.auths) { + const normalized = normalizeImageHost(entry.host); + if (!referencedSet.has(normalized) && !coveredSet.has(normalized)) { + throw new Error('Delivery includes undeclared registry host'); + } + if (merged[normalized]) { + continue; + } + const targetResolution = await registry.resolveDockerConfigForHostDetailed(entry.host); + if (targetResolution.state === 'unavailable') { + throw new Error(`Registry credentials unavailable for ${entry.host}`); + } + if (targetResolution.state === 'available' && targetResolution.auth) { + merged[normalized] = encodeAuth( + targetResolution.auth.username, + targetResolution.auth.password, + ); + continue; + } + merged[normalized] = encodeAuth(entry.username, entry.password); + } + + return { auths: merged, prepId }; +} + +export type { JwtPayload }; diff --git a/backend/src/helpers/registryOpClassifier.ts b/backend/src/helpers/registryOpClassifier.ts new file mode 100644 index 00000000..bd425493 --- /dev/null +++ b/backend/src/helpers/registryOpClassifier.ts @@ -0,0 +1,98 @@ +export type RegistryDeliveryStage = + | 'stack-deploy' + | 'stack-update' + | 'stack-pull-update' + | 'service-update' + | 'service-pull-update' + | 'webhook-deploy' + | 'scheduler-auto-update' + | 'scheduler-auto-start' + | 'mesh-redeploy' + | 'blueprint-apply' + | 'fleet-label' + | 'fleet-snapshot' + | 'template-deploy' + | 'from-git-deploy-now' + | 'git-apply-auto-deploy'; + +export interface RegistryDeliveryClassification { + eligible: boolean; + stage?: RegistryDeliveryStage; + stack?: string; + service?: string; +} + +function stackNameFromPath(apiPath: string): string | undefined { + const match = apiPath.match(/^\/api\/stacks\/([^/]+)/); + return match?.[1]; +} + +/** + * Classify whether an API request is eligible for registry credential delivery. + * Permission checks remain in stackRouteAuth; this only identifies delivery stages. + */ +export function classifyRegistryDeliveryOp(method: string, apiPath: string): RegistryDeliveryClassification { + const upper = method.toUpperCase(); + if (upper !== 'POST' && upper !== 'PUT' && upper !== 'PATCH') { + return { eligible: false }; + } + + const stack = stackNameFromPath(apiPath); + + if (apiPath.match(/^\/api\/stacks\/[^/]+\/deploy$/)) { + return { eligible: true, stage: 'stack-deploy', stack }; + } + if (apiPath.match(/^\/api\/stacks\/[^/]+\/update$/)) { + return { eligible: true, stage: 'stack-update', stack }; + } + if (apiPath.match(/^\/api\/stacks\/[^/]+\/pull-update$/)) { + return { eligible: true, stage: 'stack-pull-update', stack }; + } + + const serviceMatch = apiPath.match(/^\/api\/stacks\/([^/]+)\/services\/([^/]+)\/(update|pull-update)$/); + if (serviceMatch) { + return { + eligible: true, + stage: serviceMatch[3] === 'pull-update' ? 'service-pull-update' : 'service-update', + stack: serviceMatch[1], + service: serviceMatch[2], + }; + } + + if (apiPath === '/api/templates/deploy') { + return { eligible: true, stage: 'template-deploy' }; + } + if (apiPath === '/api/stacks/from-git') { + return { eligible: true, stage: 'from-git-deploy-now' }; + } + if (apiPath.match(/^\/api\/stacks\/[^/]+\/git-source\/apply$/)) { + return { eligible: true, stage: 'git-apply-auto-deploy', stack }; + } + if (apiPath.match(/^\/api\/fleet\/[^/]+\/snapshot$/)) { + return { eligible: true, stage: 'fleet-snapshot' }; + } + if (apiPath.match(/^\/api\/labels\/[^/]+\/action$/)) { + return { eligible: true, stage: 'fleet-label' }; + } + if (apiPath.match(/^\/api\/scheduled-tasks\/[^/]+\/execute$/)) { + return { eligible: true, stage: 'scheduler-auto-update' }; + } + if (apiPath.match(/^\/api\/image-updates\/selector/)) { + return { eligible: true, stage: 'scheduler-auto-update' }; + } + if (apiPath.match(/^\/api\/mesh\/[^/]+\/redeploy$/)) { + return { eligible: true, stage: 'mesh-redeploy' }; + } + if (apiPath.match(/^\/api\/stacks\/[^/]+\/rollback$/)) { + return { eligible: true, stage: 'stack-deploy', stack }; + } + + if (apiPath === '/api/blueprints/apply-local') { + return { eligible: true, stage: 'blueprint-apply' }; + } + if (apiPath.match(/^\/api\/blueprints\/[^/]+\/apply/)) { + return { eligible: true, stage: 'blueprint-apply' }; + } + + return { eligible: false }; +} diff --git a/backend/src/helpers/trustedProxyCidrs.ts b/backend/src/helpers/trustedProxyCidrs.ts new file mode 100644 index 00000000..daca5c2a --- /dev/null +++ b/backend/src/helpers/trustedProxyCidrs.ts @@ -0,0 +1,102 @@ +import net from 'net'; + +const ENV_KEY = 'SENCHO_TRUSTED_PROXY_CIDRS'; + +let cachedBlockList: net.BlockList | null | undefined; + +function parseCidrEntry(raw: string): { family: 4 | 6; address: string; prefix: number } | null { + const trimmed = raw.trim(); + if (!trimmed) return null; + + const slash = trimmed.lastIndexOf('/'); + if (slash <= 0) return null; + + const addressPart = trimmed.slice(0, slash); + const prefixPart = trimmed.slice(slash + 1); + const prefix = Number(prefixPart); + if (!Number.isInteger(prefix)) return null; + + const family = net.isIP(addressPart); + if (family === 4) { + if (prefix < 0 || prefix > 32) return null; + return { family: 4, address: addressPart, prefix }; + } + if (family === 6) { + if (prefix < 0 || prefix > 128) return null; + return { family: 6, address: addressPart, prefix }; + } + return null; +} + +/** + * Parse SENCHO_TRUSTED_PROXY_CIDRS once at process start. Invalid or duplicate + * entries fail closed by returning null (treat all upgrades as non-confidential). + */ +export function getTrustedProxyBlockList(): net.BlockList | null { + if (cachedBlockList !== undefined) { + return cachedBlockList; + } + + const raw = process.env[ENV_KEY]?.trim(); + if (!raw) { + cachedBlockList = null; + return cachedBlockList; + } + + const entries = raw.split(',').map(part => part.trim()).filter(Boolean); + if (entries.length === 0) { + cachedBlockList = null; + return cachedBlockList; + } + + const seen = new Set(); + const blockList = new net.BlockList(); + + for (const entry of entries) { + if (seen.has(entry)) { + cachedBlockList = null; + return cachedBlockList; + } + seen.add(entry); + + const parsed = parseCidrEntry(entry); + if (!parsed) { + cachedBlockList = null; + return cachedBlockList; + } + + try { + if (parsed.family === 4) { + blockList.addSubnet(parsed.address, parsed.prefix, 'ipv4'); + } else { + blockList.addSubnet(parsed.address, parsed.prefix, 'ipv6'); + } + } catch { + cachedBlockList = null; + return cachedBlockList; + } + } + + cachedBlockList = blockList; + return cachedBlockList; +} + +/** Reset cached parser (tests only). */ +export function resetTrustedProxyBlockListCache(): void { + cachedBlockList = undefined; +} + +export function isTrustedProxyPeer(peerAddress: string | undefined): boolean { + if (!peerAddress) return false; + const blockList = getTrustedProxyBlockList(); + if (!blockList) return false; + + const family = net.isIP(peerAddress); + if (family === 4) { + return blockList.check(peerAddress, 'ipv4'); + } + if (family === 6) { + return blockList.check(peerAddress, 'ipv6'); + } + return false; +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 3ed6faaa..2fe9e10d 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -3,6 +3,7 @@ import './types/express'; import { authGate, auditLog } from './middleware/authGate'; import { enforceApiTokenScope } from './middleware/apiTokenScope'; import { hubOnlyGuard } from './middleware/hubOnlyGuard'; +import { registryDeliveryMiddleware } from './middleware/registryDelivery'; import { errorHandler } from './middleware/errorHandler'; import { createApp } from './app'; import { createRemoteProxyMiddleware } from './proxy/remoteNodeProxy'; @@ -59,6 +60,7 @@ import { secretsRouter } from './routes/secrets'; import { diagnosticsRouter } from './routes/diagnostics'; import { dependencyMapRouter } from './routes/dependencyMap'; import { networkingRouter } from './routes/networking'; +import { registryDeliveryRouter } from './routes/registryDelivery'; // Suppress [DEP0060] DeprecationWarning emitted by http-proxy@1.18.1 which calls // util._extend internally. The warning fires at runtime when createProxyServer() is @@ -100,6 +102,9 @@ app.use('/api', enforceApiTokenScope); // for the prefix list and middleware/hubOnlyGuard.ts for the rationale. app.use('/api', hubOnlyGuard); +// Registry delivery envelope capture/scrub on classified target routes. +app.use('/api', registryDeliveryMiddleware); + // Remote Node HTTP Proxy (see proxy/remoteNodeProxy.ts). Mounted BEFORE the // per-group routers so a request targeting a remote node short-circuits into // the proxy instead of hitting a local handler that would read local state. @@ -153,6 +158,7 @@ app.use('/api/dashboard', dashboardRouter); app.use('/api/diagnostics', diagnosticsRouter); app.use('/api/dependency-map', dependencyMapRouter); app.use('/api/networking', networkingRouter); +app.use('/api/registry-delivery', registryDeliveryRouter); app.use('/api/nodes', nodesRouter); app.use('/api/stacks', stackActivityRouter); app.use('/api/stacks', stacksRouter); diff --git a/backend/src/middleware/registryDelivery.ts b/backend/src/middleware/registryDelivery.ts new file mode 100644 index 00000000..88cc5628 --- /dev/null +++ b/backend/src/middleware/registryDelivery.ts @@ -0,0 +1,119 @@ +import type { Request, Response, NextFunction } from 'express'; +import { REGISTRY_DELIVERY_BODY_FIELD } from '../helpers/registryDeliveryBodyLimits'; +import type { RegistryDeliveryEnvelope } from '../helpers/registryDeliveryContext'; +import { runWithRegistryDeliveryContext } from '../helpers/registryDeliveryContext'; +import { classifyRegistryDeliveryOp } from '../helpers/registryOpClassifier'; +import { resolveDeliveryStack } from '../helpers/registryDeliveryDiscoverPayload'; +import { + attestationJtiFromToken, + hashPrepId, + recordRegistryDeliveryEvent, +} from '../helpers/registryDeliveryEvidence'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; + +/** Express strips the /api mount prefix from req.path; classifiers expect /api/... */ +export function registryDeliveryApiPath(req: Pick): string { + return `/api${req.path}`; +} + +function scrubDeliveryField(body: unknown): void { + if (!body || typeof body !== 'object') return; + if (REGISTRY_DELIVERY_BODY_FIELD in (body as Record)) { + delete (body as Record)[REGISTRY_DELIVERY_BODY_FIELD]; + } +} + +/** + * Capture and verify registry delivery envelopes on classified target routes. + * Installed after authGate and before remoteNodeProxy. + */ +export function registryDeliveryMiddleware(req: Request, res: Response, next: NextFunction): void { + const apiPath = registryDeliveryApiPath(req); + const classification = classifyRegistryDeliveryOp(req.method, apiPath); + if (!classification.eligible) { + next(); + return; + } + + const rawBody = req.body as Record | undefined; + const delivery = rawBody?.[REGISTRY_DELIVERY_BODY_FIELD] as RegistryDeliveryEnvelope | undefined; + scrubDeliveryField(rawBody); + + if (!delivery) { + next(); + return; + } + + try { + RegistryDeliveryService.getInstance().parseAttestation(delivery.attestation); + req.registryDeliveryEnvelope = delivery; + + const abortController = new AbortController(); + req.registryDeliveryAbortController = abortController; + + const onReqAborted = () => { + if (!abortController.signal.aborted) { + abortController.abort(); + } + }; + const onResClose = () => { + if (!res.writableEnded && !abortController.signal.aborted) { + abortController.abort(); + } + }; + + let abortListenersDetached = false; + const detachAbortListeners = () => { + if (abortListenersDetached) return; + abortListenersDetached = true; + req.off('aborted', onReqAborted); + res.off('close', onResClose); + }; + + req.on('aborted', onReqAborted); + res.on('close', onResClose); + + let operationEvidenceRecorded = false; + const recordOperationEvidence = () => { + if (operationEvidenceRecorded) return; + operationEvidenceRecorded = true; + const stack = resolveDeliveryStack(req.method, apiPath, rawBody) ?? classification.stack; + if (!stack || !classification.stage) return; + recordRegistryDeliveryEvent({ + deliverySourceId: delivery.deliverySourceId, + eventType: abortController.signal.aborted ? 'operation_aborted' : 'operation_completed', + stack, + op: classification.stage, + attestationJti: attestationJtiFromToken(delivery.attestation), + prepIdSha256: delivery.prepId ? hashPrepId(delivery.prepId) : null, + }); + }; + + res.once('finish', () => { + recordOperationEvidence(); + detachAbortListeners(); + }); + res.once('close', detachAbortListeners); + + const stack = resolveDeliveryStack(req.method, apiPath, rawBody) ?? classification.stack; + if (!stack || !classification.stage) { + detachAbortListeners(); + res.status(400).json({ error: 'Invalid registry delivery route classification' }); + return; + } + + runWithRegistryDeliveryContext( + { + envelope: delivery, + nodeId: req.nodeId, + stack, + stage: classification.stage, + service: classification.service, + abortSignal: abortController.signal, + }, + () => next(), + ); + } catch { + res.status(400).json({ error: 'Invalid registry delivery envelope' }); + } +} diff --git a/backend/src/pilot/agent.ts b/backend/src/pilot/agent.ts index 484a7855..c0506a6e 100644 --- a/backend/src/pilot/agent.ts +++ b/backend/src/pilot/agent.ts @@ -131,7 +131,7 @@ export class PilotAgent { private ws: WebSocket | null = null; private pingTimer?: NodeJS.Timeout; private reconnectTimer?: NodeJS.Timeout; - private readonly httpStreams = new Map(); + private readonly httpStreams = new Map(); private readonly wsStreams = new Map(); /** Per-connection mesh frame handler. Created on `connect()`, cleaned up on disconnect. */ private switchboard: TcpStreamSwitchboard | null = null; @@ -486,6 +486,7 @@ export class PilotAgent { } case 'http_req': this.onHttpReq(frame); break; case 'http_req_end': this.onHttpReqEnd(frame.s); break; + case 'http_cancel': this.onHttpCancel(frame.s); break; case 'ws_open': this.onWsOpen(frame); break; case 'ws_msg_text': this.onWsMsgText(frame.s, frame.data); break; case 'ws_close': this.onWsClose(frame.s, frame.code, frame.reason); break; @@ -500,7 +501,7 @@ export class PilotAgent { switch (frame.type) { case BinaryFrameType.HttpReqBody: { const entry = this.httpStreams.get(frame.streamId); - if (!entry) return; + if (!entry || entry.cancelled) return; try { entry.req.write(frame.payload); } catch { /* ignore */ } this.refreshIdleTimer(frame.streamId); break; @@ -591,11 +592,20 @@ export class PilotAgent { private onHttpReqEnd(streamId: number): void { const entry = this.httpStreams.get(streamId); - if (!entry) return; + if (!entry || entry.cancelled) return; try { entry.req.end(); } catch { /* ignore */ } this.refreshIdleTimer(streamId); } + private onHttpCancel(streamId: number): void { + const entry = this.httpStreams.get(streamId); + if (!entry || entry.cancelled) return; + entry.cancelled = true; + try { entry.req.destroy(); } catch { /* ignore */ } + this.httpStreams.delete(streamId); + this.clearIdleTimer(streamId); + } + // --- WebSocket dispatch (tunnel -> loopback) --- private onWsOpen(frame: Extract, { t: 'ws_open' }>): void { diff --git a/backend/src/pilot/protocol.ts b/backend/src/pilot/protocol.ts index 2759af23..f000abba 100644 --- a/backend/src/pilot/protocol.ts +++ b/backend/src/pilot/protocol.ts @@ -78,6 +78,7 @@ export type JsonFrame = | HttpReqEndFrame | HttpResFrame | HttpResEndFrame + | HttpCancelFrame | HttpErrorFrame | WsOpenFrame | WsAcceptFrame @@ -122,6 +123,12 @@ export interface HttpResEndFrame { s: number; } +/** Primary -> agent: cancel an in-flight loopback HTTP request (client disconnect). */ +export interface HttpCancelFrame { + t: 'http_cancel'; + s: number; +} + export interface HttpErrorFrame { t: 'http_err'; s: number; diff --git a/backend/src/proxy/remoteNodeProxy.ts b/backend/src/proxy/remoteNodeProxy.ts index 9890fcf7..66b05d27 100644 --- a/backend/src/proxy/remoteNodeProxy.ts +++ b/backend/src/proxy/remoteNodeProxy.ts @@ -51,6 +51,12 @@ import { import type { PermissionAction } from '../middleware/permissions'; import { SETTING_WRITE_PERMISSIONS } from '../routes/settings'; import { rejectApiTokenScope } from '../middleware/apiTokenScope'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; +import { + classifyRegistryDeliveryRouteClass, + getRegistryDeliveryTotalBodyLimit, +} from '../helpers/registryDeliveryBodyLimits'; +import { augmentRemoteProxyWithRegistryDelivery, evaluateRegistryDeliveryProxyGate } from '../helpers/registryDeliveryProxy'; /** * Per-request hop timing for the critical hydration GETs, kept off the Request @@ -693,6 +699,68 @@ export function createRemoteProxyMiddleware(): RequestHandler { req.proxyElevatedRole = 'node-admin'; } + // Registry credential delivery: when capability and confidential + // transport are present, run hop-1 discover and attach the envelope to + // the forwarded JSON body. Otherwise forward unchanged (AUD-30). + const deliveryApiPath = `/api${req.path}`; + if (RegistryDeliveryService.getInstance().isDeliveryEligibleRoute(req.method, deliveryApiPath)) { + const gate = await evaluateRegistryDeliveryProxyGate( + req, + res, + req.nodeId, + node, + req.method, + deliveryApiPath, + ); + if (gate.outcome === 'stop') { + return; + } + if (gate.outcome === 'run-delivery') { + if (hasNonIdentityContentEncoding(req)) { + await drainRequestBody(req); + res.status(415).json({ + error: 'Compressed request bodies are not supported for remote registry delivery', + code: 'encoding_unsupported', + }); + return; + } + const routeClass = classifyRegistryDeliveryRouteClass(req.method, deliveryApiPath); + if (routeClass) { + const bodyLimit = getRegistryDeliveryTotalBodyLimit(routeClass); + try { + req.rawBody = await bufferRequestBody(req, bodyLimit); + } catch (err) { + const status = Number((err as { status?: number }).status); + if (status === 413) { + res.status(413).json({ error: 'Request body too large for registry delivery' }); + return; + } + if (status === 400) { + res.status(400).json({ error: 'Incomplete request body' }); + return; + } + throw err; + } + const deliveryResult = await augmentRemoteProxyWithRegistryDelivery( + req, + req.nodeId, + node, + target, + req.rawBody, + ); + if (!deliveryResult.forward) { + if (req.registryDeliveryAbortController?.signal.aborted) { + return; + } + res.status(deliveryResult.status ?? 500).json({ + error: deliveryResult.error ?? 'Registry delivery failed', + }); + return; + } + } + } + } + req.proxyTarget = target; // Identity routes are reworked before the hop, not during it: a request @@ -716,6 +784,9 @@ export function createRemoteProxyMiddleware(): RequestHandler { } beginProxyTiming(req, res); + if (req.registryDeliveryAbortController?.signal.aborted) { + return; + } proxy(req, res, next); }; diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts index 2bd91b95..059e181c 100644 --- a/backend/src/routes/fleet.ts +++ b/backend/src/routes/fleet.ts @@ -32,6 +32,7 @@ import { validateStackPatternForRedos } from '../helpers/stackPattern'; export { validateStackPatternForRedos } from '../helpers/stackPattern'; import { getErrorMessage } from '../utils/errors'; +import { prepareOutboundRegistryDeliveryBody } from '../helpers/registryDeliveryOutbound'; import { parseIntParam } from '../utils/parseIntParam'; import { parseRequestedTargetVersion, pickCompareTarget } from '../utils/targetVersion'; import { buildTargetImageRef, isRepinBlocked, type ImagePinKind } from '../helpers/selfUpdateCompose'; @@ -2813,12 +2814,23 @@ async function redeploySnapshotStack(node: Node, stackName: string): Promise { + try { + if (req.machineAuthScope !== 'node_proxy' && req.machineAuthScope !== 'pilot_tunnel') { + res.status(403).json({ error: 'Machine authentication required' }); + return; + } + + const service = RegistryDeliveryService.getInstance(); + const result = await service.discoverOnTarget(req.body); + res.json(result); + } catch (error) { + console.error('[registry-delivery] discover failed'); + res.status(500).json({ error: 'Registry delivery discovery failed' }); + } +}); + +registryDeliveryRouter.get('/evidence', (req: Request, res: Response) => { + if (req.machineAuthScope !== 'node_proxy' && req.machineAuthScope !== 'pilot_tunnel') { + res.status(403).json({ error: 'Machine authentication required' }); + return; + } + + const service = RegistryDeliveryService.getInstance(); + const limit = Math.min(Math.max(parseInt(String(req.query.limit ?? '100'), 10) || 100, 1), 500); + const cursor = Math.max(parseInt(String(req.query.cursor ?? '0'), 10) || 0, 0); + const deliverySourceId = service.getDeliverySourceId(); + + res.json(listRegistryDeliveryEvidencePage(deliverySourceId, cursor, limit)); +}); + +registryDeliveryRouter.use((_req, res) => { + res.status(404).json({ error: 'Not found' }); +}); + +// Configure prepared-source store at module load when delivery source id exists. +try { + const deliverySourceId = RegistryDeliveryService.getInstance().getDeliverySourceId(); + PreparedSourceStore.getInstance().configure(deliverySourceId); +} catch { + /* configured on first use */ +} diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 0479b9ca..e4deba6d 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -26,6 +26,7 @@ import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../se import { repoUrlRejectionMessage } from '../services/gitops/repoIdentity'; import { REF_MAX_LEN } from '../services/git/nativeGitTransport'; import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; +import { getRegistryDeliveryContext, getRegistryDeliveryLockContext } from '../helpers/registryDeliveryContext'; import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService'; import { DriftLedgerService, type DriftTemporal } from '../services/DriftLedgerService'; import { ComposeDoctorService } from '../services/ComposeDoctorService'; @@ -144,7 +145,14 @@ function tryAcquireStackOpLock( action: StackOpAction, ): boolean { const user = req.user?.username ?? 'system'; - const result = StackOpLockService.getInstance().tryAcquire(req.nodeId, stackName, action, user); + const lockContext = getRegistryDeliveryLockContext(); + const result = StackOpLockService.getInstance().tryAcquire( + req.nodeId, + stackName, + action, + user, + lockContext, + ); if (!result.acquired) { res.status(409).json({ error: `${stackName} is already ${STACK_OP_PRESENT_PARTICIPLE[result.existing.action]}`, @@ -2597,7 +2605,13 @@ stacksRouter.post('/:stackName/rollback', async (req: Request, res: Response) => // without this a blocked rollback would leave disk rolled back while the // deployed state is unchanged. revertRestore = await fsSvc.snapshotStackFiles(stackName); - await fsSvc.restoreStackFiles(stackName); + const deliveryPrepId = getRegistryDeliveryContext()?.envelope.prepId; + if (deliveryPrepId) { + const { materializePreparedSourceToStack } = await import('../helpers/registryDeliveryMaterialize'); + await materializePreparedSourceToStack(deliveryPrepId, req.nodeId, stackName); + } else { + await fsSvc.restoreStackFiles(stackName); + } if (!(await runPolicyGate(req, res, stackName, req.nodeId))) { try { await revertRestore(); diff --git a/backend/src/routes/templates.ts b/backend/src/routes/templates.ts index ae635a9e..3d377722 100644 --- a/backend/src/routes/templates.ts +++ b/backend/src/routes/templates.ts @@ -14,8 +14,45 @@ import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; import { runPolicyGate, triggerPostDeployScan } from '../helpers/policyGate'; import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; +import { getRegistryDeliveryContext, getRegistryDeliveryLockContext } from '../helpers/registryDeliveryContext'; +import { materializePreparedSourceToStack } from '../helpers/registryDeliveryMaterialize'; +import { StackOpLockService } from '../services/StackOpLockService'; import { getTerminalWs, DEPLOY_SESSION_HEADER } from '../websocket/generic'; +function tryAcquireRegistryDeliveryLock( + req: Request, + res: Response, + stackName: string, +): boolean { + if (!getRegistryDeliveryContext()) return true; + const lockContext = getRegistryDeliveryLockContext(); + if (!lockContext) { + res.status(400).json({ error: 'Registry delivery lock context missing' }); + return false; + } + const user = req.user?.username ?? 'system'; + const result = StackOpLockService.getInstance().tryAcquire( + req.nodeId, + stackName, + 'deploy', + user, + lockContext, + ); + if (!result.acquired) { + res.status(409).json({ + error: `${stackName} is already busy`, + code: 'stack_op_in_progress', + inProgress: { + action: result.existing.action, + startedAt: result.existing.startedAt, + user: result.existing.user, + }, + }); + return false; + } + return true; +} + export const templatesRouter = Router(); templatesRouter.get('/', authMiddleware, async (req: Request, res: Response) => { @@ -68,6 +105,8 @@ templatesRouter.post('/refresh-cache', authMiddleware, (req: Request, res: Respo templatesRouter.post('/deploy', authMiddleware, async (req: Request, res: Response) => { if (!requirePermission(req, res, 'stack:create')) return; if (!requirePermission(req, res, 'stack:deploy')) return; + let needsDeliveryLock = false; + let lockedStackName: string | null = null; try { const { stackName, template, envVars, skip_scan } = req.body; @@ -86,6 +125,12 @@ templatesRouter.post('/deploy', authMiddleware, async (req: Request, res: Respon return res.status(400).json({ error: 'Invalid stack path' }); } + needsDeliveryLock = Boolean(getRegistryDeliveryContext()); + if (needsDeliveryLock && !tryAcquireRegistryDeliveryLock(req, res, stackName)) { + return; + } + lockedStackName = stackName; + try { await fsPromises.access(stackPath); @@ -118,13 +163,17 @@ templatesRouter.post('/deploy', authMiddleware, async (req: Request, res: Respon await fsService.createStack(stackName); - const composeYaml = templateService.generateComposeFromTemplate(template, stackName); - await fsService.saveStackContent(stackName, composeYaml); + const deliveryPrepId = getRegistryDeliveryContext()?.envelope.prepId; + if (deliveryPrepId) { + await materializePreparedSourceToStack(deliveryPrepId, req.nodeId, stackName); + } else { + const composeYaml = templateService.generateComposeFromTemplate(template, stackName); + await fsService.saveStackContent(stackName, composeYaml); - if (envVars && Object.keys(envVars).length > 0) { - const envString = templateService.generateEnvString(envVars); - const defaultEnvPath = path.join(stackPath, '.env'); - await fsPromises.writeFile(defaultEnvPath, envString, 'utf-8'); + if (envVars && Object.keys(envVars).length > 0) { + const envString = templateService.generateEnvString(envVars); + await fsService.saveEnvContent(stackName, envString); + } } try { @@ -190,5 +239,9 @@ templatesRouter.post('/deploy', authMiddleware, async (req: Request, res: Respon const message = getErrorMessage(error, 'Failed to deploy template'); console.error('[Templates] Deploy error:', message); res.status(500).json({ error: message }); + } finally { + if (needsDeliveryLock && lockedStackName) { + StackOpLockService.getInstance().release(req.nodeId, lockedStackName); + } } }); diff --git a/backend/src/services/BlueprintService.ts b/backend/src/services/BlueprintService.ts index 5c9fd39f..59c06610 100644 --- a/backend/src/services/BlueprintService.ts +++ b/backend/src/services/BlueprintService.ts @@ -16,6 +16,8 @@ import { NodeRegistry } from './NodeRegistry'; import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers'; import { LicenseService } from './LicenseService'; import { assertPolicyGateAllows, buildSystemPolicyGateOptions, describePolicyBlock, triggerPostDeployScan } from '../helpers/policyGate'; +import { prepareOutboundRegistryDeliveryBody } from '../helpers/registryDeliveryOutbound'; +import { getRegistryDeliveryLockContext } from '../helpers/registryDeliveryContext'; import { enforcePolicyForImageRefs } from './PolicyEnforcement'; import { BlueprintAnalyzer } from './BlueprintAnalyzer'; import { sanitizeForLog } from '../utils/safeLog'; @@ -545,6 +547,16 @@ export class BlueprintService { await fs.createStack(stackName); createdStack = true; } + let previousComposeContent: string | null = null; + if (!createdStack) { + const prior = await fs.readStackFile(stackName, COMPOSE_FILENAME); + if (prior.oversized || prior.binary || prior.content === undefined) { + throw new Error( + `Cannot snapshot existing compose for blueprint apply on "${stackName}"`, + ); + } + previousComposeContent = prior.content; + } await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent); // Clear lower-priority compose siblings so discovery cannot shadow compose.yaml. await fs.removeAlternateRootComposeFiles(stackName); @@ -572,10 +584,21 @@ export class BlueprintService { sanitizeForLog(BlueprintService.formatError(cleanupErr)), ); } + } else if (previousComposeContent !== null) { + try { + await fs.writeStackFile(stackName, COMPOSE_FILENAME, previousComposeContent); + } catch (restoreErr) { + console.warn( + '[BlueprintService] Failed to restore prior compose for "%s" after apply error: %s', + sanitizeForLog(stackName), + sanitizeForLog(BlueprintService.formatError(restoreErr)), + ); + } } throw err; } }, + getRegistryDeliveryLockContext(), ); return lock.ran ? { ran: true } : { ran: false, existingAction: lock.existing.action }; } @@ -617,14 +640,25 @@ export class BlueprintService { const baseUrl = target.apiUrl.replace(/\/$/, ''); const headers = this.remoteHeaders(target.apiToken); + const applyBody = { + stackName: blueprint.name, + composeContent: blueprint.compose_content, + markerContent: JSON.stringify(marker, null, 2), + }; + const augmented = await prepareOutboundRegistryDeliveryBody({ + method: 'POST', + apiPath: '/api/blueprints/apply-local', + nodeId: node.id, + body: applyBody, + }); + if (!augmented.ok) { + throw new Error(augmented.error); + } + // Atomic apply: the remote validates ownership and writes under its stack lock. const res = await axios.post( `${baseUrl}/api/blueprints/apply-local`, - { - stackName: blueprint.name, - composeContent: blueprint.compose_content, - markerContent: JSON.stringify(marker, null, 2), - }, + augmented.body, { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, ); if (res.status === 404) { diff --git a/backend/src/services/CapabilityRegistry.ts b/backend/src/services/CapabilityRegistry.ts index d4d9d22e..cf541f51 100644 --- a/backend/src/services/CapabilityRegistry.ts +++ b/backend/src/services/CapabilityRegistry.ts @@ -64,6 +64,7 @@ export const CAPABILITIES = [ 'service-scoped-update', 'service-scoped-stack-alert', 'scoped-stack-auth-evidence', + 'remote-registry-credentials', ] as const; /** @@ -119,6 +120,10 @@ export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY = export const SCOPED_STACK_AUTH_EVIDENCE_CAPABILITY = 'scoped-stack-auth-evidence' as const satisfies Capability; +/** Remotes that accept hub-delivered registry credentials for Compose operations. */ +export const REMOTE_REGISTRY_CREDENTIALS_CAPABILITY = + 'remote-registry-credentials' as const satisfies Capability; + /** Returns true when the string is a usable semver version. */ export function isValidVersion(v: string | null | undefined): v is string { return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v); diff --git a/backend/src/services/ComposeService.ts b/backend/src/services/ComposeService.ts index 3899c442..5cc0d43f 100644 --- a/backend/src/services/ComposeService.ts +++ b/backend/src/services/ComposeService.ts @@ -1,6 +1,5 @@ import { spawn } from 'child_process'; import fs from 'fs'; -import os from 'os'; import path from 'path'; import WebSocket from 'ws'; import DockerController from './DockerController'; @@ -10,6 +9,12 @@ import { MeshService } from './MeshService'; import { LogFormatter } from './LogFormatter'; import { NodeRegistry } from './NodeRegistry'; import { RegistryService } from './RegistryService'; +import { RegistryDeliveryService } from './RegistryDeliveryService'; +import { createDockerAuthTempDir } from '../helpers/dockerAuthTempDir'; +import { getRegistryDeliveryContext } from '../helpers/registryDeliveryContext'; +import { PreparedSourceStore } from '../services/preparedSourceStore'; +import { recordRegistryDeliveryEvent } from '../helpers/registryDeliveryEvidence'; +import { resolveRegistryAuthAtSeam } from '../helpers/registryDeliverySeam'; import { DriftLedgerService } from './DriftLedgerService'; import SelfIdentityService from './SelfIdentityService'; import { parseEffectiveModel } from './preflight/effectiveModel'; @@ -273,8 +278,16 @@ export class ComposeService { // When set, terminate the child if it emits no output for this long while // still running (idle stall backstop). Appended last so the existing // registry-auth call sites that pass `env` are unaffected. - idleTimeoutMs?: number + idleTimeoutMs?: number, + abortSignal?: AbortSignal, ): Promise { + const deliveryAbortSignal = getRegistryDeliveryContext()?.abortSignal; + const effectiveAbortSignal = abortSignal ?? deliveryAbortSignal; + + if (effectiveAbortSignal?.aborted) { + return Promise.reject(new Error('OPERATION_ABORTED: client disconnected')); + } + return new Promise((resolve, reject) => { const child = spawn(command, args, { cwd, @@ -299,28 +312,6 @@ export class ComposeService { } }; - const cleanup = () => { - if (timeout) { - clearTimeout(timeout); - timeout = null; - } - if (forceKillTimeout) { - clearTimeout(forceKillTimeout); - forceKillTimeout = null; - } - if (idleTimeout) { - clearTimeout(idleTimeout); - idleTimeout = null; - } - }; - - const finish = (complete: () => void) => { - if (settled) return; - settled = true; - cleanup(); - complete(); - }; - const terminateChild = (error: Error) => { pendingTerminationError = pendingTerminationError ?? error; if (exited) return; @@ -339,6 +330,38 @@ export class ComposeService { }, 5000); }; + const onAbort = effectiveAbortSignal + ? () => { + sendOutput('=== Operation cancelled (client disconnected) ===\n'); + terminateChild(new Error('OPERATION_ABORTED: client disconnected')); + } + : undefined; + + const cleanup = () => { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + if (forceKillTimeout) { + clearTimeout(forceKillTimeout); + forceKillTimeout = null; + } + if (idleTimeout) { + clearTimeout(idleTimeout); + idleTimeout = null; + } + if (effectiveAbortSignal && onAbort) { + effectiveAbortSignal.removeEventListener('abort', onAbort); + } + }; + + const finish = (complete: () => void) => { + if (settled) return; + settled = true; + cleanup(); + complete(); + }; + // Idle stall backstop. Armed once below and reset on every output chunk; // if it ever fires, the step has been silent for idleTimeoutMs while still // running, so terminate it. Never rearmed after a termination is pending or @@ -367,6 +390,10 @@ export class ComposeService { armIdleTimeout(); + if (effectiveAbortSignal && onAbort) { + effectiveAbortSignal.addEventListener('abort', onAbort); + } + const onData = (data: Buffer) => { const text = data.toString(); errorLog += text; @@ -417,37 +444,101 @@ export class ComposeService { fn: (env: Record) => Promise, sendOutput?: (data: string) => void, ): Promise { - const registries = DatabaseService.getInstance().getRegistries(); - if (registries.length === 0) { + const deliveryContext = getRegistryDeliveryContext(); + + const mergedAuths: Record = {}; + + if (deliveryContext) { + if (!deliveryContext.seamResult) { + deliveryContext.seamResult = await resolveRegistryAuthAtSeam({ + envelope: deliveryContext.envelope, + nodeId: deliveryContext.nodeId, + stack: deliveryContext.stack, + stage: deliveryContext.stage, + service: deliveryContext.service, + }); + deliveryContext.seamSettled = true; + } + Object.assign(mergedAuths, deliveryContext.seamResult.auths); + } else { + const registries = DatabaseService.getInstance().getRegistries(); + if (registries.length > 0) { + const { config, warnings } = await RegistryService.getInstance().resolveDockerConfig(); + if (warnings.length > 0 && sendOutput) { + for (const warning of warnings) { + sendOutput(`[Sencho] Warning: ${warning}\n`); + } + } + Object.assign(mergedAuths, config.auths); + } + } + + if (Object.keys(mergedAuths).length === 0) { return fn({ ...process.env, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', }); } - const { config, warnings } = await RegistryService.getInstance().resolveDockerConfig(); - if (warnings.length > 0 && sendOutput) { - for (const warning of warnings) { - sendOutput(`[Sencho] Warning: ${warning}\n`); - } - } - - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-docker-')); - const configPath = path.join(tmpDir, 'config.json'); + const deliverySourceId = RegistryDeliveryService.getInstance().getDeliverySourceId(); + const handle = createDockerAuthTempDir( + deliverySourceId, + deliveryContext ? 'delivered' : 'local', + { auths: mergedAuths }, + ); try { - fs.writeFileSync(configPath, JSON.stringify(config), { mode: 0o600 }); return await fn({ ...process.env, - DOCKER_CONFIG: tmpDir, + DOCKER_CONFIG: handle.dirPath, PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', }); } finally { - // Best-effort cleanup; each step runs independently so a file that was never - // written (e.g., writeFileSync threw) does not prevent the directory removal. - try { fs.unlinkSync(configPath); } catch { /* file may not exist */ } - try { fs.rmdirSync(tmpDir); } catch (e) { - console.warn('[ComposeService] Could not remove temp Docker config dir:', (e as Error).message); + try { + handle.cleanup(); + } catch (cleanupErr) { + const cleanupMessage = getErrorMessage(cleanupErr, 'unknown error'); + if (deliveryContext) { + try { + recordRegistryDeliveryEvent({ + deliverySourceId, + eventType: 'cleanup_failed', + tempDirId: path.basename(handle.dirPath), + stack: deliveryContext.stack ?? null, + op: deliveryContext.stage ?? null, + }); + console.error( + 'Registry delivery temp dir cleanup failed for %s:', + sanitizeForLog(deliverySourceId), + cleanupMessage, + ); + } catch (evidenceErr) { + console.error( + 'Registry delivery cleanup and evidence both failed for %s:', + sanitizeForLog(deliverySourceId), + getErrorMessage(evidenceErr, 'unknown error'), + cleanupMessage, + ); + } + } else { + console.error( + 'Registry delivery temp dir cleanup failed for %s:', + sanitizeForLog(deliverySourceId), + cleanupMessage, + ); + } + } + const prepId = deliveryContext?.seamResult?.prepId ?? deliveryContext?.envelope.prepId; + if (prepId) { + try { + PreparedSourceStore.getInstance().finalize(prepId); + } catch (finalizeErr) { + console.error( + 'Registry delivery prepared-source finalize failed for %s:', + sanitizeForLog(prepId), + getErrorMessage(finalizeErr, 'unknown error'), + ); + } } } } diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index ca858cc6..881bc1de 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -1,6 +1,7 @@ import Database from 'better-sqlite3'; import path from 'path'; import fs from 'fs'; +import crypto from 'crypto'; import { CryptoService } from './CryptoService'; import { isSeverityAtLeast } from '../utils/severity'; import { evaluatePolicyRisk, policyInputs, type PolicyBlockReason } from '../utils/policy-risk'; @@ -1146,6 +1147,7 @@ export class DatabaseService { this.migrateEncryptNodeTokens(); this.migrateSSOColumns(); this.migrateRegistries(); + this.migrateRegistryDelivery(); this.migrateRoleAssignments(); this.migrateNotificationRoutes(); this.migrateNotificationRoutesNodeId(); @@ -2335,6 +2337,62 @@ stmt.run('gitops_schema_version', '1'); `); } + private migrateRegistryDelivery(): void { + this.db.exec(` + CREATE TABLE IF NOT EXISTS registry_delivery_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT UNIQUE NOT NULL, + delivery_source_id TEXT NOT NULL, + stack TEXT, + op TEXT, + attestation_jti TEXT, + prep_id_sha256 TEXT, + temp_dir_id TEXT, + event_type TEXT NOT NULL, + source_hash TEXT, + pruned_through_seq INTEGER, + created_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_registry_delivery_events_source_seq + ON registry_delivery_events(delivery_source_id, seq); + + CREATE TABLE IF NOT EXISTS registry_delivery_imported_events ( + event_id TEXT NOT NULL, + delivery_source_id TEXT NOT NULL, + seq INTEGER NOT NULL, + hub_node_id_snapshot INTEGER NOT NULL, + stack TEXT, + op TEXT, + attestation_jti TEXT, + prep_id_sha256 TEXT, + temp_dir_id TEXT, + event_type TEXT NOT NULL, + source_hash TEXT, + pruned_through_seq INTEGER, + created_at INTEGER NOT NULL, + imported_at INTEGER NOT NULL, + PRIMARY KEY (delivery_source_id, event_id) + ); + + CREATE TABLE IF NOT EXISTS registry_delivery_import_cursor ( + delivery_source_id TEXT PRIMARY KEY, + last_seq INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + `); + + const existing = this.db.prepare( + 'SELECT value FROM global_settings WHERE key = ?', + ).get('delivery_source_id') as { value: string } | undefined; + if (!existing) { + const id = crypto.randomUUID(); + this.db.prepare( + 'INSERT INTO global_settings (key, value) VALUES (?, ?)', + ).run('delivery_source_id', id); + this.cachedGlobalSettings = null; + } + } + private migrateRoleAssignments(): void { this.db.exec(` CREATE TABLE IF NOT EXISTS role_assignments ( @@ -6072,6 +6130,149 @@ stmt.run('gitops_schema_version', '1'); this.db.prepare('DELETE FROM audit_log WHERE timestamp < ?').run(cutoff); } + public insertRegistryDeliveryEvent(params: { + deliverySourceId: string; + eventType: string; + stack?: string | null; + op?: string | null; + attestationJti?: string | null; + prepIdSha256?: string | null; + tempDirId?: string | null; + sourceHash?: string | null; + prunedThroughSeq?: number | null; + }): number { + const eventId = crypto.randomUUID(); + const createdAt = Date.now(); + const result = this.db.prepare(` + INSERT INTO registry_delivery_events ( + event_id, delivery_source_id, stack, op, attestation_jti, + prep_id_sha256, temp_dir_id, event_type, source_hash, pruned_through_seq, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + eventId, + params.deliverySourceId, + params.stack ?? null, + params.op ?? null, + params.attestationJti ?? null, + params.prepIdSha256 ?? null, + params.tempDirId ?? null, + params.eventType, + params.sourceHash ?? null, + params.prunedThroughSeq ?? null, + createdAt, + ); + return Number(result.lastInsertRowid); + } + + public listRegistryDeliveryEvents( + deliverySourceId: string, + afterSeq: number, + limit: number, + ): import('../types/registryDeliveryEvidence').RegistryDeliveryEventRow[] { + return this.db.prepare(` + SELECT seq, event_id, delivery_source_id, stack, op, attestation_jti, + prep_id_sha256, temp_dir_id, event_type, source_hash, pruned_through_seq, created_at + FROM registry_delivery_events + WHERE delivery_source_id = ? AND seq > ? + ORDER BY seq ASC + LIMIT ? + `).all(deliverySourceId, afterSeq, limit) as import('../types/registryDeliveryEvidence').RegistryDeliveryEventRow[]; + } + + public getRegistryDeliveryImportCursor(deliverySourceId: string): number { + const row = this.db.prepare( + 'SELECT last_seq FROM registry_delivery_import_cursor WHERE delivery_source_id = ?', + ).get(deliverySourceId) as { last_seq: number } | undefined; + return row?.last_seq ?? 0; + } + + public importRegistryDeliveryEventPage( + hubNodeIdSnapshot: number, + deliverySourceId: string, + events: import('../types/registryDeliveryEvidence').RegistryDeliveryEventRow[], + ): { imported: number; lastSeq: number } { + if (events.length === 0) { + return { + imported: 0, + lastSeq: this.getRegistryDeliveryImportCursor(deliverySourceId), + }; + } + + const importPage = this.db.transaction((rows: import('../types/registryDeliveryEvidence').RegistryDeliveryEventRow[]) => { + const insert = this.db.prepare(` + INSERT OR IGNORE INTO registry_delivery_imported_events ( + event_id, delivery_source_id, seq, hub_node_id_snapshot, + stack, op, attestation_jti, prep_id_sha256, temp_dir_id, + event_type, source_hash, pruned_through_seq, created_at, imported_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + let imported = 0; + let maxSeq = this.getRegistryDeliveryImportCursor(deliverySourceId); + const now = Date.now(); + for (const row of rows) { + const result = insert.run( + row.event_id, + deliverySourceId, + row.seq, + hubNodeIdSnapshot, + row.stack, + row.op, + row.attestation_jti, + row.prep_id_sha256, + row.temp_dir_id, + row.event_type, + row.source_hash, + row.pruned_through_seq, + row.created_at, + now, + ); + if (result.changes > 0) imported += 1; + if (row.seq > maxSeq) maxSeq = row.seq; + } + this.db.prepare(` + INSERT INTO registry_delivery_import_cursor (delivery_source_id, last_seq, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(delivery_source_id) DO UPDATE SET + last_seq = CASE + WHEN excluded.last_seq > registry_delivery_import_cursor.last_seq + THEN excluded.last_seq + ELSE registry_delivery_import_cursor.last_seq + END, + updated_at = excluded.updated_at + `).run(deliverySourceId, maxSeq, now); + return { imported, lastSeq: maxSeq }; + }); + + return importPage(events); + } + + public cleanupOldDeliveryEvents(daysToKeep = 90): number { + const deliverySourceId = this.getGlobalSettings().delivery_source_id; + if (!deliverySourceId) return 0; + + const cutoff = Date.now() - (daysToKeep * 24 * 60 * 60 * 1000); + const maxPruned = this.db.prepare(` + SELECT MAX(seq) as maxSeq FROM registry_delivery_events + WHERE delivery_source_id = ? AND created_at < ? + `).get(deliverySourceId, cutoff) as { maxSeq: number | null } | undefined; + const prunedThrough = maxPruned?.maxSeq ?? null; + if (prunedThrough === null) return 0; + + const result = this.db.prepare(` + DELETE FROM registry_delivery_events + WHERE delivery_source_id = ? AND created_at < ? + `).run(deliverySourceId, cutoff); + + if (result.changes > 0) { + this.insertRegistryDeliveryEvent({ + deliverySourceId, + eventType: 'retention_gap', + prunedThroughSeq: prunedThrough, + }); + } + return result.changes; + } + public getAuditLogsInRange(from: number, to: number, limit?: number): AuditLogEntry[] { this.flushAuditLogBuffer(); if (limit !== undefined) { diff --git a/backend/src/services/FileSystemService.ts b/backend/src/services/FileSystemService.ts index 8a69300a..961931df 100644 --- a/backend/src/services/FileSystemService.ts +++ b/backend/src/services/FileSystemService.ts @@ -1486,6 +1486,37 @@ export class FileSystemService { } } + /** + * Copy managed backup-slot files into destDir for registry-delivery preparation. + * Skips integrity markers; only regular compose and env files are copied. + */ + async copyBackupSlotToDir(stackName: string, destDir: string): Promise { + const backupRoot = path.resolve(getBackupBaseDir()); + const backupDir = path.resolve(backupRoot, String(this.nodeId), stackName); + if (!backupDir.startsWith(backupRoot + path.sep)) { + throw Object.assign(new Error('Path escapes backup directory'), { code: 'INVALID_PATH' }); + } + await fsPromises.mkdir(destDir, { recursive: true, mode: 0o700 }); + const items = await fsPromises.readdir(backupDir); + for (const item of items) { + if (item === '.checksums' || item === '.timestamp') continue; + const src = path.resolve(backupDir, item); + if (!src.startsWith(backupDir + path.sep)) continue; + const stat = await fsPromises.lstat(src); + if (!stat.isFile() || stat.isSymbolicLink()) continue; + const dest = path.join(destDir, item); + await fsPromises.copyFile(src, dest, fsPromises.constants.COPYFILE_EXCL).catch(async (err: NodeJS.ErrnoException) => { + if (err.code === 'EEXIST') { + await fsPromises.unlink(dest); + await fsPromises.copyFile(src, dest); + return; + } + throw err; + }); + await fsPromises.chmod(dest, 0o600); + } + } + // --------------------------------------------------------------------------- // Stack-scoped file explorer methods // --------------------------------------------------------------------------- diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index 5363719e..13f635a2 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -1,5 +1,4 @@ import { promises as fsPromises, existsSync } from 'fs'; -import { spawn } from 'child_process'; import crypto from 'crypto'; import os from 'os'; import path from 'path'; @@ -43,9 +42,12 @@ import { stackManagedRoot, } from './gitops/directApplication'; import type { GitOpsApplicationRow } from './gitops/types'; -import { appliedRelPathFor, candidateRelPathForSha, deleteStagingMarker, readStagingMarker, writeStagingMarker } from './gitops/createStagingMarker'; +import { appliedRelPathFor, candidateRelPathForSha, deleteStagingMarker, readStagingMarker, validateCandidateRelPath, writeStagingMarker } from './gitops/createStagingMarker'; import { cleanupUnclaimedManagedRoot, removeOperationOwnedPaths } from './gitops/createCleanup'; import { managedAreaBase } from './gitops/managedPaths'; +import { getRegistryDeliveryContext, getRegistryDeliveryLockContext } from '../helpers/registryDeliveryContext'; +import { copyPreparedPayloadDirectory } from '../helpers/registryDeliveryMaterialize'; +import { runDockerCompose as spawnDockerCompose } from '../helpers/dockerComposeRunner'; /** * GitSourceService - fetch compose files from a Git repository and apply @@ -1222,9 +1224,9 @@ export class GitSourceService { const startedAt = Date.now(); const diag = isDebugEnabled(); if (diag) { - console.log( - `[GitSource:diag] fetch start host=${sanitizeForLog(repoHost(repoUrl))} branch=${sanitizeForLog(branch)} files=${composePaths.length} envSync=${envPath ? 'true' : 'false'}` - ); + console.log(sanitizeForLog( + `[GitSource:diag] fetch start host=${repoHost(repoUrl)} branch=${branch} files=${composePaths.length} envSync=${envPath ? 'true' : 'false'}`, + )); } try { @@ -1544,26 +1546,8 @@ export class GitSourceService { }; } - private runDockerCompose(args: string[], cwd: string, timeoutMs: number): Promise<{ code: number; stdout: string; stderr: string }> { - return new Promise((resolve) => { - const child = spawn('docker', args, { cwd }); - let stdout = ''; - let stderr = ''; - const timer = setTimeout(() => { - try { child.kill('SIGKILL'); } catch { /* best effort */ } - resolve({ code: -1, stdout, stderr: stderr + '\nValidation timed out.' }); - }, timeoutMs); - child.stdout.on('data', d => { stdout += d.toString(); }); - child.stderr.on('data', d => { stderr += d.toString(); }); - child.on('close', (code) => { - clearTimeout(timer); - resolve({ code: code ?? -1, stdout, stderr }); - }); - child.on('error', (err) => { - clearTimeout(timer); - resolve({ code: -1, stdout, stderr: stderr + '\n' + err.message }); - }); - }); + private runDockerCompose(args: string[], cwd: string, timeoutMs: number) { + return spawnDockerCompose(args, cwd, timeoutMs); } // ─── Hashing + diff ────────────────────────────────────────────────────── @@ -2205,6 +2189,7 @@ export class GitSourceService { 'git_apply', opts.actor ?? 'system:git-source', () => this.applyLocked(stackName, commitSha, opts), + getRegistryDeliveryLockContext(), ); if (!lock.ran) { throw new GitSourceError( @@ -2348,6 +2333,15 @@ export class GitSourceService { // The staged candidate must still exist and be complete; a deleted // candidate (or a node restart that swept it) invalidates the pull. + const deliveryPrepId = getRegistryDeliveryContext()?.envelope.prepId; + if (deliveryPrepId) { + await this.restoreApplyFromPreparedGitCandidate( + deliveryPrepId, + stackName, + commitSha, + pending.candidateRelPath, + ); + } const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data'); const candidateAbs = path.join(dataDir, 'git-managed', String(nodeId), stackName, pending.candidateRelPath); try { @@ -2801,6 +2795,7 @@ export class GitSourceService { // the complete-project candidate inside the clone lifecycle. const manifestSvc = GitProjectManifestService.getInstance(); const materialization: { value: MaterializationResult | null } = { value: null }; + const deliveryPrepId = getRegistryDeliveryContext()?.envelope.prepId; let fetched: FetchResult; const createFetchAuth = input.authType === 'token' ? { token: input.token } @@ -2813,31 +2808,43 @@ export class GitSourceService { } : { token: null }; try { - fetched = await this.fetchFromGit({ - repoUrl: input.repoUrl, - branch: input.branch, - composePaths: input.composePaths, - envPath: input.syncEnv ? input.envPath : null, - ...createFetchAuth, - onClone: async (cloneDir, commitSha, envContent) => { - // The candidate path is recorded before the build that - // creates it, so a crash mid-build still names exactly one - // directory this operation owns. - staged.candidateRelPath = candidateRelPathForSha(commitSha); - await writeStagingMarker(managedRoot, { - schemaVersion: 1, - operationId: gitopsOperationId, - rootPreexisted, - candidateRelPath: staged.candidateRelPath, - createdAt: Date.now(), - }); - materialization.value = await this.buildMaterialization(input.stackName, cloneDir, commitSha, { - compose_paths: input.composePaths, - context_dir: input.contextDir, - sync_env: input.syncEnv, - }, envContent); - }, - }); + if (deliveryPrepId) { + const restored = await this.restoreCreateFromPreparedGitCandidate( + deliveryPrepId, + managedRoot, + rootPreexisted, + gitopsOperationId, + staged, + ); + fetched = restored.fetched; + materialization.value = restored.materialization; + } else { + fetched = await this.fetchFromGit({ + repoUrl: input.repoUrl, + branch: input.branch, + composePaths: input.composePaths, + envPath: input.syncEnv ? input.envPath : null, + ...createFetchAuth, + onClone: async (cloneDir, commitSha, envContent) => { + // The candidate path is recorded before the build that + // creates it, so a crash mid-build still names exactly one + // directory this operation owns. + staged.candidateRelPath = candidateRelPathForSha(commitSha); + await writeStagingMarker(managedRoot, { + schemaVersion: 1, + operationId: gitopsOperationId, + rootPreexisted, + candidateRelPath: staged.candidateRelPath, + createdAt: Date.now(), + }); + materialization.value = await this.buildMaterialization(input.stackName, cloneDir, commitSha, { + compose_paths: input.composePaths, + context_dir: input.contextDir, + sync_env: input.syncEnv, + }, envContent); + }, + }); + } } catch (e) { // Materialization refuses routinely, not just on crashes. The // marker has to come off with the staged files, or it would @@ -3684,6 +3691,201 @@ export class GitSourceService { } } + // ─── Registry delivery preparation ───────────────────────────────────── + + private async loadPreparedGitCandidate( + prepId: string, + managedRoot: string, + expectations?: { commitSha?: string; candidateRelPath?: string }, + ): Promise { + const { PreparedSourceStore } = await import('./preparedSourceStore'); + const { + installGitCandidatePayloadToManagedRoot, + readGitCandidatePreparedMeta, + } = await import('../helpers/registryDeliveryGitCandidate'); + const payloadPath = PreparedSourceStore.getInstance().peekPayloadPath(prepId); + const meta = await readGitCandidatePreparedMeta(payloadPath); + if (!meta.materialization.validation.ok) { + throw new GitSourceError( + 'GIT_ERROR', + `Compose validation failed: ${meta.materialization.validation.error ?? 'unknown'}`, + ); + } + if (expectations?.commitSha && meta.commitSha !== expectations.commitSha) { + throw new GitSourceError('GIT_ERROR', 'Prepared git candidate commit mismatch'); + } + if (expectations?.candidateRelPath && meta.candidateRelPath !== expectations.candidateRelPath) { + throw new GitSourceError('GIT_ERROR', 'Prepared git candidate path mismatch'); + } + await installGitCandidatePayloadToManagedRoot(payloadPath, managedRoot, meta.candidateRelPath); + return meta; + } + + private async restoreCreateFromPreparedGitCandidate( + prepId: string, + managedRoot: string, + rootPreexisted: boolean, + gitopsOperationId: string, + staged: { candidateRelPath: string | null }, + ): Promise<{ fetched: FetchResult; materialization: MaterializationResult }> { + const { fetchResultFromPreparedMeta } = await import('../helpers/registryDeliveryGitCandidate'); + const meta = await this.loadPreparedGitCandidate(prepId, managedRoot); + staged.candidateRelPath = meta.candidateRelPath; + await writeStagingMarker(managedRoot, { + schemaVersion: 1, + operationId: gitopsOperationId, + rootPreexisted, + candidateRelPath: staged.candidateRelPath, + createdAt: Date.now(), + }); + return { + fetched: fetchResultFromPreparedMeta(meta), + materialization: meta.materialization, + }; + } + + private async restoreApplyFromPreparedGitCandidate( + prepId: string, + stackName: string, + commitSha: string, + candidateRelPath: string, + ): Promise { + const managedRoot = path.resolve(stackManagedRoot(stackName)); + await this.loadPreparedGitCandidate(prepId, managedRoot, { commitSha, candidateRelPath }); + } + + public async prepareRegistryDeliveryFromGit( + input: CreateStackFromGitInput, + ): Promise<{ prepId: string; sourceHash: string }> { + const materialization: { value: MaterializationResult | null } = { value: null }; + const fetched = await this.fetchFromGit({ + repoUrl: input.repoUrl, + branch: input.branch, + composePaths: input.composePaths, + envPath: input.syncEnv ? input.envPath : null, + token: input.token, + onClone: async (cloneDir, commitSha, envContent) => { + materialization.value = await this.buildMaterialization( + input.stackName, + cloneDir, + commitSha, + { + compose_paths: input.composePaths, + context_dir: input.contextDir, + sync_env: input.syncEnv, + }, + envContent, + ); + }, + }); + if (!materialization.value?.validation.ok) { + throw new GitSourceError( + 'GIT_ERROR', + `Compose validation failed: ${materialization.value?.validation.error ?? 'unknown'}`, + ); + } + const managedRoot = path.resolve(stackManagedRoot(input.stackName)); + const candidateRel = materialization.value.candidateRelPath; + const pathReason = validateCandidateRelPath(candidateRel, managedRoot); + if (pathReason) { + throw new GitSourceError('GIT_ERROR', pathReason); + } + const candidateAbs = path.resolve(managedRoot, candidateRel); + if (!candidateAbs.startsWith(managedRoot + path.sep)) { + throw new GitSourceError('GIT_ERROR', 'Invalid candidate path'); + } + const stagingDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-regprep-')); + try { + await copyPreparedPayloadDirectory(candidateAbs, stagingDir); + const { writeGitCandidatePreparedMeta } = await import('../helpers/registryDeliveryGitCandidate'); + await writeGitCandidatePreparedMeta(stagingDir, { + version: 1, + commitSha: fetched.commitSha, + resolvedRefKind: fetched.resolvedRefKind, + candidateRelPath: materialization.value.candidateRelPath, + composeFiles: fetched.composeFiles, + envContent: fetched.envContent, + materialization: materialization.value, + warnings: fetched.warnings, + }); + const { hashDeliverySourceDir } = await import('../helpers/registryDeliveryHashes'); + const { PreparedSourceStore } = await import('./preparedSourceStore'); + const sourceHash = hashDeliverySourceDir(stagingDir); + const entry = await PreparedSourceStore.getInstance().prepareFromDirectory( + 'git-candidate', + sourceHash, + stagingDir, + ); + return { prepId: entry.prepId, sourceHash }; + } catch (error) { + await fsPromises.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); + throw error; + } finally { + const rmTarget = path.resolve(candidateAbs); + if (rmTarget.startsWith(managedRoot + path.sep)) { + // Canonical js/path-injection barrier inline with the rm sink. + await fsPromises.rm(rmTarget, { recursive: true, force: true }).catch(() => undefined); + } + } + } + + public async prepareRegistryDeliveryFromPending( + stackName: string, + ): Promise<{ prepId: string; sourceHash: string }> { + const src = DatabaseService.getInstance().getGitSource(stackName); + if (!src?.pending_commit_sha || !src.pending_compose_content) { + throw new GitSourceError('GIT_ERROR', 'No pending pull to prepare'); + } + const pending = this.decodePendingCompose(src.pending_compose_content); + if (!pending.candidateRelPath || pending.inventory === null) { + throw new GitSourceError('GIT_ERROR', 'No staged candidate for pending apply'); + } + const envContent = src.pending_env_content !== null + ? this.crypto.decrypt(src.pending_env_content) + : null; + const managedRoot = path.resolve(stackManagedRoot(stackName)); + const pathReason = validateCandidateRelPath(pending.candidateRelPath, managedRoot); + if (pathReason) { + throw new GitSourceError('GIT_ERROR', pathReason); + } + const candidateAbs = path.resolve(managedRoot, pending.candidateRelPath); + if (!candidateAbs.startsWith(managedRoot + path.sep)) { + throw new GitSourceError('GIT_ERROR', 'Invalid candidate path'); + } + const stagingDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-regprep-')); + try { + await copyPreparedPayloadDirectory(candidateAbs, stagingDir); + const { writeGitCandidatePreparedMeta } = await import('../helpers/registryDeliveryGitCandidate'); + await writeGitCandidatePreparedMeta(stagingDir, { + version: 1, + commitSha: src.pending_commit_sha, + resolvedRefKind: priorFetchIdentity(this.gitopsApplicationFor(stackName))?.kind ?? 'branch', + candidateRelPath: pending.candidateRelPath, + composeFiles: pending.files, + envContent, + materialization: { + inventory: pending.inventory, + contextCopyPlans: [], + candidateRelPath: pending.candidateRelPath, + validation: { ok: true }, + }, + warnings: [], + }); + const { hashDeliverySourceDir } = await import('../helpers/registryDeliveryHashes'); + const { PreparedSourceStore } = await import('./preparedSourceStore'); + const sourceHash = hashDeliverySourceDir(stagingDir); + const entry = await PreparedSourceStore.getInstance().prepareFromDirectory( + 'git-candidate', + sourceHash, + stagingDir, + ); + return { prepId: entry.prepId, sourceHash }; + } catch (error) { + await fsPromises.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); + throw error; + } + } + // ─── Concurrency ───────────────────────────────────────────────────────── private async withStackLock(stackName: string, fn: () => Promise): Promise { diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index ca87971d..ce995088 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -24,6 +24,7 @@ import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '.. import { getErrorMessage } from '../utils/errors'; import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants'; import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate'; +import { prepareOutboundRegistryDeliveryBody } from '../helpers/registryDeliveryOutbound'; const ACTIVITY_BUFFER_SIZE = 1000; const ALIAS_REFRESH_INTERVAL_MS = 60_000; @@ -2497,10 +2498,30 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { if (target.apiToken) headers['Authorization'] = `Bearer ${target.apiToken}`; const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); headers[PROXY_TIER_HEADER] = proxyHeaders.tier; + + let bodyToSend: unknown = body; + if (method === 'POST' || method === 'PUT') { + const bodyRecord = body === undefined || body === null + ? {} + : (typeof body === 'object' && !Array.isArray(body) ? body as Record : null); + if (bodyRecord !== null) { + const augmented = await prepareOutboundRegistryDeliveryBody({ + method, + apiPath, + nodeId, + body: bodyRecord, + }); + if (!augmented.ok) { + throw new MeshError('push_failed', augmented.error); + } + bodyToSend = augmented.body; + } + } + return await fetch(url, { method, headers, - body: body === undefined ? undefined : JSON.stringify(body), + body: bodyToSend === undefined ? undefined : JSON.stringify(bodyToSend), signal: AbortSignal.timeout(timeoutMs), }); } diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index 54268c63..7b86be5f 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -706,6 +706,7 @@ export class MonitorService { const notifSummary = db.cleanupOldNotifications(isNaN(retentionDays) ? 30 : retentionDays); const auditRetentionDays = parseInt(settings['audit_retention_days'] || '90', 10); db.cleanupOldAuditLogs(isNaN(auditRetentionDays) ? 90 : auditRetentionDays); + db.cleanupOldDeliveryEvents(isNaN(auditRetentionDays) ? 90 : auditRetentionDays); const scanPerImage = parseInt(settings['scan_history_per_image_limit'] || '50', 10); const scanPruned = db.pruneScanHistoryPerImage(isNaN(scanPerImage) ? 50 : scanPerImage); if (isDebugEnabled()) console.log(`[Monitor:diag] Cleanup: metrics ${isNaN(retentionHours) ? 24 : retentionHours}h, notifications ${isNaN(retentionDays) ? 30 : retentionDays}d (ttl=${notifSummary.ttl} perStack=${notifSummary.perStack} perNode=${notifSummary.perNode}), audit ${isNaN(auditRetentionDays) ? 90 : auditRetentionDays}d, scans pruned ${scanPruned}`); diff --git a/backend/src/services/PilotTunnelBridge.ts b/backend/src/services/PilotTunnelBridge.ts index 04a318b1..4ae67fee 100644 --- a/backend/src/services/PilotTunnelBridge.ts +++ b/backend/src/services/PilotTunnelBridge.ts @@ -368,16 +368,28 @@ export class PilotTunnelBridge extends EventEmitter implements MeshTunnelHandle if (s) this.teardownStream(s); this.removeStream(streamId); }); + req.on('aborted', () => { + if (this.streams.has(streamId)) { + this.notifyHttpClientAbort(streamId); + this.removeStream(streamId); + } + }); res.on('close', () => { // Client disconnected before response finished. if (this.streams.has(streamId)) { + this.notifyHttpClientAbort(streamId); this.removeStream(streamId); - this.sendJson({ t: 'http_err', s: streamId, code: 'tunnel_down', message: 'client aborted' }); } }); } + /** Notify the agent that the loopback HTTP client disconnected mid-request. */ + private notifyHttpClientAbort(streamId: number): void { + this.sendJson({ t: 'http_cancel', s: streamId }); + this.sendJson({ t: 'http_err', s: streamId, code: 'tunnel_down', message: 'client aborted' }); + } + private handleLoopbackUpgrade(req: IncomingMessage, socket: Socket, head: Buffer): void { if (this.closed || this.tunnelWs.readyState !== WebSocket.OPEN) { socket.write('HTTP/1.1 502 Bad Gateway\r\n\r\n'); diff --git a/backend/src/services/PilotTunnelManager.ts b/backend/src/services/PilotTunnelManager.ts index cea8888e..b36ada21 100644 --- a/backend/src/services/PilotTunnelManager.ts +++ b/backend/src/services/PilotTunnelManager.ts @@ -79,6 +79,7 @@ export class PilotTunnelManager extends EventEmitter { private static instance: PilotTunnelManager; private bridges: Map = new Map(); private bridgeKinds: Map = new Map(); + private tunnelConfidential: Map = new Map(); private softWarned = false; private constructor() { @@ -104,6 +105,7 @@ export class PilotTunnelManager extends EventEmitter { } PilotTunnelManager.instance.bridges.clear(); PilotTunnelManager.instance.bridgeKinds.clear(); + PilotTunnelManager.instance.tunnelConfidential.clear(); } PilotTunnelManager.instance = undefined as unknown as PilotTunnelManager; } @@ -125,13 +127,19 @@ export class PilotTunnelManager extends EventEmitter { * * Resolves once the loopback HTTP server is listening. */ - public async registerTunnel(nodeId: number, ws: WebSocket, agentVersion?: string): Promise { + public async registerTunnel( + nodeId: number, + ws: WebSocket, + agentVersion?: string, + tunnelConfidential = false, + ): Promise { const existing = this.bridges.get(nodeId); const replaced = existing != null; if (existing) { existing.close(PilotCloseCode.Replaced, 'replaced by newer tunnel'); this.bridges.delete(nodeId); this.bridgeKinds.delete(nodeId); + this.tunnelConfidential.delete(nodeId); } // Hard cap: only counts tunnels for *other* nodes since we just @@ -158,6 +166,7 @@ export class PilotTunnelManager extends EventEmitter { if (this.bridges.get(nodeId) === bridge) { this.bridges.delete(nodeId); this.bridgeKinds.delete(nodeId); + this.tunnelConfidential.delete(nodeId); DatabaseService.getInstance().updateNodeStatus(nodeId, 'offline'); this.emit('tunnel-down', nodeId); } @@ -166,6 +175,7 @@ export class PilotTunnelManager extends EventEmitter { this.bridges.set(nodeId, bridge); this.bridgeKinds.set(nodeId, 'pilot'); + this.tunnelConfidential.set(nodeId, tunnelConfidential); const db = DatabaseService.getInstance(); db.updateNodeStatus(nodeId, 'online'); db.updateNode(nodeId, { @@ -179,6 +189,11 @@ export class PilotTunnelManager extends EventEmitter { this.emit('tunnel-up', nodeId); } + /** Whether the active pilot tunnel was established over confidential transport. */ + public isTunnelConfidential(nodeId: number): boolean { + return this.tunnelConfidential.get(nodeId) === true; + } + /** * Per-tunnel breakdown for the metrics endpoint. Includes the * loopback-relative connectedAt and bufferedAmount so one-bad-node cases diff --git a/backend/src/services/RegistryDeliveryReconciler.ts b/backend/src/services/RegistryDeliveryReconciler.ts new file mode 100644 index 00000000..68290c5c --- /dev/null +++ b/backend/src/services/RegistryDeliveryReconciler.ts @@ -0,0 +1,179 @@ +import axios from 'axios'; + +import { + importRegistryDeliveryEvidencePage, +} from '../helpers/registryDeliveryEvidence'; +import type { RegistryDeliveryEvidencePage } from '../types/registryDeliveryEvidence'; +import { getErrorMessage } from '../utils/errors'; +import { isDebugEnabled } from '../utils/debug'; +import { DatabaseService } from './DatabaseService'; +import { NodeRegistry } from './NodeRegistry'; +import { PilotTunnelManager } from './PilotTunnelManager'; + +const RECONCILE_INTERVAL_MS = 5 * 60 * 1000; +const RECONCILE_INITIAL_DELAY_MS = 30_000; +const EVIDENCE_PAGE_LIMIT = 100; +const NODE_FAILURE_BACKOFF_MS = 15 * 60 * 1000; + +interface NodeBackoffState { + until: number; + failures: number; +} + +export class RegistryDeliveryReconciler { + private static instance: RegistryDeliveryReconciler | null = null; + private intervalHandle: ReturnType | null = null; + private initialTimer: ReturnType | null = null; + private running = false; + private stopped = false; + private readonly nodeBackoff = new Map(); + private readonly lastSourceIdByNode = new Map(); + + static getInstance(): RegistryDeliveryReconciler { + if (!this.instance) this.instance = new RegistryDeliveryReconciler(); + return this.instance; + } + + static resetForTests(): void { + this.instance?.stop(); + this.instance = null; + } + + private constructor() { /* singleton */ } + + start(): void { + if (this.intervalHandle || this.initialTimer) return; + this.stopped = false; + this.initialTimer = setTimeout(() => { + this.initialTimer = null; + if (this.stopped) return; + void this.tick(); + this.intervalHandle = setInterval(() => void this.tick(), RECONCILE_INTERVAL_MS); + if (typeof this.intervalHandle.unref === 'function') { + this.intervalHandle.unref(); + } + }, RECONCILE_INITIAL_DELAY_MS); + if (typeof this.initialTimer.unref === 'function') { + this.initialTimer.unref(); + } + } + + stop(): void { + this.stopped = true; + if (this.initialTimer) { + clearTimeout(this.initialTimer); + this.initialTimer = null; + } + if (this.intervalHandle) { + clearInterval(this.intervalHandle); + this.intervalHandle = null; + } + } + + async tick(): Promise { + if (this.running || this.stopped) return; + this.running = true; + try { + const nodes = DatabaseService.getInstance().getNodes() + .filter((node) => node.type === 'remote'); + for (const node of nodes) { + if (node.mode === 'pilot_agent' && !PilotTunnelManager.getInstance().hasActiveTunnel(node.id!)) { + continue; + } + await this.reconcileNode(node.id!); + } + } finally { + this.running = false; + } + } + + async reconcileNode(nodeId: number): Promise { + const backoff = this.nodeBackoff.get(nodeId); + if (backoff && Date.now() < backoff.until) { + return; + } + + const target = NodeRegistry.getInstance().getProxyTarget(nodeId); + if (!target) { + this.markNodeFailure(nodeId); + return; + } + + try { + let deliverySourceId = this.lastSourceIdByNode.get(nodeId); + let cursor = deliverySourceId + ? DatabaseService.getInstance().getRegistryDeliveryImportCursor(deliverySourceId) + : 0; + let pages = 0; + + while (pages < 50) { + const page = await this.fetchEvidencePage(target, cursor, EVIDENCE_PAGE_LIMIT); + deliverySourceId = page.deliverySourceId; + this.lastSourceIdByNode.set(nodeId, deliverySourceId); + if (page.events.length === 0) { + break; + } + + const hubNodeId = nodeId; + importRegistryDeliveryEvidencePage(hubNodeId, deliverySourceId, page.events); + + cursor = page.nextCursor; + pages += 1; + if (page.events.length < EVIDENCE_PAGE_LIMIT) { + break; + } + } + + this.nodeBackoff.delete(nodeId); + if (isDebugEnabled() && deliverySourceId) { + console.log( + `[RegistryDeliveryReconciler:diag] imported evidence from node ${nodeId} source=${deliverySourceId} cursor=${cursor}`, + ); + } + } catch (error) { + this.markNodeFailure(nodeId); + console.warn( + `[RegistryDeliveryReconciler] evidence import failed for node ${nodeId}:`, + getErrorMessage(error, 'unknown'), + ); + } + } + + private async fetchEvidencePage( + target: { apiUrl: string; apiToken: string }, + cursor: number, + limit: number, + ): Promise { + const base = target.apiUrl.replace(/\/$/, ''); + const headers: Record = {}; + if (target.apiToken) { + headers.Authorization = `Bearer ${target.apiToken}`; + } + + const res = await axios.get(`${base}/api/registry-delivery/evidence`, { + headers, + params: { cursor, limit }, + timeout: 30_000, + validateStatus: () => true, + }); + + if (res.status < 200 || res.status >= 300) { + const message = typeof res.data?.error === 'string' + ? res.data.error + : 'Registry delivery evidence fetch failed'; + throw Object.assign(new Error(message), { status: res.status }); + } + + return res.data as RegistryDeliveryEvidencePage; + } + + private markNodeFailure(nodeId: number): void { + const existing = this.nodeBackoff.get(nodeId); + const failures = (existing?.failures ?? 0) + 1; + const backoffMs = Math.min(NODE_FAILURE_BACKOFF_MS * failures, 60 * 60 * 1000); + this.nodeBackoff.set(nodeId, { + failures, + until: Date.now() + backoffMs, + }); + } +} diff --git a/backend/src/services/RegistryDeliveryService.ts b/backend/src/services/RegistryDeliveryService.ts new file mode 100644 index 00000000..8335c61b --- /dev/null +++ b/backend/src/services/RegistryDeliveryService.ts @@ -0,0 +1,348 @@ +import crypto from 'crypto'; +import jwt from 'jsonwebtoken'; +import path from 'path'; +import { DatabaseService } from './DatabaseService'; +import { NodeRegistry } from './NodeRegistry'; +import { + RegistryService, + type DockerConfigHostResolution, +} from './RegistryService'; +import { discoverRegistryReferences } from './registryReferenceDiscovery'; +import { remoteAdvertisesCapability } from '../helpers/remoteCapabilities'; +import { REMOTE_REGISTRY_CREDENTIALS_CAPABILITY } from './CapabilityRegistry'; +import { isTrustedProxyPeer } from '../helpers/trustedProxyCidrs'; +import type { RegistryDeliveryEnvelope, RegistryDeliveryAuthEntry } from '../helpers/registryDeliveryContext'; +import { classifyRegistryDeliveryOp } from '../helpers/registryOpClassifier'; +import { prepareSourceForDiscover, resolveBlueprintPostApplyDiscovery } from '../helpers/registryDeliveryPrepare'; +import { PreparedSourceStore } from './preparedSourceStore'; +import { hashProjectSource } from '../helpers/registryDeliveryHashes'; +import { isValidStackName } from '../utils/validation'; +import { + resolveComposeEnvForDiscovery, +} from '../helpers/registryDeliveryComposeEnv'; + +const ATTESTATION_AUD = 'registry-delivery'; +const ATTESTATION_TTL_SECONDS = 900; + +export interface RegistryDeliveryDiscoverRequest { + stack?: string; + op: string; + service?: string; + sourceKind: string; + sourceHash?: string; + actionSetHash: string; + prepId?: string; + envVars?: Record; + template?: unknown; + stackName?: string; + git?: Record; + gitApply?: boolean; + restoreVariant?: string; + composeContent?: string; +} + +export interface RegistryDeliveryDiscoverResponse { + prepId?: string; + referencedHosts: string[]; + coveredHosts: string[]; + sourceHash: string; + actionSetHash: string; + deliverySourceId: string; + attestation: string; +} + +export class RegistryDeliveryService { + private static instance: RegistryDeliveryService | null = null; + private readonly targetSessionId = crypto.randomBytes(16).toString('hex'); + private readonly consumedJtis = new Map(); + private maxConsumedJtis = 10_000; + + static getInstance(): RegistryDeliveryService { + if (!this.instance) this.instance = new RegistryDeliveryService(); + return this.instance; + } + + static resetForTests(): void { + this.instance = null; + } + + /** @internal Narrow replay-store capacity for unit tests. */ + setReplayStoreCapacityForTests(capacity: number): void { + this.maxConsumedJtis = capacity; + this.consumedJtis.clear(); + } + + getTargetSessionId(): string { + return this.targetSessionId; + } + + getDeliverySourceId(): string { + const settings = DatabaseService.getInstance().getGlobalSettings(); + const id = settings.delivery_source_id; + if (!id) { + throw new Error('delivery_source_id is not configured'); + } + return id; + } + + private getJwtSecret(): string { + const secret = DatabaseService.getInstance().getGlobalSettings().auth_jwt_secret; + if (!secret) { + throw new Error('auth_jwt_secret is not configured'); + } + return secret; + } + + hashHostList(hosts: string[]): string { + return crypto.createHash('sha256').update(hosts.slice().sort().join('\n')).digest('hex'); + } + + signAttestation(payload: { + nodeIdClaim: number; + stack?: string; + op: string; + service?: string; + sourceHash: string; + referencedHostsHash: string; + coveredHostsHash: string; + actionSetHash: string; + prepId?: string; + }): string { + const jti = crypto.randomBytes(16).toString('hex'); + return jwt.sign( + { + aud: ATTESTATION_AUD, + nodeIdClaim: payload.nodeIdClaim, + stack: payload.stack, + op: payload.op, + service: payload.service, + sourceHash: payload.sourceHash, + referencedHostsHash: payload.referencedHostsHash, + coveredHostsHash: payload.coveredHostsHash, + actionSetHash: payload.actionSetHash, + prepId: payload.prepId, + jti_t: jti, + target_session_id: this.targetSessionId, + }, + this.getJwtSecret(), + { expiresIn: ATTESTATION_TTL_SECONDS }, + ); + } + + parseAttestation(token: string): jwt.JwtPayload { + const decoded = jwt.verify(token, this.getJwtSecret(), { audience: ATTESTATION_AUD }); + if (typeof decoded === 'string') { + throw new Error('Invalid attestation payload'); + } + if (decoded.target_session_id !== this.targetSessionId) { + throw new Error('Attestation session mismatch'); + } + return decoded; + } + + private evictExpiredJtis(now = Date.now()): void { + for (const [jti, expiresAt] of this.consumedJtis) { + if (expiresAt <= now) { + this.consumedJtis.delete(jti); + } + } + } + + consumeAttestationJti(jti: string, expiresAtMs?: number): void { + const now = Date.now(); + this.evictExpiredJtis(now); + if (this.consumedJtis.has(jti)) { + throw new Error('Attestation already consumed'); + } + if (this.consumedJtis.size >= this.maxConsumedJtis) { + throw new Error('Attestation replay store at capacity'); + } + const expiresAt = expiresAtMs ?? now + ATTESTATION_TTL_SECONDS * 1000; + this.consumedJtis.set(jti, expiresAt); + } + + /** @deprecated Use parseAttestation at the middleware and consumeAttestationJti at the seam. */ + verifyAttestation(token: string): jwt.JwtPayload { + const decoded = this.parseAttestation(token); + const jti = decoded.jti_t; + if (typeof jti !== 'string' || !jti) { + throw new Error('Attestation missing jti'); + } + this.consumeAttestationJti(jti); + return decoded; + } + + async discoverOnTarget(request: RegistryDeliveryDiscoverRequest): Promise { + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + if ( + request.sourceKind === 'restore-candidate' + || request.sourceKind === 'live-project' + || request.sourceKind === 'body-content' + || request.gitApply === true + ) { + const stack = request.stack ?? request.stackName; + if (typeof stack !== 'string' || !isValidStackName(stack)) { + throw new Error('Invalid stack name'); + } + request.stack = stack; + } + + let referencedHosts: string[] = []; + let sourceHash = request.sourceHash; + let prepId = request.prepId; + + const prepared = await prepareSourceForDiscover(request); + if (prepared) { + prepId = prepared.prepId; + sourceHash = prepared.sourceHash; + const payloadPath = PreparedSourceStore.getInstance().peekPayloadPath(prepId); + const discovery = discoverRegistryReferences( + payloadPath, + resolveComposeEnvForDiscovery(payloadPath, request.envVars), + ); + referencedHosts = discovery.referencedHosts; + } else if (request.sourceKind === 'body-content' && typeof request.composeContent === 'string') { + const MAX_COMPOSE_CONTENT_BYTES = 2 * 1024 * 1024; + if (Buffer.byteLength(request.composeContent, 'utf8') > MAX_COMPOSE_CONTENT_BYTES) { + throw new Error('Compose content exceeds size limit'); + } + const stack = request.stack ?? request.stackName; + if (typeof stack !== 'string') { + throw new Error('Invalid stack name'); + } + const discovery = await resolveBlueprintPostApplyDiscovery( + stack, + request.composeContent, + nodeId, + ); + sourceHash = discovery.sourceHash; + referencedHosts = discovery.referencedHosts; + } else if (request.stack) { + if (!isValidStackName(request.stack)) { + throw new Error('Invalid stack name'); + } + const { FileSystemService } = await import('./FileSystemService'); + const fs = FileSystemService.getInstance(nodeId); + const baseResolved = path.resolve(fs.getBaseDir()); + const projectDir = path.resolve(baseResolved, request.stack); + if (!projectDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + if (!sourceHash || request.sourceKind === 'live-project') { + sourceHash = hashProjectSource(projectDir); + } + const discovery = discoverRegistryReferences( + projectDir, + resolveComposeEnvForDiscovery(projectDir, request.envVars), + ); + referencedHosts = discovery.referencedHosts; + } + + if (!sourceHash) { + sourceHash = crypto.createHash('sha256').update('').digest('hex'); + } + + const registry = RegistryService.getInstance(); + const coveredHosts: string[] = []; + for (const host of referencedHosts) { + const resolution = await registry.resolveDockerConfigForHostDetailed(host); + if (resolution.state === 'unavailable') { + throw new Error(`Registry credentials unavailable for ${host}`); + } + if (resolution.state === 'available') { + coveredHosts.push(host); + } + } + + const referencedHostsHash = this.hashHostList(referencedHosts); + const coveredHostsHash = this.hashHostList(coveredHosts); + const deliverySourceId = this.getDeliverySourceId(); + + const attestation = this.signAttestation({ + nodeIdClaim: nodeId, + stack: request.stack, + op: request.op, + service: request.service, + sourceHash, + referencedHostsHash, + coveredHostsHash, + actionSetHash: request.actionSetHash, + prepId, + }); + + return { + prepId, + referencedHosts, + coveredHosts, + sourceHash, + actionSetHash: request.actionSetHash, + deliverySourceId, + attestation, + }; + } + + async buildHubEnvelope( + nodeId: number, + discover: RegistryDeliveryDiscoverResponse, + ): Promise { + const deltaHosts = discover.referencedHosts.filter(host => { + return !discover.coveredHosts.includes(host); + }); + + const registry = RegistryService.getInstance(); + const auths: RegistryDeliveryAuthEntry[] = []; + + for (const host of deltaHosts) { + const hubResolution: DockerConfigHostResolution = await registry.resolveDockerConfigForHostDetailed(host); + if (hubResolution.state === 'unavailable') { + throw new Error(`Hub registry credentials unavailable for ${host}`); + } + if (hubResolution.state === 'missing') { + continue; + } + if (!hubResolution.auth) continue; + auths.push({ + host, + username: hubResolution.auth.username, + password: hubResolution.auth.password, + expiresAt: hubResolution.expiresAt, + }); + } + + const now = Date.now(); + const envelopeExp = now + ATTESTATION_TTL_SECONDS * 1000; + const providerExpiries = auths.map(a => a.expiresAt).filter((v): v is number => typeof v === 'number'); + const notAfter = Math.min(envelopeExp, ...providerExpiries.length > 0 ? providerExpiries : [envelopeExp]); + + return { + attestation: discover.attestation, + prepId: discover.prepId, + auths, + notAfter, + deliverySourceId: discover.deliverySourceId, + }; + } + + isProxyTransportConfidential(nodeId: number): boolean { + const node = NodeRegistry.getInstance().getNode(nodeId); + if (!node?.api_url) return false; + return node.api_url.trim().toLowerCase().startsWith('https://'); + } + + isPilotTransportConfidential(socketEncrypted: boolean, forwardedProto: string | undefined, peerAddress: string | undefined): boolean { + if (socketEncrypted) return true; + if (forwardedProto?.toLowerCase() === 'https' && isTrustedProxyPeer(peerAddress)) { + return true; + } + return false; + } + + async shouldAttemptDelivery(nodeId: number, confidential: boolean): Promise { + if (!confidential) return false; + return remoteAdvertisesCapability(nodeId, REMOTE_REGISTRY_CREDENTIALS_CAPABILITY); + } + + isDeliveryEligibleRoute(method: string, apiPath: string): boolean { + return classifyRegistryDeliveryOp(method, apiPath).eligible; + } +} diff --git a/backend/src/services/RegistryService.ts b/backend/src/services/RegistryService.ts index c909b556..38546630 100644 --- a/backend/src/services/RegistryService.ts +++ b/backend/src/services/RegistryService.ts @@ -111,7 +111,7 @@ export function hostFromStoredRegistry(reg: Pick): str } /** Normalize an image reference's host (the thing ImageUpdateService passes in). */ -function normalizeImageHost(host: string): string { +export function normalizeImageHost(host: string): string { const lower = host.trim().toLowerCase(); // Docker Hub aliases resolve to the same credential. if (lower === 'docker.io' || lower === 'registry-1.docker.io' || lower === '') { @@ -177,6 +177,14 @@ function httpGet( }); } +export type DockerConfigHostState = 'missing' | 'available' | 'unavailable'; + +export interface DockerConfigHostResolution { + state: DockerConfigHostState; + auth?: { username: string; password: string }; + expiresAt?: number; +} + // ─── Service ───────────────────────────────────────────────────────────────── export class RegistryService { @@ -394,15 +402,17 @@ export class RegistryService { /** Resolve a Docker config containing credentials for one registry host only. */ public async resolveDockerConfigForHost(registryHost: string): Promise { - const auth = await this.getAuthForRegistry(registryHost); - if (!auth) return { config: { auths: {} }, warnings: [] }; + const detailed = await this.resolveDockerConfigForHostDetailed(registryHost); + if (detailed.state !== 'available' || !detailed.auth) { + return { config: { auths: {} }, warnings: [] }; + } const normalized = normalizeImageHost(registryHost); return { config: { auths: { [normalized]: { - auth: Buffer.from(`${auth.username}:${auth.password}`).toString('base64'), + auth: Buffer.from(`${detailed.auth.username}:${detailed.auth.password}`).toString('base64'), }, }, }, @@ -410,6 +420,47 @@ export class RegistryService { }; } + /** + * Tri-state host resolution for registry delivery. Target `unavailable` + * means a configured row exists but credentials could not be resolved. + */ + public async resolveDockerConfigForHostDetailed(registryHost: string): Promise { + const db = DatabaseService.getInstance(); + const registries = db.getRegistries(); + const normalized = normalizeImageHost(registryHost); + const match = registries.find(r => hostFromStoredRegistry(r) === normalized); + + if (!match) { + return { state: 'missing' }; + } + + try { + if (match.type === 'ecr') { + const creds = await this.getEcrCredentials(match); + const cached = this.ecrCache.get(match.id); + const expiresAt = cached?.expiresAt ?? Date.now() + ECR_DEFAULT_TTL_MS; + return { + state: 'available', + auth: { username: creds.username, password: creds.password }, + expiresAt, + }; + } + return { + state: 'available', + auth: { + username: match.username, + password: this.crypto.decrypt(match.secret), + }, + }; + } catch (e) { + console.warn( + `[RegistryService] resolveDockerConfigForHostDetailed(${registryHost}) failed:`, + sanitizeForLog((e as Error).message), + ); + return { state: 'unavailable' }; + } + } + /** * Resolve credentials for a specific registry row by ID only. diff --git a/backend/src/services/RollbackGenerationStore.ts b/backend/src/services/RollbackGenerationStore.ts index 04a88315..38fd7d2d 100644 --- a/backend/src/services/RollbackGenerationStore.ts +++ b/backend/src/services/RollbackGenerationStore.ts @@ -1158,6 +1158,35 @@ export class RollbackGenerationStore { } return Buffer.from(b64, 'base64'); } + + /** + * Copy present generation compose/project files into destDir for registry-delivery + * preparation. Preserves stack-relative paths under destDir. + */ + static async copyPresentFilesToDir( + nodeId: number, + stackName: string, + generationId: string, + destDir: string, + ): Promise { + assertSafeStackName(stackName); + assertSafeGenerationId(generationId); + const genDir = this.getGenerationDir(nodeId, stackName, generationId); + const manifest = await this.readAndVerifyGeneration(genDir); + await mkdirPrivate(destDir); + for (const entry of manifest.entries) { + if (entry.state !== 'present') continue; + const rel = posixRel(entry.relativePath); + const dest = path.resolve(destDir, rel); + const destRoot = path.resolve(destDir); + if (!dest.startsWith(destRoot + path.sep) && dest !== destRoot) { + throw Object.assign(new Error('Path escapes restore staging directory'), { code: 'INVALID_PATH' }); + } + await mkdirPrivate(path.dirname(dest)); + const content = await this.readPresentEntryBytes(genDir, entry); + await writePrivate(dest, content); + } + } } export { getBackupBaseDir, getDataDir }; diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index 0d42651a..032b6be2 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -32,6 +32,7 @@ import type { ScanAllNodeImagesResult } from './TrivyService'; import TrivyInstaller from './TrivyInstaller'; import { CloudBackupService } from './CloudBackupService'; import { buildSystemPolicyGateOptions } from '../helpers/policyGate'; +import { prepareOutboundRegistryDeliveryBody } from '../helpers/registryDeliveryOutbound'; import { filterContainersByComposeService } from '../helpers/composeServiceMatch'; import { excludeSelfContainers } from '../helpers/excludeSelfContainers'; import { enforcePolicyPreDeploy } from './PolicyEnforcement'; @@ -1245,10 +1246,20 @@ export class SchedulerService { const proxyTarget = this.requireRemoteProxyTarget(nodeId); const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); + const apiPath = `/api/stacks/${routeSuffix}`; if (isDebugEnabled()) { console.log(`[SchedulerService:debug] postToRemoteStack: node=${nodeId} route=${routeSuffix}`); } try { + const augmented = await prepareOutboundRegistryDeliveryBody({ + method: 'POST', + apiPath, + nodeId, + body: {}, + }); + if (!augmented.ok) { + throw new Error(augmented.error); + } const response = await fetch(`${baseUrl}/api/stacks/${routeSuffix}`, { method: 'POST', headers: { @@ -1257,6 +1268,7 @@ export class SchedulerService { [PROXY_TIER_HEADER]: proxyHeaders.tier, ...extraHeaders, }, + body: JSON.stringify(augmented.body), signal: AbortSignal.timeout(300_000), }); if (!response.ok) { diff --git a/backend/src/services/StackOpLockService.ts b/backend/src/services/StackOpLockService.ts index 49475db2..61bb9aad 100644 --- a/backend/src/services/StackOpLockService.ts +++ b/backend/src/services/StackOpLockService.ts @@ -21,10 +21,16 @@ export function stackOpSkipMessage(stackName: string, existingAction: StackOpAct return `Skipped "${stackName}": another operation (${existingAction}) is already in progress.`; } +export interface StackOpLockContext { + opId?: string; + kind?: string; +} + export interface StackOpLock { action: StackOpAction; startedAt: number; user: string; + context?: StackOpLockContext; } interface AcquireSuccess { @@ -60,11 +66,12 @@ export class StackOpLockService { stackName: string, action: StackOpAction, user: string, + context?: StackOpLockContext, ): AcquireResult { const k = this.key(nodeId, stackName); const existing = this.locks.get(k); if (existing) return { acquired: false, existing }; - this.locks.set(k, { action, startedAt: Date.now(), user }); + this.locks.set(k, { action, startedAt: Date.now(), user, context }); return { acquired: true }; } @@ -114,8 +121,9 @@ export class StackOpLockService { action: StackOpAction, user: string, fn: () => Promise, + context?: StackOpLockContext, ): Promise<{ ran: true; result: T } | { ran: false; existing: StackOpLock }> { - const acquired = this.tryAcquire(nodeId, stackName, action, user); + const acquired = this.tryAcquire(nodeId, stackName, action, user, context); if (!acquired.acquired) return { ran: false, existing: acquired.existing }; try { const result = await fn(); diff --git a/backend/src/services/TemplateService.ts b/backend/src/services/TemplateService.ts index ba03f326..bb2dbfac 100644 --- a/backend/src/services/TemplateService.ts +++ b/backend/src/services/TemplateService.ts @@ -3,6 +3,7 @@ import YAML from 'yaml'; import { DatabaseService } from './DatabaseService'; import { CacheService } from './CacheService'; import { isDebugEnabled } from '../utils/debug'; +import { isValidStackName } from '../utils/validation'; interface TemplateEnv { @@ -365,6 +366,9 @@ export class TemplateService { } public generateComposeFromTemplate(template: Template, serviceName: string): string { + if (!isValidStackName(serviceName)) { + throw new Error('Invalid service name'); + } const service: ComposeServiceDefinition = { restart: 'unless-stopped' }; if (template.image) { diff --git a/backend/src/services/WebhookService.ts b/backend/src/services/WebhookService.ts index 43c0ed56..baa013a9 100644 --- a/backend/src/services/WebhookService.ts +++ b/backend/src/services/WebhookService.ts @@ -13,6 +13,7 @@ import { getErrorMessage } from '../utils/errors'; import { redactSensitiveText } from '../utils/safeLog'; import { isValidStackName } from '../utils/validation'; import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate'; +import { prepareOutboundRegistryDeliveryBody } from '../helpers/registryDeliveryOutbound'; type ExecutionResult = { success: boolean; error?: string; duration_ms: number }; type ExecutionStatus = 'success' | 'failure'; @@ -335,10 +336,42 @@ export class WebhookService { // Build URL from validated, server-controlled components. const url = `${protocol}//${host}/api/stacks/${encodeURIComponent(stackName)}/${endpoint}`; + const apiPath = `/api/stacks/${encodeURIComponent(stackName)}/${endpoint}`; + let bodyToSend = body; + if (method === 'POST' && body !== undefined) { + const bodyRecord = typeof body === 'object' && body !== null && !Array.isArray(body) + ? body as Record + : {}; + const augmented = await prepareOutboundRegistryDeliveryBody({ + method, + apiPath, + nodeId, + body: bodyRecord, + }); + if (!augmented.ok) { + const err = new Error(augmented.error); + (err as { status?: number }).status = augmented.status; + throw err; + } + bodyToSend = augmented.body; + } else if (method === 'POST') { + const augmented = await prepareOutboundRegistryDeliveryBody({ + method, + apiPath, + nodeId, + body: {}, + }); + if (!augmented.ok) { + const err = new Error(augmented.error); + (err as { status?: number }).status = augmented.status; + throw err; + } + bodyToSend = augmented.body; + } return await fetch(url, { method, headers, - body: method === 'GET' || body === undefined ? undefined : JSON.stringify(body), + body: method === 'GET' || bodyToSend === undefined ? undefined : JSON.stringify(bodyToSend), signal: controller.signal, }); } catch (err) { diff --git a/backend/src/services/preparedSourceStore.ts b/backend/src/services/preparedSourceStore.ts new file mode 100644 index 00000000..f85d9946 --- /dev/null +++ b/backend/src/services/preparedSourceStore.ts @@ -0,0 +1,219 @@ +import crypto from 'crypto'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { ensureTrustedRoot, validateTrustedRoot } from '../helpers/privateRootValidator'; + +export const PREPARED_SOURCE_MARKER_FILE = '.sencho-prepared-source'; +export const PREPARED_SOURCE_PARENT_PREFIX = 'sencho-registry-prepared-'; +const DEFAULT_TTL_MS = 900_000; + +export type PreparedSourceState = 'prepared' | 'claimed' | 'finalized'; + +export interface PreparedSourceEntry { + prepId: string; + sourceKind: string; + sourceHash: string; + dirPath: string; + state: PreparedSourceState; + createdAt: number; + expiresAt: number; +} + +function deliverySourceHash(deliverySourceId: string): string { + return crypto.createHash('sha256').update(`prepared-source:${deliverySourceId}`).digest('hex'); +} + +export function getPreparedSourceRootPath(deliverySourceId: string): string { + return path.join(os.tmpdir(), `${PREPARED_SOURCE_PARENT_PREFIX}${deliverySourceHash(deliverySourceId)}`); +} + +function publishMarker(childDir: string, sourceKind: string): void { + const markerPath = path.join(childDir, PREPARED_SOURCE_MARKER_FILE); + const fd = fs.openSync(markerPath, 'wx', 0o600); + try { + fs.writeSync(fd, `${sourceKind}\n`); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +export class PreparedSourceStore { + private static instance: PreparedSourceStore | null = null; + private entries = new Map(); + private expiryTimer: ReturnType | null = null; + private deliverySourceId: string | null = null; + + static getInstance(): PreparedSourceStore { + if (!this.instance) this.instance = new PreparedSourceStore(); + return this.instance; + } + + configure(deliverySourceId: string): void { + this.deliverySourceId = deliverySourceId; + } + + start(): void { + if (this.expiryTimer) return; + this.expiryTimer = setInterval(() => this.expireStaleEntries(), 60_000); + if (typeof this.expiryTimer.unref === 'function') { + this.expiryTimer.unref(); + } + } + + stop(): void { + if (this.expiryTimer) { + clearInterval(this.expiryTimer); + this.expiryTimer = null; + } + } + + private requireDeliverySourceId(): string { + if (!this.deliverySourceId) { + throw new Error('PreparedSourceStore is not configured'); + } + return this.deliverySourceId; + } + + private childPath(prepId: string): string { + const root = getPreparedSourceRootPath(this.requireDeliverySourceId()); + return path.join(root, prepId); + } + + async prepareFromDirectory( + sourceKind: string, + sourceHash: string, + stagingDir: string, + ): Promise { + const deliverySourceId = this.requireDeliverySourceId(); + const rootPath = getPreparedSourceRootPath(deliverySourceId); + const rootValidation = ensureTrustedRoot({ rootPath, kind: 'prepared-source' }); + if (!rootValidation.ok) { + throw new Error(rootValidation.reason); + } + + const prepId = crypto.randomBytes(16).toString('hex'); + const childDir = path.join(rootPath, prepId); + fs.mkdirSync(childDir, { mode: 0o700 }); + + try { + publishMarker(childDir, sourceKind); + const payloadDir = path.join(childDir, 'payload'); + await fs.promises.rename(stagingDir, payloadDir); + } catch (error) { + try { await fs.promises.rm(childDir, { recursive: true, force: true }); } catch { /* ignore */ } + throw error; + } + + const now = Date.now(); + const entry: PreparedSourceEntry = { + prepId, + sourceKind, + sourceHash, + dirPath: childDir, + state: 'prepared', + createdAt: now, + expiresAt: now + DEFAULT_TTL_MS, + }; + this.entries.set(prepId, entry); + return entry; + } + + getEntry(prepId: string): PreparedSourceEntry | undefined { + return this.entries.get(prepId); + } + + claim(prepId: string): PreparedSourceEntry { + const entry = this.entries.get(prepId); + if (!entry || entry.state !== 'prepared') { + throw new Error('Prepared source is not available for claim'); + } + if (entry.expiresAt <= Date.now()) { + throw new Error('Prepared source expired'); + } + entry.state = 'claimed'; + return entry; + } + + peekPayloadPath(prepId: string): string { + const entry = this.entries.get(prepId); + if (!entry || entry.state === 'finalized') { + throw new Error('Prepared source not found'); + } + if (entry.expiresAt <= Date.now()) { + throw new Error('Prepared source expired'); + } + return path.join(entry.dirPath, 'payload'); + } + + finalize(prepId: string): void { + const entry = this.entries.get(prepId); + if (!entry) return; + entry.state = 'finalized'; + try { + fs.rmSync(entry.dirPath, { recursive: true, force: true }); + } catch { + /* best effort */ + } + this.entries.delete(prepId); + } + + getPayloadPath(prepId: string): string { + const entry = this.entries.get(prepId); + if (!entry) { + throw new Error('Prepared source not found'); + } + return path.join(entry.dirPath, 'payload'); + } + + private expireStaleEntries(): void { + const now = Date.now(); + for (const [prepId, entry] of this.entries) { + if (entry.expiresAt <= now) { + try { + fs.rmSync(entry.dirPath, { recursive: true, force: true }); + } catch { + /* ignore */ + } + this.entries.delete(prepId); + } + } + } + + async sweepOrphans(deliverySourceId: string): Promise { + const rootPath = getPreparedSourceRootPath(deliverySourceId); + const swept: string[] = []; + const validation = validateTrustedRoot({ rootPath, kind: 'prepared-source' }); + if (!validation.ok) { + return swept; + } + + let entries: string[]; + try { + entries = await fs.promises.readdir(rootPath); + } catch { + return swept; + } + + for (const entry of entries) { + const childPath = path.join(rootPath, entry); + let stat: fs.Stats; + try { + stat = await fs.promises.lstat(childPath); + } catch { + continue; + } + if (!stat.isDirectory() || stat.isSymbolicLink()) continue; + const markerPath = path.join(childPath, PREPARED_SOURCE_MARKER_FILE); + if (!fs.existsSync(markerPath)) continue; + try { + await fs.promises.rm(childPath, { recursive: true, force: true }); + swept.push(entry); + } catch { + /* ignore */ + } + } + return swept; + } +} diff --git a/backend/src/services/registryReferenceDiscovery.ts b/backend/src/services/registryReferenceDiscovery.ts new file mode 100644 index 00000000..73767ab9 --- /dev/null +++ b/backend/src/services/registryReferenceDiscovery.ts @@ -0,0 +1,153 @@ +import fs from 'fs'; +import path from 'path'; +import { parseImageRef } from './registry-api'; +import { extractImagesFromCompose } from './ImageUpdateService'; +import { normalizeImageHost } from './RegistryService'; + +const MAX_DOCKERFILE_BYTES = 1_048_576; + +export interface RegistryReferenceDiscoveryResult { + referencedHosts: string[]; +} + +function hostFromImageRef(imageRef: string): string | null { + const parsed = parseImageRef(imageRef); + if (!parsed) return null; + return normalizeImageHost(parsed.registry); +} + +function parseDockerfileReferences(content: string): string[] { + const hosts = new Set(); + const lines = content.split('\n'); + for (const rawLine of lines) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) continue; + + const fromMatch = /^FROM\s+(--platform=[^\s]+\s+)?([^\s]+)/i.exec(line); + if (fromMatch?.[2]) { + const host = hostFromImageRef(fromMatch[2]); + if (host) hosts.add(host); + } + + const copyFromMatch = /^COPY\s+--from=([^\s]+)/i.exec(line); + if (copyFromMatch?.[1] && !/^\d+$/.test(copyFromMatch[1])) { + const host = hostFromImageRef(copyFromMatch[1]); + if (host) hosts.add(host); + } + } + return [...hosts]; +} + +function readRegularFileSync(filePath: string, baseResolved: string): Buffer | null { + const resolved = path.resolve(filePath); + if (!resolved.startsWith(baseResolved + path.sep)) return null; + const fd = fs.openSync(resolved, 'r'); + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile()) return null; + const buf = Buffer.alloc(stat.size); + fs.readSync(fd, buf, 0, stat.size, 0); + return buf; + } finally { + fs.closeSync(fd); + } +} + +function discoverFromComposeFile(baseResolved: string, fileName: string, envVars: Record): string[] { + const safePath = path.resolve(baseResolved, fileName); + if (!safePath.startsWith(baseResolved + path.sep)) { + return []; + } + const content = readRegularFileSync(safePath, baseResolved); + if (!content) return []; + const images = extractImagesFromCompose(content.toString('utf8'), envVars); + const hosts = new Set(); + for (const image of images) { + const host = hostFromImageRef(image); + if (host) hosts.add(host); + } + return [...hosts]; +} + +function discoverDockerfiles(baseResolved: string): string[] { + const hosts = new Set(); + const stack: string[] = [baseResolved]; + while (stack.length > 0) { + const current = stack.pop(); + if (!current) continue; + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const full = path.resolve(current, entry.name); + if (!full.startsWith(baseResolved + path.sep)) continue; + if (entry.isDirectory()) { + stack.push(full); + continue; + } + if (!entry.isFile()) continue; + const lower = entry.name.toLowerCase(); + if (lower !== 'dockerfile' && !lower.startsWith('dockerfile.')) continue; + const fd = fs.openSync(full, 'r'); + try { + const stat = fs.fstatSync(fd); + if (stat.size > MAX_DOCKERFILE_BYTES) { + throw new Error(`Dockerfile exceeds size limit: ${entry.name}`); + } + const buf = Buffer.alloc(stat.size); + fs.readSync(fd, buf, 0, stat.size, 0); + for (const host of parseDockerfileReferences(buf.toString('utf8'))) { + hosts.add(host); + } + } finally { + fs.closeSync(fd); + } + } + } + return [...hosts]; +} + +export function discoverRegistryReferencesFromComposeContent( + composeContent: string, + envVars: Record = {}, +): RegistryReferenceDiscoveryResult { + const hosts = new Set(); + for (const image of extractImagesFromCompose(composeContent, envVars)) { + const host = hostFromImageRef(image); + if (host) hosts.add(host); + } + return { referencedHosts: [...hosts].sort() }; +} + +/** + * Discover registry hosts referenced by compose files and Dockerfiles in a + * project directory. Returns referenced hosts, not proven-private hosts. + */ +export function discoverRegistryReferences( + projectDir: string, + envVars: Record = {}, +): RegistryReferenceDiscoveryResult { + const hosts = new Set(); + const baseResolved = path.resolve(projectDir); + + const composeNames = ['compose.yaml', 'compose.yml', 'docker-compose.yaml', 'docker-compose.yml']; + for (const name of composeNames) { + const composePath = path.resolve(baseResolved, name); + if (!composePath.startsWith(baseResolved + path.sep)) continue; + if (!fs.existsSync(composePath)) continue; + for (const host of discoverFromComposeFile(baseResolved, name, envVars)) { + hosts.add(host); + } + } + + for (const host of discoverDockerfiles(baseResolved)) { + hosts.add(host); + } + + return { referencedHosts: [...hosts].sort() }; +} + +export { parseDockerfileReferences }; diff --git a/backend/src/types/express.ts b/backend/src/types/express.ts index b0b4e206..0c06a3a1 100644 --- a/backend/src/types/express.ts +++ b/backend/src/types/express.ts @@ -58,6 +58,10 @@ declare global { * passes for a non-admin user. Resets to undefined after the hop. */ proxyElevatedRole?: 'node-admin'; + /** Verified registry delivery envelope for the current classified operation. */ + registryDeliveryEnvelope?: import('../helpers/registryDeliveryContext').RegistryDeliveryEnvelope; + /** Abort controller wired for registry delivery disconnect handling. */ + registryDeliveryAbortController?: AbortController; } } } diff --git a/backend/src/types/registryDeliveryEvidence.ts b/backend/src/types/registryDeliveryEvidence.ts new file mode 100644 index 00000000..783d0f2b --- /dev/null +++ b/backend/src/types/registryDeliveryEvidence.ts @@ -0,0 +1,44 @@ +export type RegistryDeliveryEventType = + | 'operation_completed' + | 'operation_aborted' + | 'cleanup_failed' + | 'swept_local' + | 'swept_delivered' + | 'swept_prepared' + | 'legacy_orphan_observed' + | 'retention_gap' + | 'source_reset'; + +export interface RegistryDeliveryEventRow { + seq: number; + event_id: string; + delivery_source_id: string; + stack: string | null; + op: string | null; + attestation_jti: string | null; + prep_id_sha256: string | null; + temp_dir_id: string | null; + event_type: RegistryDeliveryEventType | string; + source_hash: string | null; + pruned_through_seq: number | null; + created_at: number; +} + +export interface RegistryDeliveryEventInput { + deliverySourceId: string; + eventType: RegistryDeliveryEventType; + stack?: string | null; + op?: string | null; + attestationJti?: string | null; + prepIdSha256?: string | null; + tempDirId?: string | null; + sourceHash?: string | null; + prunedThroughSeq?: number | null; +} + +export interface RegistryDeliveryEvidencePage { + deliverySourceId: string; + events: RegistryDeliveryEventRow[]; + nextCursor: number; + limit: number; +} diff --git a/backend/src/websocket/pilotTunnel.ts b/backend/src/websocket/pilotTunnel.ts index a9c3dee6..430d1c9f 100644 --- a/backend/src/websocket/pilotTunnel.ts +++ b/backend/src/websocket/pilotTunnel.ts @@ -9,6 +9,7 @@ import { PilotMetrics } from '../services/PilotMetrics'; import { encodeJsonFrame as encodePilotJsonFrame, PROTOCOL_VERSION as PILOT_PROTOCOL_VERSION } from '../pilot/protocol'; import { getErrorMessage } from '../utils/errors'; import { rejectUpgrade as rejectSocket } from './reject'; +import { RegistryDeliveryService } from '../services/RegistryDeliveryService'; /** Diagnostic reject reason for Pilot agents (never required for enroll fallback). */ type PilotRejectReason = @@ -94,6 +95,15 @@ export async function handlePilotTunnel( const agentVersion = firstHeader(req.headers['x-sencho-agent-version']); + const socketEncrypted = (req.socket as { encrypted?: boolean }).encrypted === true; + const forwardedProto = firstHeader(req.headers['x-forwarded-proto']); + const peerAddress = req.socket.remoteAddress ?? undefined; + const tunnelConfidential = RegistryDeliveryService.getInstance().isPilotTransportConfidential( + socketEncrypted, + forwardedProto, + peerAddress, + ); + pilotTunnelWss.handleUpgrade(req, socket, head, async (ws) => { try { ws.send(encodePilotJsonFrame({ @@ -114,7 +124,12 @@ export async function handlePilotTunnel( } try { - await PilotTunnelManager.getInstance().registerTunnel(decoded.nodeId!, ws, agentVersion); + await PilotTunnelManager.getInstance().registerTunnel( + decoded.nodeId!, + ws, + agentVersion, + tunnelConfidential, + ); } catch (err) { if (err instanceof PilotTunnelCapacityError) { // 1013 (Try Again Later) signals the agent to back off rather than diff --git a/docs/features/private-registries.mdx b/docs/features/private-registries.mdx index c26ddbe8..3c98a082 100644 --- a/docs/features/private-registries.mdx +++ b/docs/features/private-registries.mdx @@ -175,6 +175,16 @@ Registry credentials are stored on the Sencho instance where you enter them, and Because the Registries section is hidden when you view another node through the node switcher, manage a remote node's registries by signing into that node's own Sencho instance directly. Configure each private registry on every instance that deploys images from it, so a stack keeps pulling no matter which node it runs on. +## Remote fleet delivery + +When you manage remote nodes from a central Sencho instance, you can store private registry credentials once on the hub and have Sencho deliver them to a compatible remote target for a single Compose operation (deploy, update, rollback, template deploy, or Git apply with auto-deploy). + +Delivery runs only when the remote target advertises the `remote-registry-credentials` capability and the hop is confidential (HTTPS proxy URL, or a Pilot tunnel terminated with TLS or a trusted reverse proxy). Otherwise the operation forwards unchanged: public images and registry rows already configured on the target keep working without hub credentials traveling to that node. + +Credentials exist only in the forwarded delivery envelope and a temporary `DOCKER_CONFIG` for the duration of the Compose child process. They are not written to the target's registry table, responses, logs, or audit trail. Target-local registry rows always win when both sides have a credential for the same host. + +If discovery or delivery fails before Compose starts, Sencho returns a clear error and does not spawn the operation. Retry the deploy after fixing hub registry configuration or upgrading the remote Sencho version. + ## Troubleshooting diff --git a/docs/getting-started/configuration.mdx b/docs/getting-started/configuration.mdx index f22d0290..ccb89064 100644 --- a/docs/getting-started/configuration.mdx +++ b/docs/getting-started/configuration.mdx @@ -62,6 +62,10 @@ These tune optional subsystems. Most deployments never set them; the defaults ar Running a remote host as a pilot agent uses four more variables (`SENCHO_MODE`, `SENCHO_PRIMARY_URL`, `SENCHO_ENROLL_TOKEN`, and `SENCHO_PILOT_CA_FILE`), set only on the remote agent container. Sencho bakes them into the enrollment Compose file it generates, so you rarely write them by hand. See [Pilot Agent](/features/pilot-agent) for the full enrollment walkthrough. +| Variable | Default | Description | +|----------|---------|-------------| +| `SENCHO_TRUSTED_PROXY_CIDRS` | *(unset)* | Comma-separated CIDRs of reverse proxies that may set `X-Forwarded-Proto` for Pilot Agent TLS termination. When unset or invalid, non-TLS Pilot upgrades are treated as non-confidential and hub registry credential delivery is skipped for that hop. Set this when a TLS-terminating proxy sits in front of the primary and pilots connect through it. | + ## ZFS ARC-aware host memory On OpenZFS hosts (TrueNAS SCALE, Proxmox, ZFS on Ubuntu or Debian) the ZFS ARC cache can hold a large share of RAM. ARC is reclaimable on demand, but the Linux kernel reports it as unavailable, so a naive reading counts ARC as used memory and can raise false host-memory alerts. diff --git a/frontend/src/lib/capabilities.ts b/frontend/src/lib/capabilities.ts index 501860cf..c21c2502 100644 --- a/frontend/src/lib/capabilities.ts +++ b/frontend/src/lib/capabilities.ts @@ -42,6 +42,7 @@ export const CAPABILITIES = [ 'service-scoped-update', 'service-scoped-stack-alert', 'scoped-stack-auth-evidence', + 'remote-registry-credentials', ] as const; export type Capability = (typeof CAPABILITIES)[number]; @@ -58,3 +59,4 @@ export const GUIDED_EXTERNAL_NETWORK_PREFLIGHT_CAPABILITY = 'guided-external-net export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY = 'service-scoped-stack-alert' as const satisfies Capability; export const SCOPED_STACK_AUTH_EVIDENCE_CAPABILITY = 'scoped-stack-auth-evidence' as const satisfies Capability; +export const REMOTE_REGISTRY_CREDENTIALS_CAPABILITY = 'remote-registry-credentials' as const satisfies Capability;