From f23b7e1bacb6f2f6c1863cc8d64ec3791e9fd860 Mon Sep 17 00:00:00 2001 From: Anso Date: Wed, 17 Jun 2026 13:24:55 -0400 Subject: [PATCH] feat: ordered multi-file Compose for Git sources (#1380) * feat: ordered multi-file Compose for Git sources Extend Git sources to deploy an ordered list of compose files merged with docker compose -f base.yaml -f override.yaml ..., plus an optional project directory. - Pick and reorder compose files from the repository tree (drag to reorder on desktop, up/down arrows on phones); manual path entry is also supported. - The ordered set drives every stack-scoped compose command (deploy, update, start/stop/restart/down, image scans, Compose Doctor) and the container lookup, so a service or image declared only in an override is handled too. - Runtime keys off the materialized set, not the saved configuration: saving a source does not change deploy args until the pull is applied, and apply materializes from the pending snapshot rather than live config. - The project directory is passed as --project-directory, with -p pinning the Compose project so container labels stay stable. - The Mesh override is layered last; single-file sources are byte-identical to before, and existing rows keep working via the single-path fallback. Docs cover the picker, ordering, project directory, and the new troubleshooting and limitations (referenced files are not materialized; the dependency graph, drift, and networking views read the primary file). * fix: harden multi-file Git source (hash, unlink, collisions, node id) - hashContent folds ordered file CONTENTS (not paths) so a clean multi-file stack is not flagged as locally edited: create/apply hash the fetched files (repo paths) while pull hashes the on-disk files (materialized paths), which previously disagreed and showed a false "local edits detected". - Block unlinking a multi-file or project-directory Git source (409): the deploy spec lives on the source row, so removing it would silently revert deploys to root compose.yaml. Single-file sources still unlink. - Reject materialized-path collisions in the selection validator: an additional file equal to or nested under compose.yaml, an ancestor/descendant overlap between selected files, and a project directory nested under a compose file (previously a 500 at materialization). - DockerController.getContainersByStack uses the controller's node compose dir and passes its node id to the authored prefix, instead of the process default. * fix: CI failures on multi-file Git source (test crash, aria query, path barrier) - GitSourceFields no longer crashes when repoUrl/branch are falsy: the canBrowse trim() is optional-chained, so a reusable field component tolerates partial props. Fixes the apply-binding panel test, which feeds a minimal source object. - GitSourcePanel tests query the footer Remove button by its exact name, so the picker's per-file "Remove " buttons no longer collide with the broad /remove/i match (the test intent, footer Remove present/absent, is unchanged). - validateCompose uses an inline resolve + startsWith barrier at the context-dir mkdir sink (CodeQL does not credit the wrapped isPathWithinBase helper), clearing the js/path-injection alert. The containment check is equivalent and contextDir is also validated upstream. * test: update Git source E2E spec for the multi-file compose picker The compose-file picker replaced the single #git-source-path input and added per-file Remove buttons, so the E2E spec drove selectors that no longer exist: - Drop the redundant compose.yaml fills (the picker defaults to compose.yaml). - Select the footer Remove button by exact name so the picker's per-file "Remove " buttons no longer make the locator ambiguous. - Set a custom compose path through the picker (add via the manual input, press Enter, then remove the default compose.yaml). * test: match the footer Remove button with an exact Playwright name Playwright's getByRole name option is a substring match by default, so { name: 'Remove' } also matched the picker's "Remove " buttons. Require an exact match so only the footer Remove button is selected. --- .../__tests__/authored-compose-args.test.ts | 138 +++++ backend/src/__tests__/compose-images.test.ts | 2 +- backend/src/__tests__/compose-service.test.ts | 29 + .../src/__tests__/docker-controller.test.ts | 1 + .../src/__tests__/git-compose-files.test.ts | 37 ++ .../src/__tests__/git-source-routes.test.ts | 129 ++++- .../__tests__/git-source-selection.test.ts | 168 ++++++ .../src/__tests__/git-source-service.test.ts | 481 ++++++++++++++-- .../__tests__/image-update-service.test.ts | 2 + .../src/__tests__/network-topology.test.ts | 2 +- .../__tests__/self-identity-service.test.ts | 2 +- .../src/__tests__/webhooks-git-source.test.ts | 64 +-- backend/src/helpers/gitSourceSelection.ts | 114 ++++ backend/src/routes/gitSources.ts | 109 +++- backend/src/routes/stacks.ts | 22 +- backend/src/services/ComposeService.ts | 59 +- backend/src/services/DatabaseService.ts | 103 +++- backend/src/services/DockerController.ts | 31 +- backend/src/services/GitSourceService.ts | 519 +++++++++++++----- backend/src/services/ImageUpdateService.ts | 81 ++- backend/src/services/MeshService.ts | 49 +- backend/src/services/UpdatePreviewService.ts | 7 + backend/src/utils/authoredComposeArgs.ts | 57 ++ backend/src/utils/gitComposeFiles.ts | 25 + docs/features/git-sources.mdx | 35 +- docs/features/overview.mdx | 2 +- docs/features/stack-management.mdx | 10 +- e2e/git-sources.spec.ts | 19 +- .../EditorLayout/CreateStackDialog.tsx | 50 +- .../components/stack/GitComposeFilePicker.tsx | 207 +++++++ .../components/stack/GitSourceDiffDialog.tsx | 2 +- .../src/components/stack/GitSourceFields.tsx | 69 +-- .../components/stack/GitSourcePanel.test.tsx | 4 +- .../src/components/stack/GitSourcePanel.tsx | 55 +- 34 files changed, 2299 insertions(+), 385 deletions(-) create mode 100644 backend/src/__tests__/authored-compose-args.test.ts create mode 100644 backend/src/__tests__/git-compose-files.test.ts create mode 100644 backend/src/__tests__/git-source-selection.test.ts create mode 100644 backend/src/helpers/gitSourceSelection.ts create mode 100644 backend/src/utils/authoredComposeArgs.ts create mode 100644 backend/src/utils/gitComposeFiles.ts create mode 100644 frontend/src/components/stack/GitComposeFilePicker.tsx diff --git a/backend/src/__tests__/authored-compose-args.test.ts b/backend/src/__tests__/authored-compose-args.test.ts new file mode 100644 index 00000000..609dbb8a --- /dev/null +++ b/backend/src/__tests__/authored-compose-args.test.ts @@ -0,0 +1,138 @@ +/** + * Unit tests for authoredComposeFileArgs: the docker compose global-flag prefix + * (`-f` per file, `-p `, optional `--project-directory`) derived from a + * stack's applied deploy spec. + * + * Contract: + * - no applied spec (single-file / non-git) -> [] so runtime stays plain auto-discovery + * - a multi-file applied spec -> ordered -f flags, then -p , then + * --project-directory when a context dir is set + * - any spec file path or context dir that is absolute / contains ".." throws + * before any args are returned (it is spliced straight into child-process argv) + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import path from 'path'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let authoredComposeFileArgs: typeof import('../utils/authoredComposeArgs').authoredComposeFileArgs; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let NodeRegistry: typeof import('../services/NodeRegistry').NodeRegistry; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ authoredComposeFileArgs } = await import('../utils/authoredComposeArgs')); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ NodeRegistry } = await import('../services/NodeRegistry')); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + const db = DatabaseService.getInstance(); + for (const s of db.getGitSources()) db.deleteGitSource(s.stack_name); +}); + +/** Seed a git-source row (no applied spec yet) so setGitSourceAppliedSpec can target it. */ +function seedSource(stackName: string, composePaths: string[]): void { + DatabaseService.getInstance().upsertGitSource({ + stack_name: stackName, + repo_url: 'https://github.com/example/repo.git', + branch: 'main', + compose_path: composePaths[0], + compose_paths: composePaths, + context_dir: null, + sync_env: false, + env_path: null, + auth_type: 'none', + encrypted_token: null, + auto_apply_on_webhook: false, + auto_deploy_on_apply: false, + last_applied_commit_sha: null, + last_applied_content_hash: null, + pending_commit_sha: null, + pending_compose_content: null, + pending_env_content: null, + pending_fetched_at: null, + last_debounce_at: null, + }); +} + +describe('authoredComposeFileArgs', () => { + it('returns [] for a stack with no git source at all', () => { + expect(authoredComposeFileArgs('no-such-stack')).toEqual([]); + }); + + it('returns [] for a git-source stack with no applied spec (single-file)', () => { + seedSource('single-stack', ['compose.yaml']); + // No setGitSourceAppliedSpec call -> applied_deploy_spec stays null. + expect(authoredComposeFileArgs('single-stack')).toEqual([]); + }); + + it('returns ordered -f / -p / --project-directory for a multi-file spec', () => { + const stackName = 'multi-stack'; + seedSource(stackName, ['infra/base.yml', 'infra/prod.yml']); + DatabaseService.getInstance().setGitSourceAppliedSpec(stackName, { + files: ['compose.yaml', 'infra/prod.yml'], + contextDir: 'app', + }); + + const args = authoredComposeFileArgs(stackName); + + const baseDir = NodeRegistry.getInstance().getComposeDir(NodeRegistry.getInstance().getDefaultNodeId()); + const expectedCtx = path.resolve(baseDir, stackName, 'app'); + expect(args).toEqual([ + '-f', 'compose.yaml', + '-f', 'infra/prod.yml', + '-p', stackName, + '--project-directory', expectedCtx, + ]); + }); + + it('omits --project-directory when the spec has no context dir', () => { + const stackName = 'multi-no-ctx'; + seedSource(stackName, ['infra/base.yml', 'infra/prod.yml']); + DatabaseService.getInstance().setGitSourceAppliedSpec(stackName, { + files: ['compose.yaml', 'infra/prod.yml'], + contextDir: null, + }); + + expect(authoredComposeFileArgs(stackName)).toEqual([ + '-f', 'compose.yaml', + '-f', 'infra/prod.yml', + '-p', stackName, + ]); + }); + + it('throws when a spec file path is absolute', () => { + const stackName = 'abs-file'; + seedSource(stackName, ['compose.yaml', 'infra/prod.yml']); + DatabaseService.getInstance().setGitSourceAppliedSpec(stackName, { + files: ['compose.yaml', '/etc/passwd'], + contextDir: null, + }); + expect(() => authoredComposeFileArgs(stackName)).toThrow(); + }); + + it('throws when a spec file path contains a ".." traversal', () => { + const stackName = 'dotdot-file'; + seedSource(stackName, ['compose.yaml', 'infra/prod.yml']); + DatabaseService.getInstance().setGitSourceAppliedSpec(stackName, { + files: ['compose.yaml', '../escape.yml'], + contextDir: null, + }); + expect(() => authoredComposeFileArgs(stackName)).toThrow(); + }); + + it('throws when the context dir contains a ".." traversal', () => { + const stackName = 'dotdot-ctx'; + seedSource(stackName, ['compose.yaml', 'infra/prod.yml']); + DatabaseService.getInstance().setGitSourceAppliedSpec(stackName, { + files: ['compose.yaml', 'infra/prod.yml'], + contextDir: '../escape', + }); + expect(() => authoredComposeFileArgs(stackName)).toThrow(); + }); +}); diff --git a/backend/src/__tests__/compose-images.test.ts b/backend/src/__tests__/compose-images.test.ts index a38ea2e3..d9653fe5 100644 --- a/backend/src/__tests__/compose-images.test.ts +++ b/backend/src/__tests__/compose-images.test.ts @@ -38,7 +38,7 @@ vi.mock('../services/DockerController', () => ({ })); vi.mock('../services/DatabaseService', () => ({ - DatabaseService: { getInstance: () => ({ getRegistries: () => [] }) }, + DatabaseService: { getInstance: () => ({ getRegistries: () => [], getGitSource: () => undefined }) }, })); vi.mock('../services/RegistryService', () => ({ diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts index 959ec6a8..5f524329 100644 --- a/backend/src/__tests__/compose-service.test.ts +++ b/backend/src/__tests__/compose-service.test.ts @@ -82,6 +82,7 @@ vi.mock('../services/DatabaseService', () => ({ getInstance: () => ({ getRegistries: mockGetRegistries, getGlobalSettings: mockGetGlobalSettings, + getGitSource: () => undefined, }), }, })); @@ -107,6 +108,15 @@ vi.mock('../services/LogFormatter', () => ({ LogFormatter: { formatLine: (line: string) => line }, })); +// runCommand and the deploy/update paths route through authoredComposeArgs, which +// resolves the (optional) mesh override. Stub it to "no override" so a single-file +// stack yields plain `docker compose ` args deterministically. +vi.mock('../services/MeshService', () => ({ + MeshService: { + getInstance: () => ({ ensureStackOverride: vi.fn().mockResolvedValue(null) }), + }, +})); + import { ComposeService, getComposeRollbackInfo } from '../services/ComposeService'; const originalComposeTimeout = process.env.SENCHO_COMPOSE_COMMAND_TIMEOUT_MS; @@ -130,6 +140,16 @@ function createMockProcess() { return proc; } +/** + * runCommand now awaits the authored compose args (mesh override resolution) before + * spawning, so the child is created one microtask after the call. Tests that drive + * the mock child's events must wait for the spawn first, or the event fires before + * any listener is attached. + */ +async function waitForSpawn() { + await vi.waitFor(() => expect(mockSpawn).toHaveBeenCalled()); +} + /** Sets up mockSpawn to auto-close with exit code 0 on next tick */ function setupAutoCloseSpawn(exitCode = 0) { mockSpawn.mockImplementation(() => { @@ -186,6 +206,7 @@ describe('ComposeService - runCommand', () => { const svc = ComposeService.getInstance(1); const promise = svc.runCommand('my-stack', 'restart'); + await waitForSpawn(); proc.emit('close', 0); await promise; @@ -202,6 +223,7 @@ describe('ComposeService - runCommand', () => { const svc = ComposeService.getInstance(1); const promise = svc.runCommand('my-stack', 'start'); + await waitForSpawn(); proc.emit('close', 0); await expect(promise).resolves.toBeUndefined(); @@ -213,6 +235,7 @@ describe('ComposeService - runCommand', () => { const svc = ComposeService.getInstance(1); const promise = svc.runCommand('my-stack', 'stop'); + await waitForSpawn(); proc.stderr.emit('data', Buffer.from('service not found')); proc.emit('close', 1); @@ -225,6 +248,7 @@ describe('ComposeService - runCommand', () => { const svc = ComposeService.getInstance(1); const promise = svc.runCommand('my-stack', 'stop'); + await waitForSpawn(); proc.stderr.emit('data', Buffer.from('token=abc123SECRET password=hunter2 Authorization: Bearer abc.def.ghi')); proc.emit('close', 1); @@ -240,6 +264,7 @@ describe('ComposeService - runCommand', () => { const svc = ComposeService.getInstance(1); const promise = svc.runCommand('my-stack', 'restart', ws); + await waitForSpawn(); proc.stdout.emit('data', Buffer.from('Restarting...')); proc.emit('close', 0); await promise; @@ -272,6 +297,7 @@ describe('ComposeService - runCommand', () => { const svc = ComposeService.getInstance(1); const promise = svc.runCommand('my-stack', 'restart', ws); + await waitForSpawn(); // The deploy is owned by its HTTP request; closing the progress socket // (panel minimized, navigated away, connection blip) must not abort it. ws.emit('close'); @@ -292,6 +318,7 @@ describe('ComposeService - runCommand', () => { const svc = ComposeService.getInstance(1); const promise = svc.runCommand('my-stack', 'restart', ws); + await waitForSpawn(); const err = Object.assign(new Error('spawn docker ENOMEM'), { code: 'ENOMEM' }); proc.emit('error', err); @@ -314,6 +341,7 @@ describe('ComposeService - runCommand', () => { const svc = ComposeService.getInstance(1); const promise = svc.runCommand('my-stack', 'restart', ws); + await waitForSpawn(); const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' }); proc.emit('error', err); @@ -335,6 +363,7 @@ describe('ComposeService - runCommand', () => { const svc = ComposeService.getInstance(1); const promise = svc.runCommand('my-stack', 'restart'); + await waitForSpawn(); const err = Object.assign(new Error('spawn docker ENOENT'), { code: 'ENOENT' }); proc.emit('error', err); diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 00494539..586fa141 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -38,6 +38,7 @@ vi.mock('../services/NodeRegistry', () => ({ // Prevent COMPOSE_DIR related issues vi.mock('child_process', () => ({ exec: vi.fn(), + execFile: vi.fn(), })); vi.mock('util', () => ({ promisify: () => vi.fn(), diff --git a/backend/src/__tests__/git-compose-files.test.ts b/backend/src/__tests__/git-compose-files.test.ts new file mode 100644 index 00000000..caa961e4 --- /dev/null +++ b/backend/src/__tests__/git-compose-files.test.ts @@ -0,0 +1,37 @@ +/** + * Unit tests for gitSourceLocalComposeFiles: the mapping from ordered repo + * compose paths to the local relative filenames Sencho materializes under the + * stack directory. + * + * Contract: + * - index 0 (primary) always maps to compose.yaml at the stack root + * - every additional file keeps its repo-relative path, with a leading "./" stripped + */ +import { describe, it, expect } from 'vitest'; +import { gitSourceLocalComposeFiles, PRIMARY_COMPOSE_FILENAME } from '../utils/gitComposeFiles'; + +describe('gitSourceLocalComposeFiles', () => { + it('maps the primary file to compose.yaml regardless of its repo name', () => { + expect(gitSourceLocalComposeFiles(['infra/base.yml'])).toEqual([PRIMARY_COMPOSE_FILENAME]); + expect(gitSourceLocalComposeFiles(['deploy/docker-compose.prod.yaml'])).toEqual(['compose.yaml']); + }); + + it('keeps additional files at their repo-relative paths', () => { + expect(gitSourceLocalComposeFiles(['infra/base.yml', 'infra/prod.yml'])) + .toEqual(['compose.yaml', 'infra/prod.yml']); + }); + + it('strips a leading "./" only from additional files', () => { + expect(gitSourceLocalComposeFiles(['./base.yml', './override/prod.yml'])) + .toEqual(['compose.yaml', 'override/prod.yml']); + }); + + it('preserves order across a larger set', () => { + expect(gitSourceLocalComposeFiles(['a/base.yml', 'b/x.yml', 'c/y.yml', 'd/z.yml'])) + .toEqual(['compose.yaml', 'b/x.yml', 'c/y.yml', 'd/z.yml']); + }); + + it('returns just compose.yaml for a single-file selection', () => { + expect(gitSourceLocalComposeFiles(['compose.yaml'])).toEqual(['compose.yaml']); + }); +}); diff --git a/backend/src/__tests__/git-source-routes.test.ts b/backend/src/__tests__/git-source-routes.test.ts index 6b683ace..bdbe92ed 100644 --- a/backend/src/__tests__/git-source-routes.test.ts +++ b/backend/src/__tests__/git-source-routes.test.ts @@ -26,6 +26,8 @@ function seedGitSource(stackName: string): void { repo_url: 'https://github.com/example/repo.git', branch: 'main', compose_path: 'compose.yaml', + compose_paths: ['compose.yaml'], + context_dir: null, sync_env: false, env_path: null, auth_type: 'none', @@ -132,7 +134,7 @@ describe('PUT /api/stacks/:stackName/git-source — max-length caps', () => { compose_path: 'c'.repeat(1100), }); expect(res.status).toBe(400); - expect(res.body.error).toMatch(/compose_path/i); + expect(res.body.error).toMatch(/compose path/i); }); it('rejects oversized env_path', async () => { @@ -191,7 +193,7 @@ describe('git-source routes — repository path validation', () => { auth_type: 'none', }); expect(res.status).toBe(400); - expect(res.body.error).toMatch(/compose_path/i); + expect(res.body.error).toMatch(/compose path/i); }); it('rejects absolute env_path on create-from-git', async () => { @@ -288,6 +290,97 @@ describe('GET /api/stacks/:stackName/git-source', () => { }); }); +describe('PUT /api/stacks/:stackName/git-source: multi-file selection', () => { + // The success-path tests forward a parsed selection to upsert(), whose dry-run + // fetch would clone a real repo. Stub upsert so the assertion stays at the + // route layer (parse + forward) without network. Rejection-path tests hit the + // parseComposeSelection 400 before upsert is ever reached, so they need no stub. + it('persists a compose_paths array and forwards it to the service', async () => { + const upsertSpy = vi.spyOn(GitSourceService.getInstance(), 'upsert') + .mockResolvedValue({} as Awaited>); + const res = await request(app) + .put('/api/stacks/existing-stack/git-source') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ + repo_url: 'https://github.com/example/repo.git', + branch: 'main', + compose_paths: ['infra/base.yml', 'infra/prod.yml'], + context_dir: 'app', + auth_type: 'none', + }); + expect(res.status).toBe(200); + expect(upsertSpy).toHaveBeenCalledWith(expect.objectContaining({ + composePaths: ['infra/base.yml', 'infra/prod.yml'], + contextDir: 'app', + })); + upsertSpy.mockRestore(); + }); + + it('still accepts the legacy compose_path string and maps it to a one-element array', async () => { + const upsertSpy = vi.spyOn(GitSourceService.getInstance(), 'upsert') + .mockResolvedValue({} as Awaited>); + const res = await request(app) + .put('/api/stacks/existing-stack/git-source') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ + repo_url: 'https://github.com/example/repo.git', + branch: 'main', + compose_path: 'stacks/web/compose.yaml', + auth_type: 'none', + }); + expect(res.status).toBe(200); + expect(upsertSpy).toHaveBeenCalledWith(expect.objectContaining({ + composePaths: ['stacks/web/compose.yaml'], + contextDir: null, + })); + upsertSpy.mockRestore(); + }); + + it('rejects a compose_paths array with more than 10 files (400)', async () => { + const tooMany = Array.from({ length: 11 }, (_, i) => `f${i}.yml`); + const res = await request(app) + .put('/api/stacks/existing-stack/git-source') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ + repo_url: 'https://github.com/example/repo.git', + branch: 'main', + compose_paths: tooMany, + auth_type: 'none', + }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/exceed/i); + }); + + it('rejects duplicate entries in compose_paths (400)', async () => { + const res = await request(app) + .put('/api/stacks/existing-stack/git-source') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ + repo_url: 'https://github.com/example/repo.git', + branch: 'main', + compose_paths: ['compose.yaml', 'infra/prod.yml', 'infra/prod.yml'], + auth_type: 'none', + }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/duplicate/i); + }); + + it('rejects a context_dir that collides with the primary compose.yaml (400)', async () => { + const res = await request(app) + .put('/api/stacks/existing-stack/git-source') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ + repo_url: 'https://github.com/example/repo.git', + branch: 'main', + compose_paths: ['compose.yaml'], + context_dir: 'compose.yaml', + auth_type: 'none', + }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/context_dir/i); + }); +}); + describe('POST /api/stacks/from-git', () => { const validBody = { stack_name: 'route-from-git', @@ -399,6 +492,38 @@ describe('POST /api/stacks/:stackName/git-source/webhook-pull status codes', () }); }); +describe('DELETE /api/stacks/:stackName/git-source — multi-file unlink guard', () => { + it('blocks unlinking a multi-file source with 409 and keeps the row', async () => { + seedGitSource('mf-unlink'); + DatabaseService.getInstance().setGitSourceAppliedSpec('mf-unlink', { files: ['compose.yaml', 'infra/prod.yml'], contextDir: null }); + const res = await request(app) + .delete('/api/stacks/mf-unlink/git-source') + .set('Authorization', `Bearer ${adminToken()}`); + expect(res.status).toBe(409); + expect(res.body.error).toMatch(/multiple compose files/i); + expect(DatabaseService.getInstance().getGitSource('mf-unlink')).toBeTruthy(); + }); + + it('blocks unlinking a context-dir source with 409', async () => { + seedGitSource('ctx-unlink'); + DatabaseService.getInstance().setGitSourceAppliedSpec('ctx-unlink', { files: ['compose.yaml'], contextDir: 'app' }); + const res = await request(app) + .delete('/api/stacks/ctx-unlink/git-source') + .set('Authorization', `Bearer ${adminToken()}`); + expect(res.status).toBe(409); + expect(DatabaseService.getInstance().getGitSource('ctx-unlink')).toBeTruthy(); + }); + + it('allows unlinking a single-file source', async () => { + seedGitSource('sf-unlink'); + const res = await request(app) + .delete('/api/stacks/sf-unlink/git-source') + .set('Authorization', `Bearer ${adminToken()}`); + expect(res.status).toBe(200); + expect(DatabaseService.getInstance().getGitSource('sf-unlink')).toBeUndefined(); + }); +}); + describe('GET /api/git-sources', () => { it('returns 200 and a JSON array for an authenticated admin', async () => { const res = await request(app) diff --git a/backend/src/__tests__/git-source-selection.test.ts b/backend/src/__tests__/git-source-selection.test.ts new file mode 100644 index 00000000..236c5eb2 --- /dev/null +++ b/backend/src/__tests__/git-source-selection.test.ts @@ -0,0 +1,168 @@ +/** + * Unit tests for the compose-selection parser used by the git-source routes. + * + * parseComposeSelection normalizes a request body into an ordered compose-path + * list plus an optional context dir, enforcing the file-count cap, duplicate + * rejection, the reserved-primary-name rule, and context-dir validation. Pure + * function, no DB or filesystem. + */ +import { describe, it, expect } from 'vitest'; +import { parseComposeSelection, defaultEnvPath, MAX_COMPOSE_FILES } from '../helpers/gitSourceSelection'; + +/** Narrow the result union to its ok branch (throws if the parse failed). */ +function expectOk(result: ReturnType) { + if (!result.ok) throw new Error(`expected ok, got error: ${result.error}`); + return result.value; +} + +describe('parseComposeSelection', () => { + it('maps a legacy single compose_path string to a one-element array', () => { + const value = expectOk(parseComposeSelection({ compose_path: 'stacks/web/compose.yaml' })); + expect(value.composePaths).toEqual(['stacks/web/compose.yaml']); + expect(value.contextDir).toBeNull(); + }); + + it('accepts an ordered compose_paths array', () => { + const value = expectOk(parseComposeSelection({ compose_paths: ['infra/base.yml', 'infra/prod.yml'] })); + expect(value.composePaths).toEqual(['infra/base.yml', 'infra/prod.yml']); + }); + + it('prefers compose_paths over a legacy compose_path when both are present', () => { + const value = expectOk(parseComposeSelection({ + compose_paths: ['a.yml', 'b.yml'], + compose_path: 'ignored.yml', + })); + expect(value.composePaths).toEqual(['a.yml', 'b.yml']); + }); + + it('rejects an empty selection', () => { + expect(parseComposeSelection({ compose_paths: [] }).ok).toBe(false); + expect(parseComposeSelection({}).ok).toBe(false); + }); + + it('rejects more than the file-count cap', () => { + const tooMany = Array.from({ length: MAX_COMPOSE_FILES + 1 }, (_, i) => `f${i}.yml`); + const result = parseComposeSelection({ compose_paths: tooMany }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/exceed/i); + }); + + it('accepts exactly the file-count cap', () => { + const exact = Array.from({ length: MAX_COMPOSE_FILES }, (_, i) => `f${i}.yml`); + expect(parseComposeSelection({ compose_paths: exact }).ok).toBe(true); + }); + + it('rejects duplicate compose paths', () => { + const result = parseComposeSelection({ compose_paths: ['compose.yaml', 'infra/prod.yml', 'infra/prod.yml'] }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/duplicate/i); + }); + + it('rejects a non-first entry named compose.yaml (reserved for the primary)', () => { + const result = parseComposeSelection({ compose_paths: ['infra/base.yml', 'compose.yaml'] }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/compose\.yaml/i); + }); + + it('rejects a non-first entry that normalizes to compose.yaml via "./"', () => { + const result = parseComposeSelection({ compose_paths: ['infra/base.yml', './compose.yaml'] }); + expect(result.ok).toBe(false); + }); + + it('rejects an additional file nested under the primary compose.yaml', () => { + // 'compose.yaml/prod.yml' would try to write under the root compose.yaml file. + const result = parseComposeSelection({ compose_paths: ['infra/base.yml', 'compose.yaml/prod.yml'] }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/collides/i); + }); + + it('rejects two selected files where one is a directory ancestor of another', () => { + const result = parseComposeSelection({ compose_paths: ['base.yml', 'sub.yml', 'sub.yml/deep.yml'] }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/collides/i); + }); + + it('allows compose.yaml as the primary (index 0)', () => { + const value = expectOk(parseComposeSelection({ compose_paths: ['compose.yaml', 'infra/prod.yml'] })); + expect(value.composePaths).toEqual(['compose.yaml', 'infra/prod.yml']); + }); + + it('rejects a path that escapes the repo with traversal', () => { + expect(parseComposeSelection({ compose_paths: ['../escape.yml'] }).ok).toBe(false); + expect(parseComposeSelection({ compose_path: '/etc/passwd' }).ok).toBe(false); + }); + + it('rejects a path targeting the .git directory', () => { + expect(parseComposeSelection({ compose_paths: ['.git/config'] }).ok).toBe(false); + }); + + it('accepts a valid multi-file selection with a context_dir', () => { + const value = expectOk(parseComposeSelection({ + compose_paths: ['infra/base.yml', 'infra/prod.yml'], + context_dir: 'app', + })); + expect(value.contextDir).toBe('app'); + }); + + it('normalizes an empty context_dir to null', () => { + expect(expectOk(parseComposeSelection({ compose_paths: ['compose.yaml'], context_dir: '' })).contextDir).toBeNull(); + expect(expectOk(parseComposeSelection({ compose_paths: ['compose.yaml'], context_dir: ' ' })).contextDir).toBeNull(); + }); + + it('rejects a context_dir with traversal', () => { + const result = parseComposeSelection({ compose_paths: ['compose.yaml'], context_dir: '../escape' }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/context_dir/i); + }); + + it('rejects a context_dir that targets the .git directory', () => { + const result = parseComposeSelection({ compose_paths: ['compose.yaml'], context_dir: '.git' }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/\.git/i); + }); + + it('rejects a context_dir equal to compose.yaml', () => { + const result = parseComposeSelection({ compose_paths: ['compose.yaml'], context_dir: 'compose.yaml' }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/context_dir/i); + }); + + it('rejects a context_dir equal to a selected additional compose path', () => { + const result = parseComposeSelection({ + compose_paths: ['compose.yaml', 'infra/prod.yml'], + context_dir: 'infra/prod.yml', + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/context_dir/i); + }); + + it('rejects a context_dir nested under the primary compose.yaml', () => { + const result = parseComposeSelection({ compose_paths: ['infra/base.yml'], context_dir: 'compose.yaml/app' }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/context_dir/i); + }); + + it('rejects a context_dir nested under a selected additional compose file', () => { + const result = parseComposeSelection({ + compose_paths: ['compose.yaml', 'infra/prod.yml'], + context_dir: 'infra/prod.yml/sub', + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toMatch(/context_dir/i); + }); +}); + +describe('defaultEnvPath', () => { + it('returns the explicit env path when provided', () => { + expect(defaultEnvPath('infra/base.yml', 'custom/.env')).toBe('custom/.env'); + }); + + it('defaults to a sibling .env of the primary compose file', () => { + expect(defaultEnvPath('infra/base.yml', undefined)).toBe('infra/.env'); + expect(defaultEnvPath('infra/base.yml', '')).toBe('infra/.env'); + }); + + it('defaults to .env at the repo root when the primary is at the root', () => { + expect(defaultEnvPath('compose.yaml', undefined)).toBe('.env'); + }); +}); diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index c30c683a..6d17203d 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -12,6 +12,7 @@ * - Webhook debounce enforcement * - Per-stack mutex serialization ordering */ +import crypto from 'crypto'; import { describe, it, expect, vi, beforeAll, afterAll, beforeEach } from 'vitest'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; @@ -67,6 +68,11 @@ function mockSuccessfulClone(options: { composePath?: string; envPath?: string | null; sha?: string; + /** + * Additional repo-relative files to write into the clone temp dir on top of + * the primary compose file. Lets multi-file tests stage base+override layouts. + */ + extraFiles?: Record; } = {}) { const { compose = 'services:\n web:\n image: nginx\n', @@ -74,6 +80,7 @@ function mockSuccessfulClone(options: { composePath = 'compose.yaml', envPath = null, sha = 'abc1234567890abc1234567890abc1234567890a', + extraFiles = {}, } = options; mockGitClone.mockImplementation(async (args: { dir: string }) => { @@ -82,6 +89,11 @@ function mockSuccessfulClone(options: { const composeAbs = path.join(args.dir, composePath); await fsp.mkdir(path.dirname(composeAbs), { recursive: true }); await fsp.writeFile(composeAbs, compose, 'utf-8'); + for (const [rel, content] of Object.entries(extraFiles)) { + const abs = path.join(args.dir, rel); + await fsp.mkdir(path.dirname(abs), { recursive: true }); + await fsp.writeFile(abs, content, 'utf-8'); + } if (env !== null && envPath) { const envAbs = path.join(args.dir, envPath); await fsp.mkdir(path.dirname(envAbs), { recursive: true }); @@ -92,71 +104,116 @@ function mockSuccessfulClone(options: { return sha; } +/** Wrap a single compose string in the ComposeFile[] shape the new APIs take. */ +function asFiles(content: string): import('../services/GitSourceService').ComposeFile[] { + return [{ path: 'compose.yaml', content }]; +} + +/** Best-effort teardown for tests that materialize a stack on disk. */ +async function cleanupStackDir(name: string) { + const { FileSystemService } = await import('../services/FileSystemService'); + try { + await FileSystemService.getInstance().deleteStack(name); + } catch { + // directory may not exist; ignore + } +} + // ── Tests ────────────────────────────────────────────────────────────── describe('GitSourceService.hashContent', () => { it('produces stable hashes for identical inputs', () => { const svc = GitSourceService.getInstance(); - const a = svc.hashContent('services:\n web: nginx\n', 'FOO=bar'); - const b = svc.hashContent('services:\n web: nginx\n', 'FOO=bar'); + const a = svc.hashContent(asFiles('services:\n web: nginx\n'), 'FOO=bar'); + const b = svc.hashContent(asFiles('services:\n web: nginx\n'), 'FOO=bar'); expect(a).toBe(b); expect(a).toMatch(/^[a-f0-9]{64}$/); }); it('distinguishes env=null from env=""', () => { const svc = GitSourceService.getInstance(); - const nullHash = svc.hashContent('x: 1', null); - const emptyHash = svc.hashContent('x: 1', ''); + const nullHash = svc.hashContent(asFiles('x: 1'), null); + const emptyHash = svc.hashContent(asFiles('x: 1'), ''); // Both hash-empty-string after null-coalesce, so they should match by design. expect(nullHash).toBe(emptyHash); }); it('changes when compose content changes', () => { const svc = GitSourceService.getInstance(); - const a = svc.hashContent('x: 1', null); - const b = svc.hashContent('x: 2', null); + const a = svc.hashContent(asFiles('x: 1'), null); + const b = svc.hashContent(asFiles('x: 2'), null); expect(a).not.toBe(b); }); it('changes when env content changes', () => { const svc = GitSourceService.getInstance(); - const a = svc.hashContent('x: 1', 'A=1'); - const b = svc.hashContent('x: 1', 'A=2'); + const a = svc.hashContent(asFiles('x: 1'), 'A=1'); + const b = svc.hashContent(asFiles('x: 1'), 'A=2'); expect(a).not.toBe(b); }); it('does not confuse compose|env boundary (uses NUL separator)', () => { const svc = GitSourceService.getInstance(); // If the separator were absent, "ab" + "cd" would equal "abc" + "d". - const a = svc.hashContent('ab', 'cd'); - const b = svc.hashContent('abc', 'd'); + const a = svc.hashContent(asFiles('ab'), 'cd'); + const b = svc.hashContent(asFiles('abc'), 'd'); expect(a).not.toBe(b); }); + + it('keeps the single-file hash stable vs the legacy content+env formula', () => { + const svc = GitSourceService.getInstance(); + // Legacy single-string hash was sha256(content + '\x00' + (env ?? '')). + const legacy = crypto + .createHash('sha256') + .update('x: 1') + .update('\x00') + .update('FOO=bar') + .digest('hex'); + expect(svc.hashContent(asFiles('x: 1'), 'FOO=bar')).toBe(legacy); + }); + + it('folds ordered contents (not paths) for a multi-file set', () => { + const svc = GitSourceService.getInstance(); + const base = { path: 'compose.yaml', content: 'a' }; + const override = { path: 'infra/prod.yml', content: 'b' }; + const ab = svc.hashContent([base, override], null); + const ba = svc.hashContent([override, base], null); + // Order-sensitive: swapping the two files changes the hash (content order). + expect(ab).not.toBe(ba); + // Path-INsensitive by design: the same contents in the same order hash equal + // regardless of path, so create (repo paths) and pull (materialized paths, + // primary -> compose.yaml) agree and a clean stack is not flagged as edited. + const repoPaths = svc.hashContent([{ path: 'infra/base.yml', content: 'a' }, { path: 'infra/prod.yml', content: 'b' }], null); + const localPaths = svc.hashContent([{ path: 'compose.yaml', content: 'a' }, { path: 'infra/prod.yml', content: 'b' }], null); + expect(repoPaths).toBe(localPaths); + // Content-sensitive: changing a file's content changes the hash. + expect(ab).not.toBe(svc.hashContent([base, { path: 'infra/prod.yml', content: 'B' }], null)); + }); }); describe('GitSourceService.validateCompose (YAML pre-check)', () => { const svc = () => GitSourceService.getInstance(); it('rejects empty content', async () => { - const r = await svc().validateCompose('', null); + const r = await svc().validateCompose(asFiles(''), null, null); expect(r.ok).toBe(false); expect(r.error).toMatch(/empty/i); }); it('rejects a YAML array at the root', async () => { - const r = await svc().validateCompose('- one\n- two\n', null); + const r = await svc().validateCompose(asFiles('- one\n- two\n'), null, null); expect(r.ok).toBe(false); expect(r.error).toMatch(/mapping/i); }); it('rejects a YAML scalar at the root', async () => { - const r = await svc().validateCompose('42', null); + const r = await svc().validateCompose(asFiles('42'), null, null); expect(r.ok).toBe(false); expect(r.error).toMatch(/mapping/i); }); it('rejects malformed YAML syntax', async () => { - const r = await svc().validateCompose('services:\n web:\n image: "unterminated\n', null); + const r = await svc().validateCompose(asFiles('services:\n web:\n image: "unterminated\n'), null, null); expect(r.ok).toBe(false); expect(r.error).toMatch(/YAML parse error/i); }); @@ -170,7 +227,8 @@ describe('GitSourceService.upsert (encryption + reachability)', () => { stackName: 'enc-stack', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'token', @@ -196,7 +254,8 @@ describe('GitSourceService.upsert (encryption + reachability)', () => { stackName: 'keep-stack', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'token', @@ -210,7 +269,8 @@ describe('GitSourceService.upsert (encryption + reachability)', () => { stackName: 'keep-stack', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'token', @@ -229,7 +289,8 @@ describe('GitSourceService.upsert (encryption + reachability)', () => { stackName: 'clear-stack', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'token', @@ -242,7 +303,8 @@ describe('GitSourceService.upsert (encryption + reachability)', () => { stackName: 'clear-stack', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -260,7 +322,8 @@ describe('GitSourceService.upsert (encryption + reachability)', () => { stackName: 'bad-matrix', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -279,7 +342,8 @@ describe('GitSourceService.upsert (encryption + reachability)', () => { stackName: 'unreachable', repoUrl: 'https://github.com/example/nope.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -296,7 +360,7 @@ describe('GitSourceService error mapping', () => { const fetchParams = { repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], }; it('maps 401 with supplied token to AUTH_FAILED', async () => { @@ -427,7 +491,7 @@ describe('GitSourceService error mapping', () => { mockGitLog.mockResolvedValue([{ oid: 'deadbeef' }]); await expect(svc().fetchFromGit({ ...fetchParams, - composePath: 'missing/compose.yaml', + composePaths: ['missing/compose.yaml'], })).rejects.toMatchObject({ code: 'FILE_NOT_FOUND' }); }); @@ -485,7 +549,7 @@ describe('GitSourceService.fetchFromGit (size limits)', () => { const fetchParams = { repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], }; it('rejects a compose file larger than the per-file read cap', async () => { @@ -541,7 +605,8 @@ describe('GitSourceService pending lifecycle', () => { stackName: 'pending-stack', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -566,7 +631,8 @@ describe('GitSourceService.handleWebhookPull debounce', () => { stackName: 'debounce-stack', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -602,7 +668,8 @@ describe('GitSourceService.handleWebhookPull debounce', () => { stackName: 'fanout-stack', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -630,7 +697,8 @@ describe('GitSourceService.handleWebhookPull debounce', () => { stackName: 'webhook-validate-fail', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -707,7 +775,7 @@ describe('GitSourceService.fetchFromGit (.git metadata guard)', () => { await expect(svc().fetchFromGit({ repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: '.git/config', + composePaths: ['.git/config'], })).rejects.toMatchObject({ code: 'FILE_NOT_FOUND' }); expect(mockGitClone).not.toHaveBeenCalled(); }); @@ -716,7 +784,7 @@ describe('GitSourceService.fetchFromGit (.git metadata guard)', () => { await expect(svc().fetchFromGit({ repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'subdir/.git/HEAD', + composePaths: ['subdir/.git/HEAD'], })).rejects.toMatchObject({ code: 'FILE_NOT_FOUND' }); }); @@ -724,7 +792,7 @@ describe('GitSourceService.fetchFromGit (.git metadata guard)', () => { await expect(svc().fetchFromGit({ repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], envPath: '.git/config', })).rejects.toMatchObject({ code: 'FILE_NOT_FOUND' }); }); @@ -734,7 +802,7 @@ describe('GitSourceService.fetchFromGit (.git metadata guard)', () => { await expect(svc().fetchFromGit({ repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'gitops.yaml', + composePaths: ['gitops.yaml'], })).resolves.toBeDefined(); }); @@ -748,7 +816,7 @@ describe('GitSourceService.fetchFromGit (.git metadata guard)', () => { await expect(svc().fetchFromGit({ repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], })).rejects.toMatchObject({ code: 'FILE_NOT_FOUND', message: expect.stringMatching(/symbolic link/i), @@ -768,7 +836,7 @@ describe('GitSourceService.fetchFromGit (LFS + submodule detection)', () => { await expect(svc().fetchFromGit({ repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], })).rejects.toMatchObject({ code: 'GIT_ERROR', message: expect.stringMatching(/LFS/i), @@ -784,7 +852,7 @@ describe('GitSourceService.fetchFromGit (LFS + submodule detection)', () => { await expect(svc().fetchFromGit({ repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], envPath: '.env', })).rejects.toMatchObject({ code: 'GIT_ERROR', @@ -808,7 +876,7 @@ describe('GitSourceService.fetchFromGit (LFS + submodule detection)', () => { const result = await svc().fetchFromGit({ repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], }); expect(result.warnings).toEqual( expect.arrayContaining([expect.stringMatching(/submodules/i)]), @@ -820,7 +888,7 @@ describe('GitSourceService.fetchFromGit (LFS + submodule detection)', () => { const result = await svc().fetchFromGit({ repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], }); expect(result.warnings).toEqual([]); }); @@ -834,15 +902,6 @@ describe('GitSourceService.pull', () => { }); describe('GitSourceService.createStackFromGit', () => { - async function cleanupStackDir(name: string) { - const { FileSystemService } = await import('../services/FileSystemService'); - try { - await FileSystemService.getInstance().deleteStack(name); - } catch { - // directory may not exist; ignore - } - } - it('creates a stack on disk, writes compose, and seeds last_applied columns', async () => { const sha = 'fedcba9876543210fedcba9876543210fedcba98'; mockSuccessfulClone({ @@ -855,7 +914,8 @@ describe('GitSourceService.createStackFromGit', () => { stackName: 'create-happy', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -876,6 +936,42 @@ describe('GitSourceService.createStackFromGit', () => { await cleanupStackDir('create-happy'); }); + it('multi-file create then pull reports no local changes (hash is path-independent)', async () => { + const sha = 'aaaa1111bbbb2222cccc3333dddd4444eeee5555'; + mockSuccessfulClone({ + composePath: 'infra/base.yml', + compose: 'services:\n web:\n image: nginx\n', + extraFiles: { 'infra/prod.yml': 'services:\n web:\n restart: always\n' }, + sha, + }); + const svc = GitSourceService.getInstance(); + await svc.createStackFromGit({ + stackName: 'mf-clean-pull', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['infra/base.yml', 'infra/prod.yml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + + // The primary is materialized as compose.yaml, the override at its repo path. + const row = DatabaseService.getInstance().getGitSource('mf-clean-pull'); + expect(row?.applied_deploy_spec?.files).toEqual(['compose.yaml', 'infra/prod.yml']); + + // Pulling the identical commit must NOT flag local edits, even though the + // stored hash was computed from repo paths while the disk read uses the + // materialized paths (primary -> compose.yaml). This was the regression. + const pull = await svc.pull('mf-clean-pull'); + expect(pull.hasLocalChanges).toBe(false); + + await cleanupStackDir('mf-clean-pull'); + }); + it('resolves a nested compose_path and nested env_path into the stack dir', async () => { const sha = 'deadbeef1234567890deadbeef1234567890abcd'; mockSuccessfulClone({ @@ -891,7 +987,8 @@ describe('GitSourceService.createStackFromGit', () => { stackName: 'create-nested', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'apps/web/compose.yaml', + composePaths: ['apps/web/compose.yaml'], + contextDir: null, syncEnv: true, envPath: 'apps/web/.env', authType: 'none', @@ -928,7 +1025,8 @@ describe('GitSourceService.createStackFromGit', () => { stackName: 'create-env', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: true, envPath: '.env', authType: 'none', @@ -951,7 +1049,8 @@ describe('GitSourceService.createStackFromGit', () => { stackName: 'create-bad-matrix', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -975,7 +1074,8 @@ describe('GitSourceService.createStackFromGit', () => { stackName: 'create-bad-yaml', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -1003,7 +1103,8 @@ describe('GitSourceService.createStackFromGit', () => { stackName: 'create-rollback', repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -1028,7 +1129,8 @@ describe('GitSourceService.apply', () => { stackName, repoUrl: 'https://github.com/example/repo.git', branch: 'main', - composePath: 'compose.yaml', + composePaths: ['compose.yaml'], + contextDir: null, syncEnv: false, envPath: null, authType: 'none', @@ -1171,3 +1273,276 @@ describe('GitSourceService.apply', () => { scanSpy.mockRestore(); }); }); + +describe('GitSourceService DB normalization (compose_paths back-compat)', () => { + it('reads back [compose_path] when a row stores compose_paths as null (legacy)', async () => { + mockSuccessfulClone({ composePath: 'stacks/web/compose.yaml' }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'legacy-null', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['stacks/web/compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + + // Simulate a row written before the multi-file column existed. + const db = DatabaseService.getInstance(); + db.getDb().prepare('UPDATE stack_git_sources SET compose_paths = NULL WHERE stack_name = ?').run('legacy-null'); + + const row = db.getGitSource('legacy-null'); + expect(row?.compose_paths).toEqual(['stacks/web/compose.yaml']); + expect(row?.compose_path).toBe('stacks/web/compose.yaml'); + }); + + it('reads back [compose_path] when compose_paths holds an empty JSON array', async () => { + mockSuccessfulClone(); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'legacy-empty', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + + const db = DatabaseService.getInstance(); + db.getDb().prepare(`UPDATE stack_git_sources SET compose_paths = '[]' WHERE stack_name = ?`).run('legacy-empty'); + + expect(db.getGitSource('legacy-empty')?.compose_paths).toEqual(['compose.yaml']); + }); + + it('backfills compose_paths to json_array(compose_path) for a NULL-column row', async () => { + // The migration runs json_array(compose_path) for any row whose + // compose_paths is NULL. Insert a NULL-column row, run the backfill SQL, + // and confirm the stored JSON is a one-element array of the legacy path. + mockSuccessfulClone({ composePath: 'deploy/compose.yaml' }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'backfill-stack', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['deploy/compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const db = DatabaseService.getInstance(); + db.getDb().prepare('UPDATE stack_git_sources SET compose_paths = NULL WHERE stack_name = ?').run('backfill-stack'); + + db.getDb().prepare( + `UPDATE stack_git_sources SET compose_paths = json_array(compose_path) WHERE compose_paths IS NULL`, + ).run(); + + const stored = db.getDb() + .prepare('SELECT compose_paths FROM stack_git_sources WHERE stack_name = ?') + .get('backfill-stack') as { compose_paths: string }; + expect(JSON.parse(stored.compose_paths)).toEqual(['deploy/compose.yaml']); + }); +}); + +describe('GitSourceService multi-file create + apply flow', () => { + it('materializes both files and persists a non-null applied_deploy_spec for a two-file create', async () => { + const sha = '1111aaa1111aaa1111aaa1111aaa1111aaa1111a'; + mockSuccessfulClone({ + compose: 'services:\n web:\n image: nginx\n', + composePath: 'infra/base.yml', + extraFiles: { 'infra/prod.yml': 'services:\n web:\n environment:\n - X=1\n' }, + sha, + }); + const svc = GitSourceService.getInstance(); + // Isolate materialize behavior from docker availability. + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + const result = await svc.createStackFromGit({ + stackName: 'multi-create', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['infra/base.yml', 'infra/prod.yml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + + expect(result.commitSha).toBe(sha); + const row = DatabaseService.getInstance().getGitSource('multi-create'); + expect(row?.compose_paths).toEqual(['infra/base.yml', 'infra/prod.yml']); + expect(row?.applied_deploy_spec).not.toBeNull(); + expect(row?.applied_deploy_spec?.files).toEqual(['compose.yaml', 'infra/prod.yml']); + + const { FileSystemService } = await import('../services/FileSystemService'); + const fsSvc = FileSystemService.getInstance(); + // Primary lands at the root compose.yaml; the additional file at its repo path. + expect(await fsSvc.getStackContent('multi-create')).toContain('image: nginx'); + const prod = await fsSvc.readStackFile('multi-create', 'infra/prod.yml'); + expect(prod.content).toContain('X=1'); + + validateSpy.mockRestore(); + await cleanupStackDir('multi-create'); + }); + + it('leaves applied_deploy_spec null for a plain single-file create', async () => { + const sha = '2222bbb2222bbb2222bbb2222bbb2222bbb2222b'; + mockSuccessfulClone({ compose: 'services:\n web:\n image: nginx\n', sha }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + await svc.createStackFromGit({ + stackName: 'single-create', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + + const row = DatabaseService.getInstance().getGitSource('single-create'); + expect(row?.compose_paths).toEqual(['compose.yaml']); + expect(row?.applied_deploy_spec).toBeNull(); + + validateSpy.mockRestore(); + await cleanupStackDir('single-create'); + }); + + it('sets applied_deploy_spec for a single-file create that has a context_dir', async () => { + const sha = '3333ccc3333ccc3333ccc3333ccc3333ccc3333c'; + mockSuccessfulClone({ compose: 'services:\n web:\n image: nginx\n', sha }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + await svc.createStackFromGit({ + stackName: 'ctx-create', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: 'app', + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + + const row = DatabaseService.getInstance().getGitSource('ctx-create'); + expect(row?.applied_deploy_spec).not.toBeNull(); + expect(row?.applied_deploy_spec?.files).toEqual(['compose.yaml']); + expect(row?.applied_deploy_spec?.contextDir).toBe('app'); + + validateSpy.mockRestore(); + await cleanupStackDir('ctx-create'); + }); + + it('pulls a multi-file v2 pending blob and applies both files to disk', async () => { + const sha = '4444ddd4444ddd4444ddd4444ddd4444ddd4444d'; + mockSuccessfulClone({ + compose: 'services:\n web:\n image: nginx\n', + composePath: 'infra/base.yml', + extraFiles: { 'infra/prod.yml': 'services:\n web:\n environment:\n - Y=2\n' }, + sha, + }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const fsSvc = FileSystemService.getInstance(); + await fsSvc.createStack('multi-pull'); + + await svc.upsert({ + stackName: 'multi-pull', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['infra/base.yml', 'infra/prod.yml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + + const pull = await svc.pull('multi-pull'); + // Pending compose blob is encrypted; assert it round-trips by applying it. + const row = DatabaseService.getInstance().getGitSource('multi-pull'); + expect(row?.pending_commit_sha).toBe(sha); + + const applied = await svc.apply('multi-pull', pull.commitSha); + expect(applied.applied).toBe(true); + + const after = DatabaseService.getInstance().getGitSource('multi-pull'); + expect(after?.applied_deploy_spec?.files).toEqual(['compose.yaml', 'infra/prod.yml']); + expect(await fsSvc.getStackContent('multi-pull')).toContain('image: nginx'); + expect((await fsSvc.readStackFile('multi-pull', 'infra/prod.yml')).content).toContain('Y=2'); + + validateSpy.mockRestore(); + await cleanupStackDir('multi-pull'); + }); + + it('keeps pending across an upsert with the SAME config and clears it when compose_paths change', async () => { + const sha = '5555eee5555eee5555eee5555eee5555eee5555e'; + mockSuccessfulClone({ + compose: 'services:\n web:\n image: nginx\n', + composePath: 'infra/base.yml', + extraFiles: { + 'infra/prod.yml': 'services:\n web:\n environment:\n - Z=3\n', + // The changed-config upsert re-fetches this path; the dry-run reads + // every configured file, so it must exist in the clone. + 'infra/staging.yml': 'services:\n web:\n environment:\n - Z=4\n', + }, + sha, + }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + await FileSystemService.getInstance().createStack('pending-config'); + + const baseInput = { + stackName: 'pending-config', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['infra/base.yml', 'infra/prod.yml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none' as const, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }; + await svc.upsert(baseInput); + await svc.pull('pending-config'); + const db = DatabaseService.getInstance(); + expect(db.getGitSource('pending-config')?.pending_commit_sha).toBe(sha); + + // Same config -> pending must survive. + await svc.upsert(baseInput); + expect(db.getGitSource('pending-config')?.pending_commit_sha).toBe(sha); + + // Changed compose_paths -> the captured pending blob no longer matches, clear it. + await svc.upsert({ ...baseInput, composePaths: ['infra/base.yml', 'infra/staging.yml'] }); + expect(db.getGitSource('pending-config')?.pending_commit_sha).toBeNull(); + + validateSpy.mockRestore(); + await cleanupStackDir('pending-config'); + }); +}); diff --git a/backend/src/__tests__/image-update-service.test.ts b/backend/src/__tests__/image-update-service.test.ts index f1bd1aa4..0e4cabe2 100644 --- a/backend/src/__tests__/image-update-service.test.ts +++ b/backend/src/__tests__/image-update-service.test.ts @@ -43,6 +43,7 @@ vi.mock('../services/DatabaseService', () => ({ getInstance: () => ({ getGlobalSettings: mockGetGlobalSettings, getNodes: () => [], + getGitSource: () => undefined, upsertStackUpdateStatus: mockUpsertStackUpdateStatus, getStackUpdateStatus: mockGetStackUpdateStatus, clearStackUpdateStatus: mockClearStackUpdateStatus, @@ -551,6 +552,7 @@ describe('ImageUpdateService - check() concurrency guard', () => { dbModule.DatabaseService.getInstance = (() => ({ getGlobalSettings: () => ({ developer_mode: developerMode }), getNodes: () => [{ type: 'local', id: 1, name: 'local', mode: 'proxy', compose_dir: '/tmp/compose', is_default: true, status: 'online', created_at: 1 }], + getGitSource: () => undefined, upsertStackUpdateStatus: mockUpsertStackUpdateStatus, getStackUpdateStatus: mockGetStackUpdateStatus, clearStackUpdateStatus: mockClearStackUpdateStatus, diff --git a/backend/src/__tests__/network-topology.test.ts b/backend/src/__tests__/network-topology.test.ts index bbd823ea..57c0a1da 100644 --- a/backend/src/__tests__/network-topology.test.ts +++ b/backend/src/__tests__/network-topology.test.ts @@ -38,7 +38,7 @@ vi.mock('../services/NodeRegistry', () => ({ // Prevent COMPOSE_DIR filesystem reads (resolveProjectNameMap reads compose files). // With these mocked, fs.readFile returns ENOENT and the map falls back to // { stackName: stackName } for each stack passed in. -vi.mock('child_process', () => ({ exec: vi.fn() })); +vi.mock('child_process', () => ({ exec: vi.fn(), execFile: vi.fn() })); vi.mock('util', () => ({ promisify: () => vi.fn() })); import DockerController from '../services/DockerController'; diff --git a/backend/src/__tests__/self-identity-service.test.ts b/backend/src/__tests__/self-identity-service.test.ts index 81676eac..dbde94ab 100644 --- a/backend/src/__tests__/self-identity-service.test.ts +++ b/backend/src/__tests__/self-identity-service.test.ts @@ -29,7 +29,7 @@ vi.mock('../services/NodeRegistry', () => ({ }, })); -vi.mock('child_process', () => ({ exec: vi.fn() })); +vi.mock('child_process', () => ({ exec: vi.fn(), execFile: vi.fn() })); vi.mock('util', () => ({ promisify: () => vi.fn() })); import SelfIdentityService from '../services/SelfIdentityService'; diff --git a/backend/src/__tests__/webhooks-git-source.test.ts b/backend/src/__tests__/webhooks-git-source.test.ts index 7fba6713..bede3302 100644 --- a/backend/src/__tests__/webhooks-git-source.test.ts +++ b/backend/src/__tests__/webhooks-git-source.test.ts @@ -13,6 +13,30 @@ function adminToken(): string { return jwt.sign({ username: TEST_USERNAME, role: 'admin' }, TEST_JWT_SECRET, { expiresIn: '1m' }); } +function seedGitSource(stackName: string): void { + DatabaseService.getInstance().upsertGitSource({ + stack_name: stackName, + repo_url: 'https://github.com/example/repo.git', + branch: 'main', + compose_path: 'compose.yaml', + compose_paths: ['compose.yaml'], + context_dir: null, + sync_env: false, + env_path: null, + auth_type: 'none', + encrypted_token: null, + auto_apply_on_webhook: false, + auto_deploy_on_apply: false, + last_applied_commit_sha: null, + last_applied_content_hash: null, + pending_commit_sha: null, + pending_compose_content: null, + pending_env_content: null, + pending_fetched_at: null, + last_debounce_at: null, + }); +} + beforeAll(async () => { tmpDir = await setupTestDb(); ({ app } = await import('../index')); @@ -35,25 +59,7 @@ describe('node-aware Git source webhooks', () => { it('persists node_id when creating a webhook', async () => { const db = DatabaseService.getInstance(); const nodeId = db.getDefaultNode()!.id; - db.upsertGitSource({ - stack_name: 'webhook-local-git', - repo_url: 'https://github.com/example/repo.git', - branch: 'main', - compose_path: 'compose.yaml', - sync_env: false, - env_path: null, - auth_type: 'none', - encrypted_token: null, - auto_apply_on_webhook: false, - auto_deploy_on_apply: false, - last_applied_commit_sha: null, - last_applied_content_hash: null, - pending_commit_sha: null, - pending_compose_content: null, - pending_env_content: null, - pending_fetched_at: null, - last_debounce_at: null, - }); + seedGitSource('webhook-local-git'); const res = await request(app) .post('/api/webhooks') @@ -117,25 +123,7 @@ describe('node-aware Git source webhooks', () => { it('rejects retargeting an existing git-pull webhook without a Git source', async () => { const db = DatabaseService.getInstance(); const nodeId = db.getDefaultNode()!.id; - db.upsertGitSource({ - stack_name: 'retarget-source-stack', - repo_url: 'https://github.com/example/repo.git', - branch: 'main', - compose_path: 'compose.yaml', - sync_env: false, - env_path: null, - auth_type: 'none', - encrypted_token: null, - auto_apply_on_webhook: false, - auto_deploy_on_apply: false, - last_applied_commit_sha: null, - last_applied_content_hash: null, - pending_commit_sha: null, - pending_compose_content: null, - pending_env_content: null, - pending_fetched_at: null, - last_debounce_at: null, - }); + seedGitSource('retarget-source-stack'); const webhookId = db.addWebhook({ node_id: nodeId, name: 'retarget git webhook', diff --git a/backend/src/helpers/gitSourceSelection.ts b/backend/src/helpers/gitSourceSelection.ts new file mode 100644 index 00000000..2d214cb6 --- /dev/null +++ b/backend/src/helpers/gitSourceSelection.ts @@ -0,0 +1,114 @@ +import path from 'path'; +import { isValidGitSourcePath, isValidRelativeStackPath } from '../utils/validation'; +import { PRIMARY_COMPOSE_FILENAME, gitSourceLocalComposeFiles } from '../utils/gitComposeFiles'; + +/** Upper bound on how many compose files one stack can order. Generous; real + * base+override layouts use a handful. */ +export const MAX_COMPOSE_FILES = 10; +const MAX_COMPOSE_PATH_LENGTH = 1024; +const MAX_CONTEXT_DIR_LENGTH = 1024; + +export interface ComposeSelection { + composePaths: string[]; + contextDir: string | null; +} + +type ParseResult = + | { ok: true; value: ComposeSelection } + | { ok: false; error: string }; + +/** + * Validate and normalize the multi-file compose selection from a request body. + * Accepts `compose_paths` (ordered array) or the legacy `compose_path` (single + * string, mapped to a one-element array). Enforces the file-count cap, rejects + * duplicates and a root `compose.yaml` collision (the primary is always + * materialized there), and validates an optional `context_dir`. + */ +export function parseComposeSelection(body: unknown): ParseResult { + const b = (body ?? {}) as Record; + + let rawPaths: unknown = b.compose_paths; + if (rawPaths === undefined && typeof b.compose_path === 'string') { + rawPaths = [b.compose_path]; + } + if (!Array.isArray(rawPaths) || rawPaths.length === 0) { + return { ok: false, error: 'compose_paths must be a non-empty array of repository file paths' }; + } + if (rawPaths.length > MAX_COMPOSE_FILES) { + return { ok: false, error: `compose_paths cannot exceed ${MAX_COMPOSE_FILES} files` }; + } + + const composePaths: string[] = []; + for (const raw of rawPaths) { + if (typeof raw !== 'string' || !raw.trim()) { + return { ok: false, error: 'each compose path must be a non-empty string' }; + } + const trimmed = raw.trim(); + if (trimmed.length > MAX_COMPOSE_PATH_LENGTH) { + return { ok: false, error: 'a compose path is too long' }; + } + if (!isValidGitSourcePath(trimmed)) { + return { ok: false, error: `compose path must be a relative repository file path: ${trimmed}` }; + } + composePaths.push(trimmed); + } + + if (new Set(composePaths).size !== composePaths.length) { + return { ok: false, error: 'compose_paths cannot contain duplicate entries' }; + } + + // Materialized local layout: the primary (index 0) is written to the root + // compose.yaml, each additional file at its repo-relative path. Reject the + // collisions that would make materialization fail at runtime (ENOTDIR / clobber): + // an additional file equal to or nested under compose.yaml, and any file path + // that is a directory ancestor of another selected file. + const materialized = gitSourceLocalComposeFiles(composePaths); + const label = (k: number) => (k === 0 ? PRIMARY_COMPOSE_FILENAME : composePaths[k]); + for (let i = 0; i < materialized.length; i++) { + for (let j = 0; j < materialized.length; j++) { + if (i === j) continue; + if (materialized[j] === materialized[i] || materialized[j].startsWith(materialized[i] + '/')) { + return { ok: false, error: `compose file "${label(j)}" collides with "${label(i)}" once materialized to disk` }; + } + } + } + + let contextDir: string | null = null; + const rawCtx = b.context_dir; + if (rawCtx !== undefined && rawCtx !== null && rawCtx !== '') { + if (typeof rawCtx !== 'string') { + return { ok: false, error: 'context_dir must be a string' }; + } + if (rawCtx.length > MAX_CONTEXT_DIR_LENGTH) { + return { ok: false, error: 'context_dir is too long' }; + } + const ctx = rawCtx.trim().replace(/^\.\//, '').replace(/\/+$/, ''); + if (ctx !== '') { + if (!isValidRelativeStackPath(ctx)) { + return { ok: false, error: 'context_dir must be a relative path within the repository' }; + } + if (ctx.split('/').some(seg => seg.toLowerCase() === '.git')) { + return { ok: false, error: 'context_dir cannot target the .git directory' }; + } + // The project dir is mkdir'd, so it cannot equal a compose file path or sit + // beneath one (the primary compose.yaml and any override are files, not dirs). + if (materialized.some(f => ctx === f || ctx.startsWith(f + '/'))) { + return { ok: false, error: 'context_dir cannot match or be nested under a compose file path' }; + } + contextDir = ctx; + } + } + + return { ok: true, value: { composePaths, contextDir } }; +} + +/** + * Default the env path to a sibling `.env` of the primary compose file when env + * sync is on and no explicit path is provided. Mirrors the prior single-file + * behavior, keyed off the primary (compose_paths[0]). + */ +export function defaultEnvPath(primaryComposePath: string, explicit: unknown): string { + if (typeof explicit === 'string' && explicit.trim()) return explicit.trim(); + const dir = path.posix.dirname(primaryComposePath.replace(/\\/g, '/')) || '.'; + return path.posix.join(dir, '.env'); +} diff --git a/backend/src/routes/gitSources.ts b/backend/src/routes/gitSources.ts index 419bdb46..9c4577b8 100644 --- a/backend/src/routes/gitSources.ts +++ b/backend/src/routes/gitSources.ts @@ -1,11 +1,12 @@ import { Router, type Request, type Response } from 'express'; -import path from 'path'; import { GitSourceService } from '../services/GitSourceService'; import { FileSystemService } from '../services/FileSystemService'; import { DatabaseService } from '../services/DatabaseService'; +import { CryptoService } from '../services/CryptoService'; import { checkPermission, requirePermission } from '../middleware/permissions'; import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; import { triggerPostDeployScan } from '../helpers/policyGate'; +import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection'; import { isValidGitSourcePath, isValidStackName } from '../utils/validation'; import { sendGitSourceError, webhookPullStatus } from '../utils/gitSourceHttp'; import { sanitizeForLog } from '../utils/safeLog'; @@ -14,10 +15,59 @@ import { sanitizeForLog } from '../utils/safeLog'; // payloads. Generous compared to anything a real Git provider emits. const MAX_REPO_URL_LENGTH = 2048; const MAX_BRANCH_LENGTH = 256; -const MAX_COMPOSE_PATH_LENGTH = 1024; const MAX_ENV_PATH_LENGTH = 1024; const MAX_TOKEN_LENGTH = 8192; +/** + * Shared handler for the "browse repository" compose-file picker: validate the + * repo target, clone it, and list its files. `storedToken` (already decrypted) + * is reused when the request omits a token, so the edit-mode flow does not force + * re-entering a stored PAT. + */ +async function handleBrowse(req: Request, res: Response, storedToken: string | null): Promise { + const { repo_url, branch, auth_type, token } = req.body ?? {}; + if (typeof repo_url !== 'string' || !repo_url.trim()) { + res.status(400).json({ error: 'repo_url is required' }); + return; + } + if (typeof branch !== 'string' || !branch.trim()) { + res.status(400).json({ error: 'branch is required' }); + return; + } + if (!/^https:\/\//i.test(repo_url)) { + res.status(400).json({ error: 'Only HTTPS repository URLs are supported' }); + return; + } + if (repo_url.length > MAX_REPO_URL_LENGTH) { + res.status(400).json({ error: 'repo_url is too long' }); + return; + } + if (branch.length > MAX_BRANCH_LENGTH) { + res.status(400).json({ error: 'branch is too long' }); + return; + } + if (auth_type !== undefined && auth_type !== 'none' && auth_type !== 'token') { + res.status(400).json({ error: 'auth_type must be "none" or "token"' }); + return; + } + if (typeof token === 'string' && token.length > MAX_TOKEN_LENGTH) { + res.status(400).json({ error: 'token is too long' }); + return; + } + const explicitToken = typeof token === 'string' && token.trim() ? token : null; + const effectiveToken = auth_type === 'none' ? null : (explicitToken ?? storedToken); + try { + const result = await GitSourceService.getInstance().listRepoTree({ + repoUrl: repo_url.trim(), + branch: branch.trim(), + token: effectiveToken, + }); + res.json(result); + } catch (error) { + sendGitSourceError(res, error); + } +} + /** Router for listing git-source configuration: `GET /api/git-sources`. */ export const gitSourcesRouter = Router(); @@ -33,6 +83,13 @@ gitSourcesRouter.get('/', async (req: Request, res: Response): Promise => } }); +// Create-mode repo browse (no stack yet): gated by the same permission as +// creating a stack from Git. +gitSourcesRouter.post('/browse', async (req: Request, res: Response): Promise => { + if (!requirePermission(req, res, 'stack:create')) return; + await handleBrowse(req, res, null); +}); + /** * Router for per-stack git-source endpoints. Mount at `/api/stacks` so the * `/:stackName/git-source*` paths work alongside other stack-scoped routes @@ -80,7 +137,6 @@ stackGitSourceRouter.put('/:stackName/git-source', async (req: Request, res: Res const { repo_url, branch, - compose_path, sync_env, env_path, auth_type, @@ -97,8 +153,9 @@ stackGitSourceRouter.put('/:stackName/git-source', async (req: Request, res: Res res.status(400).json({ error: 'branch is required' }); return; } - if (typeof compose_path !== 'string' || !compose_path.trim()) { - res.status(400).json({ error: 'compose_path is required' }); + const selection = parseComposeSelection(req.body); + if (!selection.ok) { + res.status(400).json({ error: selection.error }); return; } if (auth_type !== 'none' && auth_type !== 'token') { @@ -125,18 +182,10 @@ stackGitSourceRouter.put('/:stackName/git-source', async (req: Request, res: Res res.status(400).json({ error: 'branch is too long' }); return; } - if (compose_path.length > MAX_COMPOSE_PATH_LENGTH) { - res.status(400).json({ error: 'compose_path is too long' }); - return; - } if (typeof env_path === 'string' && env_path.length > MAX_ENV_PATH_LENGTH) { res.status(400).json({ error: 'env_path is too long' }); return; } - if (!isValidGitSourcePath(compose_path.trim())) { - res.status(400).json({ error: 'compose_path must be a relative repository file path' }); - return; - } if (typeof env_path === 'string' && env_path.trim() && !isValidGitSourcePath(env_path.trim())) { res.status(400).json({ error: 'env_path must be a relative repository file path' }); return; @@ -160,16 +209,15 @@ stackGitSourceRouter.put('/:stackName/git-source', async (req: Request, res: Res const syncEnv = Boolean(sync_env); const resolvedEnvPath = syncEnv - ? (typeof env_path === 'string' && env_path.trim() - ? env_path - : path.posix.join(path.posix.dirname(compose_path.replace(/\\/g, '/')) || '.', '.env')) + ? defaultEnvPath(selection.value.composePaths[0], env_path) : null; const source = await GitSourceService.getInstance().upsert({ stackName, repoUrl: repo_url.trim(), branch: branch.trim(), - composePath: compose_path.trim(), + composePaths: selection.value.composePaths, + contextDir: selection.value.contextDir, syncEnv, envPath: resolvedEnvPath, authType: auth_type, @@ -193,6 +241,18 @@ stackGitSourceRouter.delete('/:stackName/git-source', async (req: Request, res: } if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; try { + // The deploy spec lives on the Git-source row, so unlinking a multi-file (or + // project-directory) source would silently drop the spec and revert future + // deploys to root compose.yaml auto-discovery, ignoring the override files + // still on disk. Refuse rather than change deploy semantics out from under the + // user; deleting the stack removes it cleanly. + const spec = DatabaseService.getInstance().getGitSource(stackName)?.applied_deploy_spec; + if (spec && (spec.files.length > 1 || spec.contextDir)) { + res.status(409).json({ + error: 'This stack deploys multiple compose files configured by its Git source. Unlinking would change it to deploy only compose.yaml. Delete the stack to remove it, or keep the Git source.', + }); + return; + } GitSourceService.getInstance().delete(stackName); console.log(`[GitSource] Removed git source for ${stackName}`); res.json({ success: true }); @@ -299,3 +359,18 @@ stackGitSourceRouter.post('/:stackName/git-source/dismiss-pending', async (req: sendGitSourceError(res, error); } }); + +// Edit-mode repo browse for an existing stack: gated by stack:edit so a user who +// can edit (but not create) stacks can re-pick files, and reuses the stored token +// when the request omits one. +stackGitSourceRouter.post('/:stackName/git-source/browse', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + const src = DatabaseService.getInstance().getGitSource(stackName); + const storedToken = src?.encrypted_token ? CryptoService.getInstance().decrypt(src.encrypted_token) : null; + await handleBrowse(req, res, storedToken); +}); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 1fa96219..c64bb8f1 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -34,6 +34,7 @@ import { sendGitSourceError } from '../utils/gitSourceHttp'; import { buildPolicyGateOptions, runPolicyGate, triggerPostDeployScan } from '../helpers/policyGate'; import { parseComposePreview, type ComposePreview } from '../helpers/composePreview'; import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; +import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection'; import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants'; import { getTerminalWs, DEPLOY_SESSION_HEADER } from '../websocket/generic'; @@ -751,7 +752,6 @@ stacksRouter.post('/from-git', async (req: Request, res: Response) => { stack_name, repo_url, branch, - compose_path, sync_env, env_path, auth_type, @@ -775,8 +775,9 @@ stacksRouter.post('/from-git', async (req: Request, res: Response) => { if (typeof branch !== 'string' || !branch.trim()) { return res.status(400).json({ error: 'branch is required' }); } - if (typeof compose_path !== 'string' || !compose_path.trim()) { - return res.status(400).json({ error: 'compose_path is required' }); + const selection = parseComposeSelection(req.body); + if (!selection.ok) { + return res.status(400).json({ error: selection.error }); } if (auto_apply_on_webhook !== undefined && typeof auto_apply_on_webhook !== 'boolean') { return res.status(400).json({ error: 'auto_apply_on_webhook must be a boolean' }); @@ -794,18 +795,12 @@ stacksRouter.post('/from-git', async (req: Request, res: Response) => { if (branch.length > 256) { return res.status(400).json({ error: 'branch is too long' }); } - if (compose_path.length > 1024) { - return res.status(400).json({ error: 'compose_path is too long' }); - } if (typeof env_path === 'string' && env_path.length > 1024) { return res.status(400).json({ error: 'env_path is too long' }); } if (typeof token === 'string' && token.length > 8192) { return res.status(400).json({ error: 'token is too long' }); } - if (!isValidGitSourcePath(compose_path.trim())) { - return res.status(400).json({ error: 'compose_path must be a relative repository file path' }); - } if (typeof env_path === 'string' && env_path.trim() && !isValidGitSourcePath(env_path.trim())) { return res.status(400).json({ error: 'env_path must be a relative repository file path' }); } @@ -821,14 +816,12 @@ stacksRouter.post('/from-git', async (req: Request, res: Response) => { const syncEnv = Boolean(sync_env); const resolvedEnvPath = syncEnv - ? (typeof env_path === 'string' && env_path.trim() - ? env_path - : path.posix.join(path.posix.dirname(compose_path.replace(/\\/g, '/')) || '.', '.env')) + ? defaultEnvPath(selection.value.composePaths[0], env_path) : null; if (fromGitDiag) { dlog( - `[Stacks:diag] from-git start stack=${sanitizeForLog(stack_name)} nodeId=${req.nodeId ?? 'local'} host=${sanitizeForLog(gitRepoHost(repo_url))} branch=${sanitizeForLog(branch)} composePath=${sanitizeForLog(compose_path)} envPath=${sanitizeForLog(resolvedEnvPath ?? 'none')} authType=${sanitizeForLog(resolvedAuthType)} autoApplyOnWebhook=${autoApplyOnWebhook} autoDeployOnApply=${autoDeployOnApply} deployNow=${deploy_now === true}` + `[Stacks:diag] from-git start stack=${sanitizeForLog(stack_name)} nodeId=${req.nodeId ?? 'local'} host=${sanitizeForLog(gitRepoHost(repo_url))} branch=${sanitizeForLog(branch)} files=${selection.value.composePaths.length} envPath=${sanitizeForLog(resolvedEnvPath ?? 'none')} authType=${sanitizeForLog(resolvedAuthType)} autoApplyOnWebhook=${autoApplyOnWebhook} autoDeployOnApply=${autoDeployOnApply} deployNow=${deploy_now === true}` ); } @@ -836,7 +829,8 @@ stacksRouter.post('/from-git', async (req: Request, res: Response) => { stackName: stack_name.trim(), repoUrl: repo_url.trim(), branch: branch.trim(), - composePath: compose_path.trim(), + composePaths: selection.value.composePaths, + contextDir: selection.value.contextDir, syncEnv, envPath: resolvedEnvPath, authType: resolvedAuthType, diff --git a/backend/src/services/ComposeService.ts b/backend/src/services/ComposeService.ts index 5413f7a9..4d7f1688 100644 --- a/backend/src/services/ComposeService.ts +++ b/backend/src/services/ComposeService.ts @@ -16,6 +16,7 @@ import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; import { describeSpawnError } from '../utils/spawnErrors'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; +import { authoredComposeFileArgs } from '../utils/authoredComposeArgs'; import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; export class ComposeRollbackError extends Error { @@ -86,14 +87,24 @@ export class ComposeService { } /** - * Build the `docker compose` argument prefix for a stack, splicing in the - * Sencho Mesh override file if the stack is opted into the mesh. When no - * override applies, returns args without `-f` so docker compose's built-in - * file discovery resolves the stack's actual compose filename. The user's - * source compose file is never mutated. + * Build the authored `docker compose` argument list for a stack: the validated + * multi-file deploy prefix (ordered `-f` files + `-p ` + + * `--project-directory`) for a Git source with an applied multi-file spec, then + * the Sencho Mesh override file last (highest `-f` precedence) when the stack is + * opted into the mesh, then the action. Single-file / non-git stacks get no file + * prefix, so docker compose's built-in discovery resolves the root compose.yaml, + * byte-identical to the pre-multi-file behavior. The user's source files are + * never mutated. Lifecycle commands (deploy, update, stop/start/restart/down) + * route through this method, so they share one file prefix plus the mesh override. + * Image scans (listStackImages) and the Compose Doctor (renderConfig) reuse the + * same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh + * override, rendering the user's authored model without mesh injection. */ - private async composeArgs(stackName: string, action: string[]): Promise { + private async authoredComposeArgs(stackName: string, action: string[]): Promise { const args: string[] = ['compose']; + const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + args.push(...filePrefix); + let overridePath: string | null = null; try { overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName); @@ -101,8 +112,13 @@ export class ComposeService { console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message)); } if (overridePath) { - const baseFilename = await FileSystemService.getInstance(this.nodeId).getComposeFilename(stackName); - args.push('-f', baseFilename, '-f', overridePath); + if (filePrefix.length === 0) { + // Single-file stack: passing any -f disables auto-discovery, so name the + // base file explicitly before layering the override on top of it. + const baseFilename = await FileSystemService.getInstance(this.nodeId).getComposeFilename(stackName); + args.push('-f', baseFilename); + } + args.push('-f', overridePath); } args.push(...action); return args; @@ -323,7 +339,7 @@ export class ComposeService { const fsSvc = FileSystemService.getInstance(this.nodeId); await fsSvc.restoreStackFiles(stackName); await this.withRegistryAuth(async (env) => { - await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env); }, sendOutput); sendOutput('=== Rolled back successfully ===\n'); return true; @@ -342,7 +358,7 @@ export class ComposeService { async runCommand(stackName: string, action: 'down' | 'start' | 'stop' | 'restart', ws?: WebSocket): Promise { const stackDir = path.join(this.baseDir, stackName); - await this.execute('docker', ['compose', action], stackDir, ws); + await this.execute('docker', await this.authoredComposeArgs(stackName, [action]), stackDir, ws); } async deployStack(stackName: string, ws?: WebSocket, atomic?: boolean): Promise { @@ -371,7 +387,7 @@ export class ComposeService { } await this.withRegistryAuth(async (env) => { - await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); }, sendOutput); // Post-Deploy Health Probe @@ -548,10 +564,10 @@ export class ComposeService { await this.withRegistryAuth(async (env) => { sendOutput('=== Pulling latest images ===\n'); - await this.execute('docker', ['compose', 'pull'], stackDir, ws, true, env, getComposeStallTimeoutMs()); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull']), stackDir, ws, true, env, getComposeStallTimeoutMs()); sendOutput('=== Recreating containers ===\n'); - await this.execute('docker', await this.composeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); }, sendOutput); // Post-Update Health Probe @@ -611,7 +627,7 @@ export class ComposeService { public async downStack(stackName: string): Promise { const stackPath = path.join(this.baseDir, stackName); try { - await this.execute('docker', ['compose', 'down', '--volumes', '--remove-orphans'], stackPath, undefined, false); + await this.execute('docker', await this.authoredComposeArgs(stackName, ['down', '--volumes', '--remove-orphans']), stackPath, undefined, false); } catch (error) { console.warn(`[Teardown] Docker down failed or nothing to clean up for ${sanitizeForLog(stackName)}`); } @@ -634,7 +650,10 @@ export class ComposeService { if (!isPathWithinBase(stackDir, this.baseDir) || path.resolve(this.baseDir) === stackDir) { throw new Error('Invalid stack path'); } - const stdout = await this.captureCompose(['config', '--images'], stackDir); + // Use the authored multi-file model (no mesh override) so override-only image + // refs are scanned by the policy gate; single-file stacks get an empty prefix. + const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + const stdout = await this.captureCompose([...filePrefix, 'config', '--images'], stackDir); const seen = new Set(); const images: string[] = []; for (const raw of stdout.split(/\r?\n/)) { @@ -705,7 +724,15 @@ export class ComposeService { if (!stackDir.startsWith(baseResolved + path.sep)) { return Promise.reject(new Error('Invalid stack path')); } - const child = spawn('docker', ['compose', 'config', '--format', 'json'], { + // Render the authored multi-file model (no mesh override) so the Compose Doctor + // sees every override file; single-file stacks get an empty prefix. + let filePrefix: string[]; + try { + filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + } catch (err) { + return Promise.reject(err instanceof Error ? err : new Error(String(err))); + } + const child = spawn('docker', ['compose', ...filePrefix, 'config', '--format', 'json'], { cwd: stackDir, env: { ...process.env, diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index fbaf8b94..957dff43 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -221,12 +221,28 @@ export interface Webhook { export type GitSourceAuthType = 'none' | 'token'; +/** + * The ordered set of local compose files actually materialized on disk for a + * stack, plus the optional project directory. This is the deploy-time source of + * truth: the deploy/lifecycle path, the mesh service, and the image-update path all + * read it (never the configured `compose_paths`), so a saved-but-not-yet-applied config change does + * not alter runtime until apply writes the files. `null` (the default) means a + * plain single-file stack that uses docker compose auto-discovery, byte-identical + * to pre-multi-file behavior. + */ +export interface GitSourceAppliedSpec { + files: string[]; // ordered local relative compose filenames (files[0] is always compose.yaml) + contextDir: string | null; // optional project dir within the stack, passed as --project-directory +} + export interface StackGitSource { id?: number; stack_name: string; repo_url: string; branch: string; - compose_path: string; + compose_path: string; // primary repo path; kept in sync with compose_paths[0] for back-compat readers + compose_paths: string[]; // ordered repo-relative compose paths; normalized to [compose_path] for legacy rows + context_dir: string | null; // configured project dir within the repo sync_env: boolean; env_path: string | null; auth_type: GitSourceAuthType; @@ -240,6 +256,7 @@ export interface StackGitSource { pending_env_content: string | null; pending_fetched_at: number | null; last_debounce_at: number | null; + applied_deploy_spec: GitSourceAppliedSpec | null; // deploy-time materialized file set; null = single-file auto-discovery created_at: number; updated_at: number; } @@ -786,6 +803,7 @@ export class DatabaseService { this.migrateAutoHealNodeId(); this.migrateFleetSyncStickyError(); this.migrateStackDossierHashes(); + this.migrateGitSourceMultiFile(); // Reset the cache once at end of constructor in case any migration // populated it via getGlobalSettings() and a subsequent migration @@ -1195,6 +1213,8 @@ export class DatabaseService { repo_url TEXT NOT NULL, branch TEXT NOT NULL, compose_path TEXT NOT NULL, + compose_paths TEXT, + context_dir TEXT, sync_env INTEGER NOT NULL DEFAULT 0, env_path TEXT, auth_type TEXT NOT NULL DEFAULT 'none', @@ -1208,6 +1228,7 @@ export class DatabaseService { pending_env_content TEXT, pending_fetched_at INTEGER, last_debounce_at INTEGER, + applied_deploy_spec TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); @@ -1649,6 +1670,21 @@ export class DatabaseService { this.tryAddColumn('stack_dossiers', 'rendered_hash', 'TEXT'); } + private migrateGitSourceMultiFile(): void { + this.tryAddColumn('stack_git_sources', 'compose_paths', 'TEXT'); + this.tryAddColumn('stack_git_sources', 'context_dir', 'TEXT'); + this.tryAddColumn('stack_git_sources', 'applied_deploy_spec', 'TEXT'); + // Backfill legacy rows so compose_paths is always a JSON array. parseGitSource + // also normalizes on read, so this is belt-and-suspenders for any direct readers. + try { + this.db.prepare( + `UPDATE stack_git_sources SET compose_paths = json_array(compose_path) WHERE compose_paths IS NULL` + ).run(); + } catch (e) { + console.warn('[DatabaseService] git-source compose_paths backfill skipped:', (e as Error).message); + } + } + private migrateScanPolicyFleetColumns(): void { this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''"); this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0'); @@ -3736,14 +3772,51 @@ export class DatabaseService { // --- Stack Git Sources --- + /** + * Normalize the stored compose_paths column to a non-empty ordered array. + * Legacy rows (column null), empty arrays, or malformed JSON all fall back + * to the single compose_path so every consumer can rely on a usable list. + */ + private normalizeComposePaths(raw: unknown, composePath: string): string[] { + if (typeof raw === 'string' && raw.trim()) { + try { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) { + const paths = parsed.filter((p): p is string => typeof p === 'string' && p.trim().length > 0); + if (paths.length > 0) return paths; + } + } catch { + // fall through to the single-path fallback + } + } + return [composePath]; + } + + private parseAppliedDeploySpec(raw: unknown): GitSourceAppliedSpec | null { + if (typeof raw !== 'string' || !raw.trim()) return null; + try { + const parsed = JSON.parse(raw) as Partial; + if (!parsed || !Array.isArray(parsed.files)) return null; + const files = parsed.files.filter((f): f is string => typeof f === 'string' && f.trim().length > 0); + if (files.length === 0) return null; + return { files, contextDir: typeof parsed.contextDir === 'string' ? parsed.contextDir : null }; + } catch { + return null; + } + } + private parseGitSource(row: Record | undefined): StackGitSource | undefined { if (!row) return undefined; + const composePath = row.compose_path as string; return { id: row.id as number, stack_name: row.stack_name as string, repo_url: row.repo_url as string, branch: row.branch as string, - compose_path: row.compose_path as string, + compose_path: composePath, + compose_paths: this.normalizeComposePaths(row.compose_paths, composePath), + context_dir: (row.context_dir as string | null) ?? null, + applied_deploy_spec: this.parseAppliedDeploySpec(row.applied_deploy_spec), sync_env: Number(row.sync_env) === 1, env_path: (row.env_path as string | null) ?? null, auth_type: row.auth_type as GitSourceAuthType, @@ -3772,19 +3845,21 @@ export class DatabaseService { return rows.map(r => this.parseGitSource(r)!); } - public upsertGitSource(source: Omit): number { + public upsertGitSource(source: Omit): number { const now = Date.now(); const existing = this.getGitSource(source.stack_name); + const composePathsJson = JSON.stringify(source.compose_paths ?? [source.compose_path]); if (existing) { this.db.prepare( `UPDATE stack_git_sources SET - repo_url = ?, branch = ?, compose_path = ?, sync_env = ?, env_path = ?, + repo_url = ?, branch = ?, compose_path = ?, compose_paths = ?, context_dir = ?, + sync_env = ?, env_path = ?, auth_type = ?, encrypted_token = ?, auto_apply_on_webhook = ?, auto_deploy_on_apply = ?, updated_at = ? WHERE stack_name = ?` ).run( - source.repo_url, source.branch, source.compose_path, + source.repo_url, source.branch, source.compose_path, composePathsJson, source.context_dir, source.sync_env ? 1 : 0, source.env_path, source.auth_type, source.encrypted_token, source.auto_apply_on_webhook ? 1 : 0, source.auto_deploy_on_apply ? 1 : 0, @@ -3794,12 +3869,12 @@ export class DatabaseService { } const result = this.db.prepare( `INSERT INTO stack_git_sources - (stack_name, repo_url, branch, compose_path, sync_env, env_path, + (stack_name, repo_url, branch, compose_path, compose_paths, context_dir, sync_env, env_path, auth_type, encrypted_token, auto_apply_on_webhook, auto_deploy_on_apply, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( - source.stack_name, source.repo_url, source.branch, source.compose_path, + source.stack_name, source.repo_url, source.branch, source.compose_path, composePathsJson, source.context_dir, source.sync_env ? 1 : 0, source.env_path, source.auth_type, source.encrypted_token, source.auto_apply_on_webhook ? 1 : 0, source.auto_deploy_on_apply ? 1 : 0, @@ -3808,6 +3883,18 @@ export class DatabaseService { return result.lastInsertRowid as number; } + /** + * Persist (or clear) the deploy-time materialized compose file set. Called by + * the Git apply/create paths after files land on disk; the deploy/lifecycle + * path, the mesh service, and the image-update path read it back via + * getGitSource(). Passing null resets the stack to single-file auto-discovery. + */ + public setGitSourceAppliedSpec(stackName: string, spec: GitSourceAppliedSpec | null): void { + this.db.prepare( + `UPDATE stack_git_sources SET applied_deploy_spec = ?, updated_at = ? WHERE stack_name = ?` + ).run(spec ? JSON.stringify(spec) : null, Date.now(), stackName); + } + public deleteGitSource(stackName: string): void { this.db.prepare('DELETE FROM stack_git_sources WHERE stack_name = ?').run(stackName); } diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 9b69285a..40d1c38f 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -1,6 +1,6 @@ import Docker from 'dockerode'; import WebSocket from 'ws'; -import { exec } from 'child_process'; +import { execFile } from 'child_process'; import { promisify } from 'util'; import path from 'path'; import fs from 'fs/promises'; @@ -13,8 +13,9 @@ import { isPathWithinBase } from '../utils/validation'; import { isDebugEnabled } from '../utils/debug'; import { sanitizeForLog } from '../utils/safeLog'; import { describeSpawnError } from '../utils/spawnErrors'; +import { authoredComposeFileArgs } from '../utils/authoredComposeArgs'; -const execAsync = promisify(exec); +const execFileAsync = promisify(execFile); const COMPOSE_DIR = process.env.COMPOSE_DIR || '/app/compose'; /** Canonical compose file name variants, checked in priority order. */ @@ -1277,16 +1278,28 @@ class DockerController { } public async getContainersByStack(stackName: string) { - const stackDir = path.join(COMPOSE_DIR, stackName); + // Resolve the compose dir and the authored prefix for THIS controller's node, + // not the process default, so a non-default local node sees its own stack dir + // and deploy spec. + const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName); try { - const { stdout, stderr } = await execAsync('docker compose ps --format json -a', { - cwd: stackDir, - env: { - ...process.env, - PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + // Splice the authored multi-file prefix (-f files + -p + --project-directory) + // so a Git stack's override-only services are listed; single-file stacks get an + // empty prefix and behave exactly as before. execFile avoids shell quoting on + // the absolute --project-directory path. + const filePrefix = authoredComposeFileArgs(stackName, this.nodeId); + const { stdout, stderr } = await execFileAsync( + 'docker', + ['compose', ...filePrefix, 'ps', '--format', 'json', '-a'], + { + cwd: stackDir, + env: { + ...process.env, + PATH: process.env.PATH || '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + } } - }); + ); // Robust JSON parsing - handle both JSON array and newline-separated JSON objects // Docker Compose v2 may return either format depending on version diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index 25577ea6..67d89976 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -5,7 +5,7 @@ import os from 'os'; import path from 'path'; import YAML from 'yaml'; import { CryptoService } from './CryptoService'; -import { DatabaseService, type StackGitSource, type GitSourceAuthType } from './DatabaseService'; +import { DatabaseService, type StackGitSource, type GitSourceAuthType, type GitSourceAppliedSpec } from './DatabaseService'; import { FileSystemService } from './FileSystemService'; import { ComposeService } from './ComposeService'; import { HealthGateService } from './HealthGateService'; @@ -13,7 +13,8 @@ import { NodeRegistry } from './NodeRegistry'; import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate'; import { isDebugEnabled } from '../utils/debug'; import { sanitizeForLog } from '../utils/safeLog'; -import { isPathWithinBase } from '../utils/validation'; +import { isPathWithinBase, isValidRelativeStackPath } from '../utils/validation'; +import { gitSourceLocalComposeFiles, PRIMARY_COMPOSE_FILENAME } from '../utils/gitComposeFiles'; import type { GitHttpRequest, GitHttpResponse, HttpClient } from 'isomorphic-git/http/node'; // isomorphic-git is the heaviest dependency in the backend (~5 MB) and only @@ -170,17 +171,23 @@ export class GitSourceError extends Error { } } +/** A single compose file fetched from a repo, keyed by its repo-relative path. */ +export interface ComposeFile { + path: string; + content: string; +} + export interface FetchParams { repoUrl: string; branch: string; - composePath: string; + composePaths: string[]; envPath?: string | null; token?: string | null; timeoutMs?: number; } export interface FetchResult { - composeContent: string; + composeFiles: ComposeFile[]; envContent: string | null; commitSha: string; /** @@ -195,7 +202,8 @@ export interface UpsertInput { stackName: string; repoUrl: string; branch: string; - composePath: string; + composePaths: string[]; + contextDir: string | null; syncEnv: boolean; envPath: string | null; authType: GitSourceAuthType; @@ -208,7 +216,8 @@ export interface CreateStackFromGitInput { stackName: string; repoUrl: string; branch: string; - composePath: string; + composePaths: string[]; + contextDir: string | null; syncEnv: boolean; envPath: string | null; authType: GitSourceAuthType; @@ -240,6 +249,8 @@ export interface PublicGitSource { repo_url: string; branch: string; compose_path: string; + compose_paths: string[]; + context_dir: string | null; sync_env: boolean; env_path: string | null; auth_type: GitSourceAuthType; @@ -536,6 +547,8 @@ export class GitSourceService { repo_url: src.repo_url, branch: src.branch, compose_path: src.compose_path, + compose_paths: src.compose_paths, + context_dir: src.context_dir, sync_env: src.sync_env, env_path: src.env_path, auth_type: src.auth_type, @@ -583,23 +596,39 @@ export class GitSourceService { throw new GitSourceError('GIT_ERROR', 'Auto-deploy requires auto-apply-on-webhook to be enabled.'); } - // Dry-run reachability check before persisting. + // Dry-run reachability check before persisting. Fetches every configured + // file so a bad path in the ordered list is caught at save time. const token = encryptedToken ? this.crypto.decrypt(encryptedToken) : null; await this.fetchFromGit({ repoUrl: input.repoUrl, branch: input.branch, - composePath: input.composePath, + composePaths: input.composePaths, envPath: input.syncEnv ? input.envPath : null, token, }); + const resolvedEnvPath = input.syncEnv ? input.envPath : null; + // A pending pull captured the files/contextDir for the previous config. If + // any of those change, that pending blob would apply the wrong files, so + // clear it; the user re-pulls against the new config. + const configChanged = !!existing && ( + existing.repo_url !== input.repoUrl || + existing.branch !== input.branch || + JSON.stringify(existing.compose_paths) !== JSON.stringify(input.composePaths) || + existing.sync_env !== input.syncEnv || + (existing.env_path ?? null) !== resolvedEnvPath || + (existing.context_dir ?? null) !== input.contextDir + ); + db.upsertGitSource({ stack_name: input.stackName, repo_url: input.repoUrl, branch: input.branch, - compose_path: input.composePath, + compose_path: input.composePaths[0], + compose_paths: input.composePaths, + context_dir: input.contextDir, sync_env: input.syncEnv, - env_path: input.syncEnv ? input.envPath : null, + env_path: resolvedEnvPath, auth_type: input.authType, encrypted_token: encryptedToken, auto_apply_on_webhook: input.autoApplyOnWebhook, @@ -613,6 +642,10 @@ export class GitSourceService { last_debounce_at: existing?.last_debounce_at ?? null, }); + if (configChanged) { + db.clearGitSourcePending(input.stackName); + } + return this.get(input.stackName)!; } @@ -622,29 +655,23 @@ export class GitSourceService { // ─── Fetch ─────────────────────────────────────────────────────────────── - public async fetchFromGit(params: FetchParams): Promise { - const { repoUrl, branch, composePath, envPath, token } = params; + /** + * Clone a repo into a throwaway temp dir, run `fn` against the checkout, and + * always clean up. Centralizes the clone timeout, size cap, commit-sha read, + * and submodule warning so both fetchFromGit (reads compose/env files) and + * listRepoTree (lists the working tree) share one hardened clone path. + */ + private async withClonedRepo( + params: { repoUrl: string; branch: string; token?: string | null; timeoutMs?: number }, + fn: (dir: string, commitSha: string, warnings: string[]) => Promise, + ): Promise { + const { repoUrl, branch, token } = params; const timeoutMs = params.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS; - - // Reject any compose/env target that resolves inside the `.git` - // metadata directory BEFORE we spin up a clone. This blocks a - // caller from reading `.git/config` (which leaks the remote URL - // and any mis-configured inline credentials) via the fetch path. - assertNotGitMeta(composePath, 'compose_path'); - if (envPath) assertNotGitMeta(envPath, 'env_path'); - const dir = await createTempDir(); - const startedAt = Date.now(); - const diag = isDebugEnabled(); - if (diag) { - console.log( - `[GitSource:diag] fetch start host=${sanitizeForLog(repoHost(repoUrl))} branch=${sanitizeForLog(branch)} compose=${sanitizeForLog(composePath)} envSync=${envPath ? 'true' : 'false'} timeoutMs=${timeoutMs}` - ); - } - // isomorphic-git's onAuth callback hands credentials to the HTTP - // layer without them touching the URL string, which keeps tokens - // out of any error messages generated during the clone. + // isomorphic-git's onAuth callback hands credentials to the HTTP layer + // without them touching the URL string, keeping tokens out of any error + // messages generated during the clone. const onAuth = token ? () => ({ username: 'x-access-token', password: token }) : undefined; @@ -698,38 +725,6 @@ export class GitSourceService { } const commitSha = log[0].oid; - const composeContent = await readRepoFile(dir, composePath, 'Compose path'); - if (isLfsPointer(composeContent)) { - console.error(`[GitSource] LFS pointer detected in ${sanitizeForLog(composePath)}`); - throw new GitSourceError( - 'GIT_ERROR', - `Compose file at ${composePath} is stored in Git LFS, which is not supported. Commit the plain file or replace the LFS pointer before linking this repository.`, - ); - } - - let envContent: string | null = null; - if (envPath) { - try { - envContent = await readRepoFile(dir, envPath, 'Env path'); - } catch (e) { - if (e instanceof GitSourceError && e.code === 'FILE_NOT_FOUND' && e.message.startsWith('File not found')) { - // A missing sibling .env is legitimate (repo may not carry one - // in the requested directory). Return null so the caller can - // decide whether to warn. - envContent = null; - } else { - throw e; - } - } - if (envContent !== null && isLfsPointer(envContent)) { - console.error(`[GitSource] LFS pointer detected in ${sanitizeForLog(envPath)}`); - throw new GitSourceError( - 'GIT_ERROR', - `Env file at ${envPath} is stored in Git LFS, which is not supported. Commit the plain file or replace the LFS pointer before linking this repository.`, - ); - } - } - // Submodule detection: non-fatal, surfaced as a warning. isomorphic-git // does not recursively clone submodules, so any path that lives inside // a submodule directory will be empty after apply. Users need to know. @@ -739,12 +734,75 @@ export class GitSourceService { warnings.push(SUBMODULE_WARNING); } - if (diag) { - console.log( - `[GitSource:diag] fetch ok host=${sanitizeForLog(repoHost(repoUrl))} branch=${sanitizeForLog(branch)} sha=${commitSha.slice(0, 7)} env=${envContent !== null ? 'present' : 'absent'} warnings=${warnings.length} elapsedMs=${Date.now() - startedAt}` - ); - } - return { composeContent, envContent, commitSha, warnings }; + return await fn(dir, commitSha, warnings); + } finally { + await removeTempDir(dir); + } + } + + public async fetchFromGit(params: FetchParams): Promise { + const { repoUrl, branch, composePaths, envPath, token } = params; + + // Reject any compose/env target that resolves inside the `.git` + // metadata directory BEFORE we spin up a clone. This blocks a + // caller from reading `.git/config` (which leaks the remote URL + // and any mis-configured inline credentials) via the fetch path. + for (const composePath of composePaths) assertNotGitMeta(composePath, 'compose_path'); + if (envPath) assertNotGitMeta(envPath, 'env_path'); + + const startedAt = Date.now(); + const diag = isDebugEnabled(); + if (diag) { + console.log( + `[GitSource:diag] fetch start host=${sanitizeForLog(repoHost(repoUrl))} branch=${sanitizeForLog(branch)} files=${composePaths.length} envSync=${envPath ? 'true' : 'false'}` + ); + } + + try { + return await this.withClonedRepo({ repoUrl, branch, token, timeoutMs: params.timeoutMs }, async (dir, commitSha, warnings) => { + const composeFiles: ComposeFile[] = []; + for (const composePath of composePaths) { + const content = await readRepoFile(dir, composePath, 'Compose path'); + if (isLfsPointer(content)) { + console.error(`[GitSource] LFS pointer detected in ${sanitizeForLog(composePath)}`); + throw new GitSourceError( + 'GIT_ERROR', + `Compose file at ${composePath} is stored in Git LFS, which is not supported. Commit the plain file or replace the LFS pointer before linking this repository.`, + ); + } + composeFiles.push({ path: composePath, content }); + } + + let envContent: string | null = null; + if (envPath) { + try { + envContent = await readRepoFile(dir, envPath, 'Env path'); + } catch (e) { + if (e instanceof GitSourceError && e.code === 'FILE_NOT_FOUND' && e.message.startsWith('File not found')) { + // A missing sibling .env is legitimate (repo may not carry one + // in the requested directory). Return null so the caller can + // decide whether to warn. + envContent = null; + } else { + throw e; + } + } + if (envContent !== null && isLfsPointer(envContent)) { + console.error(`[GitSource] LFS pointer detected in ${sanitizeForLog(envPath)}`); + throw new GitSourceError( + 'GIT_ERROR', + `Env file at ${envPath} is stored in Git LFS, which is not supported. Commit the plain file or replace the LFS pointer before linking this repository.`, + ); + } + } + + if (diag) { + console.log( + `[GitSource:diag] fetch ok host=${sanitizeForLog(repoHost(repoUrl))} branch=${sanitizeForLog(branch)} sha=${commitSha.slice(0, 7)} files=${composeFiles.length} env=${envContent !== null ? 'present' : 'absent'} warnings=${warnings.length} elapsedMs=${Date.now() - startedAt}` + ); + } + return { composeFiles, envContent, commitSha, warnings }; + }); } catch (err) { if (diag) { const msg = err instanceof GitSourceError ? `${err.code}: ${err.message}` : (err as Error).message; @@ -753,11 +811,56 @@ export class GitSourceService { ); } throw err; - } finally { - await removeTempDir(dir); } } + /** + * Clone a repo and list its working-tree files (POSIX-relative, `.git` + * skipped) for the "browse repository" compose-file picker. Bounded by the + * same clone size/timeout guards as fetch, plus a file-count cap. + */ + public async listRepoTree( + params: { repoUrl: string; branch: string; token?: string | null; timeoutMs?: number }, + ): Promise<{ files: string[]; truncated: boolean; commitSha: string; warnings: string[] }> { + return this.withClonedRepo(params, async (dir, commitSha, warnings) => { + const { files, truncated } = await this.walkRepoFiles(dir); + return { files, truncated, commitSha, warnings }; + }); + } + + private async walkRepoFiles(rootDir: string): Promise<{ files: string[]; truncated: boolean }> { + const MAX_FILES = 2000; + const files: string[] = []; + let truncated = false; + const walk = async (relDir: string): Promise => { + if (truncated) return; + let entries: import('fs').Dirent[]; + try { + entries = await fsPromises.readdir(path.join(rootDir, relDir), { withFileTypes: true }); + } catch (e) { + // A readdir failure on a just-cloned subtree silently drops it from the + // picker; log so a partial listing is traceable (the user can still + // add paths manually). + console.warn(`[GitSource] repo walk skipped ${sanitizeForLog(relDir || '.')}:`, (e as Error).message); + return; + } + for (const entry of entries) { + if (truncated) return; + if (entry.name === '.git') continue; + const rel = relDir ? `${relDir}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + await walk(rel); + } else if (entry.isFile()) { + if (files.length >= MAX_FILES) { truncated = true; return; } + files.push(rel); + } + } + }; + await walk(''); + files.sort(); + return { files, truncated }; + } + private mapGitError(err: Error, hasToken: boolean, host = 'unknown'): GitSourceError { const raw = scrubCredentials(err.message || String(err)); const code = (err as Error & { code?: string }).code; @@ -833,26 +936,55 @@ export class GitSourceService { * errors, invalid `include:` references, etc., which a shallow schema * check would miss. */ - public async validateCompose(composeContent: string, envContent: string | null): Promise<{ ok: boolean; error?: string }> { - // Cheap syntax pre-check - try { - const parsed = YAML.parse(composeContent); - if (parsed === null || parsed === undefined) { - return { ok: false, error: 'Compose file is empty.' }; + public async validateCompose(composeFiles: ComposeFile[], envContent: string | null, contextDir: string | null): Promise<{ ok: boolean; error?: string }> { + if (composeFiles.length === 0) return { ok: false, error: 'No compose files provided.' }; + + // Cheap syntax pre-check per file + for (const file of composeFiles) { + try { + const parsed = YAML.parse(file.content); + if (parsed === null || parsed === undefined) { + return { ok: false, error: `Compose file ${file.path} is empty.` }; + } + if (typeof parsed !== 'object' || Array.isArray(parsed)) { + return { ok: false, error: `Compose file ${file.path} must be a YAML mapping.` }; + } + } catch (e) { + return { ok: false, error: `YAML parse error in ${file.path}: ${(e as Error).message}` }; } - if (typeof parsed !== 'object' || Array.isArray(parsed)) { - return { ok: false, error: 'Compose file must be a YAML mapping.' }; - } - } catch (e) { - return { ok: false, error: `YAML parse error: ${(e as Error).message}` }; } - // Semantic check via `docker compose config` + // Semantic check via `docker compose config` over the ordered set, written + // in the same local layout the deploy materializes (primary -> compose.yaml, + // additional files under their repo-relative paths), with each path segment + // re-sanitized for this throwaway dir. So the merge order, project directory, + // and relative cross-references resolve from the same base the real deploy uses. const dir = await createTempDir(); try { - const composeFile = path.join(dir, 'compose.yaml'); - await fsPromises.writeFile(composeFile, composeContent, 'utf-8'); - const args = ['compose', '-f', composeFile]; + const localFiles = gitSourceLocalComposeFiles(composeFiles.map(f => f.path)); + const args = ['compose']; + for (let i = 0; i < composeFiles.length; i++) { + const safeRel = localFiles[i].replace(/\\/g, '/').split('/').map(s => path.basename(s)).join('/'); + const abs = path.resolve(dir, safeRel); + if (!isPathWithinBase(abs, dir)) { + return { ok: false, error: `Compose path escapes the validation dir: ${composeFiles[i].path}` }; + } + await fsPromises.mkdir(path.dirname(abs), { recursive: true }); + await fsPromises.writeFile(abs, composeFiles[i].content, 'utf-8'); + args.push('-f', safeRel); + } + if (contextDir) { + // Inline path-injection barrier at the mkdir sink. CodeQL does not + // credit the wrapped isPathWithinBase helper, so resolve against a + // known-safe base and check containment with startsWith right here. + const baseResolved = path.resolve(dir); + const ctxAbs = path.resolve(baseResolved, contextDir); + if (!ctxAbs.startsWith(baseResolved + path.sep)) { + return { ok: false, error: 'Context directory escapes the validation dir.' }; + } + await fsPromises.mkdir(ctxAbs, { recursive: true }); + args.push('--project-directory', ctxAbs); + } if (envContent !== null) { const envFile = path.join(dir, '.env'); await fsPromises.writeFile(envFile, envContent, 'utf-8'); @@ -891,21 +1023,90 @@ export class GitSourceService { // ─── Hashing + diff ────────────────────────────────────────────────────── - public hashContent(compose: string, env: string | null): string { - return crypto.createHash('sha256') - .update(compose) - .update('\x00') - .update(env ?? '') - .digest('hex'); + public hashContent(files: ComposeFile[], env: string | null): string { + // Hash the ordered file CONTENTS (NUL-separated) plus env. Paths are + // deliberately excluded: create/apply hash the fetched files (repo paths) + // while pull hashes the on-disk files (materialized paths, primary -> + // compose.yaml), so including paths would make the two disagree and flag a + // clean multi-file stack as locally edited. Content order already changes + // the hash on reorder, and a reorder is a config change that re-applies. + // Single-file keeps the legacy (content + env) shape, byte-stable on upgrade. + const h = crypto.createHash('sha256'); + if (files.length === 1) { + h.update(files[0].content); + } else { + for (const f of files) { + h.update(f.content); + h.update('\x00'); + } + } + h.update('\x00'); + h.update(env ?? ''); + return h.digest('hex'); } - private async readDiskContent(stackName: string, syncEnv: boolean): Promise<{ compose: string; env: string | null }> { + /** Combine an ordered file set into a single path-headed preview for the diff UI. */ + private combinedComposePreview(files: ComposeFile[]): string { + if (files.length <= 1) return files[0]?.content ?? ''; + return files.map(f => `# ── ${f.path} ──\n${f.content}`).join('\n\n'); + } + + /** + * The deploy-time spec for an ordered file set. Single-file stacks with no + * context dir get `null`, so runtime stays plain `docker compose` auto-discovery. + */ + private deriveAppliedSpec(composePaths: string[], contextDir: string | null): GitSourceAppliedSpec | null { + if (composePaths.length <= 1 && !contextDir) return null; + return { files: gitSourceLocalComposeFiles(composePaths), contextDir: contextDir ?? null }; + } + + /** Encrypt the ordered compose file set as the v2 pending blob (carries contextDir). */ + private encodePendingCompose(files: ComposeFile[], contextDir: string | null): string { + return this.crypto.encrypt(JSON.stringify({ v: 2, files, contextDir })); + } + + /** + * Decrypt a stored pending compose blob into its ordered file set + contextDir. + * Detects the v2 marker; anything else is a legacy single-file plaintext string. + */ + private decodePendingCompose(stored: string): { files: ComposeFile[]; contextDir: string | null } { + const raw = this.crypto.decrypt(stored); + if (raw.startsWith('{"v":2')) { + try { + const parsed = JSON.parse(raw) as { v: number; files?: ComposeFile[]; contextDir?: string | null }; + if (Array.isArray(parsed.files) && parsed.files.length > 0) { + return { files: parsed.files, contextDir: parsed.contextDir ?? null }; + } + } catch (e) { + // The v2 marker proves this was written as multi-file, so a parse + // failure signals a corrupt pending blob, not a legacy row. Log it + // so the misleading downstream validation error is traceable; the + // re-validate in apply still blocks deploying the garbled content. + console.error('[GitSource] pending compose blob carried the v2 marker but failed to parse; treating as legacy:', (e as Error).message); + } + } + return { files: [{ path: PRIMARY_COMPOSE_FILENAME, content: raw }], contextDir: null }; + } + + private async readDiskContent(stackName: string, syncEnv: boolean, relFiles: string[]): Promise<{ files: ComposeFile[]; env: string | null }> { const fsSvc = FileSystemService.getInstance(); - let compose: string; - try { - compose = await fsSvc.getStackContent(stackName); - } catch { - compose = ''; + const files: ComposeFile[] = []; + for (let i = 0; i < relFiles.length; i++) { + const rel = relFiles[i]; + try { + // The primary uses compose discovery (compose.yaml / docker-compose.yml); + // additional files are read at their materialized relative path. + const content = i === 0 + ? await fsSvc.getStackContent(stackName) + : (await fsSvc.readStackFile(stackName, rel)).content ?? ''; + files.push({ path: rel, content }); + } catch (e) { + // Empty-on-error is a defensible default (a prior-spec file may have + // been removed by a concurrent edit), but log it so an unexpected + // "local changes detected" can be traced to an unreadable file. + console.warn(`[GitSource] could not read ${sanitizeForLog(rel)} for ${sanitizeForLog(stackName)} diff:`, (e as Error).message); + files.push({ path: rel, content: '' }); + } } let env: string | null = null; if (syncEnv) { @@ -915,7 +1116,57 @@ export class GitSourceService { env = null; } } - return { compose, env }; + return { files, env }; + } + + /** + * Write an ordered compose file set to a stack on disk: the primary to the + * root compose.yaml, each additional file to its repo-relative path. Creates + * the context dir when set, writes the env file when syncing, removes files + * that the previous applied spec materialized but the new set drops, and + * returns the deploy spec to persist. + */ + private async materialize( + stackName: string, + composeFiles: ComposeFile[], + contextDir: string | null, + syncEnv: boolean, + envContent: string | null, + prevSpec: GitSourceAppliedSpec | null, + ): Promise { + const fsSvc = FileSystemService.getInstance(); + const localFiles = gitSourceLocalComposeFiles(composeFiles.map(f => f.path)); + + await fsSvc.saveStackContent(stackName, composeFiles[0].content); + for (let i = 1; i < composeFiles.length; i++) { + await fsSvc.writeStackFile(stackName, localFiles[i], composeFiles[i].content); + } + + if (contextDir) { + await fsSvc.mkdirStackPath(stackName, contextDir); + } + + if (syncEnv && envContent !== null) { + await fsSvc.saveEnvContent(stackName, envContent); + } + + // Stale cleanup: remove additional files the previous apply wrote that are + // no longer in the set. Re-validate each as a safe relative path and never + // touch the primary compose.yaml. + if (prevSpec) { + const keep = new Set(localFiles); + for (const old of prevSpec.files) { + if (old === PRIMARY_COMPOSE_FILENAME || keep.has(old)) continue; + if (!isValidRelativeStackPath(old) || old === '') continue; + try { + await fsSvc.deleteStackPath(stackName, old); + } catch (e) { + console.warn(`[GitSource] stale file cleanup skipped ${sanitizeForLog(old)} for ${sanitizeForLog(stackName)}:`, (e as Error).message); + } + } + } + + return this.deriveAppliedSpec(composeFiles.map(f => f.path), contextDir); } // ─── Pull / apply ──────────────────────────────────────────────────────── @@ -949,24 +1200,25 @@ export class GitSourceService { const fetched = await this.fetchFromGit({ repoUrl: src.repo_url, branch: src.branch, - composePath: src.compose_path, + composePaths: src.compose_paths, envPath: src.sync_env ? src.env_path : null, token, }); - const validation = await this.validateCompose(fetched.composeContent, fetched.envContent); - const disk = await this.readDiskContent(stackName, src.sync_env); - const currentHash = this.hashContent(disk.compose, disk.env); + const validation = await this.validateCompose(fetched.composeFiles, fetched.envContent, src.context_dir); + const appliedFiles = src.applied_deploy_spec?.files ?? [PRIMARY_COMPOSE_FILENAME]; + const disk = await this.readDiskContent(stackName, src.sync_env, appliedFiles); + const currentHash = this.hashContent(disk.files, disk.env); const hasLocalChanges = src.last_applied_content_hash !== null && src.last_applied_content_hash !== currentHash; // Store pending so a subsequent apply doesn't re-fetch. Compose files // routinely contain secrets inlined as env interpolations or passwords, - // so encrypt the pending buffers at rest. + // so the v2 blob (ordered files + contextDir) is encrypted at rest. db.setGitSourcePending( stackName, fetched.commitSha, - this.crypto.encrypt(fetched.composeContent), + this.encodePendingCompose(fetched.composeFiles, src.context_dir), fetched.envContent !== null ? this.crypto.encrypt(fetched.envContent) : null, ); @@ -977,9 +1229,9 @@ export class GitSourceService { return { commitSha: fetched.commitSha, - incomingCompose: fetched.composeContent, + incomingCompose: this.combinedComposePreview(fetched.composeFiles), incomingEnv: fetched.envContent, - currentCompose: disk.compose, + currentCompose: this.combinedComposePreview(disk.files), currentEnv: disk.env, validation, hasLocalChanges, @@ -1025,28 +1277,29 @@ export class GitSourceService { throw new GitSourceError('GIT_ERROR', 'Pending commit has changed since this pull was fetched. Please review the latest diff.'); } - // Pending buffers are stored encrypted; decrypt is a no-op for any - // legacy plaintext rows (isEncrypted check inside CryptoService). - const composeContent = this.crypto.decrypt(src.pending_compose_content); + // Materialize from the pending blob (its files + contextDir), never the + // live config: a config edit between pull and apply must not change what + // gets written. The v2 blob is decoded here; legacy plaintext is treated + // as a single compose.yaml. + const pending = this.decodePendingCompose(src.pending_compose_content); const envContent = src.pending_env_content !== null ? this.crypto.decrypt(src.pending_env_content) : null; // Re-validate before writing. - const validation = await this.validateCompose(composeContent, envContent); + const validation = await this.validateCompose(pending.files, envContent, pending.contextDir); if (!validation.ok) { if (diag) console.log(`[GitSource:diag] apply validation fail stack=${stackName}`); throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`); } - const fsSvc = FileSystemService.getInstance(); - await fsSvc.saveStackContent(stackName, composeContent); - if (src.sync_env && envContent !== null) { - await fsSvc.saveEnvContent(stackName, envContent); - } + const appliedSpec = await this.materialize( + stackName, pending.files, pending.contextDir, src.sync_env, envContent, src.applied_deploy_spec, + ); - const hash = this.hashContent(composeContent, envContent); + const hash = this.hashContent(pending.files, envContent); db.markGitSourceApplied(stackName, commitSha, hash); + db.setGitSourceAppliedSpec(stackName, appliedSpec); const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply; if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy)); @@ -1110,31 +1363,29 @@ export class GitSourceService { const fetched = await this.fetchFromGit({ repoUrl: input.repoUrl, branch: input.branch, - composePath: input.composePath, + composePaths: input.composePaths, envPath: input.syncEnv ? input.envPath : null, token: input.token, }); // 2. Validate against the same `docker compose config` check the // apply path uses. Reject before creating anything on disk. - const validation = await this.validateCompose(fetched.composeContent, fetched.envContent); + const validation = await this.validateCompose(fetched.composeFiles, fetched.envContent, input.contextDir); if (!validation.ok) { throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`); } - // 3. Create directory + boilerplate, then overwrite with the - // fetched content. createStack() throws if the directory - // already exists, so a name collision is caught here. + // 3. Create directory + boilerplate, then materialize the fetched + // files. createStack() throws if the directory already exists, so a + // name collision is caught here. let stackCreated = false; try { await fsSvc.createStack(input.stackName); stackCreated = true; - await fsSvc.saveStackContent(input.stackName, fetched.composeContent); - let envWritten = false; - if (input.syncEnv && fetched.envContent !== null) { - await fsSvc.saveEnvContent(input.stackName, fetched.envContent); - envWritten = true; - } + const appliedSpec = await this.materialize( + input.stackName, fetched.composeFiles, input.contextDir, input.syncEnv, fetched.envContent, null, + ); + const envWritten = input.syncEnv && fetched.envContent !== null; // 4. Insert the git-source row, then mark it applied so future // pulls diff against the fetched commit rather than treating @@ -1142,11 +1393,14 @@ export class GitSourceService { const encryptedToken = input.authType === 'token' && input.token ? this.crypto.encrypt(input.token) : null; + const hash = this.hashContent(fetched.composeFiles, fetched.envContent); db.upsertGitSource({ stack_name: input.stackName, repo_url: input.repoUrl, branch: input.branch, - compose_path: input.composePath, + compose_path: input.composePaths[0], + compose_paths: input.composePaths, + context_dir: input.contextDir, sync_env: input.syncEnv, env_path: input.syncEnv ? input.envPath : null, auth_type: input.authType, @@ -1154,18 +1408,15 @@ export class GitSourceService { auto_apply_on_webhook: input.autoApplyOnWebhook, auto_deploy_on_apply: input.autoDeployOnApply, last_applied_commit_sha: fetched.commitSha, - last_applied_content_hash: this.hashContent(fetched.composeContent, fetched.envContent), + last_applied_content_hash: hash, pending_commit_sha: null, pending_compose_content: null, pending_env_content: null, pending_fetched_at: null, last_debounce_at: null, }); - db.markGitSourceApplied( - input.stackName, - fetched.commitSha, - this.hashContent(fetched.composeContent, fetched.envContent), - ); + db.markGitSourceApplied(input.stackName, fetched.commitSha, hash); + db.setGitSourceAppliedSpec(input.stackName, appliedSpec); const source = this.get(input.stackName); if (!source) { diff --git a/backend/src/services/ImageUpdateService.ts b/backend/src/services/ImageUpdateService.ts index e58effe2..74a4e71e 100644 --- a/backend/src/services/ImageUpdateService.ts +++ b/backend/src/services/ImageUpdateService.ts @@ -105,6 +105,54 @@ export function extractImagesFromCompose( return extractServiceImagesFromCompose(yamlContent, envVars).map(e => e.image); } +/** + * Extract service images from a `docker compose config --format json` render. + * The render is already merged + interpolated, so no env substitution is needed. + */ +export function extractServiceImagesFromRenderedConfig(renderedJson: string): ComposeServiceImage[] { + let parsed: { services?: Record }; + try { + parsed = JSON.parse(renderedJson); + } catch { + return []; + } + if (!parsed?.services || typeof parsed.services !== 'object') return []; + const out: ComposeServiceImage[] = []; + for (const [service, svc] of Object.entries(parsed.services)) { + const raw = svc?.image; + if (!raw || typeof raw !== 'string') continue; + const ref = raw.trim(); + if (!ref || ref.startsWith('sha256:')) continue; + out.push({ service, image: ref }); + } + return out; +} + +/** + * Service-name -> image refs for a stack. For a Git stack with an applied + * multi-file / context-dir spec, this comes from the effective merged model + * (docker compose config), so a service/image declared only in an override file + * is included. Returns null for single-file / non-git stacks (and on a render + * failure) so the caller falls back to its existing single-file compose parse. + */ +export async function loadEffectiveServiceImages(nodeId: number, stackName: string): Promise { + const spec = DatabaseService.getInstance().getGitSource(stackName)?.applied_deploy_spec; + if (!spec || spec.files.length === 0) return null; + // Lazy import to avoid a static module cycle (ComposeService is a heavy hub). + const { ComposeService } = await import('./ComposeService'); + const rendered = await ComposeService.getInstance(nodeId).renderConfig(stackName); + if (!rendered.rendered) { + // The effective render failed (unset var, bad include, timeout, output cap). + // Falling back to the root-compose parse misses override-only images, so log + // the reason; without this the degradation is invisible to the operator. + console.warn( + `[ImageUpdateService] effective image render failed for "${sanitizeForLog(stackName)}" (code=${rendered.code} timedOut=${rendered.timedOut}); falling back to root-compose parse: ${sanitizeForLog(rendered.stderr)}`, + ); + return null; + } + return extractServiceImagesFromRenderedConfig(rendered.rendered); +} + // ─── Service ────────────────────────────────────────────────────────────────── export class ImageUpdateService { @@ -348,6 +396,14 @@ export class ImageUpdateService { // Phase 2: Parse compose files for image refs for (const stackName of stacks) { try { + // Multi-file / context-dir Git stacks resolve images from the + // effective merged model so override-only images are captured. + const effective = await loadEffectiveServiceImages(nodeId, stackName); + if (effective) { + for (const e of effective) stackImages.get(stackName)?.add(e.image); + continue; + } + const content = await withTimeout(fs.getStackContent(stackName), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getStackContent'); // Load .env for variable resolution (best-effort) @@ -384,13 +440,26 @@ export class ImageUpdateService { try { const containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers'); for (const c of containers) { - const workingDir: string | undefined = c.Labels?.['com.docker.compose.project.working_dir']; - if (!workingDir) continue; + // Prefer the pinned project label (== stackName for Sencho-deployed + // stacks, including multi-file / context-dir ones where + // --project-directory would otherwise change the working-dir + // basename). Fall back to the working-dir basename for legacy / + // non-Sencho containers. + const project: string | undefined = c.Labels?.['com.docker.compose.project']; + let stackName: string | null = null; + if (project && stackImages.has(project)) { + stackName = project; + } else { + const workingDir: string | undefined = c.Labels?.['com.docker.compose.project.working_dir']; + if (workingDir) { + const resolved = path.resolve(workingDir); + if (resolved === composeDir || resolved.startsWith(composeDir + path.sep)) { + stackName = path.basename(resolved); + } + } + } + if (!stackName) continue; - const resolved = path.resolve(workingDir); - if (resolved !== composeDir && !resolved.startsWith(composeDir + path.sep)) continue; - - const stackName = path.basename(resolved); const imageRef: string = c.Image ?? ''; if (!imageRef || imageRef.startsWith('sha256:')) continue; diff --git a/backend/src/services/MeshService.ts b/backend/src/services/MeshService.ts index 7e66abad..1bcfbf6a 100644 --- a/backend/src/services/MeshService.ts +++ b/backend/src/services/MeshService.ts @@ -18,7 +18,7 @@ import { lookupContainerIp } from '../mesh/containerLookup'; import { STREAM_PENDING_DATA_MAX_BYTES } from '../pilot/protocol'; import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; import { isDebugEnabled } from '../utils/debug'; -import { isPathWithinBase, isValidStackName } from '../utils/validation'; +import { isPathWithinBase, isValidStackName, isValidRelativeStackPath } from '../utils/validation'; import { PORT as SENCHO_LISTEN_PORT } from '../helpers/constants'; import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate'; @@ -2080,18 +2080,23 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { const targetNodeId = nodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); try { const fsSvc = FileSystemService.getInstance(targetNodeId); - const filename = await fsSvc.getComposeFilename(stackName); const baseDir = fsSvc.getBaseDir(); - // path.basename strips any directory component as defense-in-depth - // on top of isValidStackName + isPathWithinBase. Recognized by - // CodeQL's path-injection model. - const composePath = path.join(baseDir, path.basename(stackName), filename); - if (!isPathWithinBase(composePath, baseDir)) return []; - const content = await fs.readFile(composePath, 'utf8'); - const parsed = YAML.parse(content) as { services?: Record } | null; - const services = parsed?.services && typeof parsed.services === 'object' ? parsed.services : null; - if (!services) return []; - return Object.keys(services).filter((name) => /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/.test(name)); + // For a multi-file Git stack, read every materialized compose file and + // union their service names, so a service declared only in an override + // file is still attached to the mesh. Single-file stacks read the one + // resolved compose file, byte-identical to the prior behavior. + const spec = DatabaseService.getInstance().getGitSource(stackName)?.applied_deploy_spec; + const relFiles = spec && spec.files.length > 0 + ? spec.files + : [await fsSvc.getComposeFilename(stackName)]; + const names = new Set(); + for (const relFile of relFiles) { + if (relFile === '' || !isValidRelativeStackPath(relFile)) continue; + for (const name of await this.readComposeServiceNames(baseDir, stackName, relFile)) { + names.add(name); + } + } + return Array.from(names); } catch (err) { console.warn( '[MeshService] getDeclaredStackServiceNames failed:', @@ -2101,6 +2106,26 @@ export class MeshService extends EventEmitter implements MeshForwarderHost { } } + /** + * Read one compose file under a stack directory and return its declared + * service names. The stack segment uses path.basename as defense-in-depth and + * the resolved path is re-checked against the base dir; the relative file is + * validated by the caller. Returns [] when the file is missing or unparseable. + */ + private async readComposeServiceNames(baseDir: string, stackName: string, relFile: string): Promise { + const composePath = path.join(baseDir, path.basename(stackName), relFile); + if (!isPathWithinBase(composePath, baseDir)) return []; + try { + const content = await fs.readFile(composePath, 'utf8'); + const parsed = YAML.parse(content) as { services?: Record } | null; + const services = parsed?.services && typeof parsed.services === 'object' ? parsed.services : null; + if (!services) return []; + return Object.keys(services).filter((name) => /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/.test(name)); + } catch { + return []; + } + } + /** * Parse an existing mesh override file and extract the service names * it already lists. Used by the defensive fallback so a transient diff --git a/backend/src/services/UpdatePreviewService.ts b/backend/src/services/UpdatePreviewService.ts index 30d1e90a..507449fb 100644 --- a/backend/src/services/UpdatePreviewService.ts +++ b/backend/src/services/UpdatePreviewService.ts @@ -4,6 +4,7 @@ import { RegistryService } from './RegistryService'; import { extractServiceImagesFromCompose, loadDotEnv, + loadEffectiveServiceImages, type ComposeServiceImage, } from './ImageUpdateService'; import { @@ -121,6 +122,12 @@ async function loadStackImages( nodeId: number, stackName: string, ): Promise { + // A multi-file / context-dir Git stack resolves images from the effective + // merged model so override-only services are included; single-file stacks + // fall through to the root-compose parse below. + const effective = await loadEffectiveServiceImages(nodeId, stackName); + if (effective) return effective; + const fs = FileSystemService.getInstance(nodeId); const composeContent = await fs.getStackContent(stackName); let envVars: Record = {}; diff --git a/backend/src/utils/authoredComposeArgs.ts b/backend/src/utils/authoredComposeArgs.ts new file mode 100644 index 00000000..5256ec0a --- /dev/null +++ b/backend/src/utils/authoredComposeArgs.ts @@ -0,0 +1,57 @@ +import path from 'path'; +import { DatabaseService } from '../services/DatabaseService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { isPathWithinBase, isValidRelativeStackPath } from './validation'; + +/** + * Build the `docker compose` global-flag prefix (`-f` per file, `-p `, + * `--project-directory`) for a stack's authored multi-file deploy spec. + * + * Returns `[]` for single-file / non-git stacks (no applied spec), so every + * caller stays byte-identical to the pre-multi-file behavior: docker compose + * resolves the single root compose.yaml via auto-discovery. + * + * `applied_deploy_spec` is DB state ultimately derived from user input, and this + * prefix is spliced straight into a child-process argv, so it is treated as a + * spawn sink: every file path and the context dir is re-validated with the + * relative-path rules and resolved under the stack directory. A malformed, + * absolute, or escaping entry throws here, before any `docker compose` runs. + * + * `-p ` pins the Compose project name so container labels stay + * `com.docker.compose.project=` even when `--project-directory` + * changes the directory basename Compose would otherwise derive the name from. + */ +export function authoredComposeFileArgs(stackName: string, nodeId?: number): string[] { + const resolvedNodeId = nodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); + const spec = DatabaseService.getInstance().getGitSource(stackName)?.applied_deploy_spec; + if (!spec || spec.files.length === 0) return []; + + const baseDir = NodeRegistry.getInstance().getComposeDir(resolvedNodeId); + const stackDir = path.resolve(baseDir, stackName); + + const args: string[] = []; + for (const file of spec.files) { + if (!file || !isValidRelativeStackPath(file)) { + throw new Error(`Invalid compose file path in deploy spec for stack "${stackName}"`); + } + if (!isPathWithinBase(path.resolve(stackDir, file), stackDir)) { + throw new Error(`Compose file path escapes the stack directory for stack "${stackName}"`); + } + args.push('-f', file); + } + + args.push('-p', stackName); + + if (spec.contextDir) { + if (!isValidRelativeStackPath(spec.contextDir)) { + throw new Error(`Invalid context directory in deploy spec for stack "${stackName}"`); + } + const ctxAbs = path.resolve(stackDir, spec.contextDir); + if (!isPathWithinBase(ctxAbs, stackDir)) { + throw new Error(`Context directory escapes the stack directory for stack "${stackName}"`); + } + args.push('--project-directory', ctxAbs); + } + + return args; +} diff --git a/backend/src/utils/gitComposeFiles.ts b/backend/src/utils/gitComposeFiles.ts new file mode 100644 index 00000000..15cab5f9 --- /dev/null +++ b/backend/src/utils/gitComposeFiles.ts @@ -0,0 +1,25 @@ +/** + * The local filename Sencho always writes a Git source's primary compose file + * to, at the stack root. Keeping the primary at this fixed root name preserves + * stack discovery (FileSystemService.getStacks / hasComposeFile) and the editor's + * compose tab, and matches single-file behavior where the fetched file is written + * to compose.yaml regardless of its repo name. + */ +export const PRIMARY_COMPOSE_FILENAME = 'compose.yaml'; + +/** + * Map an ordered list of repo-relative compose paths to the ordered local + * relative filenames Sencho materializes for them under the stack directory: + * - index 0 (primary) -> compose.yaml at the stack root + * - every additional file -> its repo-relative path, preserved under the stack dir + * + * The result is the deploy-time `applied_deploy_spec.files` list, also used to + * write the files to disk. Repo paths are already validated as POSIX relative + * paths upstream (isValidGitSourcePath), so they are used as-is for additional + * files; only a leading "./" is stripped for a clean on-disk layout. + */ +export function gitSourceLocalComposeFiles(composePaths: string[]): string[] { + return composePaths.map((p, index) => + index === 0 ? PRIMARY_COMPOSE_FILENAME : p.replace(/^\.\//, ''), + ); +} diff --git a/docs/features/git-sources.mdx b/docs/features/git-sources.mdx index 223a4756..693e9f5b 100644 --- a/docs/features/git-sources.mdx +++ b/docs/features/git-sources.mdx @@ -1,9 +1,9 @@ --- title: Git Sources -description: Link a stack to a Git repository and keep compose.yaml in sync via manual pulls or webhook triggers. +description: Link a stack to a Git repository and keep one or more compose files in sync via manual pulls or webhook triggers. --- -Git Sources turn any stack into a GitOps target. Point Sencho at a repository, branch, and `compose.yaml` path; pull updates on demand or from CI; and review a diff before applying changes to disk. Optional sibling `.env` sync keeps configuration consistent too. +Git Sources turn any stack into a GitOps target. Point Sencho at a repository and branch, choose one or more compose files to merge in order, pull updates on demand or from CI, and review a diff before applying changes to disk. Optional sibling `.env` sync keeps configuration consistent too. Git Sources are available on every tier, including Community. @@ -12,7 +12,7 @@ Git Sources turn any stack into a GitOps target. Point Sencho at a repository, b ## How it works 1. Open a stack and click the **Git Source** button in the editor toolbar. -2. Fill in the repository URL, branch, and compose file path. Add a token if the repo is private. +2. Fill in the repository URL and branch, then choose the compose files. Use **Browse** to pick them from the repository tree, or type a path and press Enter. Add a token if the repo is private. 3. Click **Pull now** to fetch the latest commit. Sencho opens a side-by-side diff between the on-disk files and the incoming version. 4. Click **Apply** to write the incoming content to disk. Tick **Deploy after apply** in the same dialog to redeploy in one step. @@ -27,7 +27,7 @@ Writes land in the stack's existing directory using the same storage Sencho uses The panel groups four regions: - **Pending update banner.** Appears at the top when a webhook in **Review only** mode has fetched a new commit. Click **Review** to re-fetch the incoming commit and open the diff dialog. -- **Form fields.** Repository URL, branch, compose file path, optional sibling `.env` sync, authentication toggle, and the apply behavior radio group. +- **Form fields.** Repository URL, branch, the ordered compose-file picker, an optional project directory, optional sibling `.env` sync, authentication toggle, and the apply behavior radio group. - **Last applied stat strip.** Shows the short SHA of the last commit Sencho applied to disk, plus the timestamp of the most recent successful save or pull. - **Footer actions.** **Remove** disconnects the source without touching the stack files; **Pull now** fetches the configured branch's HEAD; **Save** or **Update** persists form changes after a reachability check passes. @@ -39,7 +39,7 @@ Skip the "empty stack then link later" detour and point at a repo from the start New stack dialog with the From Git tab selected, showing stack name, repository URL, branch, compose path, sibling .env toggle, authentication toggle, apply behavior radio group, a Deploy after create checkbox, and an HTTPS REPOS ONLY footer hint -Sencho fetches the compose file, validates it with `docker compose config`, writes it to a fresh stack directory, and links the Git source in one step. The last-applied commit SHA is seeded from the fetch so the first manual pull produces a clean diff rather than a "local edits detected" warning. +Sencho fetches the compose files, validates the merged result with `docker compose config`, writes them to a fresh stack directory, and links the Git source in one step. The last-applied commit SHA is seeded from the fetch so the first manual pull produces a clean diff rather than a "local edits detected" warning. Tick **Deploy after create** to run `docker compose up -d` immediately after the files land. If the deploy fails, the stack and Git source are kept on disk so you can fix the underlying issue (missing image, port conflict, host resources) and retry the deploy from the editor. @@ -58,12 +58,19 @@ Tick **Deploy after create** to run `docker compose up -d` immediately after the |-------|-------------| | **Repository URL** | `https://github.com/your-org/your-repo.git` (HTTPS only) | | **Branch** | Branch to track (e.g. `main`) | -| **Compose file path** | Path within the repo (e.g. `deploy/compose.yaml`) | -| **Also sync sibling `.env` file** | When enabled, also pulls the `.env` from the same directory as the compose file. The form shows the resolved path inline (e.g. `deploy/.env` for a compose at `deploy/compose.yaml`). | +| **Compose files** | One or more paths within the repo, merged in the listed order (e.g. `deploy/base.yaml` then `deploy/prod.yaml`). The first file is the primary. Reorder by dragging (or the up/down arrows on a phone) and remove with the **×** button. | +| **Project directory** | Optional path within the repo passed to `docker compose --project-directory`, so relative build contexts, bind mounts, and `env_file` references resolve from that base. Leave blank to use the stack root. | +| **Also sync sibling `.env` file** | When enabled, also pulls the `.env` from the same directory as the primary compose file. The form shows the resolved path inline (e.g. `deploy/.env` for a primary at `deploy/compose.yaml`). | | **Authentication** | **Public (no auth)** for public repos, **Personal Access Token** for private repos | | **Apply behavior** | See the three modes below | -Saving runs a reachability check against the repository. If the URL is wrong, the token is invalid, the branch does not exist, or the file is missing, Sencho surfaces the error inline and nothing is persisted. +Saving runs a reachability check against the repository. If the URL is wrong, the token is invalid, the branch does not exist, or a file is missing, Sencho surfaces the error inline and nothing is persisted. + +### Multiple compose files + +Many projects split a stack into a base file plus environment overrides. List the files in the order you want them merged and Sencho deploys with `docker compose -f base.yaml -f prod.yaml ...`, applying each later file's values on top of the earlier ones, exactly as the Compose CLI does. The same ordered set is used for every action on the stack: deploy, update, restart, stop, and teardown. + +On disk, the primary file lands as the stack's `compose.yaml` and each additional file keeps its repository-relative path inside the stack directory, so you can still browse and edit them in the file explorer. A single-file source behaves exactly as before. ### Apply behavior modes @@ -178,7 +185,7 @@ Pulls, applies, and create-from-git operations on the same stack are serialized - The compose path is relative to the repository root and must point at the file, not its parent directory. If the file was moved, update the path on the panel and save. + Each compose path is relative to the repository root and must point at the file, not its parent directory. Every file in the list must exist on the tracked branch; if any is missing, the save or pull fails and names the path. If a file was moved, update its path on the panel and save. @@ -217,6 +224,14 @@ Pulls, applies, and create-from-git operations on the same stack are serialized Repositories that use Git submodules do not have their submodule contents cloned during a Git Source fetch. Sencho surfaces a warning on create when `.gitmodules` is present. If the compose file references paths inside a submodule (build contexts, volume mounts, include directives), inline the referenced files into the main repository or flatten the submodule so the paths resolve at deploy time. + + The first file in the list is always written to the stack's root `compose.yaml`. To avoid clobbering it, an additional file whose repository path is also `compose.yaml` is rejected. Rename it in the repository, or move it to the top of the list to make it the primary. + + + + A Git Source materializes only the compose and `.env` files you select, not the rest of the repository. Build contexts, bind-mount sources, and cross-file `extends:` targets referenced by relative path are not pulled, so they can be missing at deploy time. Set a **Project directory** to fix the base for relative paths, and for build-based stacks keep the referenced files in the repository alongside the compose files or build the image out of band. + + Git Sources fetch over HTTPS only. SSH clone URLs (`git@host:org/repo.git`) and custom protocols are rejected with a client-side validation error. Paste the `https://...` URL and use a Personal Access Token for authentication on private repositories. @@ -229,3 +244,5 @@ Pulls, applies, and create-from-git operations on the same stack are serialized - **No submodules.** Submodule contents are not fetched; paths inside a submodule directory will be missing at deploy time. A warning is shown on create when `.gitmodules` is present. - **Branch-tracking only.** Sources follow the head of a branch. Specific commit SHAs and tags are not pinnable. - **Clone size cap.** A clone is bounded on how much it downloads (and each compose/env file is capped on read), so very large repositories are rejected. Operators can adjust the download ceiling with `GITSOURCE_MAX_CLONE_BYTES`. +- **Referenced files are not materialized.** Only the selected compose and `.env` files are written to disk. Relative build contexts, bind-mount sources, and `extends:` targets are not pulled, so build-based stacks that depend on other repository files need those files committed alongside the compose files (or the image built out of band). +- **Some read-only views read the primary file.** The dependency graph, drift snapshot, and networking inspector summarize the primary compose file, so a service declared only in an override may not appear in those views. Deploy, update, image-update checks, and mesh attachment use the full merged set. diff --git a/docs/features/overview.mdx b/docs/features/overview.mdx index 49ea3703..ca5e72a1 100644 --- a/docs/features/overview.mdx +++ b/docs/features/overview.mdx @@ -57,7 +57,7 @@ Fleet-wide compose templates that Sencho keeps in sync across the nodes you choo ### Git sources -Link a stack to a Git repository and keep `compose.yaml` in sync via manual pulls or webhook triggers. Review a diff before applying changes, create stacks directly from a repo, and optionally sync sibling `.env` files for consistent configuration. [Learn more →](/features/git-sources) +Link a stack to a Git repository and keep one or more compose files in sync via manual pulls or webhook triggers. Merge a base file with environment overrides in order, review a diff before applying changes, create stacks directly from a repo, and optionally sync sibling `.env` files for consistent configuration. [Learn more →](/features/git-sources) ### Stack labels diff --git a/docs/features/stack-management.mdx b/docs/features/stack-management.mdx index 51d15dd6..0bf875d9 100644 --- a/docs/features/stack-management.mdx +++ b/docs/features/stack-management.mdx @@ -43,10 +43,10 @@ If a preview shows a relative volume path (for example `./data:/app/data`), Senc ### From Git -The **From Git** tab clones a public or private repository and treats its compose file as the stack's source of truth. +The **From Git** tab clones a public or private repository and treats its compose files as the stack's source of truth. - New stack dialog on the From Git tab with repository URL, branch, compose file path, authentication, and apply behavior fields + New stack dialog on the From Git tab with repository URL, branch, the ordered compose-file picker, an optional project directory, authentication, and apply behavior fields Fill in: @@ -54,8 +54,8 @@ Fill in: - **Stack Name**: same naming rules as the Empty tab. - **Repository URL**: HTTPS only. SSH URLs are not supported. - **Branch**: defaults to `main`. -- **Compose file path**: path inside the repository, defaults to `compose.yaml`. -- **Also sync sibling .env file**: when checked, a `.env` next to the compose file is pulled along with it. +- **Compose files**: one or more paths inside the repository, merged in the listed order. Defaults to `compose.yaml`. Browse the repository tree to pick them, or type a path. An optional **Project directory** sets the base for relative paths. +- **Also sync sibling .env file**: when checked, a `.env` next to the primary compose file is pulled along with it. - **Authentication**: pick **Public (no auth)** or **Personal Access Token** for private repos. - **Apply behavior**: controls what happens on future webhook pulls: - **Review only**: diffs surface in the sidebar; you apply manually. @@ -63,7 +63,7 @@ Fill in: - **Auto-deploy**: pulls write and redeploy automatically. - **Deploy after create**: deploys the stack immediately after the repo is cloned. -Click **Create from Git**. Sencho clones the repo, writes the compose file into the stack directory, and (when **Deploy after create** is checked) runs `docker compose up -d`. +Click **Create from Git**. Sencho clones the repo, writes the compose files into the stack directory, and (when **Deploy after create** is checked) runs `docker compose up -d`. ### Convert from a docker run command diff --git a/e2e/git-sources.spec.ts b/e2e/git-sources.spec.ts index fe3c1611..22f0204a 100644 --- a/e2e/git-sources.spec.ts +++ b/e2e/git-sources.spec.ts @@ -68,8 +68,6 @@ test.describe('Git Sources', () => { await page.locator('#git-source-repo').fill('git@github.com:org/repo.git'); await page.locator('#git-source-branch').fill('main'); - await page.locator('#git-source-path').fill('compose.yaml'); - await page.getByRole('dialog').getByRole('button', { name: /^Save$/ }).click(); await expect(page.getByText(/Only HTTPS repository URLs are supported/i)).toBeVisible({ timeout: 5_000 }); @@ -83,8 +81,6 @@ test.describe('Git Sources', () => { // which maps to NETWORK_TIMEOUT or REPO_NOT_FOUND. await page.locator('#git-source-repo').fill('https://git.invalid.example/nope/nope.git'); await page.locator('#git-source-branch').fill('main'); - await page.locator('#git-source-path').fill('compose.yaml'); - await page.getByRole('dialog').getByRole('button', { name: /^Save$/ }).click(); // Any of the mapped error messages is acceptable; the key is that nothing @@ -191,8 +187,10 @@ test.describe('Git Sources', () => { // Source should render with the saved repo URL. await expect(page.locator('#git-source-repo')).toHaveValue(/awesome-compose/); - // Click Remove → AlertDialog appears → confirm → source cleared. - await page.getByRole('dialog').getByRole('button', { name: /Remove/i }).click(); + // Click Remove → AlertDialog appears → confirm → source cleared. Playwright's + // name match is substring by default, so require an exact match to select the + // footer button and not the picker's per-file "Remove " buttons. + await page.getByRole('dialog').getByRole('button', { name: 'Remove', exact: true }).click(); await expect(page.getByRole('alertdialog')).toBeVisible({ timeout: 5_000 }); await page.getByRole('alertdialog').getByRole('button', { name: /^Remove$/ }).click(); @@ -249,8 +247,6 @@ test.describe('Create stack from Git', () => { await page.locator('#create-git-stack-name').fill(CREATE_FROM_GIT_STACK); await page.locator('#git-source-repo').fill('git@github.com:org/repo.git'); await page.locator('#git-source-branch').fill('main'); - await page.locator('#git-source-path').fill('compose.yaml'); - await page.getByRole('dialog').getByRole('button', { name: /Create from Git/i }).click(); await expect(page.getByText(/Only HTTPS repository URLs are supported/i)).toBeVisible({ timeout: 5_000 }); }); @@ -365,7 +361,12 @@ test.describe('Create stack from Git', () => { await page.locator('#create-git-stack-name').fill(uiName); await page.locator('#git-source-repo').fill('https://github.com/docker/awesome-compose.git'); await page.locator('#git-source-branch').fill('master'); - await page.locator('#git-source-path').fill('nginx-golang/compose.yaml'); + // Drive the compose-file picker: add the repo path, then drop the default + // compose.yaml so only the intended file is deployed. + const createDialog = page.getByRole('dialog'); + await createDialog.getByPlaceholder('path/to/compose.yaml').fill('nginx-golang/compose.yaml'); + await createDialog.getByPlaceholder('path/to/compose.yaml').press('Enter'); + await createDialog.getByRole('button', { name: 'Remove compose.yaml' }).click(); await page.getByRole('dialog').getByRole('button', { name: /Create from Git/i }).click(); diff --git a/frontend/src/components/EditorLayout/CreateStackDialog.tsx b/frontend/src/components/EditorLayout/CreateStackDialog.tsx index a75285c1..c1791dad 100644 --- a/frontend/src/components/EditorLayout/CreateStackDialog.tsx +++ b/frontend/src/components/EditorLayout/CreateStackDialog.tsx @@ -7,6 +7,7 @@ import { Label } from '../ui/label'; import { ScrollArea } from '../ui/scroll-area'; import { Checkbox } from '../ui/checkbox'; import { GitSourceFields, type ApplyMode } from '../stack/GitSourceFields'; +import type { GitBrowseResult } from '../stack/GitComposeFilePicker'; import { ImportStackPanel } from './ImportStackPanel'; import { apiFetch } from '@/lib/api'; import { toast } from '@/components/ui/toast-store'; @@ -64,7 +65,8 @@ export function CreateStackDialog({ open, onOpenChange, onStackCreated, onStacks const [creatingFromDockerRun, setCreatingFromDockerRun] = useState(false); const [gitRepoUrl, setGitRepoUrl] = useState(''); const [gitBranch, setGitBranch] = useState('main'); - const [gitComposePath, setGitComposePath] = useState('compose.yaml'); + const [gitComposePaths, setGitComposePaths] = useState(['compose.yaml']); + const [gitContextDir, setGitContextDir] = useState(''); const [gitSyncEnv, setGitSyncEnv] = useState(false); const [gitAuthType, setGitAuthType] = useState<'none' | 'token'>('none'); const [gitToken, setGitToken] = useState(''); @@ -76,7 +78,8 @@ export function CreateStackDialog({ open, onOpenChange, onStackCreated, onStacks setNewStackName(''); setGitRepoUrl(''); setGitBranch('main'); - setGitComposePath('compose.yaml'); + setGitComposePaths(['compose.yaml']); + setGitContextDir(''); setGitSyncEnv(false); setGitAuthType('none'); setGitToken(''); @@ -84,6 +87,35 @@ export function CreateStackDialog({ open, onOpenChange, onStackCreated, onStacks setGitDeployNow(false); }; + const browseGitRepo = async (): Promise => { + if (!gitRepoUrl.trim() || !gitBranch.trim()) { + toast.error('Enter a repository URL and branch first.'); + return null; + } + try { + const body: Record = { + repo_url: gitRepoUrl.trim(), + branch: gitBranch.trim(), + auth_type: gitAuthType, + }; + if (gitAuthType === 'token' && gitToken !== '') body.token = gitToken; + const res = await apiFetch('/git-sources/browse', { + method: 'POST', + body: JSON.stringify(body), + }); + if (res.ok) { + const data = await res.json(); + return { files: data.files ?? [], truncated: data.truncated ?? false }; + } + const err = await res.json().catch(() => ({})); + toast.error(err?.error || 'Failed to browse repository.'); + return null; + } catch (e) { + toast.error((e as Error)?.message || 'Network error.'); + return null; + } + }; + const resetCreateFromDockerRunForm = () => { setDockerRunInput(''); setConvertedYaml(null); @@ -142,8 +174,8 @@ export function CreateStackDialog({ open, onOpenChange, onStackCreated, onStacks toast.error('Stack name is required.'); return; } - if (!gitRepoUrl.trim() || !gitBranch.trim() || !gitComposePath.trim()) { - toast.error('Repository URL, branch, and compose path are required.'); + if (!gitRepoUrl.trim() || !gitBranch.trim() || gitComposePaths.length === 0) { + toast.error('Repository URL, branch, and at least one compose file are required.'); return; } if (!/^https:\/\//i.test(gitRepoUrl.trim())) { @@ -160,7 +192,8 @@ export function CreateStackDialog({ open, onOpenChange, onStackCreated, onStacks stack_name: stackName, repo_url: gitRepoUrl.trim(), branch: gitBranch.trim(), - compose_path: gitComposePath.trim(), + compose_paths: gitComposePaths, + context_dir: gitContextDir.trim() || null, sync_env: gitSyncEnv, auth_type: gitAuthType, auto_apply_on_webhook: autoApply, @@ -411,7 +444,8 @@ export function CreateStackDialog({ open, onOpenChange, onStackCreated, onStacks disabled={creatingFromGit} repoUrl={gitRepoUrl} branch={gitBranch} - composePath={gitComposePath} + composePaths={gitComposePaths} + contextDir={gitContextDir} syncEnv={gitSyncEnv} authType={gitAuthType} token={gitToken} @@ -419,11 +453,13 @@ export function CreateStackDialog({ open, onOpenChange, onStackCreated, onStacks applyMode={gitApplyMode} onRepoUrlChange={setGitRepoUrl} onBranchChange={setGitBranch} - onComposePathChange={setGitComposePath} + onComposePathsChange={setGitComposePaths} + onContextDirChange={setGitContextDir} onSyncEnvChange={setGitSyncEnv} onAuthTypeChange={setGitAuthType} onTokenChange={setGitToken} onApplyModeChange={setGitApplyMode} + onBrowse={browseGitRepo} />
diff --git a/frontend/src/components/stack/GitComposeFilePicker.tsx b/frontend/src/components/stack/GitComposeFilePicker.tsx new file mode 100644 index 00000000..d301d668 --- /dev/null +++ b/frontend/src/components/stack/GitComposeFilePicker.tsx @@ -0,0 +1,207 @@ +import { useState } from 'react'; +import { GripVertical, ArrowUp, ArrowDown, X, FolderGit2, Plus } from 'lucide-react'; +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { Button } from '@/components/ui/button'; +import { Checkbox } from '@/components/ui/checkbox'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { cn } from '@/lib/utils'; +import { useIsMobile } from '@/hooks/use-is-mobile'; + +export interface GitBrowseResult { + files: string[]; + truncated: boolean; +} + +interface GitComposeFilePickerProps { + composePaths: string[]; + contextDir: string; + onComposePathsChange: (paths: string[]) => void; + onContextDirChange: (value: string) => void; + /** Parent runs the correct browse endpoint (create vs edit) and returns the repo file list, or null on failure. */ + onBrowse: () => Promise; + /** True when the repo URL + branch are filled, so a browse can succeed. */ + canBrowse: boolean; + disabled?: boolean; +} + +const isComposeLike = (p: string) => /\.ya?ml$/i.test(p); + +export function GitComposeFilePicker({ + composePaths, + contextDir, + onComposePathsChange, + onContextDirChange, + onBrowse, + canBrowse, + disabled = false, +}: GitComposeFilePickerProps) { + const isMobile = useIsMobile(); + const [manualPath, setManualPath] = useState(''); + const [dragIndex, setDragIndex] = useState(null); + const [browsing, setBrowsing] = useState(false); + const [repoFiles, setRepoFiles] = useState(null); + const [truncated, setTruncated] = useState(false); + + const addPath = (raw: string) => { + const value = raw.trim().replace(/\\/g, '/').replace(/^\.\//, ''); + if (!value || composePaths.includes(value)) return; + onComposePathsChange([...composePaths, value]); + }; + + const removeAt = (index: number) => { + onComposePathsChange(composePaths.filter((_, i) => i !== index)); + }; + + const move = (from: number, to: number) => { + if (to < 0 || to >= composePaths.length || from === to) return; + const next = [...composePaths]; + const [item] = next.splice(from, 1); + next.splice(to, 0, item); + onComposePathsChange(next); + }; + + const runBrowse = async () => { + setBrowsing(true); + try { + const result = await onBrowse(); + if (result) { + // Compose-like files first so they are easy to pick out of a large repo. + const sorted = [...result.files].sort((a, b) => { + const ac = isComposeLike(a) ? 0 : 1; + const bc = isComposeLike(b) ? 0 : 1; + return ac - bc || a.localeCompare(b); + }); + setRepoFiles(sorted); + setTruncated(result.truncated); + } + } finally { + setBrowsing(false); + } + }; + + return ( +
+ + + {composePaths.length === 0 ? ( +

+ No compose files selected. Browse the repository or add a path below. +

+ ) : ( +
    + {composePaths.map((p, index) => ( +
  • setDragIndex(index)} + onDragOver={(e) => { if (dragIndex !== null) e.preventDefault(); }} + onDrop={(e) => { + e.preventDefault(); + if (dragIndex !== null) move(dragIndex, index); + setDragIndex(null); + }} + onDragEnd={() => setDragIndex(null)} + className={cn( + 'flex items-center gap-2 rounded-md border border-glass-border bg-card-bg/40 px-2 py-1.5', + dragIndex === index && 'opacity-50', + !disabled && !isMobile && 'cursor-grab', + )} + > + {isMobile ? ( +
    + + +
    + ) : ( + + )} + {p} + {index === 0 && primary} + +
    • + ))} +
+ )} + +
+ setManualPath(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { e.preventDefault(); addPath(manualPath); setManualPath(''); } + }} + disabled={disabled} + className="font-mono text-xs" + /> + + +
+ + {repoFiles && ( +
+ +
+ {repoFiles.length === 0 &&

No files found in the repository.

} + {repoFiles.map((file) => { + const selected = composePaths.includes(file); + return ( + + ); + })} +
+ + {truncated && ( +

+ Repository has many files; only the first 2000 are listed. Add other paths manually. +

+ )} +
+ )} + +
+ + onContextDirChange(e.target.value)} + disabled={disabled} + className="font-mono text-xs" + /> +

+ Sets --project-directory for relative paths. Leave blank to use the stack root. +

+
+
+ ); +} diff --git a/frontend/src/components/stack/GitSourceDiffDialog.tsx b/frontend/src/components/stack/GitSourceDiffDialog.tsx index 935b3095..cc40cc23 100644 --- a/frontend/src/components/stack/GitSourceDiffDialog.tsx +++ b/frontend/src/components/stack/GitSourceDiffDialog.tsx @@ -103,7 +103,7 @@ export function GitSourceDiffDialog({ - compose.yaml + Compose .env diff --git a/frontend/src/components/stack/GitSourceFields.tsx b/frontend/src/components/stack/GitSourceFields.tsx index 1ea9f73e..fa1ede4a 100644 --- a/frontend/src/components/stack/GitSourceFields.tsx +++ b/frontend/src/components/stack/GitSourceFields.tsx @@ -2,6 +2,7 @@ import { Input } from '@/components/ui/input'; import { Label } from '@/components/ui/label'; import { Checkbox } from '@/components/ui/checkbox'; import { cn } from '@/lib/utils'; +import { GitComposeFilePicker, type GitBrowseResult } from './GitComposeFilePicker'; export type ApplyMode = 'review' | 'auto-write' | 'auto-deploy'; @@ -9,8 +10,8 @@ export type ApplyMode = 'review' | 'auto-write' | 'auto-deploy'; * Mirror of the backend's env-path default (see `/api/stacks/from-git` and * the git-source PUT handler): if the user ticks "Sync .env" without * specifying an explicit path, the service reads `/.env` - * alongside the compose file. Surfacing this in the form saves the user - * a round-trip to figure out which directory the `.env` will come from. + * alongside the primary compose file. Surfacing this in the form saves the + * user a round-trip to figure out which directory the `.env` will come from. */ function computeDefaultEnvPath(composePath: string): string { const normalized = composePath.trim().replace(/\\/g, '/').replace(/^\.\//, ''); @@ -22,7 +23,8 @@ function computeDefaultEnvPath(composePath: string): string { export interface GitSourceFieldsState { repoUrl: string; branch: string; - composePath: string; + composePaths: string[]; + contextDir: string; syncEnv: boolean; authType: 'none' | 'token'; token: string; @@ -37,11 +39,14 @@ export interface GitSourceFieldsProps extends GitSourceFieldsState { variant: 'edit' | 'create'; onRepoUrlChange: (value: string) => void; onBranchChange: (value: string) => void; - onComposePathChange: (value: string) => void; + onComposePathsChange: (value: string[]) => void; + onContextDirChange: (value: string) => void; onSyncEnvChange: (value: boolean) => void; onAuthTypeChange: (value: 'none' | 'token') => void; onTokenChange: (value: string) => void; onApplyModeChange: (value: ApplyMode) => void; + /** Runs the correct browse endpoint (create vs edit); returns the repo file list or null on failure. */ + onBrowse: () => Promise; } const APPLY_MODE_COPY: Record<'edit' | 'create', Record> = { @@ -60,7 +65,8 @@ const APPLY_MODE_COPY: Record<'edit' | 'create', Record (
-
-
- - onBranchChange(e.target.value)} - disabled={disabled} - className="font-mono text-xs" - /> -
-
- - onComposePathChange(e.target.value)} - disabled={disabled} - className="font-mono text-xs" - /> -
+
+ + onBranchChange(e.target.value)} + disabled={disabled} + className="font-mono text-xs" + />
+ +
.env file
- {syncEnv && composePath.trim() !== '' && ( + {syncEnv && primaryComposePath.trim() !== '' && (

Will read{' '} - {computeDefaultEnvPath(composePath)} + {computeDefaultEnvPath(primaryComposePath)} {' '} from the repository.

diff --git a/frontend/src/components/stack/GitSourcePanel.test.tsx b/frontend/src/components/stack/GitSourcePanel.test.tsx index 2e298b64..6669ded8 100644 --- a/frontend/src/components/stack/GitSourcePanel.test.tsx +++ b/frontend/src/components/stack/GitSourcePanel.test.tsx @@ -101,7 +101,7 @@ describe('GitSourcePanel load', () => { await screen.findByRole('button', { name: /^save$/i }); expect(screen.queryByRole('button', { name: /update/i })).not.toBeInTheDocument(); expect(screen.queryByRole('button', { name: /pull now/i })).not.toBeInTheDocument(); - expect(screen.queryByRole('button', { name: /remove/i })).not.toBeInTheDocument(); + expect(screen.queryByRole('button', { name: 'Remove' })).not.toBeInTheDocument(); expect(screen.getByLabelText(/repository url/i)).toHaveValue(''); }); @@ -113,7 +113,7 @@ describe('GitSourcePanel load', () => { // A real source flips the primary action to Update and exposes Pull now / Remove. await screen.findByRole('button', { name: /update/i }); expect(screen.getByRole('button', { name: /pull now/i })).toBeInTheDocument(); - expect(screen.getByRole('button', { name: /remove/i })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Remove' })).toBeInTheDocument(); await waitFor(() => expect(screen.getByLabelText(/repository url/i)).toHaveValue('https://github.com/org/repo.git'), ); diff --git a/frontend/src/components/stack/GitSourcePanel.tsx b/frontend/src/components/stack/GitSourcePanel.tsx index 814205dd..5b69ebdd 100644 --- a/frontend/src/components/stack/GitSourcePanel.tsx +++ b/frontend/src/components/stack/GitSourcePanel.tsx @@ -10,6 +10,7 @@ import { useNodes } from '@/context/NodeContext'; import { toast } from '@/components/ui/toast-store'; import { GitSourceDiffDialog, type PullResult } from './GitSourceDiffDialog'; import { GitSourceFields, type ApplyMode } from './GitSourceFields'; +import type { GitBrowseResult } from './GitComposeFilePicker'; export interface GitSource { id: number; @@ -17,6 +18,8 @@ export interface GitSource { repo_url: string; branch: string; compose_path: string; + compose_paths: string[]; + context_dir: string | null; sync_env: boolean; env_path: string | null; auth_type: 'none' | 'token'; @@ -64,7 +67,8 @@ export function GitSourcePanel({ const [repoUrl, setRepoUrl] = useState(''); const [branch, setBranch] = useState('main'); - const [composePath, setComposePath] = useState('compose.yaml'); + const [composePaths, setComposePaths] = useState(['compose.yaml']); + const [contextDir, setContextDir] = useState(''); const [syncEnv, setSyncEnv] = useState(false); const [authType, setAuthType] = useState<'none' | 'token'>('none'); const [token, setToken] = useState(''); @@ -82,7 +86,8 @@ export function GitSourcePanel({ setSource(null); setRepoUrl(''); setBranch('main'); - setComposePath('compose.yaml'); + setComposePaths(['compose.yaml']); + setContextDir(''); setSyncEnv(false); setAuthType('none'); setToken(''); @@ -102,7 +107,8 @@ export function GitSourcePanel({ setSource(data); setRepoUrl(data.repo_url); setBranch(data.branch); - setComposePath(data.compose_path); + setComposePaths(data.compose_paths?.length ? data.compose_paths : [data.compose_path]); + setContextDir(data.context_dir ?? ''); setSyncEnv(data.sync_env); setAuthType(data.auth_type); setToken(''); @@ -131,8 +137,8 @@ export function GitSourcePanel({ }, [open, load]); const save = async () => { - if (!repoUrl.trim() || !branch.trim() || !composePath.trim()) { - toast.error('Repository URL, branch, and compose path are required.'); + if (!repoUrl.trim() || !branch.trim() || composePaths.length === 0) { + toast.error('Repository URL, branch, and at least one compose file are required.'); return; } if (!/^https:\/\//i.test(repoUrl.trim())) { @@ -147,7 +153,8 @@ export function GitSourcePanel({ const body: Record = { repo_url: repoUrl.trim(), branch: branch.trim(), - compose_path: composePath.trim(), + compose_paths: composePaths, + context_dir: contextDir.trim() || null, sync_env: syncEnv, auth_type: authType, auto_apply_on_webhook: autoApply, @@ -179,6 +186,35 @@ export function GitSourcePanel({ } }; + const browseRepo = async (): Promise => { + if (!repoUrl.trim() || !branch.trim()) { + toast.error('Enter a repository URL and branch first.'); + return null; + } + try { + const body: Record = { + repo_url: repoUrl.trim(), + branch: branch.trim(), + auth_type: authType, + }; + if (authType === 'token' && token !== '') body.token = token; + const res = await apiFetch(`/stacks/${encodeURIComponent(stackName)}/git-source/browse`, { + method: 'POST', + body: JSON.stringify(body), + }); + if (res.ok) { + const data = await res.json(); + return { files: data.files ?? [], truncated: data.truncated ?? false }; + } + const err = await res.json().catch(() => ({})); + toast.error(err?.error || 'Failed to browse repository.'); + return null; + } catch (e) { + toast.error((e as Error)?.message || 'Network error.'); + return null; + } + }; + const remove = async () => { if (!source) return; setRemoveConfirmOpen(false); @@ -343,7 +379,8 @@ export function GitSourcePanel({ disabled={!canEdit || saving} repoUrl={repoUrl} branch={branch} - composePath={composePath} + composePaths={composePaths} + contextDir={contextDir} syncEnv={syncEnv} authType={authType} token={token} @@ -351,11 +388,13 @@ export function GitSourcePanel({ applyMode={applyMode} onRepoUrlChange={setRepoUrl} onBranchChange={setBranch} - onComposePathChange={setComposePath} + onComposePathsChange={setComposePaths} + onContextDirChange={setContextDir} onSyncEnvChange={setSyncEnv} onAuthTypeChange={setAuthType} onTokenChange={setToken} onApplyModeChange={setApplyModeOverride} + onBrowse={browseRepo} /> {source && (