feat(stacks): guided first stack import flow (#1285)

* feat(stacks): add guided first stack import flow

Add an Import mode to the Create Stack dialog and a zero-stacks empty
state so a new user who already has compose files on disk can land their
first stack without reading the docs first.

A read-only scan of the compose directory (GET /api/stacks/import/scan)
lists the compose files it finds with a dry preview of each file's
services, ports, volumes, and env files. Each result is labelled by
placement: already a stack, loose at the root of the compose directory,
or one folder too deep, with the exact path to move misplaced files to.
The scan never writes, moves, or changes any files.

Manual stack creation (Empty, From Git, From Docker Run) is unchanged.

* fix(stacks): read import-scan candidates via a single file handle

Open the compose file once and stat plus read on the same descriptor so
the size check and the read observe the same inode, instead of resolving
the path twice (stat then readFile), which is a time-of-check/time-of-use
race. Mirrors the existing handle-based readers in FileSystemService.

* fix(stacks): confine import scan to the compose dir and refine the empty state

Harden the read-only import scan:
- Resolve symlinks and confirm the real target stays inside the compose
  directory before reading a candidate, and reject non-regular files, so a
  symlinked compose file or parent cannot expose a file outside the compose
  directory through the preview (matches resolveSafeStackPath).
- Read at most the stat-reported size (bounded by the 1 MiB cap) from the open
  handle, so a file that grows after the size check cannot exceed the cap.
- Log when the compose directory or a subdirectory cannot be read, so an access
  failure is not silently reported as "no compose files found".

Only show the first-run "No stacks yet" prompt when no filter chip is active, so
a filter that matches nothing is not mistaken for an empty fleet.
This commit is contained in:
Anso
2026-06-02 16:10:05 -04:00
committed by GitHub
parent c82a39c65a
commit 06b25262cc
11 changed files with 963 additions and 12 deletions
@@ -0,0 +1,86 @@
/**
* Unit tests for parseComposePreview: the pure compose-to-preview parser behind
* the guided import scan. Covers port/volume/env normalization, the relative-
* volume warning (1:1 path rule), and the non-throwing error paths.
*/
import { describe, it, expect } from 'vitest';
import { parseComposePreview } from '../helpers/composePreview';
describe('parseComposePreview', () => {
it('extracts services, short-syntax ports, volumes, and env_file', () => {
const yaml = `
services:
web:
image: nginx:1.27
ports:
- "8080:80"
- "53:53/udp"
volumes:
- ./data:/usr/share/nginx/html
env_file: web.env
`;
const result = parseComposePreview(yaml);
expect(result.parseError).toBeUndefined();
expect(result.services).toHaveLength(1);
const web = result.services[0];
expect(web.name).toBe('web');
expect(web.image).toBe('nginx:1.27');
expect(web.ports).toEqual(['8080->80', '53->53']);
expect(web.volumes).toEqual(['./data:/usr/share/nginx/html']);
expect(web.envFiles).toEqual(['web.env']);
// Relative bind source triggers the 1:1 path-rule warning.
expect(result.warnings.some((w) => w.includes('1:1 path rule'))).toBe(true);
});
it('normalizes long-syntax ports and ip-prefixed short ports', () => {
const yaml = `
services:
api:
ports:
- target: 3001
published: 2283
- "127.0.0.1:9000:9000"
- "5000"
`;
const result = parseComposePreview(yaml);
expect(result.services[0].ports).toEqual(['2283->3001', '9000->9000', '5000']);
});
it('handles env_file as an array of strings and {path} objects', () => {
const yaml = `
services:
app:
env_file:
- common.env
- path: secrets.env
`;
const result = parseComposePreview(yaml);
expect(result.services[0].envFiles).toEqual(['common.env', 'secrets.env']);
});
it('does not warn for named or absolute volumes', () => {
const yaml = `
services:
db:
volumes:
- pgdata:/var/lib/postgresql/data
- /opt/host/conf:/etc/conf
volumes:
pgdata:
`;
const result = parseComposePreview(yaml);
expect(result.services[0].volumes).toEqual(['pgdata:/var/lib/postgresql/data', '/opt/host/conf:/etc/conf']);
expect(result.warnings).toHaveLength(0);
});
it('reports a parseError for invalid YAML without throwing', () => {
const result = parseComposePreview('services: [unclosed');
expect(result.parseError).toBeDefined();
expect(result.services).toHaveLength(0);
});
it('reports a parseError when there are no services', () => {
const result = parseComposePreview("name: just-a-file\nversion: '3'");
expect(result.parseError).toBe('No services found in this file.');
});
});
+151
View File
@@ -0,0 +1,151 @@
/**
* Tests for FileSystemService.findImportCandidates: the read-only compose-dir
* walk behind the guided import scan. Uses a real temp directory so the nesting
* and placement-status logic is exercised against the actual filesystem.
*/
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import fs from 'fs';
import path from 'path';
const { tmpRoot } = vi.hoisted(() => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const nodeOs = require('os');
// eslint-disable-next-line @typescript-eslint/no-require-imports
const nodePath = require('path');
// eslint-disable-next-line @typescript-eslint/no-require-imports
const nodeFs = require('fs');
const tmpRoot: string = nodeFs.mkdtempSync(nodePath.join(nodeOs.tmpdir(), 'sencho-import-'));
return { tmpRoot };
});
vi.mock('../services/NodeRegistry', () => ({
NodeRegistry: {
getInstance: () => ({
getComposeDir: () => tmpRoot,
getDefaultNodeId: () => 1,
}),
},
}));
import { FileSystemService } from '../services/FileSystemService';
const COMPOSE = 'services:\n app:\n image: nginx:1.27\n';
describe('FileSystemService.findImportCandidates', () => {
beforeAll(() => {
// Already a stack: top-level subdir with a compose file.
fs.mkdirSync(path.join(tmpRoot, 'immich'), { recursive: true });
fs.writeFileSync(path.join(tmpRoot, 'immich', 'compose.yaml'), COMPOSE);
// Loose at the root: will not auto-register.
fs.writeFileSync(path.join(tmpRoot, 'docker-compose.yml'), COMPOSE);
// One directory too deep: apps/ has no compose, apps/vault/ does.
fs.mkdirSync(path.join(tmpRoot, 'apps', 'vault'), { recursive: true });
fs.writeFileSync(path.join(tmpRoot, 'apps', 'vault', 'compose.yaml'), COMPOSE);
// A directory with no compose file at all: ignored.
fs.mkdirSync(path.join(tmpRoot, 'notes'), { recursive: true });
fs.writeFileSync(path.join(tmpRoot, 'notes', 'README.md'), '# notes');
});
afterAll(() => {
fs.rmSync(tmpRoot, { recursive: true, force: true });
});
it('classifies listed, loose-root, and nested compose files and ignores the rest', async () => {
const candidates = await FileSystemService.getInstance().findImportCandidates();
const listed = candidates.find((c) => c.status === 'listed');
expect(listed).toMatchObject({ name: 'immich', composeFile: 'compose.yaml', location: 'immich/compose.yaml' });
expect(listed?.content).toContain('services:');
const loose = candidates.find((c) => c.status === 'loose-root');
expect(loose).toMatchObject({ name: '', composeFile: 'docker-compose.yml', location: 'docker-compose.yml' });
const nested = candidates.find((c) => c.status === 'nested');
expect(nested).toMatchObject({ name: 'vault', composeFile: 'compose.yaml', location: 'apps/vault/compose.yaml' });
// The directory with only a README produced no candidate.
expect(candidates.some((c) => c.name === 'notes')).toBe(false);
expect(candidates).toHaveLength(3);
});
it('flags oversized compose files instead of reading them', async () => {
const bigDir = path.join(tmpRoot, 'big');
fs.mkdirSync(bigDir, { recursive: true });
fs.writeFileSync(path.join(bigDir, 'compose.yaml'), 'x'.repeat(1_048_577));
try {
const candidates = await FileSystemService.getInstance().findImportCandidates();
const big = candidates.find((c) => c.name === 'big');
expect(big?.oversized).toBe(true);
expect(big?.content).toBeNull();
} finally {
fs.rmSync(bigDir, { recursive: true, force: true });
}
});
it('skips a non-regular file (a directory named compose.yaml) without reading it', async () => {
// A directory named compose.yaml passes the access() probe; the isFile()
// guard means it is reported as unreadable rather than read as content.
const weirdDir = path.join(tmpRoot, 'weird');
fs.mkdirSync(path.join(weirdDir, 'compose.yaml'), { recursive: true });
try {
const candidates = await FileSystemService.getInstance().findImportCandidates();
const weird = candidates.find((c) => c.name === 'weird');
expect(weird).toBeDefined();
expect(weird?.content).toBeNull();
expect(weird?.oversized).toBe(false);
} finally {
fs.rmSync(weirdDir, { recursive: true, force: true });
}
});
it('does not read a compose file that symlinks outside the compose directory', async () => {
// Sibling of the temp compose root, i.e. outside the base dir.
const outside = path.join(path.dirname(tmpRoot), `sencho-outside-${Date.now()}.yaml`);
fs.writeFileSync(outside, COMPOSE);
const escDir = path.join(tmpRoot, 'escape');
fs.mkdirSync(escDir, { recursive: true });
let linked = true;
try {
fs.symlinkSync(outside, path.join(escDir, 'compose.yaml'));
} catch {
// Creating symlinks needs privilege on some platforms; the assertion below
// runs for real on the Linux CI runners.
linked = false;
}
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
try {
if (!linked) return;
const candidates = await FileSystemService.getInstance().findImportCandidates();
const esc = candidates.find((c) => c.name === 'escape');
expect(esc).toBeDefined();
expect(esc?.content).toBeNull();
expect(warnSpy).toHaveBeenCalled();
} finally {
warnSpy.mockRestore();
fs.rmSync(escDir, { recursive: true, force: true });
fs.rmSync(outside, { force: true });
}
});
it('keeps the top-level listing and does not also descend into it', async () => {
// A directory that is already a stack and also has a compose file one level
// deeper yields only the listed entry, never the nested child.
const dir = path.join(tmpRoot, 'both');
fs.mkdirSync(path.join(dir, 'sub'), { recursive: true });
fs.writeFileSync(path.join(dir, 'compose.yaml'), COMPOSE);
fs.writeFileSync(path.join(dir, 'sub', 'compose.yaml'), COMPOSE);
try {
const candidates = await FileSystemService.getInstance().findImportCandidates();
const fromBoth = candidates.filter((c) => c.location.startsWith('both/'));
expect(fromBoth).toHaveLength(1);
expect(fromBoth[0]).toMatchObject({ name: 'both', status: 'listed', location: 'both/compose.yaml' });
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
it('truncates at maxCandidates', async () => {
const candidates = await FileSystemService.getInstance().findImportCandidates(2);
expect(candidates).toHaveLength(2);
});
});
+152
View File
@@ -0,0 +1,152 @@
import YAML from 'yaml';
// Refuse to parse anything beyond this bound so a malformed (or adversarial)
// compose file cannot exhaust heap while building an import preview. Mirrors the
// cap the stacks router applies for env_file resolution.
const MAX_COMPOSE_PARSE_BYTES = 1_048_576; // 1 MiB
export interface ServicePreview {
name: string;
image?: string;
/** Port mappings normalized to "host->container" (container-only when unpublished). */
ports: string[];
/** Bind/named volumes normalized to "source:target" (target-only when anonymous). */
volumes: string[];
/** env_file paths exactly as declared in the compose file. */
envFiles: string[];
}
export interface ComposePreview {
services: ServicePreview[];
warnings: string[];
/** Set when the file could not be turned into a usable preview; services is then empty. */
parseError?: string;
}
function asString(value: unknown): string | undefined {
if (typeof value === 'string') return value;
if (typeof value === 'number') return String(value);
return undefined;
}
function normalizePort(entry: unknown): string | null {
// Short syntax: "8080:80", "8080:80/tcp", "127.0.0.1:8080:80", "80", or a bare number.
const short = asString(entry);
if (short !== undefined) {
const [hostPort, containerPort] = splitShortPort(short);
if (hostPort && containerPort) return `${hostPort}->${containerPort}`;
if (containerPort) return containerPort; // container-only
return hostPort ?? short;
}
// Long syntax: { target, published?, protocol? }.
if (entry && typeof entry === 'object') {
const obj = entry as Record<string, unknown>;
const target = asString(obj.target);
const published = asString(obj.published);
if (target) return published ? `${published}->${target}` : target;
}
return null;
}
function splitShortPort(value: string): [string | null, string | null] {
const spec = value.split('/')[0]; // drop "/tcp" | "/udp"
const parts = spec.split(':');
if (parts.length >= 3) return [parts[1], parts[2]]; // ip:host:container
if (parts.length === 2) return [parts[0], parts[1]]; // host:container
return [null, parts[0]]; // container-only
}
function isRelativeSource(source: string): boolean {
return source.startsWith('./') || source.startsWith('../');
}
function normalizeVolume(entry: unknown, relativeFlag: { hit: boolean }): string | null {
// Short syntax: "source:target", "source:target:ro", or "/anon-target".
const short = asString(entry);
if (short !== undefined) {
const parts = short.split(':');
if (parts.length >= 2) {
const source = parts[0];
if (isRelativeSource(source)) relativeFlag.hit = true;
return `${source}:${parts[1]}`;
}
return short; // anonymous volume (target only)
}
// Long syntax: { type, source?, target }.
if (entry && typeof entry === 'object') {
const obj = entry as Record<string, unknown>;
const source = asString(obj.source);
const target = asString(obj.target);
if (target) {
if (source && isRelativeSource(source)) relativeFlag.hit = true;
return source ? `${source}:${target}` : target;
}
}
return null;
}
function collectEnvFiles(envFile: unknown): string[] {
if (typeof envFile === 'string') return [envFile];
if (Array.isArray(envFile)) {
const out: string[] = [];
for (const entry of envFile) {
const p = typeof entry === 'string' ? entry : asString((entry as Record<string, unknown>)?.path);
if (p) out.push(p);
}
return out;
}
return [];
}
/**
* Parse a compose file's text into a dry preview of its services, ports,
* volumes, and env files. Never throws: parse failures and non-compose files
* are reported through `parseError`. Used by the guided import scan to show
* the user what a detected file contains before they bring it in.
*/
export function parseComposePreview(content: string): ComposePreview {
if (content.length > MAX_COMPOSE_PARSE_BYTES) {
return { services: [], warnings: [], parseError: 'Compose file is too large to preview.' };
}
let parsed: unknown;
try {
parsed = YAML.parse(content);
} catch (error) {
return { services: [], warnings: [], parseError: `Could not parse compose file: ${(error as Error).message}` };
}
const services = (parsed as { services?: unknown })?.services;
if (!services || typeof services !== 'object') {
return { services: [], warnings: [], parseError: 'No services found in this file.' };
}
const relativeFlag = { hit: false };
const previews: ServicePreview[] = [];
for (const [name, raw] of Object.entries(services as Record<string, unknown>)) {
const svc = (raw ?? {}) as Record<string, unknown>;
const ports = Array.isArray(svc.ports)
? svc.ports.map((p) => normalizePort(p)).filter((p): p is string => p !== null)
: [];
const volumes = Array.isArray(svc.volumes)
? svc.volumes.map((v) => normalizeVolume(v, relativeFlag)).filter((v): v is string => v !== null)
: [];
previews.push({
name,
image: asString(svc.image),
ports,
volumes,
envFiles: collectEnvFiles(svc.env_file),
});
}
const warnings: string[] = [];
if (relativeFlag.hit) {
warnings.push(
'Uses relative volume paths. For these to resolve, your host mount path must match the container compose directory (the 1:1 path rule).',
);
}
return { services: previews, warnings };
}
+36
View File
@@ -23,6 +23,7 @@ import { isDebugEnabled } from '../utils/debug';
import { sanitizeForLog } from '../utils/safeLog';
import { sendGitSourceError } from '../utils/gitSourceHttp';
import { buildPolicyGateOptions, runPolicyGate, triggerPostDeployScan } from '../helpers/policyGate';
import { parseComposePreview, type ComposePreview } from '../helpers/composePreview';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants';
import { getTerminalWs, DEPLOY_SESSION_HEADER } from '../websocket/generic';
@@ -255,6 +256,41 @@ stacksRouter.get('/statuses', async (req: Request, res: Response) => {
}
});
// Read-only scan of the compose directory for the guided first-import flow.
// Surfaces compose files found on disk (loose at the root, already a stack, or
// one directory too deep) with a dry preview so a new user can land their first
// stack without reading the docs first. Performs no writes.
stacksRouter.get('/import/scan', async (req: Request, res: Response) => {
if (!requirePermission(req, res, 'stack:read')) return;
try {
const fsSvc = FileSystemService.getInstance(req.nodeId);
const raw = await fsSvc.findImportCandidates();
const candidates = raw.map((c) => {
let preview: ComposePreview;
if (c.oversized) {
preview = { services: [], warnings: [], parseError: 'Compose file is too large to preview.' };
} else if (c.content !== null) {
preview = parseComposePreview(c.content);
} else {
preview = { services: [], warnings: [], parseError: 'Could not read compose file.' };
}
return {
name: c.name,
composeFile: c.composeFile,
location: c.location,
status: c.status,
services: preview.services,
warnings: preview.warnings,
parseError: preview.parseError,
};
});
res.json({ composeDir: fsSvc.getBaseDir(), candidates });
} catch (error) {
console.error('Failed to scan compose directory:', error);
res.status(500).json({ error: 'Failed to scan compose directory' });
}
});
type BulkLifecycleAction = 'start' | 'stop' | 'restart' | 'update';
const VALID_BULK_ACTIONS: ReadonlySet<BulkLifecycleAction> = new Set(['start', 'stop', 'restart', 'update']);
const BULK_PARALLELISM = 4;
+141
View File
@@ -2,6 +2,7 @@ import path from 'path';
import os from 'os';
import crypto from 'crypto';
import { promises as fsPromises, createReadStream } from 'fs';
import type { Dirent } from 'fs';
import type { Readable } from 'stream';
import { NodeRegistry } from './NodeRegistry';
import { isPathWithinBase, isValidStackName } from '../utils/validation';
@@ -36,6 +37,29 @@ const PROTECTED_STACK_FILES = new Set([
'.env',
]);
// Compose filenames Sencho recognizes, in resolution-priority order. Mirrors the
// list FileSystemService uses elsewhere; named here for the import scan.
const IMPORT_COMPOSE_FILENAMES = ['compose.yaml', 'compose.yml', 'docker-compose.yaml', 'docker-compose.yml'] as const;
const IMPORT_COMPOSE_FILENAME_SET = new Set<string>(IMPORT_COMPOSE_FILENAMES);
// Skip reading compose files larger than this into the import preview.
const IMPORT_MAX_PREVIEW_BYTES = 1_048_576; // 1 MiB
/**
* A compose file discovered on disk during the guided import scan. `status`
* records placement: a top-level subdirectory with a compose file is already a
* stack (`listed`); a compose file loose at the compose-dir root (`loose-root`)
* or one directory too deep (`nested`) will not auto-register and needs the user
* to move it. `content` is null when the file was oversized or unreadable.
*/
export interface ImportCandidateRaw {
name: string;
composeFile: string;
location: string;
status: 'listed' | 'loose-root' | 'nested';
content: string | null;
oversized: boolean;
}
// Strips at most one trailing slash. The upstream validator
// (isValidRelativeStackPath) rejects any '//' sequence, so a string reaching
// this helper can carry at most one trailing slash, and a single slice is
@@ -423,6 +447,123 @@ export class FileSystemService {
return this.baseDir;
}
private async firstComposeFilename(dir: string): Promise<string | null> {
this.assertWithinBase(dir);
for (const file of IMPORT_COMPOSE_FILENAMES) {
try {
await fsPromises.access(path.join(dir, file));
return file;
} catch {
// continue
}
}
return null;
}
private async readComposeCandidate(filePath: string): Promise<{ content: string | null; oversized: boolean }> {
this.assertWithinBase(filePath);
let fh: import('fs/promises').FileHandle | null = null;
try {
// Resolve symlinks and confirm the real target is still inside the compose
// directory before reading (matches resolveSafeStackPath). A symlinked
// compose file or symlinked parent must not expose a file outside the
// compose dir through the preview.
const realPath = await fsPromises.realpath(filePath);
if (!isPathWithinBase(realPath, this.baseDir)) {
console.warn('[FileSystemService] Skipping import candidate that escapes the compose directory:', sanitizeForLog(filePath));
return { content: null, oversized: false };
}
// Open the canonical path once and stat/read on the same descriptor so the
// size check and the read observe the same inode (no time-of-check/use race).
fh = await fsPromises.open(realPath, 'r');
const stat = await fh.stat();
if (!stat.isFile()) return { content: null, oversized: false };
if (stat.size > IMPORT_MAX_PREVIEW_BYTES) return { content: null, oversized: true };
// Read at most stat.size (<= cap) bytes so a file that grows after the
// stat cannot push this buffer past the cap.
const buffer = Buffer.alloc(stat.size);
const { bytesRead } = await fh.read(buffer, 0, stat.size, 0);
return { content: buffer.subarray(0, bytesRead).toString('utf-8'), oversized: false };
} catch (error) {
// The file existed at probe time, so a failure here (permission, I/O) is
// worth a server-side line even though the scan degrades gracefully and the
// route reports it to the user.
console.warn('[FileSystemService] Failed to read import candidate:', sanitizeForLog((error as Error)?.message ?? String(error)));
return { content: null, oversized: false };
} finally {
if (fh) await fh.close();
}
}
/**
* Scan the compose directory for compose files to surface in the guided import
* flow: loose files at the root, top-level stack subdirectories, and compose
* files one directory too deep. Read-only. Bounded by `maxCandidates` and by a
* single level of nesting so a deep tree cannot make this walk unbounded.
*/
async findImportCandidates(maxCandidates = 100): Promise<ImportCandidateRaw[]> {
const candidates: ImportCandidateRaw[] = [];
let entries: Dirent[];
try {
entries = await fsPromises.readdir(this.baseDir, { withFileTypes: true });
} catch (error) {
// The compose dir itself is unreadable (missing, permissions). The scan
// degrades to an empty list, so log it rather than report "no files found"
// for what is really an access failure.
console.warn('[FileSystemService] Failed to scan compose directory for import:', sanitizeForLog((error as Error)?.message ?? String(error)));
return candidates;
}
for (const entry of entries) {
if (candidates.length >= maxCandidates) break;
if (!entry.name || typeof entry.name !== 'string') continue;
if (entry.isFile()) {
if (IMPORT_COMPOSE_FILENAME_SET.has(entry.name)) {
const loaded = await this.readComposeCandidate(path.join(this.baseDir, entry.name));
candidates.push({ name: '', composeFile: entry.name, location: entry.name, status: 'loose-root', ...loaded });
}
continue;
}
if (!entry.isDirectory()) continue;
const dir = path.join(this.baseDir, entry.name);
const topCompose = await this.firstComposeFilename(dir);
if (topCompose) {
const loaded = await this.readComposeCandidate(path.join(dir, topCompose));
candidates.push({ name: entry.name, composeFile: topCompose, location: `${entry.name}/${topCompose}`, status: 'listed', ...loaded });
continue;
}
// No compose at the top level: peek exactly one level deeper.
let children: Dirent[];
try {
children = await fsPromises.readdir(dir, { withFileTypes: true });
} catch (error) {
console.warn('[FileSystemService] Failed to read subdirectory during import scan:', sanitizeForLog((error as Error)?.message ?? String(error)));
continue;
}
for (const child of children) {
if (candidates.length >= maxCandidates) break;
if (!child.isDirectory() || !child.name || typeof child.name !== 'string') continue;
const childDir = path.join(dir, child.name);
const childCompose = await this.firstComposeFilename(childDir);
if (childCompose) {
const loaded = await this.readComposeCandidate(path.join(childDir, childCompose));
candidates.push({
name: child.name,
composeFile: childCompose,
location: `${entry.name}/${child.name}/${childCompose}`,
status: 'nested',
...loaded,
});
}
}
}
return candidates;
}
async migrateFlatToDirectory(): Promise<void> {
try {
try {