mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-25 01:37:05 +00:00
feat(stacks): per-stack environment inventory and secret-safe guardrails (#1397)
* feat(stacks): per-stack environment inventory and secret-safe guardrails
Add an Environment tab to Stack Anatomy that derives a per-stack inventory
of environment variables from the compose files and env files. Each variable
shows its source, whether Compose interpolates it or injects it into a
container, and a status (present, missing, unused, duplicate, or shell-only),
plus likely-secret classification. The inventory works from variable names
only: a value is never read, returned, or logged, and a likely secret shows
presence only. A copy env checklist action exports names and status without
values.
Surface a missing required env_file as a Compose Doctor preflight finding,
and add an opt-in node setting that refuses a deploy or update when a
required ${VAR:?...} variable is unset or empty, before any backup, pull, or
up runs. Default off.
The Environment tab is capability-gated so it hides on older remote nodes.
* fix(stacks): harden env-file reader against a stat-then-open race
Open the env-file handle first and fstat the open handle instead of
stat-ing the path before opening, removing the check-then-use window in
readEnvFileKeys. Use a secure mkdtemp directory for the out-of-base test
path instead of a predictable name in the temp root.
* fix(stacks): resolve nested env_file paths per compose file, reconcile inline keys per service
Resolve each env_file relative to the directory of the compose file that
declared it, so a nested multi-file Git override (infra/prod.yml referencing
./prod.env) lands next to that file instead of the stack root. The root
compose file is unaffected, since its directory is the stack directory.
Reconcile inline environment provenance per service, so a key an override
removed from one service's effective env is not labeled compose-inline just
because another service injects the same name from a different source.
This commit is contained in:
Anso
@@ -0,0 +1,253 @@
|
||||
/**
|
||||
* Authored-compose env analysis for a stack: the env_file declarations (with
|
||||
* existence metadata), the project `.env` interpolation source, the inline
|
||||
* `environment:` KEY names per service, and the `${}` interpolation references.
|
||||
*
|
||||
* This is the single reader of the authored compose file set, so the multi-file
|
||||
* Git case is handled once and every consumer (the route's env-file wrapper that
|
||||
* Fleet Secrets calls, the Compose Doctor preflight, and the env inventory) sees
|
||||
* the same env_file set. It surfaces NAMES and structural
|
||||
* facts only: an env-file value is never read here, and inline environment values
|
||||
* are dropped immediately after their key names are taken.
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import YAML from 'yaml';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { isPathWithinBase, isValidRelativeStackPath } from '../utils/validation';
|
||||
import { parseInterpolationRefs, type InterpolationRef } from './envVarParse';
|
||||
|
||||
const MAX_COMPOSE_PARSE_BYTES = 1_048_576; // 1 MiB, matches the routes/stacks.ts bound
|
||||
const ROOT_COMPOSE_CANDIDATES = ['compose.yaml', 'compose.yml', 'docker-compose.yaml', 'docker-compose.yml'];
|
||||
|
||||
export type EnvFileExistence = 'present' | 'missing' | 'unverifiable';
|
||||
|
||||
/**
|
||||
* One physical env file, deduped by resolved absolute path. The project `.env`
|
||||
* doubling as an `env_file: .env` entry is ONE physical file carrying both roles,
|
||||
* so it is never double-counted as a duplicate definition.
|
||||
*/
|
||||
export interface PhysicalEnvFile {
|
||||
/** Absolute path, or null when the raw path is interpolated or escapes the stack dir. */
|
||||
resolvedPath: string | null;
|
||||
/** Raw paths as written (or '.env' for the implicit project source). */
|
||||
rawPaths: string[];
|
||||
existence: EnvFileExistence;
|
||||
/** A missing file matters only when at least one declaration required it. */
|
||||
required: boolean;
|
||||
/** True when this is the project `.env` Compose reads for `${}` interpolation. */
|
||||
isInterpolationSource: boolean;
|
||||
/** True when a service `env_file:` injects this file into a container. */
|
||||
isInjectionSource: boolean;
|
||||
/** Services that declared this file via `env_file:`. */
|
||||
declaringServices: string[];
|
||||
}
|
||||
|
||||
export interface StackEnvSources {
|
||||
stackDir: string;
|
||||
baseDir: string;
|
||||
/** Absolute authored compose files actually read (multi-file Git aware). */
|
||||
composeFiles: string[];
|
||||
/** Project `.env` + every declared env_file, deduped by resolved path. */
|
||||
envFiles: PhysicalEnvFile[];
|
||||
/** Authored `environment:` KEY names per service (union across authored files). */
|
||||
inlineEnvKeysByService: Record<string, string[]>;
|
||||
/** `${}` references found across the authored compose source. */
|
||||
interpolationRefs: InterpolationRef[];
|
||||
}
|
||||
|
||||
interface EnvFileEntry {
|
||||
rawPath: string;
|
||||
required: boolean;
|
||||
}
|
||||
|
||||
/** Normalize a service `env_file:` field (string, array of strings, or long-form objects). */
|
||||
function normalizeEnvFileField(envFile: unknown): EnvFileEntry[] {
|
||||
if (typeof envFile === 'string') return [{ rawPath: envFile, required: true }];
|
||||
if (!Array.isArray(envFile)) return [];
|
||||
const out: EnvFileEntry[] = [];
|
||||
for (const entry of envFile) {
|
||||
if (typeof entry === 'string') {
|
||||
out.push({ rawPath: entry, required: true });
|
||||
} else if (entry && typeof entry === 'object') {
|
||||
const p = (entry as Record<string, unknown>).path;
|
||||
if (typeof p === 'string') {
|
||||
out.push({ rawPath: p, required: (entry as Record<string, unknown>).required !== false });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Inline `environment:` KEY names (object / `KEY=value` array / bare `KEY`), never values. */
|
||||
function inlineEnvKeysOf(environment: unknown): string[] {
|
||||
if (Array.isArray(environment)) {
|
||||
return environment
|
||||
.filter((e): e is string => typeof e === 'string')
|
||||
.map(e => e.split('=')[0].trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
if (environment && typeof environment === 'object') {
|
||||
return Object.keys(environment as Record<string, unknown>);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
async function existenceOf(fsService: FileSystemService, abs: string, baseDir: string): Promise<EnvFileExistence> {
|
||||
if (!isPathWithinBase(abs, baseDir)) return 'unverifiable';
|
||||
try {
|
||||
await fsService.access(abs);
|
||||
return 'present';
|
||||
} catch (err) {
|
||||
return (err as NodeJS.ErrnoException).code === 'ENOENT' ? 'missing' : 'unverifiable';
|
||||
}
|
||||
}
|
||||
|
||||
/** The authored compose files Compose would read: the applied Git deploy spec, else the root file. */
|
||||
async function discoverAuthoredComposeFiles(
|
||||
fsService: FileSystemService,
|
||||
stackName: string,
|
||||
stackDir: string,
|
||||
): Promise<string[]> {
|
||||
const spec = DatabaseService.getInstance().getGitSource(stackName)?.applied_deploy_spec;
|
||||
if (spec && Array.isArray(spec.files) && spec.files.length > 0) {
|
||||
const files: string[] = [];
|
||||
for (const f of spec.files) {
|
||||
if (typeof f !== 'string' || !isValidRelativeStackPath(f)) continue;
|
||||
const abs = path.resolve(stackDir, f);
|
||||
if (isPathWithinBase(abs, stackDir)) files.push(abs);
|
||||
}
|
||||
if (files.length > 0) return files;
|
||||
}
|
||||
for (const name of ROOT_COMPOSE_CANDIDATES) {
|
||||
const abs = path.resolve(stackDir, name);
|
||||
try {
|
||||
await fsService.access(abs);
|
||||
return [abs];
|
||||
} catch {
|
||||
// try next candidate
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
async function parseComposeServices(fsService: FileSystemService, absPath: string): Promise<{
|
||||
services: Record<string, unknown>;
|
||||
text: string;
|
||||
} | null> {
|
||||
let content: string;
|
||||
try {
|
||||
content = await fsService.readFile(absPath, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (content.length > MAX_COMPOSE_PARSE_BYTES) return null;
|
||||
try {
|
||||
const parsed = YAML.parse(content) as Record<string, unknown> | null;
|
||||
const services = (parsed?.services && typeof parsed.services === 'object')
|
||||
? parsed.services as Record<string, unknown>
|
||||
: {};
|
||||
return { services, text: content };
|
||||
} catch {
|
||||
return { services: {}, text: content };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve every authored env source for a stack. Reads each authored compose file
|
||||
* once and returns env_file existence, inline key names, and interpolation refs.
|
||||
*/
|
||||
export async function resolveStackEnvSources(nodeId: number, stackName: string): Promise<StackEnvSources> {
|
||||
const fsService = FileSystemService.getInstance(nodeId);
|
||||
const baseDir = fsService.getBaseDir();
|
||||
const stackDir = path.join(baseDir, stackName);
|
||||
|
||||
const composeFiles = await discoverAuthoredComposeFiles(fsService, stackName, stackDir);
|
||||
|
||||
// Physical env files, deduped by resolved absolute path. Seed with the project
|
||||
// `.env`: always the interpolation source, regardless of any env_file entry.
|
||||
const byPath = new Map<string, PhysicalEnvFile>();
|
||||
const dotenvPath = path.resolve(stackDir, '.env');
|
||||
const dotenv: PhysicalEnvFile = {
|
||||
resolvedPath: dotenvPath,
|
||||
rawPaths: ['.env'],
|
||||
existence: await existenceOf(fsService, dotenvPath, baseDir),
|
||||
required: false,
|
||||
isInterpolationSource: true,
|
||||
isInjectionSource: false,
|
||||
declaringServices: [],
|
||||
};
|
||||
byPath.set(dotenvPath, dotenv);
|
||||
|
||||
const unresolved: PhysicalEnvFile[] = [];
|
||||
const inlineEnvKeysByService: Record<string, string[]> = {};
|
||||
let authoredText = '';
|
||||
|
||||
for (const file of composeFiles) {
|
||||
const parsed = await parseComposeServices(fsService, file);
|
||||
if (!parsed) continue;
|
||||
authoredText += parsed.text + '\n';
|
||||
|
||||
for (const [serviceName, svcRaw] of Object.entries(parsed.services)) {
|
||||
const svc = (svcRaw ?? {}) as Record<string, unknown>;
|
||||
|
||||
const inlineKeys = inlineEnvKeysOf(svc.environment);
|
||||
if (inlineKeys.length > 0) {
|
||||
const existing = inlineEnvKeysByService[serviceName] ?? [];
|
||||
inlineEnvKeysByService[serviceName] = [...new Set([...existing, ...inlineKeys])];
|
||||
}
|
||||
|
||||
for (const entry of normalizeEnvFileField(svc.env_file)) {
|
||||
const interpolated = entry.rawPath.includes('${');
|
||||
// Resolve relative to the directory of the compose file that declared it,
|
||||
// so an env_file in a nested multi-file override (e.g. infra/prod.yml ->
|
||||
// ./prod.env) lands next to that file, not at the stack root. For the root
|
||||
// compose file this dir is the stack dir, so the common case is unchanged.
|
||||
const abs = interpolated ? null : path.resolve(path.dirname(file), entry.rawPath);
|
||||
const within = abs !== null && isPathWithinBase(abs, stackDir);
|
||||
const resolvedPath = within ? abs : null;
|
||||
|
||||
if (resolvedPath) {
|
||||
const existing = byPath.get(resolvedPath);
|
||||
if (existing) {
|
||||
existing.isInjectionSource = true;
|
||||
existing.required ||= entry.required;
|
||||
if (!existing.rawPaths.includes(entry.rawPath)) existing.rawPaths.push(entry.rawPath);
|
||||
if (!existing.declaringServices.includes(serviceName)) existing.declaringServices.push(serviceName);
|
||||
} else {
|
||||
byPath.set(resolvedPath, {
|
||||
resolvedPath,
|
||||
rawPaths: [entry.rawPath],
|
||||
existence: await existenceOf(fsService, resolvedPath, baseDir),
|
||||
required: entry.required,
|
||||
isInterpolationSource: false,
|
||||
isInjectionSource: true,
|
||||
declaringServices: [serviceName],
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Interpolated or escaping path: unverifiable, kept so the inventory can show it.
|
||||
unresolved.push({
|
||||
resolvedPath: null,
|
||||
rawPaths: [entry.rawPath],
|
||||
existence: 'unverifiable',
|
||||
required: entry.required,
|
||||
isInterpolationSource: false,
|
||||
isInjectionSource: true,
|
||||
declaringServices: [serviceName],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
stackDir,
|
||||
baseDir,
|
||||
composeFiles,
|
||||
envFiles: [...byPath.values(), ...unresolved],
|
||||
inlineEnvKeysByService,
|
||||
interpolationRefs: parseInterpolationRefs(authoredText),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
/**
|
||||
* Name-only parsing of Compose interpolation and env-file keys, plus the shared
|
||||
* stderr parsers Compose Doctor and the deploy guard both use. Every function
|
||||
* here surfaces variable NAMES only; an env-file value is never returned or
|
||||
* retained, and the bounded reader never loads a whole large file into memory.
|
||||
*/
|
||||
|
||||
import { promises as fsp } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
/** A `${...}` reference found in authored compose source. */
|
||||
export interface InterpolationRef {
|
||||
name: string;
|
||||
/** `${VAR:?e}` / `${VAR?e}`: Compose errors if unset (`:?` also if empty). */
|
||||
required: boolean;
|
||||
/** `${VAR:-d}` / `${VAR-d}`: a default makes the value optional. */
|
||||
hasDefault: boolean;
|
||||
/** `${VAR:+x}` / `${VAR+x}`: alternate value; an unset VAR is intentional. */
|
||||
alternate: boolean;
|
||||
}
|
||||
|
||||
// ${VAR}, ${VAR:-d}, ${VAR-d}, ${VAR:?e}, ${VAR?e}, ${VAR:+x}, ${VAR+x}.
|
||||
// The leading (?<!\$) skips Compose's `$${VAR}` escape (a literal, not a ref).
|
||||
// Group 2 is the operator (':-','-',':?','?',':+','+') or undefined for a bare ref.
|
||||
const INTERPOLATION_RE = /(?<!\$)\$\{([A-Za-z_][A-Za-z0-9_]*)(?:(:?[-?+])[^}]*)?\}/g;
|
||||
|
||||
/**
|
||||
* Extract every distinct `${...}` reference from authored compose text, with its
|
||||
* operator semantics. Operates on raw text (no YAML/value construction), so it
|
||||
* never materializes an env value.
|
||||
*/
|
||||
export function parseInterpolationRefs(source: string): InterpolationRef[] {
|
||||
const byName = new Map<string, InterpolationRef>();
|
||||
for (const m of source.matchAll(INTERPOLATION_RE)) {
|
||||
const name = m[1];
|
||||
const op = m[2];
|
||||
const required = op === ':?' || op === '?';
|
||||
const hasDefault = op === ':-' || op === '-';
|
||||
const alternate = op === ':+' || op === '+';
|
||||
const existing = byName.get(name);
|
||||
if (existing) {
|
||||
existing.required ||= required;
|
||||
existing.hasDefault ||= hasDefault;
|
||||
existing.alternate ||= alternate;
|
||||
} else {
|
||||
byName.set(name, { name, required, hasDefault, alternate });
|
||||
}
|
||||
}
|
||||
return [...byName.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the KEY name from a single env-file line, or null for a blank/comment line.
|
||||
* Handles `KEY=value`, `export KEY=value`, and a bare `KEY` (value sourced from the
|
||||
* shell). The value after `=` is never read or returned.
|
||||
*/
|
||||
export function extractEnvKeyFromLine(line: string): string | null {
|
||||
let s = line.trim();
|
||||
if (!s || s.startsWith('#')) return null;
|
||||
if (s.startsWith('export ')) s = s.slice('export '.length).trim();
|
||||
const eq = s.indexOf('=');
|
||||
const key = (eq === -1 ? s : s.slice(0, eq)).trim();
|
||||
return /^[A-Za-z_][A-Za-z0-9_]*$/.test(key) ? key : null;
|
||||
}
|
||||
|
||||
export interface EnvKeyReadLimits {
|
||||
maxBytes: number;
|
||||
maxLines: number;
|
||||
maxLineLen: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_ENV_KEY_LIMITS: EnvKeyReadLimits = {
|
||||
maxBytes: 256 * 1024,
|
||||
maxLines: 5000,
|
||||
maxLineLen: 8192,
|
||||
};
|
||||
|
||||
export interface EnvKeyReadResult {
|
||||
/** Distinct key names, in first-seen order. Never includes a value. */
|
||||
keys: string[];
|
||||
/** True when the file exceeded a limit and was only partially read. */
|
||||
truncated: boolean;
|
||||
/** True when the path escaped the base or the file could not be read/statted. */
|
||||
unverifiable: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read env-file KEY names from a file under `baseDir`, bounded by `limits`. The
|
||||
* path containment barrier is inlined at the read sink (CodeQL does not credit a
|
||||
* wrapped helper), and the read is capped so a large or adversarial file cannot
|
||||
* exhaust heap. Values are never materialized: only the slice before the first
|
||||
* `=` of each line is kept.
|
||||
*/
|
||||
export async function readEnvFileKeys(
|
||||
filePath: string,
|
||||
baseDir: string,
|
||||
limits: EnvKeyReadLimits = DEFAULT_ENV_KEY_LIMITS,
|
||||
): Promise<EnvKeyReadResult> {
|
||||
const resolved = path.resolve(filePath);
|
||||
const baseResolved = path.resolve(baseDir);
|
||||
if (resolved !== baseResolved && !resolved.startsWith(baseResolved + path.sep)) {
|
||||
return { keys: [], truncated: false, unverifiable: true };
|
||||
}
|
||||
|
||||
let handle: fsp.FileHandle | undefined;
|
||||
try {
|
||||
// Open first, then fstat the open handle (not the path), so there is no
|
||||
// check-then-use window between a path stat and the open.
|
||||
handle = await fsp.open(resolved, 'r');
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile()) return { keys: [], truncated: false, unverifiable: true };
|
||||
const truncated = stat.size > limits.maxBytes;
|
||||
const len = Math.min(stat.size, limits.maxBytes);
|
||||
const buf = Buffer.alloc(len);
|
||||
if (len > 0) await handle.read(buf, 0, len, 0);
|
||||
|
||||
const seen = new Set<string>();
|
||||
const keys: string[] = [];
|
||||
const lines = buf.toString('utf-8').split(/\r?\n/);
|
||||
const lineBudget = Math.min(lines.length, limits.maxLines);
|
||||
for (let i = 0; i < lineBudget; i++) {
|
||||
const line = lines[i];
|
||||
if (line.length > limits.maxLineLen) continue;
|
||||
const key = extractEnvKeyFromLine(line);
|
||||
if (key && !seen.has(key)) {
|
||||
seen.add(key);
|
||||
keys.push(key);
|
||||
}
|
||||
}
|
||||
return { keys, truncated: truncated || lines.length > limits.maxLines, unverifiable: false };
|
||||
} catch {
|
||||
return { keys: [], truncated: false, unverifiable: true };
|
||||
} finally {
|
||||
await handle?.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Collect the deduplicated capture-group-1 matches of a global regex over stderr. */
|
||||
function collectNames(stderr: string, re: RegExp): string[] {
|
||||
const names = new Set<string>();
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(stderr)) !== null) names.add(m[1]);
|
||||
return [...names];
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the names of variables Compose reported as unset from its stderr.
|
||||
* Compose prints this in logfmt (`msg="The \"VAR\" variable is not set..."`),
|
||||
* so the name is wrapped in an escaped quote; the pattern tolerates the
|
||||
* escaped, plain-quoted, and unquoted forms across Compose versions.
|
||||
*/
|
||||
export function parseUnsetEnvVars(stderr: string): string[] {
|
||||
return collectNames(stderr, /([A-Za-z_][A-Za-z0-9_]*)\\?"?\s+variable is not set/gi);
|
||||
}
|
||||
|
||||
/** Names of required (${VAR:?...}) variables Compose reported as missing. Names only, never values. */
|
||||
export function parseMissingRequiredVars(stderr: string): string[] {
|
||||
return collectNames(stderr, /required variable\s+\\?"?([A-Za-z_][A-Za-z0-9_]*)\\?"?\s+is missing/gi);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* Deterministic "is this env key likely a secret" classification by key NAME only.
|
||||
*
|
||||
* Used to decide whether an env-inventory row is redacted (presence shown, value
|
||||
* never read or rendered). The heuristic is segment-aware: a key is split on
|
||||
* non-alphanumeric boundaries and each segment is matched against a known set, so
|
||||
* `API_KEY` / `DB_PASSWORD` / `CLIENT_SECRET` match while a key that merely
|
||||
* contains a secret word as part of a larger token (e.g. `KEYCLOAK_URL`, where the
|
||||
* segment is `KEYCLOAK`, not `KEY`) does not. Over-flagging is safe: classification
|
||||
* only hides a value the inventory already never reads.
|
||||
*/
|
||||
|
||||
/** Whole-segment matches. Split on `_`/non-alnum, so `KEYCLOAK` never matches `KEY`. */
|
||||
const SECRET_SEGMENTS = new Set([
|
||||
'PASSWORD', 'PASSWD', 'PASS', 'PASSPHRASE',
|
||||
'SECRET', 'SECRETS',
|
||||
'TOKEN', 'KEY', 'APIKEY',
|
||||
'CREDENTIAL', 'CREDENTIALS', 'AUTH',
|
||||
]);
|
||||
|
||||
/** Connection strings whose value is sensitive but whose segments are innocuous. */
|
||||
const SECRET_FULL_KEYS = new Set([
|
||||
'DATABASE_URL', 'DATABASE_DSN', 'REDIS_URL', 'MONGO_URI', 'MONGODB_URI', 'AMQP_URL', 'DSN',
|
||||
]);
|
||||
|
||||
/** True when the key name suggests its value is a secret. Names only, never values. */
|
||||
export function isLikelySecretKey(rawKey: string): boolean {
|
||||
const key = rawKey.trim().toUpperCase();
|
||||
if (!key) return false;
|
||||
if (SECRET_FULL_KEYS.has(key)) return true;
|
||||
const segments = key.split(/[^A-Z0-9]+/).filter(Boolean);
|
||||
return segments.some(seg => SECRET_SEGMENTS.has(seg));
|
||||
}
|
||||
Reference in New Issue
Block a user