feat(stacks): per-stack environment inventory and secret-safe guardrails (#1397)

* feat(stacks): per-stack environment inventory and secret-safe guardrails

Add an Environment tab to Stack Anatomy that derives a per-stack inventory
of environment variables from the compose files and env files. Each variable
shows its source, whether Compose interpolates it or injects it into a
container, and a status (present, missing, unused, duplicate, or shell-only),
plus likely-secret classification. The inventory works from variable names
only: a value is never read, returned, or logged, and a likely secret shows
presence only. A copy env checklist action exports names and status without
values.

Surface a missing required env_file as a Compose Doctor preflight finding,
and add an opt-in node setting that refuses a deploy or update when a
required ${VAR:?...} variable is unset or empty, before any backup, pull, or
up runs. Default off.

The Environment tab is capability-gated so it hides on older remote nodes.

* fix(stacks): harden env-file reader against a stat-then-open race

Open the env-file handle first and fstat the open handle instead of
stat-ing the path before opening, removing the check-then-use window in
readEnvFileKeys. Use a secure mkdtemp directory for the out-of-base test
path instead of a predictable name in the temp root.

* fix(stacks): resolve nested env_file paths per compose file, reconcile inline keys per service

Resolve each env_file relative to the directory of the compose file that
declared it, so a nested multi-file Git override (infra/prod.yml referencing
./prod.env) lands next to that file instead of the stack root. The root
compose file is unaffected, since its directory is the stack directory.

Reconcile inline environment provenance per service, so a key an override
removed from one service's effective env is not labeled compose-inline just
because another service injects the same name from a different source.

backend/src/services/CapabilityRegistry.ts

@@ -38,6 +38,7 @@ export const CAPABILITIES = [
   'compose-doctor',
   'update-guard',
   'compose-networking',
+  'env-inventory',
 ] as const;
 
 export type Capability = (typeof CAPABILITIES)[number];

backend/src/services/ComposeDoctorService.ts

@@ -13,38 +13,17 @@ import { parseAccessUrlPorts } from './network/normalize';
 import type { ExposureIntent } from './network/types';
 import { runRules, SEVERITY_RANK, RULE_IDS, RENDER_FAILED_RULE_ID } from './preflight/rules';
 import type {
-  BindCheck, NodePortBinding, PreflightContext, PreflightFinding, PreflightReport, PreflightSeverity, PreflightStatus,
+  BindCheck, NodePortBinding, PreflightContext, PreflightFinding, PreflightReport, PreflightSeverity, PreflightStatus, MissingEnvFile,
 } from './preflight/types';
 
 import { isPathWithinBase } from '../utils/validation';
 import { getErrorMessage } from '../utils/errors';
 import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
+import { parseUnsetEnvVars, parseMissingRequiredVars } from '../helpers/envVarParse';
+import { resolveStackEnvSources } from '../helpers/envFileResolution';
 
 const MAX_RENDER_ERROR = 600; // chars kept from a (redacted) render error
 
-/** Collect the deduplicated capture-group-1 matches of a global regex over stderr. */
-function collectNames(stderr: string, re: RegExp): string[] {
-  const names = new Set<string>();
-  let m: RegExpExecArray | null;
-  while ((m = re.exec(stderr)) !== null) names.add(m[1]);
-  return [...names];
-}
-
-/**
- * Pull the names of variables Compose reported as unset from its stderr.
- * Compose prints this in logfmt (`msg="The \"VAR\" variable is not set..."`),
- * so the name is wrapped in an escaped quote; the pattern tolerates the
- * escaped, plain-quoted, and unquoted forms across Compose versions.
- */
-export function parseUnsetEnvVars(stderr: string): string[] {
-  return collectNames(stderr, /([A-Za-z_][A-Za-z0-9_]*)\\?"?\s+variable is not set/gi);
-}
-
-/** Names of required (${VAR:?...}) variables Compose reported as missing. Names only, never values. */
-export function parseMissingRequiredVars(stderr: string): string[] {
-  return collectNames(stderr, /required variable\s+\\?"?([A-Za-z_][A-Za-z0-9_]*)\\?"?\s+is missing/gi);
-}
-
 const ruleOrder = new Map(RULE_IDS.map((id, i) => [id, i]));
 /** Severity descending, then registry order, so output is deterministic. */
 function sortFindings(findings: PreflightFinding[]): PreflightFinding[] {
@@ -200,6 +179,20 @@ export class ComposeDoctorService {
     const bindChecks = model ? await this.resolveBindChecks(model, baseDir) : [];
     const { stackIntent, serviceIntents, accessUrlPorts, hasAccessUrls } = this.exposureState(nodeId, stackName);
 
+    // Required `env_file:` declarations whose file is absent. Optional
+    // (required: false) and interpolated/escaping paths are excluded. Fail-soft:
+    // a resolution error simply yields no env-file findings.
+    let missingEnvFiles: MissingEnvFile[] = [];
+    try {
+      const envSources = await resolveStackEnvSources(nodeId, stackName);
+      missingEnvFiles = envSources.envFiles
+        .filter(f => f.isInjectionSource && f.required && f.existence === 'missing')
+        .map(f => ({ rawPath: f.rawPaths[0], services: f.declaringServices }));
+    } catch (err) {
+      console.warn('[ComposeDoctor] env-file resolution failed for %s:',
+        sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown')));
+    }
+
     return {
       stackName,
       platform: process.platform,
@@ -207,6 +200,7 @@ export class ComposeDoctorService {
       renderable,
       renderError,
       unsetEnvVars,
+      missingEnvFiles,
       sourceServiceNames,
       sourceReadable,
       nodePorts,

backend/src/services/ComposeService.ts

@@ -17,6 +17,7 @@ import { getErrorMessage } from '../utils/errors';
 import { describeSpawnError } from '../utils/spawnErrors';
 import { isPathWithinBase, isValidStackName } from '../utils/validation';
 import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
+import { parseMissingRequiredVars } from '../helpers/envVarParse';
 import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
 
 export class ComposeRollbackError extends Error {
@@ -364,7 +365,34 @@ export class ComposeService {
     await this.execute('docker', await this.authoredComposeArgs(stackName, [action]), stackDir, ws);
   }
 
+  /**
+   * Opt-in guard: when `env_block_deploy_on_missing_required` is enabled, refuse a
+   * deploy whose required `${VAR:?err}` variables are unset OR empty, before any
+   * backup, cleanup, pull, or `up` runs. Compose's own resolution is authoritative
+   * (it passes process.env), and on the failing path it emits no rendered model, so
+   * no env value is materialized. Default off and any settings-read failure both
+   * fall through without blocking.
+   */
+  private async assertRequiredEnvPresent(stackName: string): Promise<void> {
+    let enabled = false;
+    try {
+      enabled = DatabaseService.getInstance().getGlobalSettings()['env_block_deploy_on_missing_required'] === '1';
+    } catch {
+      return; // safe default: a settings-read failure never blocks a deploy
+    }
+    if (!enabled) return;
+    const result = await this.renderConfig(stackName);
+    const missing = parseMissingRequiredVars(result.stderr);
+    if (missing.length === 0) return;
+    const plural = missing.length > 1;
+    throw new Error(
+      `Deploy blocked: required environment variable${plural ? 's' : ''} ${missing.join(', ')} ` +
+      `${plural ? 'are' : 'is'} missing. Define ${plural ? 'them' : 'it'} in a .env or env_file, then deploy again.`,
+    );
+  }
+
   async deployStack(stackName: string, ws?: WebSocket, atomic?: boolean): Promise<void> {
+    await this.assertRequiredEnvPresent(stackName);
     const stackDir = path.join(this.baseDir, stackName);
     const debug = isDebugEnabled();
     const t0 = Date.now();
@@ -541,6 +569,7 @@ export class ComposeService {
   }
 
   async updateStack(stackName: string, ws?: WebSocket, atomic?: boolean): Promise<void> {
+    await this.assertRequiredEnvPresent(stackName);
     const stackDir = path.join(this.baseDir, stackName);
     const debug = isDebugEnabled();
     const t0 = Date.now();

backend/src/services/DatabaseService.ts

@@ -1493,6 +1493,7 @@ export class DatabaseService {
         stmt.run('health_gate_enabled', '1');
         stmt.run('health_gate_window_seconds', '90');
         stmt.run('image_update_check_interval_minutes', '120');
+        stmt.run('env_block_deploy_on_missing_required', '0');
 
         // Seed the default local node if none exists
         const nodeCount = (this.db.prepare('SELECT COUNT(*) as count FROM nodes').get() as any)?.count || 0;

backend/src/services/EnvInventoryService.ts

@@ -0,0 +1,228 @@
+/**
+ * Per-stack environment inventory: which env vars a stack references, where they
+ * come from, whether Compose interpolates them or injects them into a container,
+ * and whether each is likely a secret. Names, sources, and status ONLY: an env
+ * value is never read, returned, retained, or logged here.
+ *
+ * Compose env semantics this encodes:
+ *  - `${VAR}` interpolation resolves from the project `.env` + the shell only.
+ *  - `env_file:` and inline `environment:` inject values into a container.
+ *  - `${VAR:?err}` fails when unset OR empty, so the unset/missing signal comes
+ *    from Compose's own stderr (authoritative), not a key-only guess.
+ *
+ * Injected keys come from the merge-correct effective model; interpolation refs
+ * and inline-vs-env-file provenance come from the authored source. `process.env`
+ * is consulted ONLY to resolve interpolation refs the stack already references, so
+ * unrelated host/system env names never appear as inventory rows.
+ */
+
+import { ComposeService } from './ComposeService';
+import { parseEffectiveModel } from './preflight/effectiveModel';
+import { resolveStackEnvSources, type EnvFileExistence } from '../helpers/envFileResolution';
+import { parseUnsetEnvVars, parseMissingRequiredVars, readEnvFileKeys } from '../helpers/envVarParse';
+import { isLikelySecretKey } from '../helpers/secretClassification';
+
+export type EnvSource = 'compose-inline' | 'env-file' | 'dotenv' | 'process-env' | 'compose-ref';
+export type EnvItemStatus = 'present' | 'missing' | 'unused' | 'duplicate' | 'unpersisted';
+
+export interface EnvInventoryItem {
+  key: string;
+  sources: EnvSource[];
+  /** Consumed by Compose `${}` interpolation. */
+  usedForInterpolation: boolean;
+  /** Injected into a container (effective model is authoritative). */
+  injectedIntoService: boolean;
+  required: boolean;
+  hasDefault: boolean;
+  likelySecret: boolean;
+  status: EnvItemStatus;
+}
+
+export interface EnvFileInfo {
+  /** Raw paths as written in compose (or '.env' for the project source). No absolute path. */
+  rawPaths: string[];
+  existence: EnvFileExistence;
+  required: boolean;
+  isInterpolationSource: boolean;
+  isInjectionSource: boolean;
+  declaringServices: string[];
+}
+
+export interface EnvInventory {
+  stackName: string;
+  /** False when the effective model could not be rendered; the inventory is then partial. */
+  renderable: boolean;
+  items: EnvInventoryItem[];
+  envFiles: EnvFileInfo[];
+  summary: {
+    total: number;
+    missing: number;
+    unused: number;
+    duplicate: number;
+    unpersisted: number;
+    likelySecret: number;
+  };
+}
+
+/** Build the env inventory for a stack on a node. Key names only; never any value. */
+export async function buildEnvInventory(nodeId: number, stackName: string): Promise<EnvInventory> {
+  const sources = await resolveStackEnvSources(nodeId, stackName);
+  const refByName = new Map(sources.interpolationRefs.map(r => [r.name, r]));
+
+  // Render the effective model: the authoritative injected-key set and the
+  // unset/missing-required signal. Failure path materializes no values.
+  const result = await ComposeService.getInstance(nodeId).renderConfig(stackName);
+  const missingRequired = new Set(parseMissingRequiredVars(result.stderr));
+  let renderable = false;
+  let unsetVars = new Set<string>();
+  const effectiveKeys = new Set<string>();
+  const effectiveKeysByService = new Map<string, Set<string>>();
+  if (result.rendered !== null) {
+    unsetVars = new Set(parseUnsetEnvVars(result.stderr));
+    try {
+      const model = parseEffectiveModel(JSON.parse(result.rendered), stackName);
+      for (const svc of model.services) {
+        const svcKeys = new Set(svc.envKeys);
+        effectiveKeysByService.set(svc.name, svcKeys);
+        for (const k of svc.envKeys) effectiveKeys.add(k);
+      }
+      renderable = true;
+    } catch {
+      renderable = false;
+    }
+  }
+
+  const names = new Set<string>();
+  const locations = new Map<string, Set<string>>(); // distinct physical definition locations
+  const itemSources = new Map<string, Set<EnvSource>>();
+  const dotenvKeys = new Set<string>();
+  const injectionFileKeys = new Set<string>();
+
+  const addLocation = (key: string, location: string) => {
+    const set = locations.get(key) ?? new Set<string>();
+    set.add(location);
+    locations.set(key, set);
+  };
+  const addSource = (key: string, source: EnvSource) => {
+    const set = itemSources.get(key) ?? new Set<EnvSource>();
+    set.add(source);
+    itemSources.set(key, set);
+  };
+
+  for (const ref of sources.interpolationRefs) names.add(ref.name);
+
+  // Env-file provenance: each physical file contributes ONE source label and ONE
+  // location, so the project `.env` doubling as `env_file: .env` is not a duplicate.
+  for (const file of sources.envFiles) {
+    if (!file.resolvedPath || file.existence !== 'present') continue;
+    const { keys, unverifiable } = await readEnvFileKeys(file.resolvedPath, sources.baseDir);
+    if (unverifiable) {
+      // The existence probe said present, but the key read failed (a permission
+      // change, a race, or transient I/O). Surface that rather than silently
+      // reporting zero keys for a file the inventory claims is present.
+      file.existence = 'unverifiable';
+      continue;
+    }
+    const label: EnvSource = file.isInterpolationSource ? 'dotenv' : 'env-file';
+    for (const key of keys) {
+      names.add(key);
+      addLocation(key, file.resolvedPath);
+      addSource(key, label);
+      if (file.isInterpolationSource) dotenvKeys.add(key);
+      if (file.isInjectionSource) injectionFileKeys.add(key);
+    }
+  }
+
+  // Inline `environment:` keys, reconciled against the effective model PER SERVICE
+  // so a key an override removed from one service is not reported as inline just
+  // because another service defines the same name elsewhere.
+  const inlineAll = new Set<string>();
+  for (const keys of Object.values(sources.inlineEnvKeysByService)) for (const k of keys) inlineAll.add(k);
+  for (const [service, keys] of Object.entries(sources.inlineEnvKeysByService)) {
+    const svcEffective = effectiveKeysByService.get(service);
+    for (const key of keys) {
+      if (renderable && !svcEffective?.has(key)) continue;
+      names.add(key);
+      addLocation(key, 'inline');
+      addSource(key, 'compose-inline');
+    }
+  }
+
+  for (const key of effectiveKeys) names.add(key);
+
+  const injectedKeys = renderable
+    ? effectiveKeys
+    : new Set<string>([...inlineAll, ...injectionFileKeys]);
+
+  const shellHas = (name: string): boolean => Object.prototype.hasOwnProperty.call(process.env, name);
+
+  const items: EnvInventoryItem[] = [];
+  for (const key of [...names].sort()) {
+    const ref = refByName.get(key);
+    const usedForInterpolation = !!ref;
+    const required = ref?.required ?? false;
+    const hasDefault = ref?.hasDefault ?? false;
+    const alternate = ref?.alternate ?? false;
+    const injected = injectedKeys.has(key);
+    const locationCount = locations.get(key)?.size ?? 0;
+    const sourceSet = new Set<EnvSource>(itemSources.get(key) ?? []);
+
+    // A referenced var defined in no stack-local source resolves from the shell or
+    // is missing. Surface that provenance without adding unreferenced shell keys.
+    if (usedForInterpolation && locationCount === 0 && !injected) {
+      if (shellHas(key)) sourceSet.add('process-env');
+      else sourceSet.add('compose-ref');
+    }
+
+    // Compose's own resolution is authoritative for unset/empty required vars; the
+    // heuristic only fills in when the model could not be rendered for other reasons.
+    const refUndefinedUnshelled = usedForInterpolation && !hasDefault && !alternate && locationCount === 0 && !shellHas(key);
+    const missing = missingRequired.has(key)
+      || (renderable && unsetVars.has(key))
+      || (!renderable && refUndefinedUnshelled);
+    const unpersisted = usedForInterpolation && locationCount === 0 && !missing && shellHas(key);
+    const unused = dotenvKeys.has(key) && !usedForInterpolation && !injected;
+
+    let status: EnvItemStatus;
+    if (missing) status = 'missing';
+    else if (locationCount >= 2) status = 'duplicate';
+    else if (unpersisted) status = 'unpersisted';
+    else if (unused) status = 'unused';
+    else status = 'present';
+
+    items.push({
+      key,
+      sources: [...sourceSet],
+      usedForInterpolation,
+      injectedIntoService: injected,
+      required,
+      hasDefault,
+      likelySecret: isLikelySecretKey(key),
+      status,
+    });
+  }
+
+  const envFiles: EnvFileInfo[] = sources.envFiles.map(f => ({
+    rawPaths: f.rawPaths,
+    existence: f.existence,
+    required: f.required,
+    isInterpolationSource: f.isInterpolationSource,
+    isInjectionSource: f.isInjectionSource,
+    declaringServices: f.declaringServices,
+  }));
+
+  return {
+    stackName,
+    renderable,
+    items,
+    envFiles,
+    summary: {
+      total: items.length,
+      missing: items.filter(i => i.status === 'missing').length,
+      unused: items.filter(i => i.status === 'unused').length,
+      duplicate: items.filter(i => i.status === 'duplicate').length,
+      unpersisted: items.filter(i => i.status === 'unpersisted').length,
+      likelySecret: items.filter(i => i.likelySecret).length,
+    },
+  };
+}

backend/src/services/effectiveAnatomy.ts

@@ -13,7 +13,7 @@
  * resolved secret VALUE in the rendered model can never reach this payload.
  */
 import { ComposeService } from './ComposeService';
-import { parseMissingRequiredVars } from './ComposeDoctorService';
+import { parseMissingRequiredVars } from '../helpers/envVarParse';
 import { getErrorMessage } from '../utils/errors';
 import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog';
 

backend/src/services/network/composeNetworkInspector.ts

@@ -10,7 +10,7 @@ import DockerController, { type DependencySnapshot } from '../DockerController';
 import { ComposeService } from '../ComposeService';
 import { FileSystemService } from '../FileSystemService';
 import { parseEffectiveModel, type EffectiveModel } from '../preflight/effectiveModel';
-import { parseMissingRequiredVars } from '../ComposeDoctorService';
+import { parseMissingRequiredVars } from '../../helpers/envVarParse';
 import {
   compareStackNetworks, fromEffectiveModel, isAllInterfaces, isLoopback,
 } from './normalize';

backend/src/services/preflight/rules.ts

@@ -87,6 +87,21 @@ const envUnset: PreflightRule = {
   },
 };
 
+const envFileMissing: PreflightRule = {
+  id: 'env-file-missing',
+  run(ctx) {
+    return ctx.missingEnvFiles.map(f => ({
+      ruleId: 'env-file-missing',
+      severity: 'high' as const,
+      title: `Missing env file ${f.rawPath}`,
+      message: `The Compose file declares env_file "${f.rawPath}"${f.services.length ? ` for service ${f.services.join(', ')}` : ''}, but no such file exists in the stack directory. Compose fails to start the stack when a required env_file is absent.`,
+      sourcePath: f.rawPath,
+      remediation: `Create ${f.rawPath} in the stack directory, fix the path, or mark the entry optional with "required: false".`,
+      service: f.services[0],
+    }));
+  },
+};
+
 const portConflictNode: PreflightRule = {
   id: 'port-conflict-node',
   run(ctx) {
@@ -671,6 +686,7 @@ const sensitiveServiceBroadExposure: PreflightRule = {
 export const PREFLIGHT_RULES: PreflightRule[] = [
   renderFailed,
   envUnset,
+  envFileMissing,
   portConflictNode,
   portConflictInternal,
   portExposedAllInterfaces,

backend/src/services/preflight/types.ts

@@ -43,6 +43,12 @@ export interface PreflightReport {
   findings: PreflightFinding[];
 }
 
+/** A declared `env_file:` that is required and absent on disk (names only). */
+export interface MissingEnvFile {
+  rawPath: string;
+  services: string[];
+}
+
 /** A host port bound by a running container on the target node. */
 export interface NodePortBinding {
   publishedPort: number;
@@ -83,6 +89,8 @@ export interface PreflightContext {
   renderError: string | null;
   /** Variable names Compose reported as unset (defaulted to empty string). */
   unsetEnvVars: string[];
+  /** Declared `env_file:` paths that are required but absent on disk (names only). */
+  missingEnvFiles: MissingEnvFile[];
   /** Service names parsed from the literal source file (pre-render). */
   sourceServiceNames: string[];
   /** Whether the source file could be read; gates source-derived checks so an