diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f33d04f..36ab389c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +* **compose:** fix atomic backup `EACCES` for stacks whose compose folder is owned by another UID. The Skipper/Admiral atomic deploy/update path used to create `.sencho-backup/` *inside* the user's stack folder, which silently failed (and broke auto-rollback and the manual rollback endpoint) for any stack where a container had chowned its bind mount (e.g. `swag`, `tautulli`, `linuxserver/*` images). Stack backups now live under `/backups//` next to `sencho.db`, which is always writable by the Sencho user. **One-time transition note:** any pre-existing `.sencho-backup/` folders inside compose dirs become orphaned (safe to delete manually); rollback for the most recent pre-upgrade deploy is unavailable until the next deploy/update creates a new backup in the new location. +* **monitor:** stop flooding the error log with `Error parsing stats for container ... 404 no such container` during stack updates. `MonitorService` polls per-container stats on a 30s tick, and a container can be removed by `docker compose up` between the `getRunningContainers()` call and the per-container stats fetch. Dockerode returns a 404 for the gone container, which is expected churn during normal stack updates, not a real error, so it's now skipped silently. Non-404 stats failures still log at error level. * **fleet:** surface real errors from failed local self-updates. Previously the helper container that runs `docker compose up --force-recreate` was launched with `docker run -d`, so `docker run` returned the container ID immediately and any failure happening inside the helper (bad compose file, image mismatch, permission issue) was invisible. The UI only saw the generic 3-minute "Local update did not complete" heuristic fallback. The helper now runs attached so `execFile` captures its exit code and stderr directly, AND it persists failure details to `/app/data/.sencho-update-error` before exiting so the error survives the gateway's death. On startup, `SelfUpdateService.recoverPreviousError()` reads that file, surfaces the real error through `getLastError()`, and deletes it, so the freshly restarted gateway reports exactly why the previous attempt failed instead of the generic timeout message. * **deps:** migrate SSO OIDC integration from `openid-client` v5 to v6. The v5 `Issuer` / `Client` / `generators` API was removed upstream; the service now uses `Configuration`, `discovery`, `buildAuthorizationUrl`, `authorizationCodeGrant`, and `fetchUserInfo`. Discovery metadata is cached for 5 minutes via `CacheService` so a single login flow does not pay the HTTPS round trip twice, and the cache is invalidated on every SSO config write. `fetchUserInfo` failures now log the underlying error instead of silently falling back to id_token claims. diff --git a/backend/src/__tests__/filesystem-backup.test.ts b/backend/src/__tests__/filesystem-backup.test.ts new file mode 100644 index 00000000..91065f94 --- /dev/null +++ b/backend/src/__tests__/filesystem-backup.test.ts @@ -0,0 +1,102 @@ +/** + * Verifies that FileSystemService stores stack backups under + * /backups// rather than inside the user's compose + * folder. The old in-stack-folder location failed with EACCES whenever a + * container had chowned the bind mount, breaking the atomic rollback + * feature for those stacks. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import path from 'path'; +import os from 'os'; +import { promises as fsPromises } from 'fs'; + +// Mutable state the mocked NodeRegistry reads. Each test rewrites these +// before instantiating FileSystemService. +const mockState = { composeDir: '' }; + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getComposeDir: () => mockState.composeDir, + getDefaultNodeId: () => 1, + }), + }, +})); + +import { FileSystemService } from '../services/FileSystemService'; + +describe('FileSystemService backup location', () => { + let composeDir: string; + let dataDir: string; + let originalDataDir: string | undefined; + + beforeEach(async () => { + composeDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-compose-')); + dataDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'sencho-data-')); + mockState.composeDir = composeDir; + originalDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = dataDir; + }); + + afterEach(async () => { + if (originalDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = originalDataDir; + await fsPromises.rm(composeDir, { recursive: true, force: true }); + await fsPromises.rm(dataDir, { recursive: true, force: true }); + }); + + it('writes backups under /backups//, not inside the stack folder', async () => { + const stackName = 'web'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n', 'utf-8'); + await fsPromises.writeFile(path.join(stackDir, '.env'), 'FOO=bar\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + await service.backupStackFiles(stackName); + + const newBackupDir = path.join(dataDir, 'backups', stackName); + const oldBackupDir = path.join(stackDir, '.sencho-backup'); + + // New location has every backed-up file + await expect(fsPromises.access(path.join(newBackupDir, 'compose.yaml'))).resolves.toBeUndefined(); + await expect(fsPromises.access(path.join(newBackupDir, '.env'))).resolves.toBeUndefined(); + const ts = await fsPromises.readFile(path.join(newBackupDir, '.timestamp'), 'utf-8'); + expect(parseInt(ts, 10)).toBeGreaterThan(0); + + // Old location must NOT be created + await expect(fsPromises.access(oldBackupDir)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('getBackupInfo reads from the new location', async () => { + const stackName = 'api'; + await fsPromises.mkdir(path.join(composeDir, stackName), { recursive: true }); + await fsPromises.writeFile(path.join(composeDir, stackName, 'compose.yaml'), 'services: {}\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + const before = await service.getBackupInfo(stackName); + expect(before).toEqual({ exists: false, timestamp: null }); + + await service.backupStackFiles(stackName); + const after = await service.getBackupInfo(stackName); + expect(after.exists).toBe(true); + expect(typeof after.timestamp).toBe('number'); + }); + + it('restoreStackFiles copies files from the new location back to the stack dir', async () => { + const stackName = 'db'; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'version: original\n', 'utf-8'); + + const service = FileSystemService.getInstance(); + await service.backupStackFiles(stackName); + + // Mutate the live stack file, then restore + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'version: mutated\n', 'utf-8'); + await service.restoreStackFiles(stackName); + + const restored = await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8'); + expect(restored).toBe('version: original\n'); + }); +}); diff --git a/backend/src/services/FileSystemService.ts b/backend/src/services/FileSystemService.ts index 4a1eb29b..2f18fbd3 100644 --- a/backend/src/services/FileSystemService.ts +++ b/backend/src/services/FileSystemService.ts @@ -3,6 +3,16 @@ import { promises as fsPromises } from 'fs'; import { spawn } from 'child_process'; import { NodeRegistry } from './NodeRegistry'; +/** + * Resolves the writable Sencho data directory (same one DatabaseService / + * CryptoService use). Recomputed lazily so test harnesses that override + * `process.env.DATA_DIR` after module load still take effect. + */ +function getBackupBaseDir(): string { + const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data'); + return path.join(dataDir, 'backups'); +} + /** * FileSystemService - local-only file I/O for compose stack management. * @@ -289,11 +299,17 @@ export class FileSystemService { } /** - * Backup stack files (compose.yaml + .env) to .sencho-backup/ within the stack dir. + * Backup stack files (compose.yaml + .env) into Sencho's data dir. + * + * Backups live at /backups// (NOT inside the user's + * compose folder) so the operation always succeeds even when the stack + * folder is owned by another UID (e.g., a container running as root has + * chowned its bind mount). DATA_DIR is the same writable location that + * holds sencho.db and encryption.key. */ async backupStackFiles(stackName: string): Promise { const stackDir = path.join(this.baseDir, stackName); - const backupDir = path.join(stackDir, '.sencho-backup'); + const backupDir = path.join(getBackupBaseDir(), stackName); await fsPromises.mkdir(backupDir, { recursive: true }); // Copy compose file @@ -327,12 +343,9 @@ export class FileSystemService { await fsPromises.writeFile(path.join(backupDir, '.timestamp'), Date.now().toString(), 'utf-8'); } - /** - * Restore stack files from .sencho-backup/ back to the stack dir. - */ async restoreStackFiles(stackName: string): Promise { const stackDir = path.join(this.baseDir, stackName); - const backupDir = path.join(stackDir, '.sencho-backup'); + const backupDir = path.join(getBackupBaseDir(), stackName); const items = await fsPromises.readdir(backupDir); for (const item of items) { @@ -341,11 +354,8 @@ export class FileSystemService { } } - /** - * Get backup info for a stack. - */ async getBackupInfo(stackName: string): Promise<{ exists: boolean; timestamp: number | null }> { - const backupDir = path.join(this.baseDir, stackName, '.sencho-backup'); + const backupDir = path.join(getBackupBaseDir(), stackName); try { await fsPromises.access(backupDir); const tsFile = path.join(backupDir, '.timestamp'); diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index 56fa932e..30d9b392 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -294,6 +294,14 @@ export class MonitorService { } } } catch (e) { + // Containers can be removed between getRunningContainers() and the + // per-container stats call (e.g., during a stack update). Dockerode + // throws a 404 in that case. That's expected churn, not a real + // error, so skip silently rather than flooding the logs. + const err = e as { statusCode?: number; reason?: string }; + if (err?.statusCode === 404 || err?.reason === 'no such container') { + continue; + } console.error(`Error parsing stats for container ${container.Id} on node ${node.name}`, e); } } diff --git a/docs/features/atomic-deployments.mdx b/docs/features/atomic-deployments.mdx index 4e1e742f..5131db30 100644 --- a/docs/features/atomic-deployments.mdx +++ b/docs/features/atomic-deployments.mdx @@ -11,7 +11,7 @@ Sencho wraps every deployment in a safety net on Skipper and Admiral tiers. Befo ## How it works -1. **Backup** - Before a deploy or update, Sencho copies your `compose.yaml` and `.env` files to a backup directory inside the stack folder +1. **Backup** - Before a deploy or update, Sencho copies your `compose.yaml` and `.env` files to a safe internal location 2. **Deploy** - Sencho runs the requested compose operation (up, pull + recreate, etc.) 3. **Health probe** - After deployment, Sencho waits briefly, then checks whether any containers exited with a non-zero exit code 4. **Auto-rollback** - If a crash is detected, Sencho restores the backed-up files and re-deploys automatically