gsadmin/sencho
1
0
Fork 0
mirror of https://github.com/Studio-Saelix/sencho.git synced 2026-07-26 11:49:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge main to pick up grpc Trivy fix

Browse Source
This commit is contained in:
SaelixCode
2026-07-22 19:47:01 -04:00
parent 7d4499a083 394e68671d
commit efb65f6d6c
107 changed files with 8002 additions and 1762 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+9 -5
View File
@@ -28,7 +28,7 @@ jobs:
&& github.head_ref != 'release-please--branches--main--components--sencho'
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6
@@ -49,6 +49,10 @@ jobs:
working-directory: ./backend
run: npm test
- name: Docker integration tests (dedicated Vitest config)
working-directory: ./backend
run: npm run test:docker-integration
- name: Lint (ESLint)
working-directory: ./backend
run: npm run lint
@@ -78,7 +82,7 @@ jobs:
&& github.head_ref != 'release-please--branches--main--components--sencho'
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6
@@ -117,7 +121,7 @@ jobs:
&& github.head_ref != 'release-please--branches--main--components--sencho'
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
@@ -183,7 +187,7 @@ jobs:
&& github.head_ref != 'release-please--branches--main--components--sencho'
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Reuse the dist/ that the `backend` job produced and verified, instead
# of running `tsc` a second time against the same source tree. The
@@ -230,7 +234,7 @@ jobs:
github.event_name == 'pull_request'
&& github.head_ref != 'release-please--branches--main--components--sencho'
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6
with:
.github/workflows/codeql.yml
+4 -4
View File
@@ -30,19 +30,19 @@ jobs:
language: [javascript-typescript, actions]
steps:
- name: Checkout repo
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Initialize CodeQL
uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
with:
languages: ${{ matrix.language }}
queries: security-extended
config-file: .github/codeql/codeql-config.yml
- name: Autobuild
uses: github/codeql-action/autobuild@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
uses: github/codeql-action/autobuild@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
- name: Perform CodeQL analysis
uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
with:
category: /language:${{ matrix.language }}
.github/workflows/dependency-review.yml
+1 -1
View File
@@ -23,7 +23,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Run dependency review
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0
with:
.github/workflows/docker-dev.yml
+1 -1
View File
@@ -53,7 +53,7 @@ jobs:
timeout-minutes: 30
steps:
- name: Check out the repo
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
.github/workflows/docker-preview.yml
+1 -1
View File
@@ -72,7 +72,7 @@ jobs:
echo "Building PR #${PR_NUMBER} at ${head_sha}"
- name: Check out the repo
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: refs/pull/${{ inputs.pr_number }}/head
.github/workflows/docker-publish.yml
+1 -1
View File
@@ -41,7 +41,7 @@ jobs:
packages: write
steps:
- name: Check out the repo
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
.github/workflows/docs-sync.yml
+2 -2
View File
@@ -35,12 +35,12 @@ jobs:
permission-contents: write
- name: Checkout Sencho repo
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
path: sencho
- name: Checkout sencho-docs repo
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository_owner }}/sencho-docs
token: ${{ steps.app-token.outputs.token }}
.github/workflows/release-please.yml
+1 -1
View File
@@ -43,7 +43,7 @@ jobs:
permission-pull-requests: write
permission-issues: read
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
token: ${{ steps.app-token.outputs.token }}
.github/workflows/security-scan.yml
+4 -4
View File
@@ -29,7 +29,7 @@ jobs:
security-events: write
steps:
- name: Checkout (trivy.yaml + VEX)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
# Same SHA-pinned action used by the PR-blocking and release-blocking
# scans. The trivy binary version is whatever this action SHA bundles;
@@ -50,7 +50,7 @@ jobs:
# scan-main job below keeps the two result sets distinct in the UI.
- name: Upload SARIF to code scanning
if: always()
uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
with:
sarif_file: trivy-published.sarif
category: trivy-published-image
@@ -64,7 +64,7 @@ jobs:
security-events: write
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
@@ -98,7 +98,7 @@ jobs:
- name: Upload SARIF to code scanning
if: always()
uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
with:
sarif_file: trivy-main.sarif
category: trivy-main-head
.github/workflows/visual-regression.yml
+2 -2
View File
@@ -37,7 +37,7 @@ jobs:
timeout-minutes: 15
steps:
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Check for committed baselines
id: baselines
@@ -86,7 +86,7 @@ jobs:
exit 1
- name: Checkout Code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Start app & install Playwright
uses: ./.github/actions/start-app
.gitignore
+3
View File
@@ -70,3 +70,6 @@ docs/internal/
# Transient temp dir a file-roots test creates under the backend working dir
backend/sfr-app-*/
# Local-only (was tracked; keep on disk, do not publish)
hardened-pipeline-skeleton/
.pr-body-tmp.md
-27
View File
@@ -1,27 +0,0 @@
## Summary
- Add a History API sync layer (`useUrlSync`) that maps shell state to `/nodes/<slug>/...` paths without react-router.
- Gate URL normalization on permissions and license readiness; preserve the current URL on authz read failures (fail closed).
- Add stack-list frozen error UI with retry, transition-specific dirty guards for popstate, and user docs at `docs/features/deep-links.mdx`.
- Fix cold-load/refresh regressions: skip dashboard reset on initial node mount; defer URL writer until hydrated `activeView` matches the route.
- Mobile URL contract: `/dashboard` opens Home (content surface); `/stacks` opens the stack list. List surface always writes `/stacks` even when `activeView` is Fleet or Resources.
- Mobile stack deep links: hydrate `pendingDetailStack`, use `loadFileForRoute` + `routeDetailError` for compose failures (frozen URL, `MobileDetailError` + Retry).
- **QA follow-up (`20b35df7`):** Bootstrap `activeView`/tabs from URL via `readUrlRouteState` so Fleet/Security/Resources cold-load correctly; settle route phase when state already matches (fixes in-app URL writes); normalize unknown view segments; open Monaco editor tabs from `/compose`/`/env`/`/files` deep links.
## Test plan
- [x] `cd frontend && npx tsc -b --noEmit`
- [x] Unit: `readUrlRouteState`, `useUrlSync`, `senchoRoute`, `useViewNavigationState` (52+ tests)
- [x] E2E routing spec expanded (shell view content assertions, mobile cases) — re-run on CI / fresh preview
- [ ] Manual QA on rebuilt preview image:
- [ ] Cold load `/nodes/local/fleet`, `/security/images`, `/resources` (correct view, not Dashboard)
- [ ] Dashboard → Fleet → Security updates address bar
- [ ] `/stacks/<name>/env` opens Monaco env tab (not Anatomy-only)
- [ ] `/nodes/local/not-a-view` normalizes to dashboard
- [ ] Mobile stack deep link + compose failure retry (from prior commit)
## Notes
- Internal architecture docs are local-only (gitignored), not in this PR.
- Anatomy panel sub-tabs remain URL-opaque; only Monaco editor tabs update the path.
- Remote node slug suffix (`name-id`) is existing `nodeSlug` behavior; not changed in this PR.
Dockerfile
+20 -12
View File
@@ -102,13 +102,14 @@ RUN if [ "$TARGETARCH" = "$BUILDARCH" ]; then \
# The fetch pulls only the v29.4.1 commit, minimising transfer size.
# docker/cli uses CalVer and ships vendor.mod instead of go.mod to avoid SemVer
# compliance requirements. We copy vendor.mod -> go.mod, drop the committed vendor
# tree, bump golang.org/x/net to v0.55.0, and build with -mod=mod so the patched
# module is resolved from the module proxy instead of the pinned v0.53.0 that the
# image scan flags for six HIGH advisories (CVE-2026-25680, -25681, -27136, -39821,
# -42502, -42506; x/net/html parsing and x/net/idna). Removing vendor/ keeps -mod=mod
# from reading the stale copy, and avoids `go mod tidy` (which does not run cleanly
# against docker/cli's vendor.mod manifest). This stage now fetches modules at build
# time rather than building fully offline.
# tree, bump golang.org/x/net to v0.55.0 and google.golang.org/grpc to v1.82.1,
# and build with -mod=mod so the patched modules are resolved from the module
# proxy. x/net v0.53.0 is flagged for six HIGH advisories (CVE-2026-25680,
# -25681, -27136, -39821, -42502, -42506; x/net/html parsing and x/net/idna).
# grpc v1.80.0 is flagged for GHSA-hrxh-6v49-42gf (xDS RBAC / HTTP/2). Removing
# vendor/ keeps -mod=mod from reading the stale copy, and avoids `go mod tidy`
# (which does not run cleanly against docker/cli's vendor.mod manifest). This
# stage now fetches modules at build time rather than building fully offline.
# Base image pinned by digest so the Go toolchain that compiles the static
# Docker CLI binary cannot change without an explicit Dependabot bump.
FROM --platform=$BUILDPLATFORM golang:1.27rc2-alpine@sha256:dcbb18cc5fa1082364dc6aa95224b6b55429d09cbb9631a053d8064c1c367300 AS cli-builder
@@ -132,7 +133,8 @@ RUN mkdir -p /build
RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \
rm -rf vendor && \
go get golang.org/x/net@v0.55.0 && \
go get golang.org/x/net@v0.55.0 \
google.golang.org/grpc@v1.82.1 && \
CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \
-mod=mod \
-ldflags "-extldflags=-static \
@@ -168,6 +170,9 @@ RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \
# patch-level fixes with no breaking API changes. Every vulnerable code path is
# daemon-side (containerd's CRI service) and is not reached by compose at all,
# so this is defense-in-depth rather than a live exposure.
#
# The same go get also bumps google.golang.org/grpc from v1.80.0 to v1.82.1 to
# clear GHSA-hrxh-6v49-42gf (xDS RBAC fail-open and HTTP/2 transport issues).
# Base image pinned by digest (same image as cli-builder above) so both
# source builds share an identical, immutable Go toolchain.
FROM --platform=$BUILDPLATFORM golang:1.27rc2-alpine@sha256:dcbb18cc5fa1082364dc6aa95224b6b55429d09cbb9631a053d8064c1c367300 AS compose-builder
@@ -190,9 +195,11 @@ WORKDIR /src/docker-compose
RUN mkdir -p /build
# Patch otel/sdk and exporters from v1.42.0 → v1.43.0 to clear CVE-2026-39883
# and CVE-2026-39882, and bump containerd/v2 from v2.2.3 → v2.2.5 to clear
# CVE-2026-46680 plus the CVE-2026-53488 / 53489 / 53492 cluster. All are
# targeted patch-level security bumps with no breaking API changes.
# and CVE-2026-39882, bump containerd/v2 from v2.2.3 → v2.2.5 to clear
# CVE-2026-46680 plus the CVE-2026-53488 / 53489 / 53492 cluster, and bump
# google.golang.org/grpc to v1.82.1 to clear GHSA-hrxh-6v49-42gf. The otel and
# containerd bumps are patch-level; the grpc bump is a minor security release.
# None introduce breaking API changes.
RUN --mount=type=cache,id=go-mod,sharing=locked,target=/go/pkg/mod \
go get go.opentelemetry.io/otel@v1.43.0 \
go.opentelemetry.io/otel/sdk@v1.43.0 \
@@ -204,7 +211,8 @@ RUN --mount=type=cache,id=go-mod,sharing=locked,target=/go/pkg/mod \
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp@v1.43.0 \
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc@v1.43.0 \
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp@v1.43.0 \
github.com/containerd/containerd/v2@v2.2.5 && \
github.com/containerd/containerd/v2@v2.2.5 \
google.golang.org/grpc@v1.82.1 && \
go mod tidy
# Build target is ./cmd (the package main with plugin.Run), per docker/compose's
backend/package-lock.json
Generated
+280 -375
View File
File diff suppressed because it is too large Load Diff
backend/package.json
+2 -1
View File
@@ -10,6 +10,7 @@
"start": "node dist/index.js",
"predev": "node scripts/generate-version.js",
"dev": "nodemon --watch src --ext ts,json --exec ts-node src/index.ts",
"test:docker-integration": "vitest run --config vitest.docker-integration.config.ts",
"test": "vitest run",
"lint": "eslint src",
"reset-mfa": "node dist/cli/resetMfa.js",
@@ -65,7 +66,7 @@
"dependencies": {
"axios": "^1.15.0",
"bcrypt": "^6.0.0",
"better-sqlite3": "^12.6.2",
"better-sqlite3": "^13.0.1",
"composerize": "^1.7.5",
"compression": "^1.8.1",
"cookie-parser": "^1.4.7",
backend/src/__tests__/blueprints-compose-apply.test.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* Regression: Blueprint apply must write canonical compose.yaml (overwriting
* createStack's nginx scaffold) and remove alternate root Compose filenames so
* Docker Compose discovery cannot prefer a leftover docker-compose.yml.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import { promises as fsPromises } from 'fs';
import path from 'path';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let BlueprintService: typeof import('../services/BlueprintService').BlueprintService;
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
let ComposeService: typeof import('../services/ComposeService').ComposeService;
let StackOpLockService: typeof import('../services/StackOpLockService').StackOpLockService;
let counter = 0;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ BlueprintService } = await import('../services/BlueprintService'));
({ FileSystemService } = await import('../services/FileSystemService'));
({ ComposeService } = await import('../services/ComposeService'));
({ StackOpLockService } = await import('../services/StackOpLockService'));
});
afterAll(() => cleanupTestDb(tmpDir));
beforeEach(() => {
counter += 1;
StackOpLockService.resetForTests();
vi.restoreAllMocks();
});
function seedLocalNode(): number {
const composeDir = process.env.COMPOSE_DIR!;
const result = DatabaseService.getInstance().getDb().prepare(
`INSERT INTO nodes (name, type, mode, compose_dir, is_default, status, created_at)
VALUES (?, 'local', 'proxy', ?, 0, 'online', ?)`,
).run(`bp-compose-node-${counter}`, composeDir, Date.now());
return result.lastInsertRowid as number;
}
function defaultNodeId(): number {
const id = DatabaseService.getInstance().getNodes().find(n => n.is_default)?.id;
if (id === undefined) throw new Error('default node missing');
return id;
}
/** Point the default node at the test COMPOSE_DIR (FileSystemService.getInstance uses it). */
function bindDefaultComposeDir(): number {
const composeDir = process.env.COMPOSE_DIR!;
DatabaseService.getInstance().getDb()
.prepare('UPDATE nodes SET compose_dir = ? WHERE is_default = 1')
.run(composeDir);
return defaultNodeId();
}
async function expectMissing(filePath: string): Promise<void> {
await expect(fsPromises.access(filePath)).rejects.toMatchObject({ code: 'ENOENT' });
}
async function anyExists(...paths: string[]): Promise<boolean> {
const results = await Promise.all(
paths.map((p) => fsPromises.access(p).then(() => true, () => false)),
);
return results.some(Boolean);
}
describe('Blueprint compose apply (real filesystem)', () => {
it('first-time apply overwrites createStack nginx scaffold with blueprint content in compose.yaml', async () => {
const nodeId = seedLocalNode();
const stackName = `bp-first-${counter}`;
const composeContent = 'services:\n web:\n image: traefik:v3\n';
const markerContent = JSON.stringify({ blueprintId: 1, revision: 1, lastApplied: Date.now() }, null, 2);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
const outcome = await BlueprintService.getInstance().applyLocalUnderLock(
nodeId,
stackName,
composeContent,
markerContent,
'/api/blueprints/test/apply',
);
expect(outcome).toEqual({ ran: true });
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(composeContent);
await expectMissing(path.join(stackDir, 'docker-compose.yml'));
const resolved = await FileSystemService.getInstance(nodeId).getComposeFilename(stackName);
expect(resolved).toBe('compose.yaml');
expect(deploySpy).toHaveBeenCalledTimes(1);
});
it('re-apply on a dual-file stack replaces compose.yaml and removes alternate root Compose files before deploy', async () => {
const nodeId = seedLocalNode();
const stackName = `bp-heal-${counter}`;
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
await fsPromises.mkdir(stackDir, { recursive: true });
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n stale:\n image: nginx:latest\n');
await fsPromises.writeFile(path.join(stackDir, 'compose.yml'), 'services:\n a:\n image: a\n');
await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yaml'), 'services:\n b:\n image: b\n');
await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yml'), 'services:\n c:\n image: c\n');
const composeContent = 'services:\n app:\n image: redis:7\n';
const markerContent = JSON.stringify({ blueprintId: 2, revision: 3, lastApplied: Date.now() }, null, 2);
let alternatesPresentAtDeploy = true;
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockImplementation(async () => {
alternatesPresentAtDeploy = await anyExists(
path.join(stackDir, 'compose.yml'),
path.join(stackDir, 'docker-compose.yaml'),
path.join(stackDir, 'docker-compose.yml'),
);
});
const outcome = await BlueprintService.getInstance().applyLocalUnderLock(
nodeId,
stackName,
composeContent,
markerContent,
'/api/blueprints/test/apply',
);
expect(outcome).toEqual({ ran: true });
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(composeContent);
expect(alternatesPresentAtDeploy).toBe(false);
await expectMissing(path.join(stackDir, 'compose.yml'));
await expectMissing(path.join(stackDir, 'docker-compose.yaml'));
await expectMissing(path.join(stackDir, 'docker-compose.yml'));
expect(deploySpy).toHaveBeenCalledTimes(1);
});
});
describe('FileSystemService.removeAlternateRootComposeFiles', () => {
it('removes all three alternate filenames and leaves compose.yaml', async () => {
const nodeId = bindDefaultComposeDir();
const stackName = `bp-alts-${counter}`;
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
await fsPromises.mkdir(stackDir, { recursive: true });
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n keep:\n image: keep\n');
await fsPromises.writeFile(path.join(stackDir, 'compose.yml'), 'x');
await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yaml'), 'y');
await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yml'), 'z');
await FileSystemService.getInstance(nodeId).removeAlternateRootComposeFiles(stackName);
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toContain('image: keep');
await expectMissing(path.join(stackDir, 'compose.yml'));
await expectMissing(path.join(stackDir, 'docker-compose.yaml'));
await expectMissing(path.join(stackDir, 'docker-compose.yml'));
});
it('treats absent alternate files as success', async () => {
const nodeId = bindDefaultComposeDir();
const stackName = `bp-absent-${counter}`;
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
await fsPromises.mkdir(stackDir, { recursive: true });
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n keep:\n image: keep\n');
await expect(
FileSystemService.getInstance(nodeId).removeAlternateRootComposeFiles(stackName),
).resolves.toBeUndefined();
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toContain('image: keep');
});
it('logs and continues when a non-ENOENT unlink failure occurs', async () => {
const nodeId = bindDefaultComposeDir();
const stackName = `bp-eacces-${counter}`;
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
await fsPromises.mkdir(stackDir, { recursive: true });
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n keep:\n image: keep\n');
await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yml'), 'services:\n stale:\n image: stale\n');
await fsPromises.writeFile(path.join(stackDir, 'compose.yml'), 'services:\n other:\n image: other\n');
const originalUnlink = fsPromises.unlink.bind(fsPromises);
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
vi.spyOn(fsPromises, 'unlink').mockImplementation(async (target) => {
const asString = String(target);
if (asString.endsWith(`${path.sep}docker-compose.yml`) || asString.endsWith('/docker-compose.yml')) {
const err = new Error('permission denied') as NodeJS.ErrnoException;
err.code = 'EACCES';
throw err;
}
return originalUnlink(target);
});
await FileSystemService.getInstance(nodeId).removeAlternateRootComposeFiles(stackName);
expect(warnSpy.mock.calls.some((args) => {
const label = String(args[0] ?? '');
const detail = String(args[1] ?? '');
return label.includes('Could not remove alternate compose file')
&& label.includes('docker-compose.yml')
&& detail.includes('permission denied');
})).toBe(true);
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toContain('image: keep');
await expectMissing(path.join(stackDir, 'compose.yml'));
expect(await fsPromises.readFile(path.join(stackDir, 'docker-compose.yml'), 'utf-8')).toContain('image: stale');
});
});
backend/src/__tests__/blueprints-preview-apply.test.ts
+125
View File
@@ -144,6 +144,128 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => {
expect(stored.updated_at).toBe(created.body.updated_at);
});
it('returns PREVIEW_STALE when compose drifts between preview and approval persist', async () => {
const node = seedNode();
counter += 1;
const created = await request(app)
.post('/api/blueprints')
.set('Cookie', adminCookie)
.send(validBlueprintBody(node.id));
expect(created.status).toBe(201);
const preview = await request(app)
.get(`/api/blueprints/${created.body.id}/preview`)
.set('Cookie', adminCookie);
expect(preview.status).toBe(200);
const db = DatabaseService.getInstance();
const originalSet = db.setBlueprintApproval.bind(db);
vi.spyOn(db, 'setBlueprintApproval').mockImplementation((id, input) => {
// Concurrent edit landed compose v2 before approval was written; Apply
// still persists the v1 fingerprint onto that row.
db.getDb().prepare('UPDATE blueprints SET compose_content = ? WHERE id = ?').run(
'services:\n app:\n image: nginx:evil\n',
id,
);
return originalSet(id, input);
});
const reconcileSpy = vi.mocked(BlueprintReconciler.getInstance().reconcileConfirmedPlan);
const res = await request(app)
.post(`/api/blueprints/${created.body.id}/apply`)
.set('Cookie', adminCookie)
.send({
planFingerprint: preview.body.planFingerprint,
actions: preview.body.confirmableActions,
});
expect(res.status).toBe(409);
expect(res.body.code).toBe('PREVIEW_STALE');
expect(reconcileSpy).not.toHaveBeenCalled();
const stored = DatabaseService.getInstance().getBlueprint(created.body.id)!;
expect(stored.approval_status).toBe('pending');
expect(stored.approved_intent_fingerprint).toBeNull();
});
it('returns PREVIEW_STALE when reconcileConfirmedPlan refuses after approval', async () => {
const node = seedNode();
counter += 1;
const created = await request(app)
.post('/api/blueprints')
.set('Cookie', adminCookie)
.send(validBlueprintBody(node.id));
expect(created.status).toBe(201);
const preview = await request(app)
.get(`/api/blueprints/${created.body.id}/preview`)
.set('Cookie', adminCookie);
expect(preview.status).toBe(200);
vi.mocked(BlueprintReconciler.getInstance().reconcileConfirmedPlan).mockResolvedValueOnce({
outcomes: [],
refused: true,
});
const res = await request(app)
.post(`/api/blueprints/${created.body.id}/apply`)
.set('Cookie', adminCookie)
.send({
planFingerprint: preview.body.planFingerprint,
actions: preview.body.confirmableActions,
});
expect(res.status).toBe(409);
expect(res.body.code).toBe('PREVIEW_STALE');
const stored = DatabaseService.getInstance().getBlueprint(created.body.id)!;
expect(stored.approval_status).toBe('pending');
expect(stored.approved_intent_fingerprint).toBeNull();
});
it('reports live pending approval when approval is cleared during reconcile', async () => {
const node = seedNode();
counter += 1;
const created = await request(app)
.post('/api/blueprints')
.set('Cookie', adminCookie)
.send(validBlueprintBody(node.id));
expect(created.status).toBe(201);
const preview = await request(app)
.get(`/api/blueprints/${created.body.id}/preview`)
.set('Cookie', adminCookie);
expect(preview.status).toBe(200);
vi.mocked(BlueprintReconciler.getInstance().reconcileConfirmedPlan).mockImplementationOnce(async (id) => {
// Concurrent edit invalidated approval while the confirmed snapshot ran.
DatabaseService.getInstance().clearBlueprintApproval(id);
return {
outcomes: [{
nodeId: node.id,
nodeName: node.name,
action: 'create',
status: 'ok',
}],
};
});
const res = await request(app)
.post(`/api/blueprints/${created.body.id}/apply`)
.set('Cookie', adminCookie)
.send({
planFingerprint: preview.body.planFingerprint,
actions: preview.body.confirmableActions,
});
expect(res.status).toBe(200);
expect(res.body.effectiveApproval).toBe('pending');
expect(res.body.message).toMatch(/approval is no longer current/i);
expect(res.body.outcomes).toHaveLength(1);
expect(res.body.outcomes[0].status).toBe('ok');
expect(res.body.outcomeSummary.ok).toBe(1);
const stored = DatabaseService.getInstance().getBlueprint(created.body.id)!;
expect(stored.approval_status).toBe('pending');
});
it('returns failed outcomes without pretending the rollout was clean', async () => {
const node = seedNode();
counter += 1;
@@ -215,6 +337,9 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => {
expect(change.action).toBe('blocked_name_conflict');
expect(change.severity).toBe('blocker');
expect(preview.body.summary.blocker).toBeGreaterThan(0);
// Name-conflict detection must not mutate unmanaged alternate compose files.
expect(fs.readFileSync(path.join(stackDir, 'docker-compose.yml'), 'utf-8'))
.toBe('services:\n app:\n image: nginx\n');
});
it('blocks rename while a non-withdrawn deployment exists', async () => {
backend/src/__tests__/blueprints-reconciler-gate.test.ts
+65
View File
@@ -246,6 +246,71 @@ describe('reconcileOne approval gate (real path)', () => {
});
});
describe('reconcileConfirmedPlan fingerprint gate', () => {
it('does not deploy when approval fingerprint no longer matches live compose', async () => {
const node = seedNode();
const bp = createBp({ nodeIds: [node.id] });
// Simulate an approved row whose fingerprint lags a concurrent compose edit.
const staleFp = intentFingerprint(bp);
DatabaseService.getInstance().getDb().prepare(
`UPDATE blueprints SET compose_content = ?,
approval_status = 'approved',
approved_intent_fingerprint = ?,
approved_blast_json = ?,
approved_at = ?,
approved_by = 'admin'
WHERE id = ?`,
).run(
'services:\n app:\n image: nginx:evil\n',
staleFp,
serializeApprovedBlast([{ nodeId: node.id, outcome: 'place' }]),
Date.now(),
bp.id,
);
const deploySpy = vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({ status: 'active' });
const withdrawSpy = vi.spyOn(BlueprintService.getInstance(), 'withdrawFromNode').mockResolvedValue({ status: 'withdrawn' });
const result = await BlueprintReconciler.getInstance().reconcileConfirmedPlan(bp.id, [
{ nodeId: node.id, action: 'create' },
]);
expect(result.outcomes).toEqual([]);
expect(result.refused).toBe(true);
expect(deploySpy).not.toHaveBeenCalled();
expect(withdrawSpy).not.toHaveBeenCalled();
});
it('deploys authorized actions when approval fingerprint matches live intent', async () => {
const authorized = seedNode();
const unauthorized = seedNode();
const bp = createBp({ nodeIds: [authorized.id, unauthorized.id] });
approvePlace(bp.id, [authorized.id]);
const deploySpy = vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({ status: 'active' });
const withdrawSpy = vi.spyOn(BlueprintService.getInstance(), 'withdrawFromNode').mockResolvedValue({ status: 'withdrawn' });
const result = await BlueprintReconciler.getInstance().reconcileConfirmedPlan(bp.id, [
{ nodeId: authorized.id, action: 'create' },
{ nodeId: unauthorized.id, action: 'create' },
]);
expect(result.refused).toBeFalsy();
expect(deploySpy).toHaveBeenCalledTimes(1);
expect(deploySpy).toHaveBeenCalledWith(
expect.objectContaining({ id: bp.id, compose_content: bp.compose_content }),
expect.objectContaining({ id: authorized.id }),
);
expect(withdrawSpy).not.toHaveBeenCalled();
expect(result.outcomes).toEqual([{
nodeId: authorized.id,
nodeName: authorized.name,
action: 'create',
status: 'ok',
}]);
});
});
describe('Accept/Evict STALE_GUARD', () => {
it('refuses Accept without a valid place approval', async () => {
const node = seedNode();
backend/src/__tests__/blueprints-remote-deploy.test.ts
+8
View File
@@ -120,7 +120,15 @@ describe('BlueprintService remote deploy', () => {
expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/apply-local$/);
expect(postSpy.mock.calls[1][0]).toMatch(/\/api\/stacks$/);
expect(putSpy).toHaveBeenCalledTimes(2); // compose + marker
const composePutUrl = String(putSpy.mock.calls[0][0]);
const markerPutUrl = String(putSpy.mock.calls[1][0]);
expect(composePutUrl).toMatch(/[?&]path=compose\.yaml(?:&|$)/);
expect(markerPutUrl).toMatch(/[?&]path=\.blueprint\.json(?:&|$)/);
expect(postSpy.mock.calls[2][0]).toMatch(/\/deploy$/);
const [composePutOrder, markerPutOrder] = putSpy.mock.invocationCallOrder;
const deployOrder = postSpy.mock.invocationCallOrder[2];
expect(composePutOrder).toBeLessThan(markerPutOrder);
expect(markerPutOrder).toBeLessThan(deployOrder);
});
it('maps a remote apply lock-conflict (409) to status=failed', async () => {
backend/src/__tests__/blueprints-withdraw-route.test.ts
+1 -1
View File
@@ -137,7 +137,7 @@ describe('POST /api/blueprints/:id/withdraw/:nodeId', () => {
.all(res.body.snapshotId) as Array<{ stack_name: string; filename: string; content: string; node_id: number }>;
expect(fileRows).toHaveLength(1);
expect(fileRows[0].stack_name).toBe(bp.name);
expect(fileRows[0].filename).toBe('docker-compose.yml');
expect(fileRows[0].filename).toBe('compose.yaml');
// Content is encrypted at rest; it decrypts back to the captured compose.
const { CryptoService } = await import('../services/CryptoService');
expect(CryptoService.getInstance().isEncrypted(fileRows[0].content)).toBe(true);
backend/src/__tests__/blueprints.test.ts
+8 -2
View File
@@ -407,6 +407,7 @@ describe('BlueprintService per-stack lock', () => {
// without touching the real filesystem or Docker.
vi.spyOn(FileSystemService.prototype, 'createStack').mockResolvedValue(undefined);
const writeSpy = vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockResolvedValue(undefined);
const cleanupSpy = vi.spyOn(FileSystemService.prototype, 'removeAlternateRootComposeFiles').mockResolvedValue(undefined);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
const outcome = await BlueprintService.getInstance().deployToNode(bp, node);
@@ -416,14 +417,19 @@ describe('BlueprintService per-stack lock', () => {
source: 'blueprint',
actor: 'system:blueprint',
});
// Compose is written first, then the marker, both before the deploy.
// Compose is written first, then the marker, both before sibling cleanup and deploy.
expect(writeSpy).toHaveBeenCalledTimes(2);
expect(writeSpy.mock.calls[0][1]).toBe('compose.yaml');
expect(writeSpy.mock.calls[0][2]).toBe(bp.compose_content);
expect(writeSpy.mock.calls[1][1]).toBe('.blueprint.json');
expect(writeSpy.mock.calls[1][2]).toContain('"blueprintId"');
expect(cleanupSpy).toHaveBeenCalledWith(bp.name);
const [composeOrder, markerOrder] = writeSpy.mock.invocationCallOrder;
const [cleanupOrder] = cleanupSpy.mock.invocationCallOrder;
const [deployOrder] = deploySpy.mock.invocationCallOrder;
expect(composeOrder).toBeLessThan(markerOrder);
expect(markerOrder).toBeLessThan(deployOrder);
expect(markerOrder).toBeLessThan(cleanupOrder);
expect(cleanupOrder).toBeLessThan(deployOrder);
});
it('deploy skips, writes no stack files, and records failed when the stack lock is held', async () => {
backend/src/__tests__/compose-service.test.ts
+129 -17
View File
@@ -24,6 +24,17 @@ const {
mockResolveMissingExternalNetworks,
mockCreateNetwork,
mockAddNotificationHistory,
mockIsMeshStackEnabled,
mockClassifyLegacyOrphansForUpdate,
mockCaptureCandidate,
mockMarkAcquired,
mockHandoff,
mockMarkReconciling,
mockMarkImmediateVerified,
mockAbandon,
mockCompensateWithCandidate,
mockBuildUnifiedHeldImagePredicate,
mockGetRecovery,
} = vi.hoisted(() => ({
mockSpawn: vi.fn(),
mockGetContainersByStack: vi.fn().mockResolvedValue([]),
@@ -58,6 +69,24 @@ const {
}),
mockCreateNetwork: vi.fn().mockResolvedValue({ id: 'net-1' }),
mockAddNotificationHistory: vi.fn(),
mockIsMeshStackEnabled: vi.fn().mockReturnValue(false),
mockClassifyLegacyOrphansForUpdate: vi.fn().mockResolvedValue({ status: 'none' }),
mockCaptureCandidate: vi.fn().mockResolvedValue({
id: 'recovery-1',
node_id: 1,
stack_name: 'my-stack',
status: 'candidate',
phase: 'captured',
override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml',
}),
mockMarkAcquired: vi.fn().mockReturnValue(true),
mockHandoff: vi.fn().mockReturnValue(true),
mockMarkReconciling: vi.fn().mockReturnValue(true),
mockMarkImmediateVerified: vi.fn().mockReturnValue(true),
mockAbandon: vi.fn().mockResolvedValue(true),
mockCompensateWithCandidate: vi.fn().mockResolvedValue(true),
mockBuildUnifiedHeldImagePredicate: vi.fn().mockReturnValue(() => false),
mockGetRecovery: vi.fn().mockReturnValue(undefined),
}));
vi.mock('child_process', () => ({ spawn: mockSpawn, execFile: vi.fn() }));
@@ -89,6 +118,7 @@ vi.mock('../services/DockerController', () => ({
getInstance: () => ({
getContainersByStack: mockGetContainersByStack,
getLegacyOrphanContainersByStack: mockGetLegacyOrphanContainersByStack,
classifyLegacyOrphansForUpdate: mockClassifyLegacyOrphansForUpdate,
removeContainers: mockRemoveContainers,
pruneDanglingImages: mockPruneDanglingImages,
createNetwork: mockCreateNetwork,
@@ -111,6 +141,7 @@ vi.mock('../services/DatabaseService', () => ({
getGitSource: () => undefined,
getStackProjectEnvFiles: () => [],
addNotificationHistory: mockAddNotificationHistory,
isMeshStackEnabled: (...args: unknown[]) => mockIsMeshStackEnabled(...args),
}),
},
}));
@@ -150,6 +181,24 @@ vi.mock('../services/MeshService', () => ({
},
}));
vi.mock('../services/StackUpdateRecoveryService', () => ({
StackUpdateRecoveryService: {
getInstance: () => ({
captureCandidate: mockCaptureCandidate,
markAcquired: mockMarkAcquired,
handoff: mockHandoff,
markReconciling: mockMarkReconciling,
markImmediateVerified: mockMarkImmediateVerified,
abandon: mockAbandon,
compensateWithCandidate: mockCompensateWithCandidate,
buildUnifiedHeldImagePredicate: mockBuildUnifiedHeldImagePredicate,
get: mockGetRecovery,
linkGateOrRetain: vi.fn(),
}),
},
}));
vi.mock('../services/SelfIdentityService', () => ({
default: {
getInstance: () => ({ getBindMounts: mockGetBindMounts }),
@@ -247,6 +296,23 @@ beforeEach(() => {
declaredExternalCount: 0,
});
mockCreateNetwork.mockResolvedValue({ id: 'net-1' });
mockClassifyLegacyOrphansForUpdate.mockResolvedValue({ status: 'none' });
mockCaptureCandidate.mockResolvedValue({
id: 'recovery-1',
node_id: 1,
stack_name: 'my-stack',
status: 'candidate',
phase: 'captured',
override_path: '/test/compose/my-stack/.sencho-recovery-abc.yml',
});
mockMarkAcquired.mockReturnValue(true);
mockHandoff.mockReturnValue(true);
mockMarkReconciling.mockReturnValue(true);
mockMarkImmediateVerified.mockReturnValue(true);
mockAbandon.mockResolvedValue(true);
mockIsMeshStackEnabled.mockReturnValue(false);
mockCompensateWithCandidate.mockResolvedValue(true);
mockBuildUnifiedHeldImagePredicate.mockReturnValue(() => false);
delete process.env.SENCHO_MODE;
vi.useFakeTimers({ shouldAdvanceTime: true });
});
@@ -586,6 +652,16 @@ describe('ComposeService - authoredComposeArgs mesh override', () => {
// ── deployStack ────────────────────────────────────────────────────────
it('fails closed when a mesh-enabled stack cannot generate its override', async () => {
mockIsMeshStackEnabled.mockReturnValue(true);
mockEnsureStackOverride.mockRejectedValue(new Error('mesh override write failed'));
const svc = ComposeService.getInstance(1);
await expect(svc.validateExactComposeInvocation('my-stack')).rejects.toThrow(/mesh override write failed/i);
expect(mockSpawn).not.toHaveBeenCalled();
});
describe('ComposeService - deployStack', () => {
it('blocks a pilot deploy with relative binds before replacing containers when path mapping differs', async () => {
process.env.SENCHO_MODE = 'pilot';
@@ -987,7 +1063,7 @@ describe('ComposeService - updateStack build-aware', () => {
expect(spawnArgs.some(args => args.includes('pull') && !args.includes('--ignore-buildable'))).toBe(true);
});
it('rolls back compose files when a build step fails during atomic update', async () => {
it('abandons recovery and leaves runtime untouched when acquire fails before handoff', async () => {
mockLoadStackBuildServices.mockResolvedValueOnce(['app']);
let spawnCount = 0;
mockSpawn.mockImplementation(() => {
@@ -1004,8 +1080,12 @@ describe('ComposeService - updateStack build-aware', () => {
const error = await result;
expect(error).not.toBeNull();
expect(mockRestoreStackFiles).toHaveBeenCalled();
expect(getComposeRollbackInfo(error)?.attempted).toBe(true);
expect(mockAbandon).toHaveBeenCalledWith('recovery-1');
expect(mockHandoff).not.toHaveBeenCalled();
expect(mockRemoveContainers).not.toHaveBeenCalled();
expect(mockCompensateWithCandidate).not.toHaveBeenCalled();
const spawnArgs = mockSpawn.mock.calls.map(c => c[1] as string[]);
expect(spawnArgs.some(args => args.includes('up'))).toBe(false);
});
});
@@ -1106,7 +1186,7 @@ describe('ComposeService - updateStack prune-on-update', () => {
// The update already succeeded before the prune ran, so a prune failure
// must neither reject nor trigger the atomic restore.
await expect(promise).resolves.toBeUndefined();
await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' });
expect(mockRestoreStackFiles).not.toHaveBeenCalled();
});
@@ -1120,7 +1200,7 @@ describe('ComposeService - updateStack prune-on-update', () => {
const promise = svc.updateStack('my-stack');
await vi.advanceTimersByTimeAsync(3100);
await expect(promise).resolves.toBeUndefined();
await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' });
});
});
@@ -1428,19 +1508,10 @@ describe('ComposeService - idle-output stall backstop', () => {
await promise;
});
it('preserves STACK_STALLED_OUTPUT through the atomic rollback wrapper', async () => {
it('surfaces STACK_STALLED_OUTPUT on acquire stall without runtime compensation', async () => {
process.env.SENCHO_COMPOSE_STALL_TIMEOUT_MS = '1000';
const pullProc = createMockProcess();
let call = 0;
// First spawn is the stalling pull; later spawns (the rollback restore's
// `up`) close cleanly, proving the restore is not idle-timeout armed.
mockSpawn.mockImplementation(() => {
call += 1;
if (call === 1) return pullProc;
const p = createMockProcess();
Promise.resolve().then(() => p.emit('close', 0));
return p;
});
mockSpawn.mockImplementation(() => pullProc);
const svc = ComposeService.getInstance(1);
const result = svc.updateStack('my-stack', undefined, true).then(() => null, (e: Error) => e);
@@ -1451,7 +1522,10 @@ describe('ComposeService - idle-output stall backstop', () => {
const error = await result;
expect(error).not.toBeNull();
expect(error!.message).toContain('STACK_STALLED_OUTPUT');
expect(getComposeRollbackInfo(error)).toMatchObject({ attempted: true });
// Acquire failure is pre-handoff: abandon candidate, do not wrap as ComposeRollbackError.
expect(getComposeRollbackInfo(error)).toBeNull();
expect(mockAbandon).toHaveBeenCalledWith('recovery-1');
expect(mockCompensateWithCandidate).not.toHaveBeenCalled();
});
});
@@ -1580,3 +1654,41 @@ describe('ComposeService - streamLogs', () => {
);
});
});
describe('ComposeService - updateStack safe ordering', () => {
it('never broad-removes Compose-managed containers before up', async () => {
setupAutoCloseSpawn();
mockListContainers.mockResolvedValue([]);
mockGetContainersByStack.mockResolvedValue([{ Id: 'should-not-remove' }]);
const svc = ComposeService.getInstance(1);
const promise = svc.updateStack('my-stack');
await vi.advanceTimersByTimeAsync(3100);
await promise;
expect(mockGetContainersByStack).not.toHaveBeenCalled();
expect(mockRemoveContainers).not.toHaveBeenCalled();
expect(mockCaptureCandidate).toHaveBeenCalled();
expect(mockClassifyLegacyOrphansForUpdate).toHaveBeenCalledWith('my-stack');
expect(mockHandoff).toHaveBeenCalled();
});
it('removes only classified orphans after handoff', async () => {
setupAutoCloseSpawn();
mockListContainers.mockResolvedValue([]);
mockClassifyLegacyOrphansForUpdate.mockResolvedValueOnce({
status: 'orphans',
ids: ['orphan-1'],
});
mockRemoveContainers.mockResolvedValueOnce([{ id: 'orphan-1', success: true }]);
const svc = ComposeService.getInstance(1);
const promise = svc.updateStack('my-stack');
await vi.advanceTimersByTimeAsync(3100);
await promise;
expect(mockHandoff).toHaveBeenCalled();
expect(mockRemoveContainers).toHaveBeenCalledWith(['orphan-1']);
});
});
backend/src/__tests__/database-metrics.test.ts
+43 -35
View File
@@ -225,23 +225,25 @@ describe('DatabaseService - notification history cap (periodic)', () => {
// A chatty stack writes 600 events.
const base = Date.now();
for (let i = 0; i < 600; i++) {
db.addNotificationHistory(0, {
level: 'info',
message: `chatty-${i}`,
timestamp: base + i,
stack_name: 'chatty',
});
}
// A quiet stack writes 3 events long before the chatty burst.
for (let i = 0; i < 3; i++) {
db.addNotificationHistory(0, {
level: 'info',
message: `quiet-${i}`,
timestamp: base - 10_000 + i,
stack_name: 'quiet',
});
}
db.transaction(() => {
for (let i = 0; i < 600; i++) {
db.addNotificationHistory(0, {
level: 'info',
message: `chatty-${i}`,
timestamp: base + i,
stack_name: 'chatty',
});
}
// A quiet stack writes 3 events long before the chatty burst.
for (let i = 0; i < 3; i++) {
db.addNotificationHistory(0, {
level: 'info',
message: `quiet-${i}`,
timestamp: base - 10_000 + i,
stack_name: 'quiet',
});
}
});
// No per-insert prune: every row is present.
const beforeCleanup = db.getNotificationHistory(0, 2000);
@@ -261,13 +263,15 @@ describe('DatabaseService - notification history cap (periodic)', () => {
db.deleteAllNotifications(0);
const base = Date.now();
for (let i = 0; i < 1200; i++) {
db.addNotificationHistory(0, {
level: 'info',
message: `system-${i}`,
timestamp: base + i,
});
}
db.transaction(() => {
for (let i = 0; i < 1200; i++) {
db.addNotificationHistory(0, {
level: 'info',
message: `system-${i}`,
timestamp: base + i,
});
}
});
db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
@@ -279,14 +283,16 @@ describe('DatabaseService - notification history cap (periodic)', () => {
it('keeps the newest entries per (node, stack) after periodic cap', () => {
db.deleteAllNotifications(0);
const base = Date.now();
for (let i = 0; i < 600; i++) {
db.addNotificationHistory(0, {
level: 'info',
message: `ordered-${i}`,
timestamp: base + i * 10,
stack_name: 'ordered',
});
}
db.transaction(() => {
for (let i = 0; i < 600; i++) {
db.addNotificationHistory(0, {
level: 'info',
message: `ordered-${i}`,
timestamp: base + i * 10,
stack_name: 'ordered',
});
}
});
db.cleanupOldNotifications(30, { perStackCap: 500, perNodeUnattachedCap: 1000 });
@@ -301,9 +307,11 @@ describe('DatabaseService - notification history cap (periodic)', () => {
it('uses safe defaults when called with only the retention argument', () => {
db.deleteAllNotifications(0);
const base = Date.now();
for (let i = 0; i < 600; i++) {
db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
}
db.transaction(() => {
for (let i = 0; i < 600; i++) {
db.addNotificationHistory(0, { level: 'info', message: `d-${i}`, timestamp: base + i, stack_name: 'default' });
}
});
// Production caller (MonitorService) only passes daysToKeep; the cap defaults must enforce the per-stack 500 limit.
const summary = db.cleanupOldNotifications(30);
const after = db.getNotificationHistory(0, 2000).filter((n: any) => n.stack_name === 'default');
backend/src/__tests__/deployed-stack-deletion-service.test.ts
+156
View File
@@ -0,0 +1,156 @@
/**
* Deployed-stack deletion: ready transaction retires both recovery models;
* blocking intents gate same-name create.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
import { randomUUID } from 'crypto';
import path from 'path';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import type {
StackUpdateRecoveryGenerationRow,
ServiceUpdateRecoveryRow,
StackUpdateCleanupPendingRow,
} from '../services/DatabaseService';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let DeployedStackDeletionService: typeof import('../services/DeployedStackDeletionService').DeployedStackDeletionService;
let overrideDeletionContainmentBase: typeof import('../services/DeployedStackDeletionService').overrideDeletionContainmentBase;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ DeployedStackDeletionService, overrideDeletionContainmentBase } = await import('../services/DeployedStackDeletionService'));
});
afterAll(() => cleanupTestDb(tmpDir));
function db() {
return DatabaseService.getInstance();
}
beforeEach(() => {
const raw = (db() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db;
raw.prepare('DELETE FROM stack_update_recovery_generations').run();
raw.prepare('DELETE FROM service_update_recovery').run();
raw.prepare('DELETE FROM stack_update_cleanup_pending').run();
});
const NODE = 1;
describe('DeployedStackDeletionService ready transaction', () => {
it('commitStackDeletionReadyTransaction deletes full-stack and service recovery rows', () => {
const now = Date.now();
const stackName = 'del-stack';
const gen: StackUpdateRecoveryGenerationRow = {
id: randomUUID(),
node_id: NODE,
stack_name: stackName,
status: 'active',
phase: 'immediate_verified',
is_current: 1,
backup_slot_id: null,
override_path: null,
services_json: '[]',
health_gate_id: null,
gate_retain_until: null,
artifact_expires_at: null,
operation_lease_expires_at: null,
created_at: now,
updated_at: now,
created_by: null,
artifacts_retired: 0,
};
db().insertStackUpdateRecoveryGeneration(gen);
const svc: ServiceUpdateRecoveryRow = {
id: randomUUID(),
node_id: NODE,
stack_name: stackName,
service_name: 'web',
replicas_json: '[]',
majority_image_id: 'sha256:abc',
declared_image_ref: 'nginx:latest',
weak_floating_tag: 0,
health_gate_id: null,
status: 'active',
expires_at: now + 60_000,
claim_expires_at: null,
created_at: now,
created_by: null,
};
db().insertServiceUpdateRecovery(svc);
const intentId = randomUUID();
const intent: StackUpdateCleanupPendingRow = {
id: intentId,
node_id: NODE,
stack_name: stackName,
status: 'prepared',
target_kind: 'local_socket',
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
created_at: now,
updated_at: now,
};
db().insertCleanupPending(intent);
expect(db().commitStackDeletionReadyTransaction(intentId, NODE, stackName)).toBe(true);
expect(db().listStackUpdateRecoveryForStack(NODE, stackName)).toHaveLength(0);
expect(db().listActiveServiceUpdateRecoveries(NODE, stackName, 'web')).toHaveLength(0);
expect(db().getCleanupPending(intentId)?.status).toBe('ready');
});
it('hasBlockingDeletionIntent is true for prepared intents', () => {
const now = Date.now();
db().insertCleanupPending({
id: randomUUID(),
node_id: NODE,
stack_name: 'blocked',
status: 'prepared',
target_kind: 'local_socket',
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
created_at: now,
updated_at: now,
});
expect(db().hasBlockingDeletionIntent(NODE, 'blocked')).toBe(true);
expect(db().hasBlockingDeletionIntent(NODE, 'other')).toBe(false);
});
it('assertNoBlockingDeletionIntent throws for prepared stacks', () => {
const now = Date.now();
db().insertCleanupPending({
id: randomUUID(),
node_id: NODE,
stack_name: 'prep',
status: 'prepared',
target_kind: 'local_socket',
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
created_at: now,
updated_at: now,
});
expect(() => {
DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(NODE, 'prep');
}).toThrow(/deletion in progress/i);
});
});
describe('overrideDeletionContainmentBase', () => {
it('confines stack-scoped intents to the stack directory', () => {
expect(overrideDeletionContainmentBase('/app/compose', 'my-stack')).toBe(
path.resolve('/app/compose', 'my-stack'),
);
expect(overrideDeletionContainmentBase('/app/compose', null)).toBe(
path.resolve('/app/compose'),
);
});
it('rejects invalid stack names that could traverse', () => {
expect(overrideDeletionContainmentBase('/app/compose', '../other')).toBeNull();
expect(overrideDeletionContainmentBase('/app/compose', 'bad/name')).toBeNull();
});
});
backend/src/__tests__/docker-controller.test.ts
+20
View File
@@ -1714,3 +1714,23 @@ describe('DockerController - getLegacyOrphanContainersByStack', () => {
fallbackSpy.mockRestore();
});
});
describe('DockerController - classifyLegacyOrphansForUpdate', () => {
it('returns none when compose ps already manages containers', async () => {
// Reuse the same mocks as getLegacyOrphanContainersByStack tests in this file.
const { default: DockerController } = await import('../services/DockerController');
const dc = DockerController.getInstance(1);
vi.spyOn(dc as any, 'fetchComposePsContainers').mockResolvedValue([{ ID: 'c1' }]);
await expect(dc.classifyLegacyOrphansForUpdate('my-stack')).resolves.toEqual({ status: 'none' });
});
it('returns classification_failed when compose ps and fallback both fail', async () => {
const { default: DockerController } = await import('../services/DockerController');
const dc = DockerController.getInstance(1);
vi.spyOn(dc as any, 'fetchComposePsContainers').mockRejectedValue(new Error('compose boom'));
vi.spyOn(dc as any, 'smartFallback').mockRejectedValue(new Error('fallback boom'));
const result = await dc.classifyLegacyOrphansForUpdate('my-stack');
expect(result.status).toBe('classification_failed');
});
});
backend/src/__tests__/docker-integration/failed-pull-keeps-stack-running.test.ts
+112
View File
@@ -0,0 +1,112 @@
/**
* Docker-backed integration: a failed pull must leave the running stack untouched.
* Skipped automatically when Docker is unavailable.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { execFileSync } from 'child_process';
import fs from 'fs';
import path from 'path';
import { setupTestDb, cleanupTestDb } from '../helpers/setupTestDb';
function dockerAvailable(): boolean {
try {
execFileSync('docker', ['info'], { stdio: 'ignore' });
return true;
} catch {
return false;
}
}
const hasDocker = dockerAvailable();
const STACK = 'fpkeep';
function compose(args: string[], cwd: string): string {
return execFileSync('docker', ['compose', '-p', STACK, ...args], {
cwd,
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe'],
});
}
describe.skipIf(!hasDocker)('failed pull keeps stack running', () => {
let tmpDir: string;
let composeDir: string;
let stackDir: string;
let nodeId: number;
beforeAll(async () => {
tmpDir = await setupTestDb();
composeDir = process.env.COMPOSE_DIR!;
stackDir = path.join(composeDir, STACK);
fs.mkdirSync(stackDir, { recursive: true });
const { DatabaseService } = await import('../../services/DatabaseService');
const db = DatabaseService.getInstance();
const local = db.getDefaultNode();
if (!local?.id) throw new Error('Test DB has no default local node');
nodeId = local.id;
db.updateGlobalSetting('prune_on_update', '0');
fs.writeFileSync(
path.join(stackDir, 'compose.yaml'),
[
'services:',
' web:',
' image: busybox:1.36.1',
' command: ["sleep", "3600"]',
'',
].join('\n'),
'utf8',
);
compose(['up', '-d', '--pull', 'always'], stackDir);
}, 180_000);
afterAll(async () => {
try {
if (stackDir && fs.existsSync(stackDir)) {
compose(['down', '--remove-orphans'], stackDir);
}
} catch {
// Best-effort cleanup.
}
if (tmpDir) cleanupTestDb(tmpDir);
}, 120_000);
it('leaves the original container running when ComposeService updateStack pull fails', async () => {
const beforeId = compose(['ps', '-q'], stackDir).trim();
expect(beforeId.length).toBeGreaterThan(0);
const beforeInspect = JSON.parse(
execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
) as Array<{ State: { Running: boolean; Status: string } }>;
expect(beforeInspect[0].State.Running).toBe(true);
fs.writeFileSync(
path.join(stackDir, 'compose.yaml'),
[
'services:',
' web:',
' image: busybox:sencho-does-not-exist-fpkeep-xyz',
' command: ["sleep", "3600"]',
'',
].join('\n'),
'utf8',
);
const { StackUpdateRecoveryService } = await import('../../services/StackUpdateRecoveryService');
StackUpdateRecoveryService.resetForTests();
const { ComposeService } = await import('../../services/ComposeService');
await expect(
ComposeService.getInstance(nodeId).updateStack(STACK, undefined, true),
).rejects.toThrow();
const afterId = compose(['ps', '-q'], stackDir).trim();
expect(afterId).toBe(beforeId);
const afterInspect = JSON.parse(
execFileSync('docker', ['inspect', beforeId], { encoding: 'utf8' }),
) as Array<{ State: { Running: boolean } }>;
expect(afterInspect[0].State.Running).toBe(true);
}, 180_000);
});
backend/src/__tests__/image-updates-routes.test.ts
+2 -2
View File
@@ -375,7 +375,7 @@ describe('POST /api/auto-update/execute', () => {
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
.mockResolvedValue({ hasUpdate: true } as never);
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
const gateSpy = vi.spyOn(PolicyEnforcement, 'enforcePolicyPreDeploy').mockResolvedValue({
ok: false,
bypassed: false,
@@ -421,7 +421,7 @@ describe('POST /api/auto-update/execute', () => {
.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never);
const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage')
.mockResolvedValue({ hasUpdate: true } as never);
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue();
const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au');
try {
const res = await request(app)
backend/src/__tests__/notification-schedule.test.ts
+114
View File
@@ -0,0 +1,114 @@
import { describe, it, expect } from 'vitest';
import {
isScheduleActive,
parseNotificationSchedule,
parseStoredNotificationSchedule,
scheduleAllowsSuppression,
type NotificationSchedule,
} from '../helpers/notificationSchedule';
/** Build a UTC epoch for a known weekday. 2026-07-18 is Saturday (getUTCDay()===6). */
function utcMs(iso: string): number {
return Date.parse(iso);
}
const satWindow: NotificationSchedule = {
days: [6],
start_minute: 22 * 60,
end_minute: 2 * 60,
tz: 'UTC',
};
const sameDay: NotificationSchedule = {
days: [1],
start_minute: 2 * 60,
end_minute: 6 * 60,
tz: 'UTC',
};
describe('parseNotificationSchedule', () => {
it('accepts unique days in any order and returns sorted', () => {
const result = parseNotificationSchedule({
days: [3, 1, 6],
start_minute: 0,
end_minute: 60,
tz: 'UTC',
});
expect(result).toEqual({
ok: true,
schedule: { days: [1, 3, 6], start_minute: 0, end_minute: 60, tz: 'UTC' },
});
});
it('rejects duplicates, empty days, equal endpoints, and non-UTC tz', () => {
expect(parseNotificationSchedule({
days: [1, 1], start_minute: 0, end_minute: 1, tz: 'UTC',
}).ok).toBe(false);
expect(parseNotificationSchedule({
days: [], start_minute: 0, end_minute: 1, tz: 'UTC',
}).ok).toBe(false);
expect(parseNotificationSchedule({
days: [1], start_minute: 30, end_minute: 30, tz: 'UTC',
}).ok).toBe(false);
expect(parseNotificationSchedule({
days: [1], start_minute: 0, end_minute: 1, tz: 'America/New_York',
}).ok).toBe(false);
});
});
describe('isScheduleActive', () => {
it('treats null schedule as always active', () => {
expect(isScheduleActive(null, utcMs('2026-07-20T12:00:00.000Z'))).toBe(true);
});
it('same-day: start inclusive, end exclusive', () => {
// Monday 2026-07-20
expect(isScheduleActive(sameDay, utcMs('2026-07-20T02:00:00.000Z'))).toBe(true);
expect(isScheduleActive(sameDay, utcMs('2026-07-20T05:59:00.000Z'))).toBe(true);
expect(isScheduleActive(sameDay, utcMs('2026-07-20T06:00:00.000Z'))).toBe(false);
expect(isScheduleActive(sameDay, utcMs('2026-07-20T01:59:00.000Z'))).toBe(false);
});
it('cross-midnight Saturday window: Sat evening and Sun morning only', () => {
expect(isScheduleActive(satWindow, utcMs('2026-07-18T22:00:00.000Z'))).toBe(true); // Sat
expect(isScheduleActive(satWindow, utcMs('2026-07-18T23:30:00.000Z'))).toBe(true);
expect(isScheduleActive(satWindow, utcMs('2026-07-19T01:59:00.000Z'))).toBe(true); // Sun
expect(isScheduleActive(satWindow, utcMs('2026-07-19T02:00:00.000Z'))).toBe(false);
expect(isScheduleActive(satWindow, utcMs('2026-07-18T21:59:00.000Z'))).toBe(false);
expect(isScheduleActive(satWindow, utcMs('2026-07-19T22:00:00.000Z'))).toBe(false); // Sun evening
expect(isScheduleActive(satWindow, utcMs('2026-07-20T01:00:00.000Z'))).toBe(false); // Mon morning
});
it('supports multiple start days', () => {
const multi: NotificationSchedule = {
days: [1, 3],
start_minute: 10 * 60,
end_minute: 11 * 60,
tz: 'UTC',
};
expect(isScheduleActive(multi, utcMs('2026-07-20T10:30:00.000Z'))).toBe(true); // Mon
expect(isScheduleActive(multi, utcMs('2026-07-22T10:30:00.000Z'))).toBe(true); // Wed
expect(isScheduleActive(multi, utcMs('2026-07-21T10:30:00.000Z'))).toBe(false); // Tue
});
it('uses UTC independent of host-local timezone interpretation of fixed instants', () => {
const ms = utcMs('2026-07-18T22:00:00.000Z');
expect(new Date(ms).getUTCDay()).toBe(6);
expect(isScheduleActive(satWindow, ms)).toBe(true);
});
});
describe('scheduleAllowsSuppression / stored parse', () => {
it('fails closed for invalid stored schedule', () => {
expect(scheduleAllowsSuppression(null, true, Date.now())).toBe(false);
expect(parseStoredNotificationSchedule('{not-json')).toEqual({ kind: 'invalid' });
expect(parseStoredNotificationSchedule(null)).toEqual({ kind: 'null' });
expect(parseStoredNotificationSchedule(undefined)).toEqual({ kind: 'null' });
});
it('treats empty and whitespace strings as invalid, not legacy null', () => {
expect(parseStoredNotificationSchedule('')).toEqual({ kind: 'invalid' });
expect(parseStoredNotificationSchedule(' ')).toEqual({ kind: 'invalid' });
expect(parseStoredNotificationSchedule('null')).toEqual({ kind: 'invalid' });
});
});
backend/src/__tests__/notification-suppression-routes-api.test.ts
+340 -6
View File
@@ -235,7 +235,7 @@ describe('Notification suppression - CRUD', () => {
.set('Authorization', `Bearer ${token}`)
.send({
rule: {
id: 9001,
id: 910001,
name: 'replica',
applies_to: 'both',
node_id: null,
@@ -255,7 +255,7 @@ describe('Notification suppression - CRUD', () => {
.set('Authorization', `Bearer ${token}`)
.send({
rule: {
id: 9001,
id: 910001,
name: 'replica',
applies_to: 'both',
stack_patterns: ['****'],
@@ -270,14 +270,14 @@ describe('Notification suppression - CRUD', () => {
},
});
expect(redos.status).toBe(400);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(9001)).toBeUndefined();
expect(DatabaseService.getInstance().getNotificationSuppressionRule(910001)).toBeUndefined();
const ok = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({
rule: {
id: 9001,
id: 910001,
name: 'replica',
applies_to: 'both',
stack_patterns: ['prod-*'],
@@ -292,7 +292,341 @@ describe('Notification suppression - CRUD', () => {
},
});
expect(ok.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(9001)?.stack_patterns).toEqual(['prod-*']);
DatabaseService.getInstance().deleteNotificationSuppressionRule(9001);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(910001)?.stack_patterns).toEqual(['prod-*']);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(910001)?.schedule).toBeNull();
DatabaseService.getInstance().deleteNotificationSuppressionRule(910001);
const omitSched = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({
rule: {
id: 910003,
name: 'replica-omit-sched',
applies_to: 'both',
stack_patterns: [],
node_id: null,
label_ids: null,
categories: null,
levels: null,
enabled: true,
expires_at: null,
created_at: 1,
updated_at: 1,
},
});
expect(omitSched.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(910003)?.schedule).toBeNull();
DatabaseService.getInstance().deleteNotificationSuppressionRule(910003);
});
it('schedule: create omit null; PUT preserve; null clear; canonicalize days; reject invalid', async () => {
const omitted = await request(app)
.post('/api/notification-suppression-rules')
.set('Cookie', authCookie)
.send({ name: 'sched omit', applies_to: 'both' });
expect(omitted.status).toBe(201);
expect(omitted.body.schedule).toBeNull();
const id = omitted.body.id as number;
const withSched = await request(app)
.put(`/api/notification-suppression-rules/${id}`)
.set('Cookie', authCookie)
.send({
schedule: { days: [3, 1], start_minute: 60, end_minute: 120, tz: 'UTC' },
});
expect(withSched.status).toBe(200);
expect(withSched.body.schedule).toEqual({
days: [1, 3],
start_minute: 60,
end_minute: 120,
tz: 'UTC',
});
const preserved = await request(app)
.put(`/api/notification-suppression-rules/${id}`)
.set('Cookie', authCookie)
.send({ enabled: false });
expect(preserved.status).toBe(200);
expect(preserved.body.schedule).toEqual({
days: [1, 3],
start_minute: 60,
end_minute: 120,
tz: 'UTC',
});
const cleared = await request(app)
.put(`/api/notification-suppression-rules/${id}`)
.set('Cookie', authCookie)
.send({ schedule: null });
expect(cleared.status).toBe(200);
expect(cleared.body.schedule).toBeNull();
const bad = await request(app)
.put(`/api/notification-suppression-rules/${id}`)
.set('Cookie', authCookie)
.send({ schedule: { days: [1], start_minute: 10, end_minute: 10, tz: 'UTC' } });
expect(bad.status).toBe(400);
DatabaseService.getInstance().deleteNotificationSuppressionRule(id);
});
it('replica rejects invalid schedule and accepts valid schedule', async () => {
const jwt = await import('jsonwebtoken');
const { TEST_JWT_SECRET } = await import('./helpers/testConstants');
const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
const bad = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({
rule: {
id: 920002,
name: 'replica-sched',
applies_to: 'both',
stack_patterns: [],
node_id: null,
label_ids: null,
categories: null,
levels: null,
enabled: true,
expires_at: null,
schedule: { days: [1], start_minute: 0, end_minute: 0, tz: 'UTC' },
created_at: 1,
updated_at: 1,
},
});
expect(bad.status).toBe(400);
const ok = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({
rule: {
id: 920002,
name: 'replica-sched',
applies_to: 'both',
stack_patterns: [],
node_id: null,
label_ids: null,
categories: null,
levels: null,
enabled: true,
expires_at: null,
schedule: { days: [6], start_minute: 1320, end_minute: 120, tz: 'UTC' },
created_at: 1,
updated_at: 1,
},
});
expect(ok.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(920002)?.schedule).toEqual({
days: [6],
start_minute: 1320,
end_minute: 120,
tz: 'UTC',
});
DatabaseService.getInstance().deleteNotificationSuppressionRule(920002);
});
it('replica forces node_id to null regardless of the payload value', async () => {
const jwt = await import('jsonwebtoken');
const { TEST_JWT_SECRET } = await import('./helpers/testConstants');
const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
const res = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({
rule: {
id: 930004,
name: 'replica-scoped',
applies_to: 'both',
stack_patterns: [],
node_id: 5,
label_ids: null,
categories: null,
levels: null,
enabled: true,
expires_at: null,
created_at: 1,
updated_at: 1,
},
});
expect(res.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(930004)?.node_id).toBeNull();
DatabaseService.getInstance().deleteNotificationSuppressionRule(930004);
});
it('replica ignores a stale write with an older updated_at than the stored row', async () => {
const jwt = await import('jsonwebtoken');
const { TEST_JWT_SECRET } = await import('./helpers/testConstants');
const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
const replicaRule = (overrides: Record<string, unknown>) => ({
id: 940005,
name: 'replica-race',
applies_to: 'both',
stack_patterns: [],
node_id: null,
label_ids: null,
categories: null,
levels: null,
enabled: true,
expires_at: null,
created_at: 1,
...overrides,
});
const first = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({ rule: replicaRule({ name: 'v2-newer', updated_at: 2000 }) });
expect(first.status).toBe(200);
const stale = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({ rule: replicaRule({ name: 'v1-delayed-stale', updated_at: 1000 }) });
expect(stale.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(940005)?.name).toBe('v2-newer');
const tie = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({ rule: replicaRule({ name: 'v2-tie-same-timestamp', updated_at: 2000 }) });
expect(tie.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(940005)?.name).toBe('v2-newer');
const newer = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({ rule: replicaRule({ name: 'v3-newest', updated_at: 3000 }) });
expect(newer.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(940005)?.name).toBe('v3-newest');
DatabaseService.getInstance().deleteNotificationSuppressionRule(940005);
});
it('replica does not resurrect a rule after it was deleted, even with a newer updated_at', async () => {
const jwt = await import('jsonwebtoken');
const { TEST_JWT_SECRET } = await import('./helpers/testConstants');
const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
const replicaRule = (overrides: Record<string, unknown>) => ({
id: 950006,
name: 'replica-delete-race',
applies_to: 'both',
stack_patterns: [],
node_id: null,
label_ids: null,
categories: null,
levels: null,
enabled: true,
expires_at: null,
created_at: 1,
...overrides,
});
const first = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({ rule: replicaRule({ updated_at: 1000 }) });
expect(first.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(950006)).not.toBeUndefined();
const del = await request(app)
.delete('/api/notification-suppression-rules/replica/950006')
.set('Authorization', `Bearer ${token}`);
expect(del.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(950006)).toBeUndefined();
// A delayed POST arrives after the DELETE, reordered by the network. Even
// though its updated_at is newer than anything the sender ever sent before
// the delete, the delete is authoritative: this id must stay gone.
const delayed = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({ rule: replicaRule({ updated_at: 2000 }) });
expect(delayed.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(950006)).toBeUndefined();
});
it('replica DELETE tombstones an id even when the remote never had that row', async () => {
const jwt = await import('jsonwebtoken');
const { TEST_JWT_SECRET } = await import('./helpers/testConstants');
const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
// This remote never received rule 960007 (e.g. it just enrolled, or the rule
// failed capability probing before its first push). A cleanup DELETE still
// arrives unconditionally from deleteRuleOnNode. A POST reordered behind it
// must not be allowed to create the rule for the first time.
const del = await request(app)
.delete('/api/notification-suppression-rules/replica/960007')
.set('Authorization', `Bearer ${token}`);
expect(del.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(960007)).toBeUndefined();
const delayed = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({
rule: {
id: 960007,
name: 'replica-delete-before-first-post',
applies_to: 'both',
stack_patterns: [],
node_id: null,
label_ids: null,
categories: null,
levels: null,
enabled: true,
expires_at: null,
created_at: 1,
updated_at: 1,
},
});
expect(delayed.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(960007)).toBeUndefined();
});
it('replica does not resurrect a deleted rule with a schedule', async () => {
const jwt = await import('jsonwebtoken');
const { TEST_JWT_SECRET } = await import('./helpers/testConstants');
const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
const replicaRule = (overrides: Record<string, unknown>) => ({
id: 970008,
name: 'replica-scheduled-delete-race',
applies_to: 'both',
stack_patterns: [],
node_id: null,
label_ids: null,
categories: null,
levels: null,
enabled: true,
expires_at: null,
schedule: { days: [1], start_minute: 120, end_minute: 360, tz: 'UTC' },
created_at: 1,
...overrides,
});
const first = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({ rule: replicaRule({ updated_at: 1000 }) });
expect(first.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(970008)?.schedule).not.toBeNull();
// This is the worst case the fix protects: a capability-cleanup DELETE
// retracts an all-day/scheduled mute from a node that stopped supporting
// it. A delayed re-push of the scheduled rule must not undo that cleanup.
const del = await request(app)
.delete('/api/notification-suppression-rules/replica/970008')
.set('Authorization', `Bearer ${token}`);
expect(del.status).toBe(200);
const delayed = await request(app)
.post('/api/notification-suppression-rules/replica')
.set('Authorization', `Bearer ${token}`)
.send({ rule: replicaRule({ updated_at: 2000 }) });
expect(delayed.status).toBe(200);
expect(DatabaseService.getInstance().getNotificationSuppressionRule(970008)).toBeUndefined();
});
});
backend/src/__tests__/notification-suppression-schedule-migration.test.ts
+135
View File
@@ -0,0 +1,135 @@
/**
* Additive schedule column on notification_suppression_rules.
*/
import { describe, it, expect, afterEach } from 'vitest';
import fs from 'fs';
import os from 'os';
import path from 'path';
import Database from 'better-sqlite3';
import { DatabaseService } from '../services/DatabaseService';
import { scheduleAllowsSuppression } from '../helpers/notificationSchedule';
function resetDatabaseSingleton(): void {
const holder = DatabaseService as unknown as { instance?: DatabaseService };
const existing = holder.instance;
if (existing) {
try {
existing.getDb().close();
} catch {
// already closed
}
holder.instance = undefined;
}
}
describe('notification suppression schedule column migration', () => {
let scratchDir: string | null = null;
let prevDataDir: string | undefined;
afterEach(() => {
resetDatabaseSingleton();
if (prevDataDir === undefined) {
delete process.env.DATA_DIR;
} else {
process.env.DATA_DIR = prevDataDir;
}
if (scratchDir) {
try {
fs.rmSync(scratchDir, { recursive: true, force: true });
} catch {
// best-effort
}
scratchDir = null;
}
});
it('adds schedule via DatabaseService startup; legacy rows load as null', { timeout: 60_000 }, () => {
scratchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-supp-sched-mig-'));
const dbPath = path.join(scratchDir, 'sencho.db');
const seed = new Database(dbPath);
try {
seed.exec(`
CREATE TABLE notification_suppression_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
node_id INTEGER NULL,
stack_patterns TEXT NOT NULL,
label_ids TEXT NULL,
categories TEXT NULL,
levels TEXT NULL,
applies_to TEXT NOT NULL,
enabled INTEGER DEFAULT 1,
expires_at INTEGER NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
INSERT INTO notification_suppression_rules
(name, node_id, stack_patterns, label_ids, categories, levels, applies_to, enabled, expires_at, created_at, updated_at)
VALUES ('Legacy mute', NULL, '[]', NULL, NULL, NULL, 'both', 1, NULL, 1, 1);
`);
} finally {
seed.close();
}
prevDataDir = process.env.DATA_DIR;
process.env.DATA_DIR = scratchDir;
resetDatabaseSingleton();
const db = DatabaseService.getInstance();
const cols = db.getDb().prepare('PRAGMA table_info(notification_suppression_rules)').all() as Array<{ name: string }>;
expect(cols.filter((c) => c.name === 'schedule')).toHaveLength(1);
const rule = db.getNotificationSuppressionRules().find((r) => r.name === 'Legacy mute');
expect(rule).toBeDefined();
expect(rule!.schedule).toBeNull();
expect(rule!.scheduleInvalid).toBe(false);
resetDatabaseSingleton();
process.env.DATA_DIR = scratchDir;
const db2 = DatabaseService.getInstance();
const cols2 = db2.getDb().prepare('PRAGMA table_info(notification_suppression_rules)').all() as Array<{ name: string }>;
expect(cols2.filter((c) => c.name === 'schedule')).toHaveLength(1);
});
it('empty-string schedule column is invalid and does not suppress via enabled load', { timeout: 60_000 }, () => {
scratchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-supp-sched-empty-'));
const dbPath = path.join(scratchDir, 'sencho.db');
const seed = new Database(dbPath);
try {
seed.exec(`
CREATE TABLE notification_suppression_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
node_id INTEGER NULL,
stack_patterns TEXT NOT NULL,
label_ids TEXT NULL,
categories TEXT NULL,
levels TEXT NULL,
applies_to TEXT NOT NULL,
enabled INTEGER DEFAULT 1,
expires_at INTEGER NULL,
schedule TEXT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
INSERT INTO notification_suppression_rules
(name, node_id, stack_patterns, label_ids, categories, levels, applies_to, enabled, expires_at, schedule, created_at, updated_at)
VALUES ('Empty schedule mute', NULL, '[]', NULL, '["monitor_alert"]', NULL, 'both', 1, NULL, '', 1, 1);
`);
} finally {
seed.close();
}
prevDataDir = process.env.DATA_DIR;
process.env.DATA_DIR = scratchDir;
resetDatabaseSingleton();
const db = DatabaseService.getInstance();
const rule = db.getEnabledNotificationSuppressionRules().find((r) => r.name === 'Empty schedule mute');
expect(rule).toBeDefined();
expect(rule!.schedule).toBeNull();
expect(rule!.scheduleInvalid).toBe(true);
expect(scheduleAllowsSuppression(rule!.schedule, rule!.scheduleInvalid, Date.now())).toBe(false);
});
});
backend/src/__tests__/notification-suppression-sync.test.ts
+275
View File
@@ -0,0 +1,275 @@
/**
* Fleet sync for suppression rules: node_id normalize, capability gate, stale DELETE.
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
const mockFetchMeta = vi.fn();
const mockGetProxyTarget = vi.fn();
const mockGetNodes = vi.fn();
const mockGetNode = vi.fn();
const mockRemoteAdvertises = vi.fn();
vi.mock('../services/DatabaseService', () => ({
DatabaseService: {
getInstance: () => ({
getNodes: mockGetNodes,
getNode: mockGetNode,
}),
},
}));
vi.mock('../services/NodeRegistry', () => ({
NodeRegistry: {
getInstance: () => ({
getProxyTarget: mockGetProxyTarget,
fetchMetaForNode: mockFetchMeta,
}),
},
}));
vi.mock('../services/LicenseService', () => ({
LicenseService: {
getInstance: () => ({
getProxyHeaders: () => ({ tier: 'community' }),
}),
},
}));
vi.mock('../helpers/remoteCapabilities', () => ({
remoteAdvertisesCapability: (...args: unknown[]) => mockRemoteAdvertises(...args),
}));
const mockFetch = vi.fn();
vi.stubGlobal('fetch', mockFetch);
import {
syncSuppressionRuleToFleet,
syncSuppressionRuleUpdateToFleet,
replicationTargetIds,
} from '../helpers/notificationSuppressionSync';
import type { NotificationSuppressionRule } from '../services/DatabaseService';
function makeRule(overrides: Partial<NotificationSuppressionRule> = {}): NotificationSuppressionRule {
return {
id: 42,
name: 'Fleet mute',
node_id: null,
stack_patterns: [],
label_ids: null,
categories: null,
levels: null,
applies_to: 'both',
enabled: true,
expires_at: null,
schedule: null,
scheduleInvalid: false,
created_at: 1,
updated_at: 1,
...overrides,
};
}
const remoteA = { id: 10, name: 'remote-a', type: 'remote' as const };
const remoteB = { id: 11, name: 'remote-b', type: 'remote' as const };
describe('notificationSuppressionSync', () => {
beforeEach(() => {
vi.clearAllMocks();
mockGetNodes.mockReturnValue([remoteA, remoteB]);
mockGetNode.mockImplementation((id: number) =>
[remoteA, remoteB].find((n) => n.id === id),
);
mockGetProxyTarget.mockImplementation((id: number) => ({
apiUrl: `http://node-${id}.example:1852`,
apiToken: 'tok',
}));
mockFetch.mockResolvedValue({ ok: true, status: 200, text: async () => '' });
mockRemoteAdvertises.mockResolvedValue(true);
});
afterEach(() => {
vi.restoreAllMocks();
});
it('replicationTargetIds: fleet-wide all remotes; scoped one remote', () => {
expect(replicationTargetIds(makeRule({ node_id: null }))).toEqual([10, 11]);
expect(replicationTargetIds(makeRule({ node_id: 10 }))).toEqual([10]);
});
it('unscheduled push sends node_id null without capability probe', async () => {
syncSuppressionRuleToFleet(makeRule({ schedule: null, node_id: 10 }));
await vi.waitFor(() => expect(mockFetch).toHaveBeenCalled());
expect(mockRemoteAdvertises).not.toHaveBeenCalled();
const body = JSON.parse((mockFetch.mock.calls[0][1] as { body: string }).body);
expect(body.rule.node_id).toBeNull();
expect(body.rule.schedule).toBeNull();
});
it('supported remote receives scheduled rule with node_id null', async () => {
mockRemoteAdvertises.mockResolvedValue(true);
syncSuppressionRuleToFleet(makeRule({
node_id: 10,
schedule: { days: [6], start_minute: 120, end_minute: 360, tz: 'UTC' },
}));
await vi.waitFor(() => expect(mockFetch).toHaveBeenCalled());
expect(mockRemoteAdvertises).toHaveBeenCalled();
const [url, init] = mockFetch.mock.calls[0] as [string, { method: string; body: string }];
expect(url).toContain('/api/notification-suppression-rules/replica');
expect(init.method).toBe('POST');
const body = JSON.parse(init.body);
expect(body.rule.node_id).toBeNull();
expect(body.rule.schedule.days).toEqual([6]);
});
it('probe false + DELETE success: no POST; cleanup logged as removed', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const error = vi.spyOn(console, 'error').mockImplementation(() => {});
mockRemoteAdvertises.mockResolvedValue(false);
mockFetch.mockResolvedValue({ ok: true, status: 200, text: async () => '' });
syncSuppressionRuleToFleet(makeRule({
node_id: 10,
schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' },
}));
await vi.waitFor(() => expect(mockFetch).toHaveBeenCalled());
expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true);
expect(warn.mock.calls.some((c) => String(c[0]).includes('replica was removed'))).toBe(true);
expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(false);
});
it('probe false + no proxy target: no successful-cleanup claim', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const error = vi.spyOn(console, 'error').mockImplementation(() => {});
mockRemoteAdvertises.mockResolvedValue(false);
mockGetProxyTarget.mockReturnValue(null);
syncSuppressionRuleToFleet(makeRule({
node_id: 10,
schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' },
}));
await vi.waitFor(() => expect(error).toHaveBeenCalled());
expect(mockFetch).not.toHaveBeenCalled();
expect(warn.mock.calls.some((c) => String(c[0]).includes('replica was removed'))).toBe(false);
expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(true);
});
it('scheduleInvalid: DELETE success, no POST', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const error = vi.spyOn(console, 'error').mockImplementation(() => {});
mockFetch.mockResolvedValue({ ok: true, status: 200, text: async () => '' });
syncSuppressionRuleToFleet(makeRule({
node_id: 10,
schedule: null,
scheduleInvalid: true,
}));
await vi.waitFor(() => {
expect(warn.mock.calls.some((c) => String(c[0]).includes('replica removed'))).toBe(true);
});
expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true);
expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(false);
});
it('scheduleInvalid: DELETE 404 counts as cleanup success, no POST', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
mockFetch.mockResolvedValue({ ok: false, status: 404, text: async () => 'gone' });
syncSuppressionRuleToFleet(makeRule({
node_id: 10,
schedule: null,
scheduleInvalid: true,
}));
await vi.waitFor(() => {
expect(warn.mock.calls.some((c) => String(c[0]).includes('replica removed'))).toBe(true);
});
expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true);
});
it('scheduleInvalid: DELETE failure logs pending cleanup, no POST', async () => {
const error = vi.spyOn(console, 'error').mockImplementation(() => {});
mockFetch.mockResolvedValue({ ok: false, status: 503, text: async () => 'down' });
syncSuppressionRuleToFleet(makeRule({
node_id: 10,
schedule: null,
scheduleInvalid: true,
}));
await vi.waitFor(() => expect(error).toHaveBeenCalled());
expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true);
expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(true);
});
it('scheduleInvalid: no proxy target logs pending, no POST', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const error = vi.spyOn(console, 'error').mockImplementation(() => {});
mockGetProxyTarget.mockReturnValue(null);
syncSuppressionRuleToFleet(makeRule({
node_id: 10,
schedule: null,
scheduleInvalid: true,
}));
await vi.waitFor(() => expect(error).toHaveBeenCalled());
expect(mockFetch).not.toHaveBeenCalled();
expect(warn.mock.calls.some((c) => String(c[0]).includes('replica removed'))).toBe(false);
expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(true);
});
it('unscheduled-to-scheduled on unsupported target attempts DELETE', async () => {
mockRemoteAdvertises.mockResolvedValue(false);
mockFetch.mockResolvedValue({ ok: true, status: 200, text: async () => '' });
const previous = makeRule({ node_id: 10, schedule: null });
const updated = makeRule({
node_id: 10,
schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' },
});
syncSuppressionRuleUpdateToFleet(previous, updated);
await vi.waitFor(() => expect(mockFetch).toHaveBeenCalled());
expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true);
});
it('probe false + DELETE failure: no POST; logs cleanup pending', async () => {
const error = vi.spyOn(console, 'error').mockImplementation(() => {});
mockRemoteAdvertises.mockResolvedValue(false);
mockFetch.mockResolvedValue({ ok: false, status: 503, text: async () => 'down' });
syncSuppressionRuleToFleet(makeRule({
node_id: 10,
schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' },
}));
await vi.waitFor(() => expect(error).toHaveBeenCalled());
expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true);
expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(true);
expect(error.mock.calls.some((c) => String(c[0]).includes('rule 42'))).toBe(true);
});
it('scheduled-to-unscheduled POST refresh does not require capability', async () => {
mockRemoteAdvertises.mockResolvedValue(false);
const previous = makeRule({
node_id: 10,
schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' },
});
const updated = makeRule({ node_id: 10, schedule: null });
syncSuppressionRuleUpdateToFleet(previous, updated);
await vi.waitFor(() => expect(mockFetch).toHaveBeenCalled());
expect(mockRemoteAdvertises).not.toHaveBeenCalled();
const body = JSON.parse((mockFetch.mock.calls[0][1] as { body: string }).body);
expect(body.rule.schedule).toBeNull();
});
it('stale targets receive DELETE on scope change', async () => {
const previous = makeRule({ node_id: null, schedule: null });
const updated = makeRule({ node_id: 10, schedule: null });
syncSuppressionRuleUpdateToFleet(previous, updated);
await vi.waitFor(() => expect(mockFetch.mock.calls.length).toBeGreaterThanOrEqual(2));
const deletes = mockFetch.mock.calls.filter((c) => (c[1] as { method: string }).method === 'DELETE');
expect(deletes.some((c) => String(c[0]).includes('node-11'))).toBe(true);
const posts = mockFetch.mock.calls.filter((c) => (c[1] as { method: string }).method === 'POST');
expect(posts.some((c) => String(c[0]).includes('node-10'))).toBe(true);
});
});
backend/src/__tests__/notification-suppression.test.ts
+72
View File
@@ -73,6 +73,8 @@ function makeSuppressionRule(overrides: Record<string, unknown> = {}) {
applies_to: 'both' as const,
enabled: true,
expires_at: null as number | null,
schedule: null as null,
scheduleInvalid: false,
created_at: Date.now(),
updated_at: Date.now(),
...overrides,
@@ -176,4 +178,74 @@ describe('NotificationService - suppression logic', () => {
expect.objectContaining({ method: 'POST' }),
);
});
it('suppresses inside weekly window and records match', async () => {
// Monday 2026-07-20 03:00 UTC inside 02:00-06:00 Mon window
vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-07-20T03:00:00.000Z'));
mockGetEnabledNotificationSuppressionRules.mockReturnValue([
makeSuppressionRule({
categories: ['monitor_alert'],
applies_to: 'both',
schedule: { days: [1], start_minute: 120, end_minute: 360, tz: 'UTC' },
}),
]);
mockGetEnabledAgents.mockReturnValue([{ type: 'slack', url: 'https://hooks.slack.com/x', enabled: true }]);
await svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' });
expect(mockBroadcast).not.toHaveBeenCalled();
expect(mockFetch).not.toHaveBeenCalled();
expect(mockUpdateNotificationSuppressionMatch).toHaveBeenCalled();
expect(mockGetEnabledNotificationSuppressionRules).toHaveBeenCalledWith(Date.parse('2026-07-20T03:00:00.000Z'));
});
it('does not suppress outside weekly window', async () => {
vi.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-07-20T12:00:00.000Z'));
mockGetEnabledNotificationSuppressionRules.mockReturnValue([
makeSuppressionRule({
categories: ['monitor_alert'],
applies_to: 'both',
schedule: { days: [1], start_minute: 120, end_minute: 360, tz: 'UTC' },
}),
]);
mockGetEnabledAgents.mockReturnValue([{ type: 'slack', url: 'https://hooks.slack.com/x', enabled: true }]);
await svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' });
expect(mockBroadcast).toHaveBeenCalled();
expect(mockFetch).toHaveBeenCalled();
expect(mockUpdateNotificationSuppressionMatch).not.toHaveBeenCalled();
});
it('does not suppress when stored schedule is invalid', async () => {
mockGetEnabledNotificationSuppressionRules.mockReturnValue([
makeSuppressionRule({
categories: ['monitor_alert'],
applies_to: 'both',
schedule: null,
scheduleInvalid: true,
}),
]);
mockGetEnabledAgents.mockReturnValue([{ type: 'slack', url: 'https://hooks.slack.com/x', enabled: true }]);
await svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' });
expect(mockBroadcast).toHaveBeenCalled();
expect(mockFetch).toHaveBeenCalled();
expect(mockUpdateNotificationSuppressionMatch).not.toHaveBeenCalled();
});
it('does not suppress when scheduled rule is expired relative to atMs', async () => {
const atMs = Date.parse('2026-07-20T03:00:00.000Z');
vi.spyOn(Date, 'now').mockReturnValue(atMs);
// getEnabled already filters expiry; empty list simulates expired
mockGetEnabledNotificationSuppressionRules.mockReturnValue([]);
mockGetEnabledAgents.mockReturnValue([{ type: 'slack', url: 'https://hooks.slack.com/x', enabled: true }]);
await svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' });
expect(mockGetEnabledNotificationSuppressionRules).toHaveBeenCalledWith(atMs);
expect(mockBroadcast).toHaveBeenCalled();
expect(mockFetch).toHaveBeenCalled();
});
});
backend/src/__tests__/scheduler-service.test.ts
+1 -1
View File
@@ -54,7 +54,7 @@ const {
mockStartContainer: vi.fn().mockResolvedValue(undefined),
mockStopContainer: vi.fn().mockResolvedValue(undefined),
mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }),
mockUpdateStack: vi.fn().mockResolvedValue(undefined),
mockUpdateStack: vi.fn().mockResolvedValue({ recoveryId: null }),
mockGetStacks: vi.fn().mockResolvedValue([]),
mockGetStackContent: vi.fn().mockResolvedValue(''),
mockGetEnvContent: vi.fn().mockResolvedValue(''),
backend/src/__tests__/stack-bulk-routes.test.ts
+2 -2
View File
@@ -260,7 +260,7 @@ describe('POST /api/stacks/bulk execution', () => {
});
it('handles update action (paid tier) by calling ComposeService.updateStack', async () => {
mockUpdateStack.mockResolvedValue(undefined);
mockUpdateStack.mockResolvedValue({ recoveryId: null });
const { LicenseService } = await import('../services/LicenseService');
const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
try {
@@ -287,7 +287,7 @@ describe('POST /api/stacks/bulk execution', () => {
policy: { id: 1, name: 'block-criticals', node_id: null, node_identity: '', stack_pattern: null, max_severity: 'HIGH', block_on_deploy: 1, block_on_severity: 1, block_on_kev: 0, block_on_fixable: 0, enabled: 1, replicated_from_control: 0, created_at: Date.now(), updated_at: Date.now() },
violations: [{ imageRef: 'nginx:latest', severity: 'CRITICAL', criticalCount: 3, highCount: 0, kevCount: 0, fixableCount: 0, reasons: ['severity'], scanId: 1 }],
});
mockUpdateStack.mockResolvedValue(undefined);
mockUpdateStack.mockResolvedValue({ recoveryId: null });
try {
const res = await request(app)
.post('/api/stacks/bulk')
backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts
+3 -1
View File
@@ -124,7 +124,9 @@ describe('DELETE /api/stacks/:stackName mesh opt-out cascade (F-1 / F-14)', () =
expect(res.status).toBe(200);
const sawCascadeWarning = warnSpy.mock.calls.some(args =>
typeof args[0] === 'string' && args[0].includes('Mesh opt-out cascade failed')
typeof args[0] === 'string'
&& (args[0].includes('Mesh opt-out cascade failed')
|| args[0].includes('Mesh opt-out failed'))
);
expect(sawCascadeWarning).toBe(true);
});
backend/src/__tests__/stack-self-protected-routes.test.ts
+2 -2
View File
@@ -164,7 +164,7 @@ describe('self stack lifecycle refusal', () => {
});
it('allows update on a non-self stack', async () => {
mockUpdateStack.mockResolvedValue(undefined);
mockUpdateStack.mockResolvedValue({ recoveryId: null });
const res = await request(app)
.post('/api/stacks/web/update')
.set('Cookie', authCookie);
@@ -177,7 +177,7 @@ describe('self stack lifecycle refusal', () => {
describe('POST /api/stacks/bulk self stack skip', () => {
beforeEach(() => {
stubSelfProject('sencho');
mockUpdateStack.mockResolvedValue(undefined);
mockUpdateStack.mockResolvedValue({ recoveryId: null });
mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]);
});
backend/src/__tests__/stack-update-orchestrator.test.ts
+3 -2
View File
@@ -35,7 +35,7 @@ function spec(name: string): EffectiveServiceSpec {
beforeEach(() => {
state.updateStack.mockReset();
state.updateStack.mockResolvedValue(undefined);
state.updateStack.mockResolvedValue({ recoveryId: null });
state.model = null;
});
@@ -85,8 +85,9 @@ describe('StackUpdateOrchestrator stack branch', () => {
{ nodeId: 0, stackName: 'web', target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' },
{ atomic: true, terminalWs: null },
);
expect(result).toEqual({ kind: 'stack_compose_done' });
expect(result).toEqual({ kind: 'stack_compose_done', recoveryId: null });
expect(state.updateStack).toHaveBeenCalledWith('web', undefined, true);
// recoveryId is forwarded from ComposeService.updateStack
});
});
backend/src/__tests__/stack-update-recovery-service.test.ts
+365
View File
@@ -0,0 +1,365 @@
/**
* Direct tests for StackUpdateRecoveryService artifact lifecycle and compensation probe.
*/
import { beforeEach, describe, expect, it, vi } from 'vitest';
const mockTag = vi.fn().mockResolvedValue(undefined);
const mockRemove = vi.fn().mockResolvedValue(undefined);
const mockListContainers = vi.fn().mockResolvedValue([]);
const mockInspectContainer = vi.fn();
const mockGetContainer = vi.fn(() => ({ inspect: mockInspectContainer }));
const mockGetImage = vi.fn((ref: string) => ({
tag: mockTag,
remove: mockRemove,
inspect: vi.fn().mockResolvedValue({ RepoDigests: [] }),
_ref: ref,
}));
vi.mock('../services/DockerController', () => ({
default: {
getInstance: () => ({
getDocker: () => ({
listContainers: mockListContainers,
getContainer: mockGetContainer,
getImage: mockGetImage,
}),
}),
},
}));
vi.mock('../services/FileSystemService', () => ({
FileSystemService: {
getInstance: () => ({
getBaseDir: () => '/test/compose',
backupStackFiles: vi.fn().mockResolvedValue(undefined),
restoreStackFiles: vi.fn().mockResolvedValue(undefined),
}),
},
}));
vi.mock('../services/composeProjectContext', () => ({
classifyReferenceKind: () => 'moving_tag',
resolveComposeProjectContext: vi.fn().mockResolvedValue({
validateForMutation: vi.fn().mockResolvedValue(undefined),
backupFromContext: vi.fn().mockResolvedValue('backup-1'),
restoreFromContext: vi.fn().mockResolvedValue(undefined),
}),
}));
vi.mock('../services/effectiveServiceModel', () => ({
buildEffectiveServiceModel: vi.fn().mockResolvedValue({
renderable: true,
services: [{ name: 'web', declaredImage: 'nginx:latest', hasBuild: false, expectedReplicas: 1 }],
}),
}));
const mockValidateExact = vi.fn().mockResolvedValue(undefined);
vi.mock('../services/ComposeService', async () => {
const actual = await vi.importActual<typeof import('../services/ComposeService')>('../services/ComposeService');
return {
...actual,
ComposeService: {
getInstance: () => ({
validateExactComposeInvocation: mockValidateExact,
buildAuthoredComposeArgs: vi.fn(),
}),
},
getComposeCommandTimeoutMs: () => 60_000,
};
});
const mockUnlink = vi.fn().mockResolvedValue(undefined);
const mockWriteFile = vi.fn().mockResolvedValue(undefined);
const mockRealpath = vi.fn(async (p: string) => p);
vi.mock('fs/promises', () => ({
default: {
unlink: (p: string) => mockUnlink(p),
writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc),
realpath: (p: string) => mockRealpath(p),
},
unlink: (p: string) => mockUnlink(p),
writeFile: (p: string, data: string, enc?: string) => mockWriteFile(p, data, enc),
realpath: (p: string) => mockRealpath(p),
}));
import { DatabaseService } from '../services/DatabaseService';
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
describe('StackUpdateRecoveryService', () => {
beforeEach(() => {
vi.clearAllMocks();
StackUpdateRecoveryService.resetForTests();
mockListContainers.mockResolvedValue([
{ Id: 'c1', State: 'running', Labels: {} },
]);
mockInspectContainer.mockResolvedValue({
State: { Status: 'running', ExitCode: 0 },
Image: 'sha256:abc',
});
mockValidateExact.mockResolvedValue(undefined);
mockUnlink.mockResolvedValue(undefined);
mockWriteFile.mockResolvedValue(undefined);
mockRealpath.mockImplementation(async (p: string) => p);
mockRemove.mockResolvedValue(undefined);
mockTag.mockResolvedValue(undefined);
});
it('validates exact invocation before tagging images', async () => {
const order: string[] = [];
mockValidateExact.mockImplementation(async () => { order.push('validate'); });
mockTag.mockImplementation(async () => { order.push('tag'); });
mockWriteFile.mockImplementation(async () => { order.push('write'); });
const spyInsert = vi.spyOn(DatabaseService.prototype, 'insertStackUpdateRecoveryGeneration')
.mockImplementation(() => { order.push('insert'); });
vi.spyOn(DatabaseService.prototype, 'getGlobalSettings').mockReturnValue({});
await StackUpdateRecoveryService.getInstance().captureCandidate({
nodeId: 1,
stackName: 'my-stack',
createdBy: 'test',
});
expect(order.indexOf('validate')).toBeLessThan(order.indexOf('tag'));
expect(order.indexOf('tag')).toBeLessThan(order.indexOf('write'));
expect(order.indexOf('write')).toBeLessThan(order.indexOf('insert'));
spyInsert.mockRestore();
});
it('retires opaque tags and override on abandon', async () => {
const row = {
id: 'gen-1',
node_id: 1,
stack_name: 'my-stack',
status: 'candidate' as const,
phase: 'captured' as const,
is_current: 0,
backup_slot_id: null,
override_path: '/test/compose/my-stack/.sencho-recovery-aaaaaaaaaaaa.yml',
services_json: JSON.stringify([{
serviceName: 'web',
scale: 1,
hasBuild: false,
declaredImageRef: 'nginx:latest',
referenceKind: 'moving_tag',
replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold' }],
}]),
health_gate_id: null,
gate_retain_until: null,
artifact_expires_at: null,
operation_lease_expires_at: null,
created_at: Date.now(),
updated_at: Date.now(),
created_by: null,
artifacts_retired: 0,
};
vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
vi.spyOn(DatabaseService.prototype, 'abandonStackUpdateRecoveryGeneration').mockReturnValue(true);
const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired')
.mockReturnValue(true);
const ok = await StackUpdateRecoveryService.getInstance().abandon('gen-1');
expect(ok).toBe(true);
expect(mockRemove).toHaveBeenCalled();
expect(mockUnlink).toHaveBeenCalledWith(row.override_path);
expect(markRetired).toHaveBeenCalledWith('gen-1');
});
it('does not mark restored_current when recovery probe finds crashed containers', async () => {
const servicesJson = JSON.stringify([{
serviceName: 'web',
scale: 1,
hasBuild: false,
declaredImageRef: 'nginx:latest',
referenceKind: 'moving_tag',
replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/x/web:hold' }],
}]);
const row = {
id: 'gen-2',
node_id: 1,
stack_name: 'my-stack',
status: 'active' as const,
phase: 'reconciling' as const,
is_current: 1,
backup_slot_id: 'b1',
override_path: '/test/compose/my-stack/.sencho-recovery-bbbbbbbbbbbb.yml',
services_json: servicesJson,
health_gate_id: null,
gate_retain_until: null,
artifact_expires_at: null,
operation_lease_expires_at: null,
created_at: Date.now(),
updated_at: Date.now(),
created_by: null,
artifacts_retired: 0,
};
vi.spyOn(DatabaseService.prototype, 'getStackUpdateRecoveryGeneration').mockReturnValue(row);
const update = vi.spyOn(DatabaseService.prototype, 'updateStackUpdateRecoveryGeneration')
.mockImplementation(() => undefined);
mockListContainers.mockResolvedValue([{ Id: 'c1', State: 'exited', Labels: { 'com.docker.compose.service': 'web' } }]);
mockInspectContainer.mockResolvedValue({ State: { ExitCode: 0 } });
vi.useFakeTimers();
const promise = StackUpdateRecoveryService.getInstance().compensateWithCandidate(
'gen-2',
async () => undefined,
);
await vi.advanceTimersByTimeAsync(3100);
const ok = await promise;
vi.useRealTimers();
expect(ok).toBe(false);
expect(update).toHaveBeenCalledWith('gen-2', expect.objectContaining({ status: 'recovery_required' }));
expect(update).not.toHaveBeenCalledWith(
'gen-2',
expect.objectContaining({ status: 'restored_current' }),
);
});
const capturedWebReplica = {
containerId: 'c1',
imageId: 'sha256:oldimg',
repoDigest: null,
state: 'running' as const,
rollbackTag: 'sencho-rb/aaaaaaaaaaaa/web:hold',
};
it('probeRecoveredStack rejects empty runtime when expected replicas were running', async () => {
const servicesJson = JSON.stringify([{
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
}]);
mockListContainers.mockResolvedValue([]);
vi.useFakeTimers();
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
await vi.advanceTimersByTimeAsync(3100);
await expect(promise).resolves.toBe(false);
vi.useRealTimers();
});
it('probeRecoveredStack rejects restarting and unhealthy expected containers', async () => {
const servicesJson = JSON.stringify([{
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
}]);
mockListContainers.mockResolvedValue([
{ Id: 'c1', State: 'restarting', Labels: { 'com.docker.compose.service': 'web' } },
]);
vi.useFakeTimers();
let promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
await vi.advanceTimersByTimeAsync(3100);
await expect(promise).resolves.toBe(false);
vi.useRealTimers();
mockListContainers.mockResolvedValue([
{ Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
]);
mockInspectContainer.mockResolvedValue({
Image: 'sha256:oldimg',
State: { Status: 'running', Health: { Status: 'unhealthy' } },
});
vi.useFakeTimers();
promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
await vi.advanceTimersByTimeAsync(3100);
await expect(promise).resolves.toBe(false);
vi.useRealTimers();
});
it('probeRecoveredStack rejects healthy containers running a mismatched image', async () => {
const servicesJson = JSON.stringify([{
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
}]);
mockListContainers.mockResolvedValue([
{ Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
]);
mockInspectContainer.mockResolvedValue({
Image: 'sha256:newfailed',
Config: { Image: 'nginx:alpine' },
State: { Status: 'running' },
});
vi.useFakeTimers();
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
await vi.advanceTimersByTimeAsync(3100);
await expect(promise).resolves.toBe(false);
vi.useRealTimers();
});
it('probeRecoveredStack rejects a running replica for a service captured at scale zero', async () => {
const servicesJson = JSON.stringify([{
serviceName: 'worker', scale: 0, hasBuild: false, declaredImageRef: 'busybox:latest',
referenceKind: 'moving_tag',
replicas: [{
containerId: 'c0', imageId: 'sha256:worker', repoDigest: null, state: 'stopped',
rollbackTag: 'sencho-rb/aaaaaaaaaaaa/worker:hold',
}],
}]);
mockListContainers.mockResolvedValue([
{ Id: 'c0', State: 'running', Labels: { 'com.docker.compose.service': 'worker' } },
]);
mockInspectContainer.mockResolvedValue({
Image: 'sha256:worker',
State: { Status: 'running' },
});
vi.useFakeTimers();
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
await vi.advanceTimersByTimeAsync(3100);
await expect(promise).resolves.toBe(false);
vi.useRealTimers();
});
it('probeRecoveredStack accepts healthy running replicas matching capture scale and image', async () => {
const servicesJson = JSON.stringify([{
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
referenceKind: 'moving_tag', replicas: [capturedWebReplica],
}]);
mockListContainers.mockResolvedValue([
{ Id: 'c1', State: 'running', Labels: { 'com.docker.compose.service': 'web' } },
]);
mockInspectContainer.mockResolvedValue({
Image: 'sha256:oldimg',
Config: { Image: 'sencho-rb/aaaaaaaaaaaa/web:hold' },
State: { Status: 'running' },
});
vi.useFakeTimers();
const promise = StackUpdateRecoveryService.getInstance().probeRecoveredStack(1, 'my-stack', servicesJson);
await vi.advanceTimersByTimeAsync(3100);
await expect(promise).resolves.toBe(true);
vi.useRealTimers();
});
it('does not mark artifacts retired when tag removal fails', async () => {
const row = {
id: 'gen-3',
node_id: 1,
stack_name: 'my-stack',
status: 'abandoned' as const,
phase: 'captured' as const,
is_current: 0,
backup_slot_id: null,
override_path: null,
services_json: JSON.stringify([{
serviceName: 'web', scale: 1, hasBuild: false, declaredImageRef: 'nginx:latest',
referenceKind: 'moving_tag',
replicas: [{ containerId: 'c1', imageId: 'sha256:abc', repoDigest: null, state: 'running', rollbackTag: 'sencho-rb/cccccccccccc/web:hold' }],
}]),
health_gate_id: null,
gate_retain_until: null,
artifact_expires_at: Date.now() - 1,
operation_lease_expires_at: null,
created_at: Date.now(),
updated_at: Date.now(),
created_by: null,
artifacts_retired: 0,
};
mockRemove.mockRejectedValueOnce(Object.assign(new Error('docker busy'), { statusCode: 500 }));
const markRetired = vi.spyOn(DatabaseService.prototype, 'markStackUpdateRecoveryArtifactsRetired')
.mockReturnValue(true);
const ok = await StackUpdateRecoveryService.getInstance().retireGenerationArtifacts(row);
expect(ok).toBe(false);
expect(markRetired).not.toHaveBeenCalled();
});
});
backend/src/__tests__/stacks-failure-notifications.test.ts
+3 -3
View File
@@ -270,7 +270,7 @@ describe('health gate begin call sites', () => {
});
it('begins a gate after a manual update and returns its id', async () => {
mockUpdateStack.mockResolvedValue(undefined);
mockUpdateStack.mockResolvedValue({ recoveryId: null });
const res = await request(app)
.post('/api/stacks/myapp/update')
.set('Cookie', authCookie)
@@ -281,7 +281,7 @@ describe('health gate begin call sites', () => {
});
it('begins a gate per stack in a bulk update and carries ids in the results', async () => {
mockUpdateStack.mockResolvedValue(undefined);
mockUpdateStack.mockResolvedValue({ recoveryId: null });
const res = await request(app)
.post('/api/stacks/bulk')
.set('Cookie', authCookie)
@@ -490,7 +490,7 @@ describe('deploy_failure notification on /update error', () => {
});
it('uses trusted proxy tier headers for remote atomic updates', async () => {
mockUpdateStack.mockResolvedValue(undefined);
mockUpdateStack.mockResolvedValue({ recoveryId: null });
const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' });
const res = await request(app)
backend/src/__tests__/update-guard-rollback.test.ts
+5 -4
View File
@@ -96,11 +96,12 @@ describe('buildRollbackItems', () => {
expect(known.detail).toContain('nginx:1.27.1');
});
it('downgrades the previous image to not_covered for a moving tag', () => {
it('treats a moving tag as ready because full-stack updates retain the prior image ID', () => {
const item = itemById(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), 'previous_images');
expect(item.state).toBe('not_covered');
expect(item.state).toBe('ready');
expect(item.detail).toContain('moving image tag');
expect(item.detail).toContain('nginx:latest');
expect(item.detail).toContain('recovery window');
});
it('does not mistake an image literally named error for a failed preview', () => {
@@ -139,9 +140,9 @@ describe('aggregateRollbackOverall', () => {
expect(aggregateRollbackOverall(items)).toBe('partial');
});
it('is partial when the rollback target is a moving tag', () => {
it('is ready when the rollback target is a moving tag (image ID capture covers it)', () => {
const items = buildRollbackItems(baseInputs({ rollbackTarget: { target: 'nginx:latest', moving: true } }), NOW);
expect(aggregateRollbackOverall(items)).toBe('partial');
expect(aggregateRollbackOverall(items)).toBe('ready');
});
it('is partial when env coverage is missing', () => {
backend/src/__tests__/update-guard-service.test.ts
+3 -4
View File
@@ -295,12 +295,11 @@ describe('UpdateGuardService.computeRollbackReadiness moving-tag wiring', () =>
mockListContainers.mockResolvedValue([]);
});
it('marks previous_images not_covered (overall partial) when any image uses a moving tag', async () => {
// Primary pinned, sidecar on a moving tag: a file rollback cannot revert it.
it('marks previous_images ready when any image uses a moving tag (full-stack image ID capture)', async () => {
mockGetPreview.mockResolvedValue(preview([{ current_tag: '1.2.3' }, { current_tag: 'latest' }]));
const report = await UpdateGuardService.getInstance().computeRollbackReadiness(0, 'app');
expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('not_covered');
expect(report.overall).toBe('partial');
expect(report.items.find(i => i.id === 'previous_images')?.state).toBe('ready');
expect(report.overall).toBe('ready');
});
it('marks previous_images ready (overall ready) when every image is pinned', async () => {
backend/src/__tests__/webhooks-trigger.test.ts
+1 -1
View File
@@ -481,7 +481,7 @@ describe('WebhookService.execute: health gate begin call sites', () => {
const { HealthGateService } = await import('../services/HealthGateService');
vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined);
vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]);
vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue(undefined);
vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null });
const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook');
const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true);
backend/src/bootstrap/startup.ts
+12
View File
@@ -12,6 +12,8 @@ import { MonitorService } from '../services/MonitorService';
import { AutoHealService } from '../services/AutoHealService';
import { HealthGateService } from '../services/HealthGateService';
import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService';
import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
import { DockerEventManager } from '../services/DockerEventManager';
import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService';
@@ -134,6 +136,16 @@ export async function startServer(server: Server): Promise<void> {
// Initialize the license service before any tier-gated code can run.
LicenseService.getInstance().initialize();
// Deletion-intent reconciliation must finish before mutation-capable
// background services or HTTP accept traffic that could recreate a stack
// name still covered by a prepared/ready tombstone.
try {
await DeployedStackDeletionService.getInstance().reconcileAtStartup();
} catch (err) {
console.error('[Startup] Deployed stack deletion reconcile failed:', (err as Error).message);
}
StackUpdateRecoveryService.getInstance().start();
// Synchronous starts: schedule background timers and continue. None of
// these fire their first tick for at least a few seconds, so they
// safely run alongside the async initializers below.
backend/src/helpers/notificationSchedule.ts
+126
View File
@@ -0,0 +1,126 @@
/**
* Weekly UTC maintenance windows for notification suppression rules.
* days are start days (Date#getUTCDay); start inclusive, end exclusive.
*/
export interface NotificationSchedule {
days: number[];
start_minute: number;
end_minute: number;
tz: 'UTC';
}
export type ParseNotificationScheduleResult =
| { ok: true; schedule: NotificationSchedule }
| { ok: false; error: string };
const SCHEDULE_KEYS = new Set(['days', 'start_minute', 'end_minute', 'tz']);
function isPlainObject(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
/** Strict write-boundary parser. Accepts days in any order; returns sorted unique days. */
export function parseNotificationSchedule(raw: unknown): ParseNotificationScheduleResult {
if (!isPlainObject(raw)) {
return { ok: false, error: 'schedule must be an object or null' };
}
const keys = Object.keys(raw);
if (keys.length !== 4 || keys.some((k) => !SCHEDULE_KEYS.has(k))) {
return { ok: false, error: 'schedule must have exactly days, start_minute, end_minute, and tz' };
}
if (raw.tz !== 'UTC') {
return { ok: false, error: 'schedule.tz must be UTC' };
}
if (!Array.isArray(raw.days) || raw.days.length === 0) {
return { ok: false, error: 'schedule.days must be a nonempty array' };
}
if (raw.days.some((d) => typeof d !== 'number' || !Number.isInteger(d) || d < 0 || d > 6)) {
return { ok: false, error: 'schedule.days must be integers 0..6' };
}
const days = [...new Set(raw.days as number[])].sort((a, b) => a - b);
if (days.length !== raw.days.length) {
return { ok: false, error: 'schedule.days must not contain duplicates' };
}
if (
typeof raw.start_minute !== 'number'
|| !Number.isInteger(raw.start_minute)
|| raw.start_minute < 0
|| raw.start_minute > 1439
) {
return { ok: false, error: 'schedule.start_minute must be an integer 0..1439' };
}
if (
typeof raw.end_minute !== 'number'
|| !Number.isInteger(raw.end_minute)
|| raw.end_minute < 0
|| raw.end_minute > 1439
) {
return { ok: false, error: 'schedule.end_minute must be an integer 0..1439' };
}
if (raw.start_minute === raw.end_minute) {
return { ok: false, error: 'schedule.start_minute and end_minute must differ' };
}
return {
ok: true,
schedule: {
days,
start_minute: raw.start_minute,
end_minute: raw.end_minute,
tz: 'UTC',
},
};
}
/**
* Parse a DB column value. Only SQL null / undefined → legacy null (always in window).
* Empty string, whitespace, or any non-null corrupt value → invalid (must not suppress).
*/
export function parseStoredNotificationSchedule(
raw: unknown,
): { kind: 'null' } | { kind: 'ok'; schedule: NotificationSchedule } | { kind: 'invalid' } {
if (raw == null) return { kind: 'null' };
if (typeof raw === 'string') {
const trimmed = raw.trim();
if (trimmed === '') return { kind: 'invalid' };
let parsed: unknown;
try {
parsed = JSON.parse(trimmed);
} catch {
return { kind: 'invalid' };
}
// JSON null is not SQL NULL; treat as corrupt rather than legacy always-on.
if (parsed == null) return { kind: 'invalid' };
const result = parseNotificationSchedule(parsed);
if (!result.ok) return { kind: 'invalid' };
return { kind: 'ok', schedule: result.schedule };
}
const result = parseNotificationSchedule(raw);
if (!result.ok) return { kind: 'invalid' };
return { kind: 'ok', schedule: result.schedule };
}
/** True when the schedule window covers atMs (UTC). Null schedule is always active. */
export function isScheduleActive(schedule: NotificationSchedule | null, atMs: number): boolean {
if (schedule == null) return true;
const date = new Date(atMs);
const day = date.getUTCDay();
const minute = date.getUTCHours() * 60 + date.getUTCMinutes();
const { days, start_minute: start, end_minute: end } = schedule;
if (start < end) {
return days.includes(day) && minute >= start && minute < end;
}
const prevDay = (day + 6) % 7;
return (days.includes(day) && minute >= start)
|| (days.includes(prevDay) && minute < end);
}
/** Whether a loaded rule may suppress at atMs (filters already matched). */
export function scheduleAllowsSuppression(
schedule: NotificationSchedule | null,
scheduleInvalid: boolean,
atMs: number,
): boolean {
if (scheduleInvalid) return false;
return isScheduleActive(schedule, atMs);
}
backend/src/helpers/notificationSuppressionSync.ts
+111 -5
View File
@@ -2,6 +2,10 @@ import { DatabaseService, type NotificationSuppressionRule, type Node } from '..
import { NodeRegistry } from '../services/NodeRegistry';
import { LicenseService } from '../services/LicenseService';
import { PROXY_TIER_HEADER } from '../services/license-headers';
import {
NOTIFICATION_SUPPRESSION_SCHEDULE_CAPABILITY,
} from '../services/CapabilityRegistry';
import { remoteAdvertisesCapability } from './remoteCapabilities';
import { getErrorMessage } from '../utils/errors';
const SYNC_TIMEOUT_MS = 15_000;
@@ -16,7 +20,12 @@ function buildRemoteHeaders(apiToken: string): Record<string, string> {
return headers;
}
function replicationTargets(rule: NotificationSuppressionRule): Node[] {
/** Hub node ids that should receive a replica for this rule (before wire identity normalize). */
export function replicationTargetIds(rule: NotificationSuppressionRule): number[] {
return replicationTargets(rule).map((n) => n.id);
}
export function replicationTargets(rule: NotificationSuppressionRule): Node[] {
const db = DatabaseService.getInstance();
const remotes = db.getNodes().filter((n) => n.type === 'remote');
if (rule.node_id != null) {
@@ -26,6 +35,10 @@ function replicationTargets(rule: NotificationSuppressionRule): Node[] {
return remotes;
}
function replicaPayload(rule: NotificationSuppressionRule): NotificationSuppressionRule {
return { ...rule, node_id: null };
}
async function pushRuleToNode(node: Node, rule: NotificationSuppressionRule): Promise<void> {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target?.apiUrl) {
@@ -36,7 +49,7 @@ async function pushRuleToNode(node: Node, rule: NotificationSuppressionRule): Pr
const res = await fetch(`${baseUrl}/api/notification-suppression-rules/replica`, {
method: 'POST',
headers: buildRemoteHeaders(target.apiToken),
body: JSON.stringify({ rule }),
body: JSON.stringify({ rule: replicaPayload(rule) }),
signal: AbortSignal.timeout(SYNC_TIMEOUT_MS),
});
if (!res.ok) {
@@ -48,8 +61,7 @@ async function pushRuleToNode(node: Node, rule: NotificationSuppressionRule): Pr
async function deleteRuleOnNode(node: Node, ruleId: number): Promise<void> {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target?.apiUrl) {
console.warn(`[SuppressionSync] Skipping node "${node.name}": no proxy target`);
return;
throw new Error(`no proxy target for node "${node.name}" (id=${node.id})`);
}
const baseUrl = target.apiUrl.replace(/\/$/, '');
const res = await fetch(`${baseUrl}/api/notification-suppression-rules/replica/${ruleId}`, {
@@ -63,6 +75,61 @@ async function deleteRuleOnNode(node: Node, ruleId: number): Promise<void> {
}
}
async function pushOrCleanupScheduled(node: Node, rule: NotificationSuppressionRule): Promise<void> {
const supportsSchedule = await remoteAdvertisesCapability(
node.id,
NOTIFICATION_SUPPRESSION_SCHEDULE_CAPABILITY,
);
if (supportsSchedule) {
await pushRuleToNode(node, rule);
return;
}
// Probe false means unsupported OR unreachable. Never POST a scheduled rule
// through the legacy contract (older remotes would mute all day). Attempt DELETE;
// only claim cleanup when DELETE succeeds.
try {
await deleteRuleOnNode(node, rule.id);
console.warn(
`[SuppressionSync] Scheduled rule ${rule.id} not applied on node "${node.name}" (id=${node.id}): ` +
`capability unsupported-or-unreachable; DELETE succeeded and replica was removed`,
);
} catch (err) {
console.error(
`[SuppressionSync] Scheduled rule ${rule.id}: cleanup pending on node "${node.name}" (id=${node.id}); ` +
`capability unsupported-or-unreachable and DELETE failed (${getErrorMessage(err, String(err))}). ` +
`Prior replica may remain until connectivity returns and the rule is re-saved`,
);
}
}
async function cleanupInvalidScheduleReplica(node: Node, rule: NotificationSuppressionRule): Promise<void> {
// Never POST an invalid schedule (would mute all day on remotes that ignore the field).
// Attempt DELETE so a prior valid/unscheduled replica cannot keep muting.
try {
await deleteRuleOnNode(node, rule.id);
console.warn(
`[SuppressionSync] Corrupt schedule on rule ${rule.id}: replica removed on node "${node.name}" (id=${node.id}); not posting`,
);
} catch (err) {
console.error(
`[SuppressionSync] Corrupt schedule on rule ${rule.id}: cleanup pending on node "${node.name}" (id=${node.id}); ` +
`DELETE failed (${getErrorMessage(err, String(err))}). Prior replica may remain until connectivity returns`,
);
}
}
async function syncRuleToNode(node: Node, rule: NotificationSuppressionRule): Promise<void> {
if (rule.scheduleInvalid) {
await cleanupInvalidScheduleReplica(node, rule);
return;
}
if (rule.schedule != null) {
await pushOrCleanupScheduled(node, rule);
return;
}
await pushRuleToNode(node, rule);
}
/** Best-effort push of a suppression rule to fleet nodes that should evaluate it. */
export function syncSuppressionRuleToFleet(rule: NotificationSuppressionRule): void {
const targets = replicationTargets(rule);
@@ -70,7 +137,7 @@ export function syncSuppressionRuleToFleet(rule: NotificationSuppressionRule): v
void Promise.allSettled(
targets.map(async (node) => {
try {
await pushRuleToNode(node, rule);
await syncRuleToNode(node, rule);
} catch (err) {
console.error(
`[SuppressionSync] Failed to push rule ${rule.id} to node "${node.name}":`,
@@ -81,6 +148,45 @@ export function syncSuppressionRuleToFleet(rule: NotificationSuppressionRule): v
);
}
/**
* After an update, DELETE replicas on targets that left the set, then refresh
* remaining / new targets. previous is the pre-update rule (for target diff).
*/
export function syncSuppressionRuleUpdateToFleet(
previous: NotificationSuppressionRule,
updated: NotificationSuppressionRule,
): void {
const oldIds = new Set(replicationTargetIds(previous));
const newIds = new Set(replicationTargetIds(updated));
const db = DatabaseService.getInstance();
const staleIds = [...oldIds].filter((id) => !newIds.has(id));
void Promise.allSettled([
...staleIds.map(async (id) => {
const node = db.getNode(id);
if (!node || node.type !== 'remote') return;
try {
await deleteRuleOnNode(node, previous.id);
} catch (err) {
console.error(
`[SuppressionSync] Failed to delete stale rule ${previous.id} on node "${node.name}":`,
getErrorMessage(err, String(err)),
);
}
}),
...replicationTargets(updated).map(async (node) => {
try {
await syncRuleToNode(node, updated);
} catch (err) {
console.error(
`[SuppressionSync] Failed to push rule ${updated.id} to node "${node.name}":`,
getErrorMessage(err, String(err)),
);
}
}),
]);
}
/** Best-effort delete of a replicated rule on fleet nodes. */
export function deleteSuppressionRuleFromFleet(rule: NotificationSuppressionRule): void {
const targets = replicationTargets(rule);
backend/src/routes/blueprints.ts
+33 -5
View File
@@ -11,6 +11,7 @@ import { BlueprintService } from '../services/BlueprintService';
import {
BlueprintReconciler,
messageForConfirmedOutcomes,
messageForSnapshotFinishedWithStaleApproval,
summarizeConfirmedOutcomes,
} from '../services/BlueprintReconciler';
import { BlueprintAnalyzer } from '../services/BlueprintAnalyzer';
@@ -18,6 +19,8 @@ import { buildBlueprintPreview, evaluateLightweightEffectiveApproval } from '../
import {
confirmableActionsEqual,
deriveBlastFromConfirmableActions,
evaluateEffectiveApproval,
intentFingerprint,
parseConfirmableActionsBody,
serializeApprovedBlast,
} from '../services/blueprintApproval';
@@ -427,17 +430,42 @@ blueprintsRouter.post('/:id/apply', async (req: Request, res: Response): Promise
}
const blast = deriveBlastFromConfirmableActions(preview.confirmableActions);
DatabaseService.getInstance().setBlueprintApproval(id, {
const approved = DatabaseService.getInstance().setBlueprintApproval(id, {
intentFingerprint: preview.planFingerprint,
blastJson: serializeApprovedBlast(blast),
approvedBy: req.user?.username ?? null,
});
const plan = await BlueprintReconciler.getInstance().reconcileConfirmedPlan(id, preview.executorActions);
// planFingerprint is from an earlier preview. Concurrent compose/selector
// edits can invalidate it before approval persists, or the reconciler can
// refuse if the live gate no longer matches. Both paths clear and 409.
const approvalMatches = !!approved && intentFingerprint(approved) === preview.planFingerprint;
const plan = approvalMatches
? await BlueprintReconciler.getInstance().reconcileConfirmedPlan(id, preview.executorActions)
: null;
if (!plan || plan.refused) {
DatabaseService.getInstance().clearBlueprintApproval(id);
const fresh = await buildBlueprintPreview(id);
res.status(409).json({
error: 'Preview is stale; refresh and confirm again',
code: 'PREVIEW_STALE',
preview: fresh,
});
return;
}
// Snapshot deploy may finish after a concurrent edit cleared approval.
// Report live effectiveApproval; do not hardcode "approved".
const live = DatabaseService.getInstance().getBlueprint(id);
const { effectiveApproval } = live
? evaluateEffectiveApproval(live, preview.executorActions)
: { effectiveApproval: 'pending' as const };
const outcomeSummary = summarizeConfirmedOutcomes(plan.outcomes);
const message = effectiveApproval === 'approved'
? messageForConfirmedOutcomes(outcomeSummary)
: messageForSnapshotFinishedWithStaleApproval(outcomeSummary);
res.json({
message: messageForConfirmedOutcomes(outcomeSummary),
message,
blueprintId: id,
effectiveApproval: 'approved',
effectiveApproval,
outcomes: plan.outcomes,
outcomeSummary,
});
@@ -507,7 +535,7 @@ blueprintsRouter.post('/:id/withdraw/:nodeId', async (req: Request, res: Respons
nodeId: node.id,
nodeName: node.name,
stackName: blueprint.name,
filename: 'docker-compose.yml',
filename: 'compose.yaml',
content: compose,
}]);
} catch (snapErr) {
backend/src/routes/imageUpdates.ts
+7 -1
View File
@@ -426,7 +426,13 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp
continue;
}
db.clearStackUpdateStatus(req.nodeId, stackName);
HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`);
const orchResult = lock.result;
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
if (recoveryId) {
const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService');
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
}
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
backend/src/routes/nodes.ts
+9 -1
View File
@@ -7,6 +7,7 @@ import { requirePermission } from '../middleware/permissions';
import { rejectApiTokenScope } from '../middleware/apiTokenScope';
import { requireAdmin, requirePaid } from '../middleware/tierGates';
import { enrollmentLimiter } from '../middleware/rateLimiters';
import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
import { DatabaseService } from '../services/DatabaseService';
import { NodeRegistry } from '../services/NodeRegistry';
import { CacheService } from '../services/CacheService';
@@ -401,7 +402,14 @@ nodesRouter.delete('/:id', async (req: Request, res: Response) => {
// local node, or one with no active connection). Mirrors the re-enroll path.
MeshProxyTunnelDialer.getInstance().closeBridge(id, 'node deleted');
PilotTunnelManager.getInstance().closeTunnel(id, PilotCloseCode.NormalClosure, 'node deleted');
DatabaseService.getInstance().deleteNode(id);
// Local-socket nodes: ready tombstone + recovery-row retirement in the same
// transaction as the node delete, then sweep tags/paths. Remote hub records
// create no Docker cleanup tombstone.
if (existing.type === 'local') {
await DeployedStackDeletionService.getInstance().deleteLocalNode(id);
} else {
DatabaseService.getInstance().deleteNode(id);
}
NodeRegistry.getInstance().evictConnection(id);
NodeRegistry.getInstance().notifyNodeRemoved(id);
CacheService.getInstance().invalidate(`${REMOTE_META_NAMESPACE}:${id}`);
backend/src/routes/notifications.ts
+67 -3
View File
@@ -17,9 +17,14 @@ import {
storedAppriseToWriteConfig,
} from '../helpers/notificationChannels';
import { parseStackPatternsInput } from '../helpers/stackPattern';
import {
parseNotificationSchedule,
type NotificationSchedule,
} from '../helpers/notificationSchedule';
import {
deleteSuppressionRuleFromFleet,
syncSuppressionRuleToFleet,
syncSuppressionRuleUpdateToFleet,
} from '../helpers/notificationSuppressionSync';
import { isDebugEnabled } from '../utils/debug';
import { logDebugTiming } from '../utils/requestTiming';
@@ -109,6 +114,27 @@ function validateExpiresAt(expires_at: unknown, res: Response): number | null |
return expires_at;
}
/**
* Resolve schedule with presence semantics.
* Create/replica omit → null; PUT omit → undefined (preserve); null clears; object validates.
*/
function resolveScheduleField(
schedule: unknown,
opts: { present: boolean; isCreate: boolean },
res: Response,
): NotificationSchedule | null | undefined | false {
if (!opts.present) {
return opts.isCreate ? null : undefined;
}
if (schedule === null) return null;
const parsed = parseNotificationSchedule(schedule);
if (!parsed.ok) {
res.status(400).json({ error: parsed.error });
return false;
}
return parsed.schedule;
}
function normalizeStoredLevels(levels: unknown): ('info' | 'warning' | 'error')[] | null {
if (!Array.isArray(levels) || levels.length === 0) return null;
return levels as ('info' | 'warning' | 'error')[];
@@ -135,7 +161,7 @@ function parseSuppressionRuleBody(
req: Request,
res: Response,
isCreate: boolean,
): Omit<NotificationSuppressionRule, 'id' | 'created_at' | 'updated_at'> | null {
): Omit<NotificationSuppressionRule, 'id' | 'created_at' | 'updated_at' | 'scheduleInvalid'> | null {
const {
name,
node_id: rawNodeId,
@@ -146,6 +172,7 @@ function parseSuppressionRuleBody(
applies_to,
enabled,
expires_at,
schedule,
} = req.body;
if (isCreate && (!name || typeof name !== 'string' || !name.trim())) {
@@ -183,6 +210,12 @@ function parseSuppressionRuleBody(
const expiresAtResult = validateExpiresAt(expires_at, res);
if (expiresAtResult === false) return null;
const scheduleResult = resolveScheduleField(schedule, {
present: 'schedule' in req.body,
isCreate,
}, res);
if (scheduleResult === false) return null;
if (enabled !== undefined && typeof enabled !== 'boolean') {
res.status(400).json({ error: 'enabled must be a boolean' });
return null;
@@ -198,6 +231,7 @@ function parseSuppressionRuleBody(
applies_to: (appliesToResult ?? 'both') as NotificationSuppressionAppliesTo,
enabled: enabled !== false,
expires_at: expiresAtResult ?? null,
schedule: scheduleResult ?? null,
};
}
@@ -508,9 +542,28 @@ notificationSuppressionRouter.post('/replica', authMiddleware, (req: Request, re
res.status(400).json({ error: patterns.error });
return;
}
if (!validateLabelIds(rule.label_ids, res)) return;
if (!validateCategories(rule.categories, res, VALID_SUPPRESSION_CATEGORIES)) return;
if (!validateLevels(rule.levels, res)) return;
const scheduleResult = resolveScheduleField(rule.schedule, {
present: 'schedule' in rule,
isCreate: true,
}, res);
if (scheduleResult === false) return;
DatabaseService.getInstance().upsertNotificationSuppressionRuleReplica({
...rule,
stack_patterns: patterns.patterns,
label_ids: Array.isArray(rule.label_ids) && rule.label_ids.length > 0 ? rule.label_ids : null,
categories: Array.isArray(rule.categories) && rule.categories.length > 0 ? rule.categories : null,
levels: normalizeStoredLevels(rule.levels),
schedule: scheduleResult ?? null,
scheduleInvalid: false,
enabled: rule.enabled !== false,
expires_at: rule.expires_at ?? null,
// Replicas are always node-agnostic on the receiving node: the hub's node_id
// is a hub-local scoping concept and never trustworthy as a foreign key here.
node_id: null,
});
res.json({ success: true });
} catch (error) {
@@ -583,6 +636,7 @@ notificationSuppressionRouter.put('/:id', authMiddleware, (req: Request, res: Re
applies_to,
enabled,
expires_at,
schedule,
} = req.body;
if (name !== undefined && (typeof name !== 'string' || !name.trim())) {
@@ -626,12 +680,21 @@ notificationSuppressionRouter.put('/:id', authMiddleware, (req: Request, res: Re
validatedExpiresAt = result;
}
let validatedSchedule: NotificationSchedule | null | undefined;
if ('schedule' in req.body) {
const result = resolveScheduleField(schedule, { present: true, isCreate: false }, res);
if (result === false) return;
validatedSchedule = result;
}
if (enabled !== undefined && typeof enabled !== 'boolean') {
res.status(400).json({ error: 'enabled must be a boolean' });
return;
}
const updates: Partial<Omit<NotificationSuppressionRule, 'id' | 'created_at'>> = { updated_at: Date.now() };
const updates: Partial<Omit<NotificationSuppressionRule, 'id' | 'created_at' | 'scheduleInvalid'>> = {
updated_at: Date.now(),
};
if (name !== undefined) updates.name = name.trim();
if (validatedNodeId !== undefined) updates.node_id = validatedNodeId;
if (cleanedPatterns !== undefined) updates.stack_patterns = cleanedPatterns;
@@ -641,11 +704,12 @@ notificationSuppressionRouter.put('/:id', authMiddleware, (req: Request, res: Re
if (validatedAppliesTo !== undefined) updates.applies_to = validatedAppliesTo;
if (enabled !== undefined) updates.enabled = enabled;
if (validatedExpiresAt !== undefined) updates.expires_at = validatedExpiresAt;
if (validatedSchedule !== undefined) updates.schedule = validatedSchedule;
const db = DatabaseService.getInstance();
db.updateNotificationSuppressionRule(id, updates);
const updated = db.getNotificationSuppressionRule(id)!;
syncSuppressionRuleToFleet(updated);
syncSuppressionRuleUpdateToFleet(existing, updated);
console.log(`[Suppression] Rule ${id} updated`);
res.json(updated);
} catch (error) {
backend/src/routes/stacks.ts
+41 -74
View File
@@ -16,7 +16,6 @@ import { ComposeService, getComposeRollbackInfo } from '../services/ComposeServi
import { StackUpdateOrchestrator, shortImageId, type OrchestratorResult } from '../services/StackUpdateOrchestrator';
import DockerController, { type BulkStackInfo } from '../services/DockerController';
import { DatabaseService, type StackDossierFields } from '../services/DatabaseService';
import { MeshService } from '../services/MeshService';
import { CacheService, type CacheFetchOutcome } from '../services/CacheService';
import { UpdatePreviewService } from '../services/UpdatePreviewService';
import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
@@ -38,6 +37,8 @@ import { labelInventoryOptionsFromRequest, requireRevealAdmin } from '../helpers
import { EXPOSURE_INTENTS, type ExposureIntent } from '../services/network/types';
import { UpdateGuardService, SingleServiceUpdateReadinessError } from '../services/UpdateGuardService';
import { HealthGateService } from '../services/HealthGateService';
import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService';
import { classifyFailure } from '../services/updateGuard/failureClassifier';
import { requirePermission, checkPermission } from '../middleware/permissions';
import { NotificationService, type NotificationCategory } from '../services/NotificationService';
@@ -99,8 +100,14 @@ const STACK_OP_PRESENT_PARTICIPLE: Record<StackOpAction, string> = {
update: 'updating',
rollback: 'rolling back',
backup: 'backing up',
delete: 'deleting',
};
function linkStackUpdateRecoveryGate(recoveryId: string | null | undefined, healthGateId: string | null): void {
if (!recoveryId) return;
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
}
function tryAcquireStackOpLock(
req: Request,
res: Response,
@@ -474,7 +481,7 @@ async function runStackBulkOp(
};
}
const atomic = true;
await StackUpdateOrchestrator.getInstance().execute(
const orchResult = await StackUpdateOrchestrator.getInstance().execute(
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'bulk', actor: user },
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
);
@@ -492,6 +499,8 @@ async function runStackBulkOp(
console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err),
);
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
return { stackName, ok: true, healthGateId };
} else {
const outcome = await containerActionForStack(req.nodeId, stackName, action);
@@ -929,6 +938,12 @@ stacksRouter.post('/', async (req: Request, res: Response) => {
if (!isValidStackName(stackName)) {
return res.status(400).json({ error: 'Stack name can only contain alphanumeric characters, hyphens, and underscores' });
}
try {
DeployedStackDeletionService.getInstance().assertNoBlockingDeletionIntent(req.nodeId, stackName);
} catch (guardError) {
res.status(409).json({ error: getErrorMessage(guardError, 'Stack deletion in progress'), code: 'stack_deletion_in_progress' });
return;
}
await FileSystemService.getInstance(req.nodeId).createStack(stackName);
invalidateNodeCaches(req.nodeId);
dlog(`[Stacks] Stack created: ${sanitizeForLog(stackName)}`);
@@ -1109,82 +1124,32 @@ stacksRouter.delete('/:stackName', async (req: Request, res: Response) => {
const sanitizedName = sanitizeForLog(stackName);
if (debug) console.debug(`[Stacks:debug] Delete starting`, { stackName: sanitizedName, pruneVolumes, nodeId: req.nodeId });
// Step 1: compose down. Best-effort: a stack with corrupt compose files or
// a temporarily-unreachable daemon should still be removable. Logged so the
// operator can investigate orphaned containers separately.
try {
await ComposeService.getInstance(req.nodeId).downStack(stackName);
if (debug) console.debug(`[Stacks:debug] Delete: down OK`, { stackName: sanitizedName });
} catch (downErr) {
console.warn('[Stacks] Compose down failed or no-op for %s:', sanitizeForLog(stackName), downErr);
}
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
nodeId: req.nodeId,
stackName,
pruneVolumes,
actor: req.user?.username ?? 'system',
});
// Step 2: volume prune (only if requested). Best-effort.
if (pruneVolumes) {
try {
const result = await DockerController.getInstance().pruneManagedOnly('volumes', [stackName]);
dlog(`[Stacks] Pruned volumes for ${sanitizeForLog(stackName)}: ${result.reclaimedBytes} bytes reclaimed`);
} catch (pruneErr) {
console.warn('[Stacks] Volume prune failed for %s, continuing delete:', sanitizeForLog(stackName), pruneErr);
if (!result.ok) {
if (result.code === 'lock_conflict') {
res.status(409).json({
error: result.error,
code: 'stack_op_in_progress',
action: result.existingAction ?? 'delete',
});
return;
}
}
// Step 3: filesystem delete. If this fails the on-disk compose files are
// still present, so we abort BEFORE touching the database — keeping DB and
// FS in sync so the operator can retry. Otherwise a half-deleted stack
// (rows gone, files remain) becomes invisible to the UI.
try {
await FileSystemService.getInstance(req.nodeId).deleteStack(stackName);
if (debug) console.debug(`[Stacks:debug] Delete: fs OK`, { stackName: sanitizedName });
} catch (fsErr) {
console.error('[Stacks] File deletion failed for %s; database state untouched:', sanitizeForLog(stackName), fsErr);
const message = getErrorMessage(fsErr, 'Failed to remove stack files');
res.status(500).json({
error: `${message}. Stack containers may have been stopped but on-disk files remain. Retry the delete or clean the files manually.`,
});
if (result.code === 'fs_failed') {
res.status(500).json({
error: `${result.error}. Stack containers may have been stopped but on-disk files remain. Retry the delete or clean the files manually.`,
});
return;
}
res.status(500).json({ error: result.error });
return;
}
// Step 4: database cleanup. Per-call idempotent; safe to run sequentially.
try {
DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName);
DatabaseService.getInstance().clearStackScanAttempts(req.nodeId, stackName);
DatabaseService.getInstance().deleteServiceUpdateRecoveries(req.nodeId, stackName);
DatabaseService.getInstance().deleteRoleAssignmentsByResource('stack', stackName);
DatabaseService.getInstance().deleteGitSource(stackName);
DatabaseService.getInstance().deleteStackDossier(req.nodeId, stackName);
DatabaseService.getInstance().deleteStackDriftFindings(req.nodeId, stackName);
DatabaseService.getInstance().deleteStackExposureIntents(req.nodeId, stackName);
DatabaseService.getInstance().deleteStackExposure(req.nodeId, stackName);
DatabaseService.getInstance().deleteStackProjectEnvFiles(req.nodeId, stackName);
DatabaseService.getInstance().deleteStackScans(req.nodeId, stackName);
if (debug) console.debug(`[Stacks:debug] Delete: db OK`, { stackName: sanitizedName });
} catch (dbErr) {
console.error('[Stacks] Database cleanup failed for %s; files already removed:', sanitizeForLog(stackName), dbErr);
const message = getErrorMessage(dbErr, 'Failed to clear stack database state');
res.status(500).json({
error: `${message}. Stack files have been removed; some database rows for this stack may remain. Recreating the stack with the same name will reuse those rows.`,
});
return;
}
// Step 5: mesh opt-out cascade. Idempotent; best-effort cleanup of derived
// aliases / override files. A mesh failure here must not flip the delete
// result, since the stack itself is already gone.
try {
await MeshService.getInstance().optOutStack(
req.nodeId,
stackName,
req.user?.username ?? 'system',
);
} catch (meshErr) {
console.warn(
'[Stacks] Mesh opt-out cascade failed for %s, continuing delete:',
sanitizeForLog(stackName),
meshErr,
);
}
invalidateNodeCaches(req.nodeId);
dlog(`[Stacks] Stack deleted: ${sanitizeForLog(stackName)}`);
res.json({ success: true });
@@ -2267,7 +2232,7 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
const debug = isDebugEnabled();
const atomic = true;
if (debug) console.debug('[Stacks:debug] Update starting', { stackName, atomic, nodeId: req.nodeId });
await StackUpdateOrchestrator.getInstance().execute(
const orchResult = await StackUpdateOrchestrator.getInstance().execute(
{ nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null },
{ atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) },
);
@@ -2285,6 +2250,8 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => {
if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`);
ok = true;
const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null);
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
linkStackUpdateRecoveryGate(recoveryId, healthGateId);
res.json({ status: 'Update completed', healthGateId });
notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system');
if (!skipScan) {
backend/src/services/BlueprintReconciler.ts
+23 -4
View File
@@ -37,6 +37,8 @@ export interface ConfirmedActionOutcome {
export interface ConfirmedPlanResult {
outcomes: ConfirmedActionOutcome[];
/** True when the approval gate refused execution (Apply must not claim success). */
refused?: boolean;
}
export interface ConfirmedOutcomeSummary {
@@ -78,6 +80,17 @@ export function messageForConfirmedOutcomes(summary: ConfirmedOutcomeSummary): s
return 'Rollout confirmed';
}
/** Apply finished a confirmed snapshot, but live approval is no longer current. */
export function messageForSnapshotFinishedWithStaleApproval(summary: ConfirmedOutcomeSummary): string {
if (summary.failed > 0) {
return 'Confirmed snapshot finished with node failures; approval is no longer current';
}
if (summary.pending > 0) {
return 'Confirmed snapshot finished with actions still in progress; approval is no longer current';
}
return 'Confirmed snapshot finished; approval is no longer current';
}
function mapDeployOutcome(
base: { nodeId: number; nodeName: string; action: PreviewAction },
result: DeployOutcome,
@@ -195,12 +208,18 @@ export class BlueprintReconciler {
): Promise<ConfirmedPlanResult> {
const blueprint = DatabaseService.getInstance().getBlueprint(blueprintId);
if (!blueprint || !blueprint.enabled) {
return { outcomes: [] };
return { outcomes: [], refused: true };
}
const parsed = parseApprovedBlastJson(blueprint.approved_blast_json);
if (!parsed.ok || blueprint.approval_status !== 'approved') {
diagnosticLog('reconcileConfirmedPlan skipped: approval missing or invalid', { blueprintId });
return { outcomes: [] };
// Same fail-closed gate as tick reconcile: never execute when approval is
// missing, invalid, or the stored fingerprint no longer matches live intent.
if (
!parsed.ok
|| blueprint.approval_status !== 'approved'
|| blueprint.approved_intent_fingerprint !== intentFingerprint(blueprint)
) {
diagnosticLog('reconcileConfirmedPlan skipped: approval missing, invalid, or drifted', { blueprintId });
return { outcomes: [], refused: true };
}
const authorized = filterAuthorizedExecutorActions(parsed.entries, executorActions);
const nodes = DatabaseService.getInstance().getNodes();
backend/src/services/BlueprintService.ts
+17 -27
View File
@@ -10,6 +10,7 @@ import {
} from './DatabaseService';
import { ComposeService } from './ComposeService';
import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService';
import { DeployedStackDeletionService } from './DeployedStackDeletionService';
import { FileSystemService } from './FileSystemService';
import { NodeRegistry } from './NodeRegistry';
import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers';
@@ -20,7 +21,8 @@ import { BlueprintAnalyzer } from './BlueprintAnalyzer';
import { sanitizeForLog } from '../utils/safeLog';
const MARKER_FILENAME = '.blueprint.json';
const COMPOSE_FILENAME = 'docker-compose.yml';
/** On-disk compose name for Blueprint applies. Must match createStack scaffold and Sencho discovery priority. */
const COMPOSE_FILENAME = 'compose.yaml';
const REMOTE_HTTP_TIMEOUT_MS = 30_000;
function isDeveloperModeEnabled(): boolean {
@@ -446,6 +448,9 @@ export class BlueprintService {
}
await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent);
await fs.writeStackFile(stackName, MARKER_FILENAME, markerContent);
// Clear lower-priority compose siblings so discovery cannot shadow compose.yaml.
// Local + modern apply-local only; legacy remote has no sibling DELETE.
await fs.removeAlternateRootComposeFiles(stackName);
await assertPolicyGateAllows(
stackName,
nodeId,
@@ -463,32 +468,17 @@ export class BlueprintService {
}
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<void> {
// Hold the per-stack lock across both the compose down and the directory
// delete so a withdraw cannot race a manual operation, nor tear the
// files out from under one that starts mid-withdraw.
const lock = await StackOpLockService.getInstance().runExclusive(
node.id, blueprint.name, 'down', 'system',
async () => {
try {
await ComposeService.getInstance(node.id).downStack(blueprint.name);
} catch (err) {
// best-effort: continue to delete the directory even if down fails
console.warn(`[BlueprintService] downStack failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`);
}
if (await this.stackDirExists(node.id, blueprint.name)) {
await FileSystemService.getInstance(node.id).deleteStack(blueprint.name);
}
// Remove the exposure descriptor so a withdrawn blueprint
// stack does not leave a stale row that escalates posture.
try {
DatabaseService.getInstance().deleteStackExposure(node.id, blueprint.name);
} catch (e) {
console.warn(`[BlueprintService] deleteStackExposure failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(e)}`);
}
},
);
if (!lock.ran) {
throw new Error(stackOpSkipMessage(blueprint.name, lock.existing.action));
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
nodeId: node.id,
stackName: blueprint.name,
pruneVolumes: false,
actor: 'system:blueprint',
});
if (!result.ok) {
if (result.code === 'lock_conflict') {
throw new Error(result.error);
}
throw new Error(result.error);
}
}
backend/src/services/CapabilityRegistry.ts
+5
View File
@@ -36,6 +36,7 @@ export const CAPABILITIES = [
'notifications',
'notification-routing',
'notification-suppression',
'notification-suppression-schedule',
'host-console',
'container-exec',
'audit-log',
@@ -70,6 +71,10 @@ export const CROSS_NODE_RBAC_CAPABILITY = 'cross-node-rbac';
export type Capability = (typeof CAPABILITIES)[number];
/** Remotes that evaluate weekly maintenance windows on mute/suppression replicas. */
export const NOTIFICATION_SUPPRESSION_SCHEDULE_CAPABILITY =
'notification-suppression-schedule' as const satisfies Capability;
/** Capability for optional `?removeVolumes=true` on POST /stacks/:name/down. */
export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability;
backend/src/services/ComposeService.ts
+226 -53
View File
@@ -143,6 +143,33 @@ export class ComposeService {
* same `authoredComposeFileArgs` prefix directly but intentionally omit the mesh
* override, rendering the user's authored model without mesh injection.
*/
/** Public wrapper for dual-arg assembly and recovery Compose invocations. */
public async buildAuthoredComposeArgs(stackName: string, action: string[]): Promise<string[]> {
return this.authoredComposeArgs(stackName, action);
}
public async validateStackForMutation(stackName: string): Promise<void> {
await this.assertRequiredEnvPresent(stackName);
await this.assertSafePilotBindMapping(stackName);
}
/**
* Render/validate the exact Compose invocation used by mutating operations
* (authored files, env pins, and generated Mesh override) before capture.
*/
public async validateExactComposeInvocation(stackName: string): Promise<void> {
if (!isValidStackName(stackName)) {
throw new Error('Invalid stack path');
}
const baseResolved = path.resolve(this.baseDir);
const stackDir = path.resolve(baseResolved, stackName);
if (!stackDir.startsWith(baseResolved + path.sep)) {
throw new Error('Invalid stack path');
}
const args = await this.authoredComposeArgs(stackName, ['config', '--quiet']);
await this.execute('docker', args, stackDir, undefined, true);
}
private async authoredComposeArgs(stackName: string, action: string[]): Promise<string[]> {
const args: string[] = ['compose'];
const filePrefix = authoredComposeFileArgs(stackName, this.nodeId);
@@ -151,12 +178,23 @@ export class ComposeService {
// directory, so deploy/update resolve the same effective config the validator did.
args.push(...await authoredComposeEnvFileArgs(stackName, this.nodeId));
const meshEnabled = DatabaseService.getInstance().isMeshStackEnabled(this.nodeId, stackName);
let overridePath: string | null = null;
try {
overridePath = await MeshService.getInstance().ensureStackOverride(this.nodeId, stackName);
} catch (err) {
if (meshEnabled) {
throw err instanceof Error
? err
: new Error(`Mesh override generation failed: ${String(err)}`);
}
console.warn('[ComposeService] mesh override skipped:', sanitizeForLog((err as Error).message));
}
if (meshEnabled && !overridePath) {
throw new Error(
`Mesh override is required for stack "${stackName}" but could not be generated`,
);
}
if (overridePath) {
if (filePrefix.length === 0) {
// Single-file stack: passing any -f disables compose's auto-discovery, so name
@@ -799,7 +837,50 @@ export class ComposeService {
startStream();
}
async updateStack(stackName: string, ws?: WebSocket, atomic?: boolean): Promise<void> {
/**
* Authored (+ mesh) compose args with an optional recovery override layered LAST.
* Used for pinned lifecycle / compensation ups (`--pull never --no-build`).
*/
public async buildComposeArgsWithRecoveryOverride(
stackName: string,
action: string[],
recoveryOverridePath: string | null,
): Promise<string[]> {
const withSentinel = await this.authoredComposeArgs(stackName, ['__SENCHO_ACTION_SENTINEL__']);
const idx = withSentinel.indexOf('__SENCHO_ACTION_SENTINEL__');
const prefix = idx >= 0 ? withSentinel.slice(0, idx) : withSentinel;
const out = [...prefix];
if (recoveryOverridePath) {
if (!out.includes('-f')) {
const fsSvc = FileSystemService.getInstance(this.nodeId);
const baseFilename = await fsSvc.getComposeFilename(stackName);
out.push('-f', baseFilename);
try {
const userOverride = await fsSvc.getOverrideFilename(stackName);
if (userOverride) out.push('-f', userOverride);
} catch (err) {
const code = (err as { code?: string }).code;
if (code === 'INVALID_STACK_NAME' || code === 'INVALID_PATH' || code === 'SYMLINK_ESCAPE') {
throw err;
}
console.warn(
'[ComposeService] could not resolve user compose override for recovery args:',
sanitizeForLog((err as Error).message),
);
}
}
out.push('-f', recoveryOverridePath);
}
out.push(...action);
return out;
}
async updateStack(
stackName: string,
ws?: WebSocket,
atomic?: boolean,
): Promise<{ recoveryId: string | null }> {
await this.assertRequiredEnvPresent(stackName);
await this.assertSafePilotBindMapping(stackName);
const stackDir = path.join(this.baseDir, stackName);
@@ -810,48 +891,106 @@ export class ComposeService {
if (ws && ws.readyState === WebSocket.OPEN) ws.send(data);
};
if (atomic) {
await this.createAtomicBackup(stackName, 'update', sendOutput);
}
// Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs).
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
const recoverySvc = StackUpdateRecoveryService.getInstance();
let recoveryId: string | null = null;
let handedOff = false;
try {
try {
const dockerController = DockerController.getInstance(this.nodeId);
const legacyContainers = await dockerController.getContainersByStack(stackName);
if (legacyContainers && legacyContainers.length > 0) {
sendOutput(`=== Cleaning up existing containers for clean update ===\n`);
await dockerController.removeContainers(legacyContainers.map((c: any) => c.Id));
}
} catch (e) {
console.warn('Failed to clean up legacy containers for %s:', sanitizeForLog(stackName), e);
}
sendOutput('=== Validating stack for update ===\n');
sendOutput('=== Capturing rollback generation ===\n');
const candidate = await recoverySvc.captureCandidate({
nodeId: this.nodeId,
stackName,
createdBy: null,
});
recoveryId = candidate.id;
const buildServices = await loadStackBuildServices(this.nodeId, stackName);
const buildAware = buildServices.length > 0;
await this.withRegistryAuth(async (env) => {
if (buildAware) {
sendOutput('=== Building images ===\n');
await this.execute('docker', await this.authoredComposeArgs(stackName, ['build', '--pull']), stackDir, ws, true, env, getComposeStallTimeoutMs());
try {
await this.withRegistryAuth(async (env) => {
if (buildAware) {
sendOutput('=== Building images ===\n');
await this.execute(
'docker',
await this.authoredComposeArgs(stackName, ['build', '--pull']),
stackDir, ws, true, env, getComposeStallTimeoutMs(),
);
sendOutput('=== Pulling registry images ===\n');
await this.execute(
'docker',
await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']),
stackDir, ws, true, env, getComposeStallTimeoutMs(),
);
} else {
sendOutput('=== Pulling latest images ===\n');
await this.execute(
'docker',
await this.authoredComposeArgs(stackName, ['pull']),
stackDir, ws, true, env, getComposeStallTimeoutMs(),
);
}
}, sendOutput);
} catch (acquireError) {
// Acquisition failure: abandon candidate; leave runtime untouched.
await recoverySvc.abandon(candidate.id);
recoveryId = null;
throw acquireError;
}
sendOutput('=== Pulling registry images ===\n');
await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull', '--ignore-buildable']), stackDir, ws, true, env, getComposeStallTimeoutMs());
} else {
sendOutput('=== Pulling latest images ===\n');
await this.execute('docker', await this.authoredComposeArgs(stackName, ['pull']), stackDir, ws, true, env, getComposeStallTimeoutMs());
}
sendOutput('=== Recreating containers ===\n');
await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs());
}, sendOutput);
// Post-Update Health Probe
await new Promise(resolve => setTimeout(resolve, 3000));
if (!recoverySvc.markAcquired(candidate.id)) {
await recoverySvc.abandon(candidate.id);
throw new Error('Failed to mark recovery generation as acquired');
}
const dockerController = DockerController.getInstance(this.nodeId);
sendOutput('=== Classifying legacy orphans ===\n');
const classified = await dockerController.classifyLegacyOrphansForUpdate(stackName);
if (classified.status === 'classification_failed') {
await recoverySvc.abandon(candidate.id);
recoveryId = null;
throw new Error(`Legacy orphan classification failed: ${classified.error}`);
}
if (!recoverySvc.handoff(candidate.id, this.nodeId, stackName)) {
await recoverySvc.abandon(candidate.id);
recoveryId = null;
throw new Error('Failed to hand off recovery generation');
}
handedOff = true;
if (!recoverySvc.markReconciling(candidate.id)) {
throw new Error('Failed to mark recovery generation as reconciling after handoff');
}
if (classified.status === 'orphans') {
sendOutput(`=== Removing ${classified.ids.length} legacy orphan container(s) ===\n`);
const results = await dockerController.removeContainers(classified.ids);
const failed = results.filter((r) => !r.success);
if (failed.length > 0) {
throw new Error(
`Failed to remove ${failed.length} legacy orphan container(s) after handoff`,
);
}
}
await this.withRegistryAuth(async (env) => {
sendOutput('=== Recreating containers ===\n');
await this.execute(
'docker',
await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']),
stackDir, ws, true, env, getComposeStallTimeoutMs(),
);
}, sendOutput);
// Immediate verification probe
await new Promise((resolve) => setTimeout(resolve, 3000));
const containers = await dockerController.getDocker().listContainers({
all: true,
filters: { label: [`com.docker.compose.project=${stackName}`] }
filters: { label: [`com.docker.compose.project=${stackName}`] },
});
for (const containerInfo of containers) {
@@ -859,56 +998,90 @@ export class ComposeService {
const container = dockerController.getDocker().getContainer(containerInfo.Id);
const inspectData = await container.inspect();
const exitCode = inspectData.State.ExitCode;
if (exitCode !== 0) {
throw this.createContainerCrashError(exitCode);
}
}
}
if (!recoverySvc.markImmediateVerified(candidate.id)) {
console.warn(
'[ComposeService] Could not CAS immediate_verified for recovery %s',
sanitizeForLog(candidate.id),
);
}
sendOutput('=== Stack updated successfully ===\n');
// Opt-out (default ON): after a clean update, prune the node's dangling
// (untagged) image layers, including the one this pull just orphaned. Read
// fresh each run so a remote node honors its own setting. Wrapped so a
// prune failure can never reach the atomic-rollback catch below.
// Defer prune until gate retention / gate link: only prune when no active holds
// would be violated. Still honor prune_on_update, but use unified holds so
// candidate/current rollback images are retained.
try {
const pruneOnUpdate = DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1';
if (pruneOnUpdate) {
// Dynamic import: ServiceUpdateRecoveryService imports getComposeCommandTimeoutMs
// from this module, so a static import here would be circular.
const { ServiceUpdateRecoveryService } = await import('./ServiceUpdateRecoveryService');
const isImageHeld = ServiceUpdateRecoveryService.getInstance().buildHeldImagePredicate(this.nodeId);
const isImageHeld = recoverySvc.buildUnifiedHeldImagePredicate(this.nodeId);
const result = await DockerController.getInstance(this.nodeId).pruneDanglingImages(isImageHeld);
// The Docker prune API does not report SpaceReclaimed on the containerd
// image store, so only show the figure when the daemon actually returns one.
const reclaimed = result.reclaimedBytes > 0
? ` · reclaimed ${(result.reclaimedBytes / (1024 * 1024)).toFixed(1)} MB`
: '';
sendOutput(`=== Pruned dangling images${reclaimed} ===\n`);
}
} catch (pruneError) {
console.warn('Failed to prune dangling images after update for %s:', sanitizeForLog(stackName), pruneError);
console.warn(
'Failed to prune dangling images after update for %s:',
sanitizeForLog(stackName),
pruneError,
);
}
if (debug) {
console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName });
}
if (debug) console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName });
} catch (updateError) {
if (atomic) {
sendOutput('\n=== Update failed - restoring previous compose and env files ===\n');
const rolledBack = await this.restoreAtomicBackup(stackName, stackDir, ws, sendOutput);
if (!handedOff && recoveryId) {
await recoverySvc.abandon(recoveryId);
recoveryId = null;
}
if (handedOff && recoveryId) {
sendOutput('\n=== Update failed - restoring previous runtime from recovery generation ===\n');
const rolledBack = await recoverySvc.compensateWithCandidate(
recoveryId,
async (overridePath) => {
await this.withRegistryAuth(async (env) => {
await this.execute(
'docker',
await this.buildComposeArgsWithRecoveryOverride(
stackName,
['up', '-d', '--remove-orphans', '--pull', 'never', '--no-build'],
overridePath,
),
stackDir,
ws,
true,
env,
getComposeStallTimeoutMs(),
);
}, sendOutput);
},
);
throw new ComposeRollbackError(updateError, true, rolledBack);
}
// Pre-handoff failure: abandon already handled on acquire/classify; runtime untouched.
throw updateError;
}
// Reached only on a successful update; re-baseline so temporal drift compares
// against what is now deployed (see deployStack for why this lives here), then
// reconcile the ledger against the updated runtime.
await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName);
await DriftLedgerService.getInstance().reconcileStack(this.nodeId, stackName);
try {
await this.refreshExposureCache(stackName);
} catch (err) {
console.warn('[ComposeService] Exposure refresh failed after update for %s:',
sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown')));
console.warn(
'[ComposeService] Exposure refresh failed after update for %s:',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(err, 'unknown')),
);
}
return { recoveryId };
}
/**
backend/src/services/DatabaseService.ts
+440 -8
View File
@@ -10,7 +10,12 @@ import { EXPOSURE_INTENTS, type ExposureIntent } from './network/types';
import { HIGH_EPSS_THRESHOLD } from './securityPosture';
import type { BackendScheduledAction } from './scheduledActionRegistry';
import { stackPatternMatches } from '../helpers/stackPattern';
import {
parseStoredNotificationSchedule,
type NotificationSchedule,
} from '../helpers/notificationSchedule';
import { readSnapshotFileRow, type SnapshotFileReadResult, type SnapshotFileRow } from '../helpers/snapshotFileDecrypt';
import { sanitizeForLog } from '../utils/safeLog';
export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt';
@@ -213,6 +218,42 @@ export interface HealthGateRunRow {
}
/** Pre-update image snapshot enabling a manual per-service restore after a service-scoped update. */
/** Full-stack update recovery generation. Separate from service_update_recovery. */
export interface StackUpdateRecoveryGenerationRow {
id: string;
node_id: number;
stack_name: string;
status: 'candidate' | 'active' | 'restored_current' | 'superseded' | 'abandoned' | 'recovery_required';
phase: 'captured' | 'acquired' | 'handoff_committed' | 'reconciling' | 'immediate_verified';
is_current: number;
backup_slot_id: string | null;
override_path: string | null;
services_json: string;
health_gate_id: string | null;
gate_retain_until: number | null;
artifact_expires_at: number | null;
operation_lease_expires_at: number | null;
created_at: number;
updated_at: number;
created_by: string | null;
artifacts_retired: number;
}
/** Durable cleanup tombstone for stack/node deletion artifact sweep. */
export interface StackUpdateCleanupPendingRow {
id: string;
node_id: number | null;
stack_name: string | null;
status: 'prepared' | 'ready' | 'cancelled';
target_kind: 'local_socket';
rollback_tags_json: string;
override_paths_json: string;
prune_volumes_requested: number;
created_at: number;
updated_at: number;
}
export interface ServiceUpdateRecoveryRow {
id: string;
node_id: number;
@@ -717,6 +758,10 @@ export interface NotificationSuppressionRule {
applies_to: NotificationSuppressionAppliesTo;
enabled: boolean;
expires_at: number | null;
/** Null = always in window (legacy). Ignored when scheduleInvalid. */
schedule: NotificationSchedule | null;
/** True when stored JSON is non-null but corrupt; must not suppress. */
scheduleInvalid: boolean;
created_at: number;
updated_at: number;
}
@@ -1647,6 +1692,50 @@ export class DatabaseService {
CREATE INDEX IF NOT EXISTS idx_service_update_recovery_status_claim_expires
ON service_update_recovery(status, claim_expires_at);
CREATE TABLE IF NOT EXISTS stack_update_recovery_generations (
id TEXT PRIMARY KEY,
node_id INTEGER NOT NULL,
stack_name TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('candidate','active','restored_current','superseded','abandoned','recovery_required')),
phase TEXT NOT NULL CHECK (phase IN ('captured','acquired','handoff_committed','reconciling','immediate_verified')),
is_current INTEGER NOT NULL DEFAULT 0,
backup_slot_id TEXT,
override_path TEXT,
services_json TEXT NOT NULL,
health_gate_id TEXT,
gate_retain_until INTEGER,
artifact_expires_at INTEGER,
operation_lease_expires_at INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
created_by TEXT,
artifacts_retired INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_node_stack_current
ON stack_update_recovery_generations(node_id, stack_name, is_current);
CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_status_expires
ON stack_update_recovery_generations(status, artifact_expires_at);
CREATE INDEX IF NOT EXISTS idx_stack_update_recovery_phase_lease
ON stack_update_recovery_generations(phase, operation_lease_expires_at);
CREATE TABLE IF NOT EXISTS stack_update_cleanup_pending (
id TEXT PRIMARY KEY,
node_id INTEGER,
stack_name TEXT,
status TEXT NOT NULL CHECK (status IN ('prepared','ready','cancelled')),
target_kind TEXT NOT NULL CHECK (target_kind IN ('local_socket')),
rollback_tags_json TEXT NOT NULL,
override_paths_json TEXT NOT NULL,
prune_volumes_requested INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_status
ON stack_update_cleanup_pending(status);
CREATE INDEX IF NOT EXISTS idx_stack_update_cleanup_node_stack
ON stack_update_cleanup_pending(node_id, stack_name);
CREATE TABLE IF NOT EXISTS secrets (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
@@ -1699,6 +1788,8 @@ export class DatabaseService {
try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); } catch (e) { /* ignore */ }
};
maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0');
// Distributed API model columns
maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''");
maybeAddCol('nodes', 'api_token', "TEXT DEFAULT ''");
@@ -2113,7 +2204,12 @@ export class DatabaseService {
);
CREATE INDEX IF NOT EXISTS idx_notification_suppression_enabled
ON notification_suppression_rules(enabled, expires_at);
CREATE TABLE IF NOT EXISTS notification_suppression_rule_tombstones (
id INTEGER PRIMARY KEY,
deleted_at INTEGER NOT NULL
);
`);
this.tryAddColumn('notification_suppression_rules', 'schedule', 'TEXT NULL');
}
private migrateNotificationHistoryContext(): void {
@@ -2704,6 +2800,12 @@ export class DatabaseService {
// --- Notification Suppression Rules ---
private parseNotificationSuppressionRule(row: Record<string, unknown>): NotificationSuppressionRule {
const stored = parseStoredNotificationSchedule(row.schedule);
if (stored.kind === 'invalid') {
console.warn(
`[DatabaseService] Ignoring corrupt notification suppression schedule on rule id=${row.id as number}`,
);
}
return {
id: row.id as number,
name: row.name as string,
@@ -2715,6 +2817,8 @@ export class DatabaseService {
applies_to: row.applies_to as NotificationSuppressionAppliesTo,
enabled: row.enabled === 1,
expires_at: row.expires_at != null ? (row.expires_at as number) : null,
schedule: stored.kind === 'ok' ? stored.schedule : null,
scheduleInvalid: stored.kind === 'invalid',
created_at: row.created_at as number,
updated_at: row.updated_at as number,
};
@@ -2740,10 +2844,10 @@ export class DatabaseService {
}
public createNotificationSuppressionRule(
rule: Omit<NotificationSuppressionRule, 'id'>,
rule: Omit<NotificationSuppressionRule, 'id' | 'scheduleInvalid'>,
): NotificationSuppressionRule {
const result = this.db.prepare(
'INSERT INTO notification_suppression_rules (name, node_id, stack_patterns, label_ids, categories, levels, applies_to, enabled, expires_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
'INSERT INTO notification_suppression_rules (name, node_id, stack_patterns, label_ids, categories, levels, applies_to, enabled, expires_at, schedule, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
).run(
rule.name,
rule.node_id ?? null,
@@ -2754,6 +2858,7 @@ export class DatabaseService {
rule.applies_to,
rule.enabled ? 1 : 0,
rule.expires_at ?? null,
rule.schedule ? JSON.stringify(rule.schedule) : null,
rule.created_at,
rule.updated_at,
);
@@ -2761,12 +2866,22 @@ export class DatabaseService {
}
public upsertNotificationSuppressionRuleReplica(rule: NotificationSuppressionRule): void {
const scheduleJson = rule.schedule ? JSON.stringify(rule.schedule) : null;
const existing = this.getNotificationSuppressionRule(rule.id);
if (existing) {
// Fleet sync is fire-and-forget with no delivery ordering guarantee: a
// push that isn't strictly newer than what's stored must never overwrite it.
if (existing.updated_at >= rule.updated_at) {
console.warn(
`[DatabaseService] Ignoring stale suppression replica write for rule id=${sanitizeForLog(rule.id)} ` +
`(incoming updated_at=${sanitizeForLog(rule.updated_at)} <= stored updated_at=${sanitizeForLog(existing.updated_at)})`,
);
return;
}
this.db.prepare(
`UPDATE notification_suppression_rules SET
name = ?, node_id = ?, stack_patterns = ?, label_ids = ?, categories = ?, levels = ?,
applies_to = ?, enabled = ?, expires_at = ?, updated_at = ?
applies_to = ?, enabled = ?, expires_at = ?, schedule = ?, updated_at = ?
WHERE id = ?`,
).run(
rule.name,
@@ -2778,15 +2893,26 @@ export class DatabaseService {
rule.applies_to,
rule.enabled ? 1 : 0,
rule.expires_at ?? null,
scheduleJson,
rule.updated_at,
rule.id,
);
return;
}
const tombstone = this.db.prepare(
'SELECT deleted_at FROM notification_suppression_rule_tombstones WHERE id = ?',
).get(rule.id) as { deleted_at: number } | undefined;
if (tombstone) {
console.warn(
`[DatabaseService] Ignoring suppression replica write for rule id=${sanitizeForLog(rule.id)}: ` +
`this id was deleted at ${sanitizeForLog(tombstone.deleted_at)} and must not be recreated`,
);
return;
}
this.db.prepare(
`INSERT INTO notification_suppression_rules
(id, name, node_id, stack_patterns, label_ids, categories, levels, applies_to, enabled, expires_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
(id, name, node_id, stack_patterns, label_ids, categories, levels, applies_to, enabled, expires_at, schedule, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
).run(
rule.id,
rule.name,
@@ -2798,6 +2924,7 @@ export class DatabaseService {
rule.applies_to,
rule.enabled ? 1 : 0,
rule.expires_at ?? null,
scheduleJson,
rule.created_at,
rule.updated_at,
);
@@ -2805,7 +2932,7 @@ export class DatabaseService {
public updateNotificationSuppressionRule(
id: number,
updates: Partial<Omit<NotificationSuppressionRule, 'id' | 'created_at'>>,
updates: Partial<Omit<NotificationSuppressionRule, 'id' | 'created_at' | 'scheduleInvalid'>>,
): void {
const fields: string[] = [];
const values: unknown[] = [];
@@ -2819,6 +2946,10 @@ export class DatabaseService {
if (updates.applies_to !== undefined) { fields.push('applies_to = ?'); values.push(updates.applies_to); }
if (updates.enabled !== undefined) { fields.push('enabled = ?'); values.push(updates.enabled ? 1 : 0); }
if ('expires_at' in updates) { fields.push('expires_at = ?'); values.push(updates.expires_at ?? null); }
if ('schedule' in updates) {
fields.push('schedule = ?');
values.push(updates.schedule ? JSON.stringify(updates.schedule) : null);
}
if (updates.updated_at !== undefined) { fields.push('updated_at = ?'); values.push(updates.updated_at); }
if (fields.length === 0) return;
@@ -2827,7 +2958,18 @@ export class DatabaseService {
}
public deleteNotificationSuppressionRule(id: number): number {
return this.db.prepare('DELETE FROM notification_suppression_rules WHERE id = ?').run(id).changes;
const changes = this.db.prepare('DELETE FROM notification_suppression_rules WHERE id = ?').run(id).changes;
// Fleet sync has no delivery ordering guarantee: a replica POST for this id can
// still be in flight. Record the delete permanently (ids are AUTOINCREMENT and
// never reused) so upsertNotificationSuppressionRuleReplica refuses to resurrect it.
// Tombstone unconditionally, even when changes is 0: deleteRuleOnNode issues this
// same DELETE for capability/invalid-schedule cleanup against remotes that may never
// have received the rule yet, and a reordered POST behind that DELETE must not create it.
this.db.prepare(
`INSERT INTO notification_suppression_rule_tombstones (id, deleted_at) VALUES (?, ?)
ON CONFLICT(id) DO UPDATE SET deleted_at = excluded.deleted_at`,
).run(id, Date.now());
return changes;
}
// --- Global Settings ---
@@ -3510,6 +3652,268 @@ export class DatabaseService {
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
}
public insertStackUpdateRecoveryGeneration(row: StackUpdateRecoveryGenerationRow): void {
this.db.prepare(
`INSERT INTO stack_update_recovery_generations (
id, node_id, stack_name, status, phase, is_current, backup_slot_id, override_path,
services_json, health_gate_id, gate_retain_until, artifact_expires_at,
operation_lease_expires_at, created_at, updated_at, created_by, artifacts_retired
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current,
row.backup_slot_id, row.override_path, row.services_json, row.health_gate_id,
row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at,
row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0,
);
}
public getStackUpdateRecoveryGeneration(id: string): StackUpdateRecoveryGenerationRow | undefined {
return this.db.prepare('SELECT * FROM stack_update_recovery_generations WHERE id = ?').get(id) as StackUpdateRecoveryGenerationRow | undefined;
}
public getCurrentStackUpdateRecovery(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined {
return this.db.prepare(
'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? AND is_current = 1 LIMIT 1'
).get(nodeId, stackName) as StackUpdateRecoveryGenerationRow | undefined;
}
public listStackUpdateRecoveryForStack(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow[] {
return this.db.prepare(
'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ? ORDER BY created_at DESC'
).all(nodeId, stackName) as StackUpdateRecoveryGenerationRow[];
}
public listStackUpdateRecoveryGenerationsForNode(nodeId: number): StackUpdateRecoveryGenerationRow[] {
return this.db.prepare(
'SELECT * FROM stack_update_recovery_generations WHERE node_id = ? ORDER BY created_at DESC'
).all(nodeId) as StackUpdateRecoveryGenerationRow[];
}
public updateStackUpdateRecoveryGeneration(
id: string,
patch: Partial<Pick<StackUpdateRecoveryGenerationRow,
'status' | 'phase' | 'is_current' | 'override_path' | 'health_gate_id' |
'gate_retain_until' | 'artifact_expires_at' | 'operation_lease_expires_at' | 'services_json'>>,
): void {
const keys = Object.keys(patch) as Array<keyof typeof patch>;
if (keys.length === 0) return;
const sets = keys.map((k) => `${String(k)} = ?`).join(', ');
const values = keys.map((k) => patch[k]);
this.db.prepare(
`UPDATE stack_update_recovery_generations SET ${sets}, updated_at = ? WHERE id = ?`
).run(...values, Date.now(), id);
}
public casStackUpdateRecoveryPhase(
id: string,
expectedPhase: StackUpdateRecoveryGenerationRow['phase'],
nextPhase: StackUpdateRecoveryGenerationRow['phase'],
): boolean {
const result = this.db.prepare(
'UPDATE stack_update_recovery_generations SET phase = ?, updated_at = ? WHERE id = ? AND phase = ?'
).run(nextPhase, Date.now(), id, expectedPhase);
return result.changes === 1;
}
public casHandoffGeneration(candidateId: string, nodeId: number, stackName: string): boolean {
const handoff = this.db.transaction(() => {
const candidate = this.getStackUpdateRecoveryGeneration(candidateId);
if (!candidate || candidate.node_id !== nodeId || candidate.stack_name !== stackName) return false;
if (candidate.status !== 'candidate' || candidate.phase !== 'acquired') return false;
this.db.prepare(
`UPDATE stack_update_recovery_generations
SET status = 'superseded', is_current = 0, artifact_expires_at = ?, updated_at = ?
WHERE node_id = ? AND stack_name = ? AND is_current = 1 AND id != ?`
).run(Date.now() + 7 * 24 * 60 * 60 * 1000, Date.now(), nodeId, stackName, candidateId);
const result = this.db.prepare(
`UPDATE stack_update_recovery_generations
SET status = 'active', is_current = 1, phase = 'handoff_committed', updated_at = ?
WHERE id = ? AND status = 'candidate' AND phase = 'acquired'`
).run(Date.now(), candidateId);
return result.changes === 1;
});
return handoff();
}
/** Generations whose Docker/FS artifacts can be retired (not actively held). */
public listStackUpdateRecoveryGenerationsForArtifactRetirement(now: number): StackUpdateRecoveryGenerationRow[] {
return this.db.prepare(
`SELECT * FROM stack_update_recovery_generations
WHERE artifacts_retired = 0
AND is_current = 0
AND status IN ('abandoned', 'superseded')
AND (artifact_expires_at IS NULL OR artifact_expires_at <= ?)
AND (gate_retain_until IS NULL OR gate_retain_until <= ?)`
).all(now, now) as StackUpdateRecoveryGenerationRow[];
}
public markStackUpdateRecoveryArtifactsRetired(id: string): boolean {
const result = this.db.prepare(
`UPDATE stack_update_recovery_generations
SET artifacts_retired = 1, override_path = NULL, updated_at = ?
WHERE id = ? AND artifacts_retired = 0`
).run(Date.now(), id);
return result.changes === 1;
}
public abandonStackUpdateRecoveryGeneration(id: string): boolean {
const now = Date.now();
const result = this.db.prepare(
`UPDATE stack_update_recovery_generations
SET status = 'abandoned', is_current = 0, artifact_expires_at = ?,
operation_lease_expires_at = NULL, updated_at = ?
WHERE id = ? AND status = 'candidate'`
).run(now, now, id);
return result.changes === 1;
}
/** Pre-handoff candidates whose operation lease has expired. */
public listStaleStackUpdateRecoveryCandidates(now: number): StackUpdateRecoveryGenerationRow[] {
return this.db.prepare(
`SELECT * FROM stack_update_recovery_generations
WHERE status = 'candidate'
AND operation_lease_expires_at IS NOT NULL
AND operation_lease_expires_at <= ?`
).all(now) as StackUpdateRecoveryGenerationRow[];
}
/** Post-handoff generations stuck before immediate_verified with an expired lease. */
public listStuckStackUpdateRecoveryGenerations(now: number): StackUpdateRecoveryGenerationRow[] {
return this.db.prepare(
`SELECT * FROM stack_update_recovery_generations
WHERE status IN ('active', 'restored_current')
AND phase IN ('handoff_committed', 'reconciling')
AND operation_lease_expires_at IS NOT NULL
AND operation_lease_expires_at <= ?`
).all(now) as StackUpdateRecoveryGenerationRow[];
}
public linkStackUpdateRecoveryHealthGate(id: string, healthGateId: string): void {
this.db.prepare(
'UPDATE stack_update_recovery_generations SET health_gate_id = ?, updated_at = ? WHERE id = ?'
).run(healthGateId, Date.now(), id);
}
public setStackUpdateRecoveryGateRetainUntil(id: string, until: number): void {
this.db.prepare(
'UPDATE stack_update_recovery_generations SET gate_retain_until = ?, updated_at = ? WHERE id = ?'
).run(until, Date.now(), id);
}
public listHeldStackUpdateRecoveryImageIds(nodeId: number, now: number): string[] {
const rows = this.db.prepare(
`SELECT services_json FROM stack_update_recovery_generations
WHERE node_id = ?
AND status IN ('candidate','active','restored_current','recovery_required')
AND (artifact_expires_at IS NULL OR artifact_expires_at > ? OR gate_retain_until > ? OR is_current = 1)`
).all(nodeId, now, now) as Array<{ services_json: string }>;
const ids = new Set<string>();
for (const row of rows) {
try {
const parsed: unknown = JSON.parse(row.services_json);
if (!Array.isArray(parsed)) continue;
for (const item of parsed) {
if (!item || typeof item !== 'object') continue;
const replicas = (item as { replicas?: unknown }).replicas;
if (Array.isArray(replicas)) {
for (const replica of replicas) {
if (replica && typeof replica === 'object'
&& typeof (replica as { imageId?: unknown }).imageId === 'string'
&& (replica as { imageId: string }).imageId.trim()) {
ids.add((replica as { imageId: string }).imageId);
}
}
} else if (typeof (item as { imageId?: unknown }).imageId === 'string') {
ids.add((item as { imageId: string }).imageId);
}
}
} catch {
// Corrupt JSON: skip.
}
}
return [...ids];
}
public deleteStackUpdateRecoveryGenerations(nodeId: number, stackName: string): void {
this.db.prepare(
'DELETE FROM stack_update_recovery_generations WHERE node_id = ? AND stack_name = ?'
).run(nodeId, stackName);
}
public insertCleanupPending(row: StackUpdateCleanupPendingRow): void {
this.db.prepare(
`INSERT INTO stack_update_cleanup_pending (
id, node_id, stack_name, status, target_kind, rollback_tags_json,
override_paths_json, prune_volumes_requested, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
row.id, row.node_id, row.stack_name, row.status, row.target_kind,
row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested,
row.created_at, row.updated_at,
);
}
public getCleanupPending(id: string): StackUpdateCleanupPendingRow | undefined {
return this.db.prepare('SELECT * FROM stack_update_cleanup_pending WHERE id = ?').get(id) as StackUpdateCleanupPendingRow | undefined;
}
public listReadyCleanupPending(): StackUpdateCleanupPendingRow[] {
return this.db.prepare(
"SELECT * FROM stack_update_cleanup_pending WHERE status = 'ready' ORDER BY created_at ASC"
).all() as StackUpdateCleanupPendingRow[];
}
public listPreparedCleanupPending(): StackUpdateCleanupPendingRow[] {
return this.db.prepare(
"SELECT * FROM stack_update_cleanup_pending WHERE status = 'prepared' ORDER BY created_at ASC"
).all() as StackUpdateCleanupPendingRow[];
}
public updateCleanupPendingStatus(id: string, status: StackUpdateCleanupPendingRow['status']): void {
this.db.prepare(
'UPDATE stack_update_cleanup_pending SET status = ?, updated_at = ? WHERE id = ?'
).run(status, Date.now(), id);
}
public deleteCleanupPending(id: string): void {
this.db.prepare('DELETE FROM stack_update_cleanup_pending WHERE id = ?').run(id);
}
public hasBlockingDeletionIntent(nodeId: number, stackName: string): boolean {
const row = this.db.prepare(
`SELECT id FROM stack_update_cleanup_pending
WHERE node_id = ? AND stack_name = ? AND status IN ('prepared','ready') LIMIT 1`
).get(nodeId, stackName);
return !!row;
}
public getPreparedDeletionIntent(nodeId: number, stackName: string): StackUpdateCleanupPendingRow | undefined {
return this.db.prepare(
`SELECT * FROM stack_update_cleanup_pending
WHERE node_id = ? AND stack_name = ? AND status = 'prepared' LIMIT 1`
).get(nodeId, stackName) as StackUpdateCleanupPendingRow | undefined;
}
public getDeletionIntentById(id: string): StackUpdateCleanupPendingRow | undefined {
return this.getCleanupPending(id);
}
public commitStackDeletionReadyTransaction(intentId: string, nodeId: number, stackName: string): boolean {
const run = this.db.transaction(() => {
const intent = this.getCleanupPending(intentId);
if (!intent || intent.status !== 'prepared') return false;
if (intent.node_id !== nodeId || intent.stack_name !== stackName) return false;
this.db.prepare(
"UPDATE stack_update_cleanup_pending SET status = 'ready', updated_at = ? WHERE id = ? AND status = 'prepared'"
).run(Date.now(), intentId);
this.deleteStackUpdateRecoveryGenerations(nodeId, stackName);
this.deleteServiceUpdateRecoveries(nodeId, stackName);
return true;
});
return run();
}
// --- Notification History ---
private mapNotificationRow(row: any): NotificationHistory {
@@ -3878,7 +4282,16 @@ export class DatabaseService {
this.db.prepare(`UPDATE nodes SET ${fields.join(', ')} WHERE id = ?`).run(...values);
}
public deleteNode(id: number): void {
/**
* Delete a node row and related per-node state.
* For a non-default local-socket node, pass localCleanup so a ready tombstone
* is inserted in the same transaction (caller enumerates tags/paths first).
* Remote hub node records must omit localCleanup (no Docker tombstone).
*/
public deleteNode(
id: number,
localCleanup?: { tombstoneId: string; tags: string[]; overridePaths: string[] },
): void {
const node = this.getNode(id);
// Protect the last local node regardless of is_default flag.
if (node && node.type === 'local' && this.getLocalNodeCount() <= 1) {
@@ -3887,7 +4300,25 @@ export class DatabaseService {
if (node?.is_default) {
throw new Error('Cannot delete the default node');
}
if (localCleanup && node?.type !== 'local') {
throw new Error('Local cleanup tombstones are only valid for local-socket nodes');
}
this.db.transaction(() => {
if (localCleanup && node?.type === 'local') {
const now = Date.now();
this.insertCleanupPending({
id: localCleanup.tombstoneId,
node_id: id,
stack_name: null,
status: 'ready',
target_kind: 'local_socket',
rollback_tags_json: JSON.stringify(localCleanup.tags),
override_paths_json: JSON.stringify(localCleanup.overridePaths),
prune_volumes_requested: 0,
created_at: now,
updated_at: now,
});
}
this.db.prepare('DELETE FROM scheduled_task_runs WHERE task_id IN (SELECT id FROM scheduled_tasks WHERE node_id = ?)').run(id);
this.db.prepare('DELETE FROM scheduled_tasks WHERE node_id = ?').run(id);
this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ?').run(id);
@@ -3901,6 +4332,7 @@ export class DatabaseService {
this.db.prepare('DELETE FROM preflight_acknowledgements WHERE node_id = ?').run(id);
this.db.prepare('DELETE FROM stack_exposure WHERE node_id = ?').run(id);
this.db.prepare('DELETE FROM health_gate_runs WHERE node_id = ?').run(id);
this.db.prepare('DELETE FROM stack_update_recovery_generations WHERE node_id = ?').run(id);
this.db.prepare('DELETE FROM service_update_recovery WHERE node_id = ?').run(id);
this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id);
this.deleteRoleAssignmentsByResource('node', String(id));
backend/src/services/DeployedStackDeletionService.ts
+521
View File
@@ -0,0 +1,521 @@
/**
* Shared deployed-stack deletion lifecycle (manual DELETE + Blueprint withdrawLocal).
*
* prepared -> (down, optional volume prune, FS delete) -> ready transaction that
* retires full-stack recovery generations and service_update_recovery rows, then
* sweeper removes ready tombstone tags/overrides.
*/
import { randomUUID } from 'crypto';
import Docker from 'dockerode';
import fs from 'fs/promises';
import path from 'path';
import {
DatabaseService,
type StackUpdateCleanupPendingRow,
} from './DatabaseService';
import { ComposeService } from './ComposeService';
import DockerController from './DockerController';
import { FileSystemService } from './FileSystemService';
import { NodeRegistry } from './NodeRegistry';
import { MeshService } from './MeshService';
import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import { isPathWithinBase, isValidStackName } from '../utils/validation';
/**
* Directory that may contain recovery override files for a tombstone sweep.
* Stack-scoped intents are limited to `<composeDir>/<stackName>/`;
* node-wide intents (null stack) use the whole compose root.
*/
export function overrideDeletionContainmentBase(
composeDir: string,
stackName: string | null,
): string | null {
const resolvedCompose = path.resolve(composeDir);
if (!stackName) return resolvedCompose;
if (!isValidStackName(stackName)) return null;
const stackDir = path.resolve(resolvedCompose, stackName);
if (!isPathWithinBase(stackDir, resolvedCompose)) return null;
return stackDir;
}
export interface DeleteDeployedStackInput {
nodeId: number;
stackName: string;
pruneVolumes: boolean;
actor: string;
/** When true, skip acquiring a new lock (caller already holds delete via continuation). */
continuationIntentId?: string;
}
export type DeleteDeployedStackResult =
| { ok: true }
| { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string };
function collectArtifactsFromGenerations(
generations: Array<{ override_path: string | null; services_json: string }>,
): { tags: string[]; overridePaths: string[] } {
const tags = new Set<string>();
const overridePaths = new Set<string>();
for (const gen of generations) {
if (gen.override_path) overridePaths.add(gen.override_path);
try {
const parsed: unknown = JSON.parse(gen.services_json);
if (!Array.isArray(parsed)) continue;
for (const svc of parsed) {
if (!svc || typeof svc !== 'object') continue;
const replicas = (svc as { replicas?: unknown }).replicas;
if (!Array.isArray(replicas)) continue;
for (const replica of replicas) {
const tag = replica && typeof replica === 'object'
? (replica as { rollbackTag?: unknown }).rollbackTag
: null;
if (typeof tag === 'string' && tag.trim()) tags.add(tag);
}
}
} catch {
// Corrupt capture JSON: skip tags for this generation.
}
}
return { tags: [...tags], overridePaths: [...overridePaths] };
}
function collectArtifacts(nodeId: number, stackName: string): { tags: string[]; overridePaths: string[] } {
return collectArtifactsFromGenerations(
DatabaseService.getInstance().listStackUpdateRecoveryForStack(nodeId, stackName),
);
}
function parseJsonStringArray(raw: string): string[] {
try {
const parsed: unknown = JSON.parse(raw);
if (!Array.isArray(parsed)) return [];
return parsed.filter((item): item is string => typeof item === 'string');
} catch {
return [];
}
}
export class DeployedStackDeletionService {
private static instance: DeployedStackDeletionService;
public static getInstance(): DeployedStackDeletionService {
if (!DeployedStackDeletionService.instance) {
DeployedStackDeletionService.instance = new DeployedStackDeletionService();
}
return DeployedStackDeletionService.instance;
}
public assertNoBlockingDeletionIntent(nodeId: number, stackName: string): void {
if (DatabaseService.getInstance().hasBlockingDeletionIntent(nodeId, stackName)) {
throw new Error(
`Stack "${stackName}" has a deletion in progress and cannot be created or mutated until cleanup finishes.`,
);
}
}
public async deleteDeployedStack(input: DeleteDeployedStackInput): Promise<DeleteDeployedStackResult> {
const { nodeId, stackName, actor } = input;
const locks = StackOpLockService.getInstance();
if (input.continuationIntentId) {
const cont = locks.tryAcquireDeletionContinuation({
intentId: input.continuationIntentId,
nodeId,
stackName,
});
if (!cont.acquired) {
return {
ok: false,
code: 'lock_conflict',
error: stackOpSkipMessage(stackName, cont.existing.action),
existingAction: cont.existing.action,
};
}
try {
return await this.runDeletionBody(input, input.continuationIntentId);
} finally {
locks.release(nodeId, stackName);
}
}
const exclusive = await locks.runExclusive(nodeId, stackName, 'delete', actor, async () => {
return this.runDeletionBody(input);
});
if (!exclusive.ran) {
return {
ok: false,
code: 'lock_conflict',
error: stackOpSkipMessage(stackName, exclusive.existing.action),
existingAction: exclusive.existing.action,
};
}
return exclusive.result;
}
private async runDeletionBody(
input: DeleteDeployedStackInput,
existingIntentId?: string,
): Promise<DeleteDeployedStackResult> {
const { nodeId, stackName, pruneVolumes } = input;
const db = DatabaseService.getInstance();
let intentId = existingIntentId;
if (!intentId) {
const { tags, overridePaths } = collectArtifacts(nodeId, stackName);
const now = Date.now();
const row: StackUpdateCleanupPendingRow = {
id: randomUUID(),
node_id: nodeId,
stack_name: stackName,
status: 'prepared',
target_kind: 'local_socket',
rollback_tags_json: JSON.stringify(tags),
override_paths_json: JSON.stringify(overridePaths),
prune_volumes_requested: pruneVolumes ? 1 : 0,
created_at: now,
updated_at: now,
};
try {
db.insertCleanupPending(row);
} catch (error) {
return {
ok: false,
code: 'tombstone_failed',
error: getErrorMessage(error, 'Failed to prepare stack deletion'),
};
}
intentId = row.id;
}
const intent = db.getDeletionIntentById(intentId);
if (!intent || intent.status !== 'prepared') {
return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
}
try {
await ComposeService.getInstance(nodeId).downStack(stackName);
} catch (downErr) {
console.warn(
'[DeployedStackDeletion] Compose down failed or no-op for %s:',
sanitizeForLog(stackName),
downErr,
);
}
if (intent.prune_volumes_requested === 1) {
try {
await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
} catch (pruneErr) {
console.warn(
'[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
sanitizeForLog(stackName),
pruneErr,
);
}
}
try {
await FileSystemService.getInstance(nodeId).deleteStack(stackName);
} catch (fsErr) {
db.updateCleanupPendingStatus(intentId, 'cancelled');
return {
ok: false,
code: 'fs_failed',
error: getErrorMessage(fsErr, 'Failed to remove stack files'),
};
}
if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) {
return {
ok: false,
code: 'db_failed',
error: 'Failed to commit stack deletion ready transaction',
};
}
try {
db.clearStackUpdateStatus(nodeId, stackName);
db.clearStackScanAttempts(nodeId, stackName);
db.deleteRoleAssignmentsByResource('stack', stackName);
db.deleteGitSource(stackName);
db.deleteStackDossier(nodeId, stackName);
db.deleteStackDriftFindings(nodeId, stackName);
db.deleteStackExposureIntents(nodeId, stackName);
db.deleteStackExposure(nodeId, stackName);
db.deleteStackProjectEnvFiles(nodeId, stackName);
db.deleteStackScans(nodeId, stackName);
} catch (dbErr) {
console.error(
'[DeployedStackDeletion] Secondary DB cleanup failed for %s; recovery rows already retired:',
sanitizeForLog(stackName),
dbErr,
);
return {
ok: false,
code: 'db_failed',
error: getErrorMessage(dbErr, 'Failed to clear stack database state'),
};
}
try {
await MeshService.getInstance().optOutStack(nodeId, stackName, input.actor);
} catch (meshErr) {
console.warn(
'[DeployedStackDeletion] Mesh opt-out failed for %s:',
sanitizeForLog(stackName),
meshErr,
);
}
await this.sweepReadyIntent(intentId);
return { ok: true };
}
/**
* Remove rollback tags + override paths for a ready tombstone, then drop the row.
* When the node row is already gone (local-node delete), pass a preserved local
* Docker handle and composeDir so we never fall back to a remote default node.
*/
public async sweepReadyIntent(
intentId: string,
opts?: { docker?: Docker; composeDir?: string },
): Promise<void> {
const db = DatabaseService.getInstance();
const intent = db.getCleanupPending(intentId);
if (!intent || intent.status !== 'ready') return;
if (intent.node_id == null) {
db.deleteCleanupPending(intentId);
return;
}
const tags = parseJsonStringArray(intent.rollback_tags_json);
const overridePaths = parseJsonStringArray(intent.override_paths_json);
let docker: Docker;
if (opts?.docker) {
docker = opts.docker;
} else if (db.getNode(intent.node_id)) {
docker = DockerController.getInstance(intent.node_id).getDocker();
} else if (intent.target_kind === 'local_socket') {
// Deleted local node: local_socket tombstones always target the host Docker socket.
docker = new Docker();
} else {
console.warn(
'[DeployedStackDeletion] Cannot sweep tombstone %s: node gone and target is not local_socket',
sanitizeForLog(intentId),
);
return;
}
let incomplete = false;
for (const tag of tags) {
try {
await docker.getImage(tag).remove({ force: true });
} catch (error) {
const status = (error as { statusCode?: number }).statusCode;
const message = getErrorMessage(error, 'unknown').toLowerCase();
if (status === 404 || message.includes('no such image') || message.includes('not found')) {
continue;
}
incomplete = true;
console.warn(
'[DeployedStackDeletion] Failed to remove rollback tag %s: %s',
sanitizeForLog(tag),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
}
}
const baseDir = opts?.composeDir
?? (db.getNode(intent.node_id)
? FileSystemService.getInstance(intent.node_id).getBaseDir()
: null);
const containmentBase = baseDir
? overrideDeletionContainmentBase(baseDir, intent.stack_name)
: null;
if (baseDir && !containmentBase) {
incomplete = true;
console.warn(
'[DeployedStackDeletion] Refusing override sweep: invalid stack containment for tombstone %s',
sanitizeForLog(intentId),
);
}
for (const overridePath of overridePaths) {
try {
const resolved = path.resolve(overridePath);
const basename = path.basename(resolved);
if (!/^\.sencho-recovery-[a-f0-9]+\.yml$/i.test(basename)) {
incomplete = true;
console.warn(
'[DeployedStackDeletion] Refusing to delete non-recovery override: %s',
sanitizeForLog(overridePath),
);
continue;
}
if (containmentBase && !isPathWithinBase(resolved, containmentBase)) {
incomplete = true;
console.warn(
'[DeployedStackDeletion] Refusing to delete override outside stack containment: %s',
sanitizeForLog(overridePath),
);
continue;
}
if (!baseDir && !path.isAbsolute(resolved)) {
incomplete = true;
console.warn(
'[DeployedStackDeletion] Refusing relative override without compose dir: %s',
sanitizeForLog(overridePath),
);
continue;
}
if (!containmentBase && baseDir) {
incomplete = true;
continue;
}
await fs.unlink(resolved);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
incomplete = true;
console.warn(
'[DeployedStackDeletion] Failed to delete override %s: %s',
sanitizeForLog(overridePath),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
}
}
}
// Keep the ready tombstone when cleanup is incomplete so startup can retry.
if (!incomplete) {
db.deleteCleanupPending(intentId);
}
}
/** Enumerate rollback tags and override paths for every stack recovery on a node. */
public collectNodeArtifacts(nodeId: number): { tags: string[]; overridePaths: string[] } {
return collectArtifactsFromGenerations(
DatabaseService.getInstance().listStackUpdateRecoveryGenerationsForNode(nodeId),
);
}
/**
* Delete a local-socket node with an atomic ready tombstone, then sweep.
* Remote node records call DatabaseService.deleteNode without cleanup.
*/
public async deleteLocalNode(nodeId: number): Promise<void> {
const db = DatabaseService.getInstance();
const node = db.getNode(nodeId);
if (!node) throw new Error('Node not found');
if (node.type !== 'local') {
db.deleteNode(nodeId);
return;
}
// Preserve Docker + compose dir before the row disappears so sweep never
// targets a remote default node.
const docker = DockerController.getInstance(nodeId).getDocker();
const composeDir = FileSystemService.getInstance(nodeId).getBaseDir();
const { tags, overridePaths } = this.collectNodeArtifacts(nodeId);
const tombstoneId = randomUUID();
db.deleteNode(nodeId, { tombstoneId, tags, overridePaths });
NodeRegistry.getInstance().evictConnection(nodeId);
try {
await this.sweepReadyIntent(tombstoneId, { docker, composeDir });
} catch (error) {
// Node row is already gone; leave the ready tombstone for startup resume.
console.error(
'[DeployedStackDeletion] Sweep after local-node delete deferred for tombstone %s: %s',
sanitizeForLog(tombstoneId),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
}
}
/**
* Startup reconciliation: resume prepared intents, complete ready transitions
* when the directory is already gone, and sweep ready tombstones.
*/
public async reconcileAtStartup(): Promise<void> {
const db = DatabaseService.getInstance();
for (const intent of db.listPreparedCleanupPending()) {
if (intent.node_id == null || !intent.stack_name) continue;
const nodeId = intent.node_id;
const stackName = intent.stack_name;
const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName);
let dirExists = true;
try {
await fs.access(stackDir);
} catch (accessError) {
const code = (accessError as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
dirExists = false;
} else {
console.warn(
'[DeployedStackDeletion] Startup access error for %s (not treating as absent): %s',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(accessError, 'unknown')),
);
continue;
}
}
if (!dirExists) {
if (!db.commitStackDeletionReadyTransaction(intent.id, nodeId, stackName)) {
console.warn(
'[DeployedStackDeletion] Startup ready commit failed for %s/%s',
nodeId,
sanitizeForLog(stackName),
);
continue;
}
try {
await MeshService.getInstance().optOutStack(nodeId, stackName, 'system:startup');
} catch (meshErr) {
console.warn(
'[DeployedStackDeletion] Startup mesh opt-out failed for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(meshErr, 'unknown')),
);
}
await this.sweepReadyIntent(intent.id);
continue;
}
const result = await this.deleteDeployedStack({
nodeId,
stackName,
pruneVolumes: intent.prune_volumes_requested === 1,
actor: 'system:startup',
continuationIntentId: intent.id,
});
if (!result.ok) {
console.warn(
'[DeployedStackDeletion] Startup resume failed for %s: %s',
sanitizeForLog(stackName),
result.error,
);
}
}
for (const intent of db.listReadyCleanupPending()) {
if (intent.node_id != null && intent.stack_name) {
try {
await MeshService.getInstance().optOutStack(intent.node_id, intent.stack_name, 'system:startup');
} catch (meshErr) {
console.warn(
'[DeployedStackDeletion] Ready-resume mesh opt-out failed for %s: %s',
sanitizeForLog(intent.stack_name),
sanitizeForLog(getErrorMessage(meshErr, 'unknown')),
);
}
}
await this.sweepReadyIntent(intent.id);
}
}
}
backend/src/services/DockerController.ts
+52
View File
@@ -24,6 +24,7 @@ import {
import type { NetworkingNetworkBase } from './network/networkingTypes';
import { isPathWithinBase } from '../utils/validation';
import { isDebugEnabled } from '../utils/debug';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import { describeSpawnError } from '../utils/spawnErrors';
import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/authoredComposeArgs';
@@ -2258,6 +2259,57 @@ class DockerController {
}
}
/**
* Strict Result API for full-stack updates. Never converts Compose-ps + fallback
* failure into empty success. Compose-managed containers are never orphan IDs.
*/
public async classifyLegacyOrphansForUpdate(
stackName: string,
): Promise<
| { status: 'none' }
| { status: 'orphans'; ids: string[] }
| { status: 'classification_failed'; error: string }
> {
const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName);
const toIds = (list: Array<{ Id?: string }>) =>
list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0)
.map((c) => c.Id);
const fallbackOrphans = async (): Promise<
| { status: 'none' }
| { status: 'orphans'; ids: string[] }
| { status: 'classification_failed'; error: string }
> => {
try {
const ids = toIds(await this.smartFallback(stackName, stackDir));
return ids.length === 0 ? { status: 'none' } : { status: 'orphans', ids };
} catch (fallbackError) {
return {
status: 'classification_failed',
error: getErrorMessage(fallbackError, 'Legacy orphan classification failed'),
};
}
};
try {
const composeContainers = await this.fetchComposePsContainers(stackName, stackDir);
// Compose already manages this stack: no legacy orphan cleanup (same as deploy).
if (composeContainers.length > 0) return { status: 'none' };
return await fallbackOrphans();
} catch (error) {
const execError = error as NodeJS.ErrnoException & { stderr?: string };
const mapped = describeSpawnError(execError, { command: 'docker compose ps' });
const detail = execError.stderr || mapped.message || getErrorMessage(error, 'docker compose ps failed');
console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail));
// Unlike getLegacyOrphanContainersByStack, never convert dual failure into empty success.
const fallback = await fallbackOrphans();
if (fallback.status === 'classification_failed') {
return { status: 'classification_failed', error: String(detail) };
}
return fallback;
}
}
public async getContainersByStack(stackName: string) {
// Resolve the compose dir and the authored prefix for THIS controller's node,
// not the process default, so a non-default local node sees its own stack dir
backend/src/services/FileSystemService.ts
+33
View File
@@ -545,6 +545,11 @@ export class FileSystemService {
}
async createStack(stackName: string): Promise<void> {
if (DatabaseService.getInstance().hasBlockingDeletionIntent(this.nodeId, stackName)) {
throw new Error(
`Stack "${stackName}" has a deletion in progress and cannot be created until cleanup finishes.`,
);
}
const stackDir = this.resolveStackDir(stackName);
await this.assertRealWithinBase(stackDir);
@@ -573,6 +578,34 @@ export class FileSystemService {
}
}
/**
* Remove non-canonical root Compose filenames so discovery cannot shadow
* compose.yaml. Filename set is fixed inside this method. ENOENT is success;
* other unlink failures are logged and skipped.
*/
async removeAlternateRootComposeFiles(stackName: string): Promise<void> {
const stackDir = this.resolveStackDir(stackName);
await this.assertRealWithinBase(stackDir);
const baseResolved = path.resolve(this.baseDir);
for (const file of IMPORT_COMPOSE_FILENAMES) {
if (file === 'compose.yaml') continue;
// Containment barrier at the unlink sink (same pattern as rollback orphan removal).
const target = path.resolve(baseResolved, path.join(stackDir, file));
if (!target.startsWith(baseResolved + path.sep)) {
throw Object.assign(new Error('Path escapes compose directory'), { code: 'INVALID_PATH' });
}
try {
await fsPromises.unlink(target);
} catch (e: unknown) {
if ((e as NodeJS.ErrnoException).code === 'ENOENT') continue;
console.warn(
`[FileSystemService] Could not remove alternate compose file ${sanitizeForLog(file)} in stack ${sanitizeForLog(stackName)}:`,
sanitizeForLog((e as Error)?.message ?? String(e)),
);
}
}
}
public async deleteStack(stackName: string): Promise<void> {
const stackDir = this.resolveStackDir(stackName);
await this.assertRealWithinBase(stackDir);
backend/src/services/NotificationService.ts
+7 -2
View File
@@ -12,6 +12,7 @@ import {
matchesNotificationFilters,
ruleNeedsStackLabels,
} from '../helpers/notificationMatchers';
import { scheduleAllowsSuppression } from '../helpers/notificationSchedule';
import {
type NotificationChannelType,
type ParsedAppriseConfig,
@@ -230,7 +231,8 @@ export class NotificationService {
});
}
const suppressionRules = this.dbService.getEnabledNotificationSuppressionRules();
const atMs = Date.now();
const suppressionRules = this.dbService.getEnabledNotificationSuppressionRules(atMs);
const routes = this.dbService.getEnabledNotificationRoutes();
const needsStackLabels = stackName !== undefined && (
ruleNeedsStackLabels(suppressionRules)
@@ -246,7 +248,10 @@ export class NotificationService {
level,
stackLabelIds,
};
const matchedSuppression = suppressionRules.filter((r) => matchesNotificationFilters(matchCtx, r));
const matchedSuppression = suppressionRules.filter((r) =>
matchesNotificationFilters(matchCtx, r)
&& scheduleAllowsSuppression(r.schedule, r.scheduleInvalid, atMs)
);
const suppressBell = matchedSuppression.some((r) => appliesToBell(r.applies_to));
const suppressExternal = matchedSuppression.some((r) => appliesToExternal(r.applies_to));
backend/src/services/SchedulerService.ts
+7 -1
View File
@@ -1104,7 +1104,13 @@ export class SchedulerService {
);
if (!lock.ran) return skipMessage(stackName, lock.existing.action);
db.clearStackUpdateStatus(nodeId, stackName);
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler');
const orchResult = lock.result;
const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
if (recoveryId) {
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
}
this.safeDispatch(
'info',
backend/src/services/ServiceUpdateRecoveryService.ts
+11 -1
View File
@@ -237,7 +237,17 @@ export class ServiceUpdateRecoveryService {
return (imageId: string) => {
const held = this.getHeldImageIds(nodeId);
if (held === null) return true;
return held.has(imageId);
if (held.has(imageId)) return true;
try {
// Dynamic import avoids a static cycle with StackUpdateRecoveryService.
// eslint-disable-next-line @typescript-eslint/no-require-imports
const { StackUpdateRecoveryService } = require('./StackUpdateRecoveryService') as typeof import('./StackUpdateRecoveryService');
const stackHeld = StackUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
if (stackHeld === null) return true;
return stackHeld.has(imageId);
} catch {
return true;
}
};
}
backend/src/services/StackOpLockService.ts
+27 -1
View File
@@ -1,3 +1,4 @@
import { DatabaseService } from './DatabaseService';
/**
* Tracks in-flight stack lifecycle operations (deploy, down, restart, stop,
* start, update, rollback, backup) per (nodeId, stackName). A second request to
@@ -9,7 +10,7 @@
* which matches the lifecycle of any in-flight `docker compose` child process.
*/
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup';
export type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'rollback' | 'backup' | 'delete';
/**
* Note returned by a background path that skipped its operation because a manual
@@ -70,6 +71,31 @@ export class StackOpLockService {
this.locks.delete(this.key(nodeId, stackName));
}
/**
* Matching-intent continuation for crash-resume of a prepared deletion.
* Succeeds only when the persisted intent matches and no other in-memory lock holds.
*/
public tryAcquireDeletionContinuation(args: {
intentId: string;
nodeId: number;
stackName: string;
}): AcquireResult {
const intent = DatabaseService.getInstance().getDeletionIntentById(args.intentId);
if (
!intent
|| intent.status !== 'prepared'
|| intent.node_id !== args.nodeId
|| intent.stack_name !== args.stackName
) {
const existing = this.locks.get(this.key(args.nodeId, args.stackName));
return {
acquired: false,
existing: existing ?? { action: 'delete', startedAt: Date.now(), user: 'system' },
};
}
return this.tryAcquire(args.nodeId, args.stackName, 'delete', 'system:deletion-continuation');
}
/**
* Acquire the per-(nodeId, stackName) lock for the duration of `fn`, then
* release it. Returns `{ ran: true, result }` when the lock was free, or
backend/src/services/StackUpdateOrchestrator.ts
+24 -3
View File
@@ -47,7 +47,7 @@ export interface ServiceUpdateOptions {
}
export type OrchestratorResult =
| { kind: 'stack_compose_done' }
| { kind: 'stack_compose_done'; recoveryId: string | null }
| {
kind: 'service_done';
serviceName: string;
@@ -182,10 +182,11 @@ export class StackUpdateOrchestrator {
ctx: UpdateOperationContext,
options: StackComposeOptions,
): Promise<OrchestratorResult> {
await ComposeService.getInstance(ctx.nodeId).updateStack(
const updateResult = await ComposeService.getInstance(ctx.nodeId).updateStack(
ctx.stackName, options.terminalWs ?? undefined, options.atomic,
);
return { kind: 'stack_compose_done' };
const recoveryId = updateResult?.recoveryId ?? null;
return { kind: 'stack_compose_done', recoveryId };
}
private async executeServiceUpdate(
@@ -195,6 +196,16 @@ export class StackUpdateOrchestrator {
): Promise<OrchestratorResult> {
const { nodeId, stackName } = ctx;
{
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) {
return serviceFailed(
'stack_recovery_pin_active',
'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.',
);
}
}
const loaded = await this.loadServiceSpec(nodeId, stackName, serviceName);
if (!loaded.ok) return loaded.result;
const { spec, services } = loaded;
@@ -308,6 +319,16 @@ export class StackUpdateOrchestrator {
const { nodeId, stackName } = ctx;
const recoveryId = options.recoveryId as string;
{
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
if (StackUpdateRecoveryService.getInstance().isRestoredCurrentPinActive(nodeId, stackName)) {
return serviceFailed(
'stack_recovery_pin_active',
'This stack is pinned to a restored recovery generation. Run a full-stack Update to continue.',
{ recoveryId },
);
}
}
const recovery = ServiceUpdateRecoveryService.getInstance().get(recoveryId);
if (
!recovery ||
backend/src/services/StackUpdateRecoveryService.ts
+767
View File
@@ -0,0 +1,767 @@
/**
* Full-stack update recovery generations: capture, opaque rollback tags,
* stack-local recovery override, handoff, compensation, and prune holds.
*
* Separate from ServiceUpdateRecoveryService (service-scoped snapshots).
* Does not run Compose; ComposeService / orchestrator own Docker mutations.
*
* Recovery fidelity contract (supported):
* - Prior images are restored via opaque hold tags (`--pull never --no-build`).
* - Observed running replica count is restored via compose `scale`.
* - Services that were fully stopped at capture are kept at `scale: 0`.
* - Authored replica counts are not used when they diverge from observed state.
*/
import { randomUUID } from 'crypto';
import fs from 'fs/promises';
import path from 'path';
import {
DatabaseService,
type StackUpdateRecoveryGenerationRow,
} from './DatabaseService';
import DockerController from './DockerController';
import { FileSystemService } from './FileSystemService';
import { buildEffectiveServiceModel } from './effectiveServiceModel';
import {
classifyReferenceKind,
type ImageReferenceKind,
resolveComposeProjectContext,
} from './composeProjectContext';
import { getComposeCommandTimeoutMs } from './ComposeService';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import { isValidStackName } from '../utils/validation';
const SWEEP_INTERVAL_MS = 5 * 60_000;
const INITIAL_SWEEP_DELAY_MS = 30_000;
const MIN_RECOVERY_WINDOW_SECONDS = 90;
const RECOVERY_TTL_BUFFER_MS = 30 * 60_000;
const GATE_RETAIN_DEFAULT_MS = 2 * 60 * 60_000;
const RECOVERY_PROBE_DELAY_MS = 3_000;
export interface StackRecoveryReplicaCapture {
containerId: string | null;
imageId: string | null;
repoDigest: string | null;
state: 'running' | 'stopped' | 'none';
rollbackTag: string | null;
}
export interface StackRecoveryServiceCapture {
serviceName: string;
/** Observed running replica count at capture (supported restore scale). */
scale: number;
hasBuild: boolean;
declaredImageRef: string | null;
referenceKind: ImageReferenceKind;
replicas: StackRecoveryReplicaCapture[];
}
export interface CaptureStackUpdateInput {
nodeId: number;
stackName: string;
createdBy: string | null;
}
function yamlQuote(value: string): string {
return JSON.stringify(value);
}
function sanitizeServiceSlug(name: string): string {
return name.replace(/[^a-zA-Z0-9._-]/g, '-').toLowerCase() || 'svc';
}
function opaqueRollbackTag(generationId: string, serviceName: string): string {
const short = generationId.replace(/-/g, '').slice(0, 12);
return `sencho-rb/${short}/${sanitizeServiceSlug(serviceName)}:hold`;
}
function parseServicesJson(raw: string): StackRecoveryServiceCapture[] {
try {
const parsed: unknown = JSON.parse(raw);
if (!Array.isArray(parsed)) return [];
return parsed as StackRecoveryServiceCapture[];
} catch {
return [];
}
}
export function collectImageIdsFromServicesJson(servicesJson: string): string[] {
const ids = new Set<string>();
for (const svc of parseServicesJson(servicesJson)) {
for (const replica of svc.replicas ?? []) {
if (replica.imageId && replica.imageId.trim()) ids.add(replica.imageId);
}
}
return [...ids];
}
function collectRollbackTags(services: StackRecoveryServiceCapture[]): string[] {
const tags = new Set<string>();
for (const svc of services) {
for (const replica of svc.replicas ?? []) {
if (replica.rollbackTag) tags.add(replica.rollbackTag);
}
}
return [...tags];
}
export class StackUpdateRecoveryService {
private static instance: StackUpdateRecoveryService;
private started = false;
private intervalId: NodeJS.Timeout | null = null;
private initialTimer: NodeJS.Timeout | null = null;
private constructor() {}
public static getInstance(): StackUpdateRecoveryService {
if (!StackUpdateRecoveryService.instance) {
StackUpdateRecoveryService.instance = new StackUpdateRecoveryService();
}
return StackUpdateRecoveryService.instance;
}
public static resetForTests(): void {
if (StackUpdateRecoveryService.instance) {
StackUpdateRecoveryService.instance.stop();
}
StackUpdateRecoveryService.instance = new StackUpdateRecoveryService();
}
public start(): void {
this.started = true;
if (this.initialTimer || this.intervalId) return;
this.initialTimer = setTimeout(() => {
void this.reconcileIncomplete();
this.intervalId = setInterval(() => {
void this.reconcileIncomplete();
}, SWEEP_INTERVAL_MS);
}, INITIAL_SWEEP_DELAY_MS);
}
public stop(): void {
this.started = false;
if (this.initialTimer) {
clearTimeout(this.initialTimer);
this.initialTimer = null;
}
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = null;
}
}
/**
* Validate exact Compose invocation, backup files, snapshot runtime, create
* opaque tags + recovery override, insert candidate generation.
*/
public async captureCandidate(input: CaptureStackUpdateInput): Promise<StackUpdateRecoveryGenerationRow> {
const { nodeId, stackName, createdBy } = input;
if (!isValidStackName(stackName)) {
throw new Error('Invalid stack name');
}
const context = await resolveComposeProjectContext(nodeId, stackName);
await context.validateForMutation();
// Exact mutating invocation (authored files + env + generated Mesh override)
// must validate before any backup, tag, or override write.
const { ComposeService } = await import('./ComposeService');
await ComposeService.getInstance(nodeId).validateExactComposeInvocation(stackName);
const backupSlotId = await context.backupFromContext('update');
const model = await buildEffectiveServiceModel(nodeId, stackName);
if (!model.renderable) {
throw new Error(model.error || 'Effective Compose model failed to render');
}
const generationId = randomUUID();
const docker = DockerController.getInstance(nodeId).getDocker();
const services: StackRecoveryServiceCapture[] = [];
const createdTags: string[] = [];
let overridePath: string | null = null;
try {
for (const spec of model.services) {
const declaredImageRef = spec.declaredImage;
const referenceKind = classifyReferenceKind(declaredImageRef);
const listed = await docker.listContainers({
all: true,
filters: {
label: [
`com.docker.compose.project=${stackName}`,
`com.docker.compose.service=${spec.name}`,
],
},
});
const replicas: StackRecoveryReplicaCapture[] = [];
for (const info of listed) {
try {
const inspect = await docker.getContainer(info.Id).inspect();
const status = inspect.State?.Status;
const state: StackRecoveryReplicaCapture['state'] =
status === 'running' ? 'running' : status ? 'stopped' : 'none';
const imageId = typeof inspect.Image === 'string' && inspect.Image.length > 0
? inspect.Image
: null;
if ((state === 'running' || state === 'stopped') && !imageId) {
throw new Error(
`Service "${spec.name}" replica ${info.Id.slice(0, 12)} has no protectable image id`,
);
}
let repoDigest: string | null = null;
if (imageId) {
try {
const image = await docker.getImage(imageId).inspect();
const digests = (image.RepoDigests ?? []) as string[];
repoDigest = digests.length > 0 ? digests[0] : null;
} catch {
repoDigest = null;
}
}
replicas.push({
containerId: info.Id,
imageId,
repoDigest,
state,
rollbackTag: null,
});
} catch (error) {
if ((error as { statusCode?: number })?.statusCode === 404) continue;
throw error;
}
}
const runningCount = replicas.filter((r) => r.state === 'running').length;
services.push({
serviceName: spec.name,
scale: runningCount,
hasBuild: spec.hasBuild,
declaredImageRef,
referenceKind,
replicas,
});
}
const taggedIds = new Set<string>();
for (const svc of services) {
const primary = svc.replicas.find((r) => r.imageId) ?? null;
if (!primary?.imageId) continue;
const tag = opaqueRollbackTag(generationId, svc.serviceName);
const tagKey = `${primary.imageId}|${tag}`;
if (!taggedIds.has(tagKey)) {
const { repo, tagName } = splitOpaqueTag(tag);
await docker.getImage(primary.imageId).tag({ repo, tag: tagName });
taggedIds.add(tagKey);
createdTags.push(tag);
}
for (const replica of svc.replicas) {
if (replica.imageId === primary.imageId) {
replica.rollbackTag = tag;
}
}
}
overridePath = await this.writeRecoveryOverride(nodeId, stackName, generationId, services);
const now = Date.now();
const row: StackUpdateRecoveryGenerationRow = {
id: generationId,
node_id: nodeId,
stack_name: stackName,
status: 'candidate',
phase: 'captured',
is_current: 0,
backup_slot_id: backupSlotId,
override_path: overridePath,
services_json: JSON.stringify(services),
health_gate_id: null,
gate_retain_until: null,
artifact_expires_at: null,
operation_lease_expires_at: now + getComposeCommandTimeoutMs() + RECOVERY_TTL_BUFFER_MS,
created_at: now,
updated_at: now,
created_by: createdBy,
artifacts_retired: 0,
};
DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row);
return row;
} catch (error) {
await this.bestEffortRemoveTags(nodeId, createdTags);
if (overridePath) {
try {
await fs.unlink(overridePath);
} catch {
// Best-effort mid-capture cleanup.
}
}
throw error;
}
}
private async writeRecoveryOverride(
nodeId: number,
stackName: string,
generationId: string,
services: StackRecoveryServiceCapture[],
): Promise<string> {
if (!isValidStackName(stackName)) {
throw new Error('Invalid stack name');
}
// Canonical inline path barrier (same pattern as ComposeService.renderConfig).
const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir());
const stackDir = path.resolve(baseResolved, stackName);
if (!stackDir.startsWith(baseResolved + path.sep)) {
throw new Error('Invalid stack path');
}
let stackDirReal: string;
let baseReal: string;
try {
[stackDirReal, baseReal] = await Promise.all([
fs.realpath(stackDir),
fs.realpath(baseResolved),
]);
} catch {
throw new Error('Stack directory not found');
}
if (stackDirReal !== baseReal && !stackDirReal.startsWith(baseReal + path.sep)) {
throw new Error('Stack directory escapes compose base');
}
const short = generationId.replace(/-/g, '').slice(0, 12);
if (!/^[a-f0-9]{12}$/i.test(short)) {
throw new Error('Invalid recovery generation id');
}
const filename = `.sencho-recovery-${short}.yml`;
const abs = path.resolve(stackDirReal, filename);
if (!abs.startsWith(stackDirReal + path.sep)) {
throw new Error('Recovery override path escapes stack directory');
}
const lines: string[] = ['services:'];
let wroteAny = false;
for (const svc of services) {
const tag = svc.replicas.find((r) => r.rollbackTag)?.rollbackTag ?? null;
const stoppedOnly =
svc.replicas.length > 0
&& svc.replicas.every((r) => r.state === 'stopped' || r.state === 'none');
if (!tag && svc.scale !== 0 && !stoppedOnly) continue;
const key = /^[a-zA-Z0-9._-]+$/.test(svc.serviceName) ? svc.serviceName : yamlQuote(svc.serviceName);
lines.push(` ${key}:`);
if (tag) {
lines.push(` image: ${yamlQuote(tag)}`);
}
// Observed running count; fully stopped services stay at scale 0.
if (svc.scale === 0 || stoppedOnly) {
lines.push(' scale: 0');
} else if (svc.scale > 0) {
lines.push(` scale: ${svc.scale}`);
}
wroteAny = true;
}
const body = wroteAny ? `${lines.join('\n')}\n` : 'services: {}\n';
await fs.writeFile(abs, body, 'utf8');
return abs;
}
public markAcquired(id: string): boolean {
return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'captured', 'acquired');
}
public handoff(candidateId: string, nodeId: number, stackName: string): boolean {
return DatabaseService.getInstance().casHandoffGeneration(candidateId, nodeId, stackName);
}
public markReconciling(id: string): boolean {
return DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'handoff_committed', 'reconciling');
}
public markImmediateVerified(id: string): boolean {
const ok = DatabaseService.getInstance().casStackUpdateRecoveryPhase(id, 'reconciling', 'immediate_verified');
if (ok) {
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(id, {
artifact_expires_at: Date.now() + this.activeRecoveryTtlMs() + RECOVERY_TTL_BUFFER_MS,
operation_lease_expires_at: null,
});
}
return ok;
}
/** Mark candidate abandoned in DB and retire Docker/FS artifacts. */
public async abandon(id: string): Promise<boolean> {
const row = this.get(id);
const ok = DatabaseService.getInstance().abandonStackUpdateRecoveryGeneration(id);
if (ok && row) {
await this.retireGenerationArtifacts({ ...row, status: 'abandoned', artifacts_retired: 0 });
}
return ok;
}
public linkHealthGate(id: string, healthGateId: string): void {
DatabaseService.getInstance().linkStackUpdateRecoveryHealthGate(id, healthGateId);
}
public setGateRetainUntil(id: string, until: number = Date.now() + GATE_RETAIN_DEFAULT_MS): void {
DatabaseService.getInstance().setStackUpdateRecoveryGateRetainUntil(id, until);
}
/** Link a health gate when present; otherwise set bounded gate_retain_until. */
public linkGateOrRetain(id: string, healthGateId: string | null): void {
try {
if (healthGateId) {
this.linkHealthGate(id, healthGateId);
} else {
this.setGateRetainUntil(id);
}
} catch (error) {
console.warn(
'[StackUpdateRecovery] linkGateOrRetain failed for %s; setting retain window:',
sanitizeForLog(id),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
try {
this.setGateRetainUntil(id);
} catch (retainError) {
console.warn(
'[StackUpdateRecovery] setGateRetainUntil failed for %s:',
sanitizeForLog(id),
sanitizeForLog(getErrorMessage(retainError, 'unknown')),
);
}
}
}
public get(id: string): StackUpdateRecoveryGenerationRow | undefined {
return DatabaseService.getInstance().getStackUpdateRecoveryGeneration(id);
}
public getCurrent(nodeId: number, stackName: string): StackUpdateRecoveryGenerationRow | undefined {
return DatabaseService.getInstance().getCurrentStackUpdateRecovery(nodeId, stackName);
}
public isRestoredCurrentPinActive(nodeId: number, stackName: string): boolean {
const current = this.getCurrent(nodeId, stackName);
return !!current && current.status === 'restored_current';
}
public getHeldImageIds(nodeId: number): Set<string> | null {
try {
const now = Date.now();
const fromStack = DatabaseService.getInstance().listHeldStackUpdateRecoveryImageIds(nodeId, now);
return new Set(fromStack);
} catch (error) {
console.warn(
'[StackUpdateRecovery] Failed to compute held image ids for node %d:',
nodeId,
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
return null;
}
}
/**
* Unified held-image predicate: service-scoped + full-stack holds.
* Fail closed (skip prune) when either lookup fails.
*/
public buildUnifiedHeldImagePredicate(nodeId: number): (imageId: string) => boolean {
// Dynamic require avoids a static cycle with ServiceUpdateRecoveryService.
// eslint-disable-next-line @typescript-eslint/no-require-imports
const { ServiceUpdateRecoveryService } = require('./ServiceUpdateRecoveryService') as typeof import('./ServiceUpdateRecoveryService');
const serviceHeld = ServiceUpdateRecoveryService.getInstance().getHeldImageIds(nodeId);
const stackHeld = this.getHeldImageIds(nodeId);
if (serviceHeld === null || stackHeld === null) {
return () => true;
}
return (imageId: string) => serviceHeld.has(imageId) || stackHeld.has(imageId);
}
/**
* Post-handoff compensation: restore files + pinned up, then probe before
* reporting restored_current / immediate_verified.
*/
public async compensateWithCandidate(
generationId: string,
composeUp: (overridePath: string) => Promise<void>,
): Promise<boolean> {
const row = this.get(generationId);
if (!row) return false;
try {
const context = await resolveComposeProjectContext(row.node_id, row.stack_name);
await context.restoreFromContext();
if (!row.override_path) {
throw new Error('Recovery generation has no override path');
}
await composeUp(row.override_path);
const probeOk = await this.probeRecoveredStack(
row.node_id,
row.stack_name,
row.services_json,
);
if (!probeOk) {
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
status: 'recovery_required',
});
return false;
}
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
status: 'restored_current',
phase: 'immediate_verified',
is_current: 1,
artifact_expires_at: null,
});
return true;
} catch (error) {
console.error(
'[StackUpdateRecovery] Compensation failed for %s: %s',
sanitizeForLog(generationId),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
DatabaseService.getInstance().updateStackUpdateRecoveryGeneration(generationId, {
status: 'recovery_required',
});
return false;
}
}
/**
* Verify recovered runtime against the captured generation.
* Rejects absent, restarting, dead, exited, or unhealthy expected replicas,
* image-id mismatches vs capture, and any running replica of a scale-0 service.
*/
public async probeRecoveredStack(
nodeId: number,
stackName: string,
servicesJson: string,
): Promise<boolean> {
await new Promise((resolve) => setTimeout(resolve, RECOVERY_PROBE_DELAY_MS));
try {
const expected = parseServicesJson(servicesJson);
const expectedRunning = new Map<string, number>();
const expectedImageIds = new Map<string, Set<string>>();
const scaleZeroServices = new Set<string>();
for (const svc of expected) {
const imageIds = new Set<string>();
for (const replica of svc.replicas ?? []) {
if (replica.imageId?.trim()) imageIds.add(replica.imageId);
}
if (svc.scale > 0) {
// Fail closed when we cannot verify image identity for expected runners.
if (imageIds.size === 0) return false;
expectedRunning.set(svc.serviceName, svc.scale);
expectedImageIds.set(svc.serviceName, imageIds);
} else {
scaleZeroServices.add(svc.serviceName);
}
}
const docker = DockerController.getInstance(nodeId).getDocker();
const containers = await docker.listContainers({
all: true,
filters: { label: [`com.docker.compose.project=${stackName}`] },
});
if (expectedRunning.size > 0 && containers.length === 0) {
return false;
}
const runningByService = new Map<string, number>();
for (const containerInfo of containers) {
const labels = (containerInfo.Labels ?? {}) as Record<string, string>;
const serviceName = labels['com.docker.compose.service'];
const state = (containerInfo.State || '').toLowerCase();
if (state === 'restarting' || state === 'dead') {
return false;
}
if (state === 'exited' || state === 'created' || state === 'removing') {
if (serviceName && expectedRunning.has(serviceName)) {
return false;
}
continue;
}
if (state !== 'running') {
if (serviceName && expectedRunning.has(serviceName)) {
return false;
}
continue;
}
// Captured at scale 0 must stay stopped; any running replica fails the probe.
if (serviceName && scaleZeroServices.has(serviceName)) {
return false;
}
const inspectData = await docker.getContainer(containerInfo.Id).inspect();
const health = inspectData.State?.Health?.Status;
if (health === 'unhealthy') {
return false;
}
if (serviceName && expectedImageIds.has(serviceName)) {
const allowedIds = expectedImageIds.get(serviceName)!;
const actualImageId = typeof inspectData.Image === 'string' ? inspectData.Image : '';
if (!actualImageId || !allowedIds.has(actualImageId)) {
return false;
}
}
if (serviceName) {
runningByService.set(serviceName, (runningByService.get(serviceName) ?? 0) + 1);
}
}
for (const [serviceName, need] of expectedRunning) {
if ((runningByService.get(serviceName) ?? 0) < need) {
return false;
}
}
return true;
} catch (error) {
console.warn(
'[StackUpdateRecovery] Recovery probe failed for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
return false;
}
}
/**
* Idempotent removal of opaque tags and override files for a generation.
* Marks artifacts_retired only when every artifact is removed or confirmed absent,
* so transient failures remain retryable via reconcileIncomplete.
*/
public async retireGenerationArtifacts(row: StackUpdateRecoveryGenerationRow): Promise<boolean> {
if (row.artifacts_retired === 1) return true;
const services = parseServicesJson(row.services_json);
const tagsOk = await this.removeRollbackTags(row.node_id, collectRollbackTags(services));
let overrideOk = true;
if (row.override_path) {
try {
await fs.unlink(row.override_path);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
overrideOk = false;
console.warn(
'[StackUpdateRecovery] Failed to delete override %s: %s',
sanitizeForLog(row.override_path),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
}
}
}
if (!tagsOk || !overrideOk) return false;
DatabaseService.getInstance().markStackUpdateRecoveryArtifactsRetired(row.id);
return true;
}
/**
* Abandon lease-expired candidates, flag stuck post-handoff generations,
* and retire expired abandoned/superseded artifacts.
*/
public async reconcileIncomplete(): Promise<void> {
if (!this.started) return;
try {
const db = DatabaseService.getInstance();
const now = Date.now();
let abandoned = 0;
for (const row of db.listStaleStackUpdateRecoveryCandidates(now)) {
if (await this.abandon(row.id)) abandoned += 1;
}
let flagged = 0;
for (const row of db.listStuckStackUpdateRecoveryGenerations(now)) {
db.updateStackUpdateRecoveryGeneration(row.id, {
status: 'recovery_required',
operation_lease_expires_at: null,
});
flagged += 1;
}
let retired = 0;
for (const row of db.listStackUpdateRecoveryGenerationsForArtifactRetirement(now)) {
// Never retire an active/current or recovery_required hold target.
if (row.is_current === 1 || row.status === 'recovery_required') continue;
if (await this.retireGenerationArtifacts(row)) retired += 1;
}
if (abandoned > 0 || flagged > 0 || retired > 0) {
console.log(
`[StackUpdateRecovery] Reconciled ${abandoned} stale candidate(s), `
+ `${flagged} stuck generation(s), retired ${retired} artifact set(s)`,
);
}
} catch (error) {
console.error(
'[StackUpdateRecovery] Reconcile failed: %s',
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
}
}
/** Returns true when every tag is removed or already absent. */
private async removeRollbackTags(nodeId: number, tags: string[]): Promise<boolean> {
if (tags.length === 0) return true;
try {
const docker = DockerController.getInstance(nodeId).getDocker();
let allOk = true;
for (const tag of tags) {
try {
await docker.getImage(tag).remove({ force: true });
} catch (error) {
const status = (error as { statusCode?: number }).statusCode;
const message = getErrorMessage(error, 'unknown').toLowerCase();
if (status === 404 || message.includes('no such image') || message.includes('not found')) {
continue;
}
allOk = false;
console.warn(
'[StackUpdateRecovery] Failed to remove rollback tag %s: %s',
sanitizeForLog(tag),
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
}
}
return allOk;
} catch (error) {
console.warn(
'[StackUpdateRecovery] Docker unavailable while removing tags: %s',
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
return false;
}
}
private async bestEffortRemoveTags(nodeId: number, tags: string[]): Promise<void> {
await this.removeRollbackTags(nodeId, tags);
}
private activeRecoveryTtlMs(): number {
return Math.max(getComposeCommandTimeoutMs(), this.readRecoveryWindowSeconds() * 1000);
}
private readRecoveryWindowSeconds(): number {
try {
const raw = parseInt(
DatabaseService.getInstance().getGlobalSettings()['health_gate_window_seconds'] ?? '',
10,
);
return Number.isFinite(raw) ? Math.max(raw, MIN_RECOVERY_WINDOW_SECONDS) : MIN_RECOVERY_WINDOW_SECONDS;
} catch (error) {
console.warn(
'[StackUpdateRecovery] Settings read failed; using default window: %s',
sanitizeForLog(getErrorMessage(error, 'unknown')),
);
return MIN_RECOVERY_WINDOW_SECONDS;
}
}
}
function splitOpaqueTag(tag: string): { repo: string; tagName: string } {
const lastColon = tag.lastIndexOf(':');
if (lastColon > 0) {
return { repo: tag.slice(0, lastColon), tagName: tag.slice(lastColon + 1) };
}
return { repo: tag, tagName: 'hold' };
}
backend/src/services/WebhookService.ts
+9 -3
View File
@@ -174,18 +174,24 @@ export class WebhookService {
case 'start':
await compose.runCommand(stackName, 'start');
break;
case 'pull':
case 'pull': {
await assertPolicyGateAllows(
stackName,
nodeId,
buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
);
await StackUpdateOrchestrator.getInstance().execute(
const orchResult = await StackUpdateOrchestrator.getInstance().execute(
{ nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' },
{ atomic: atomic ?? false, terminalWs: null },
);
HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
if (recoveryId) {
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
}
break;
}
}
},
);
backend/src/services/composeProjectContext.ts
+94
View File
@@ -0,0 +1,94 @@
/**
* Thin Compose project context for safe full-stack updates.
*
* Wraps the current authored compose argument path and atomic file backup/restore.
* When a richer shared Compose project context lands, migrate callers to that type;
* this module must not become a competing full-manifest resolver.
*/
import path from 'path';
import { randomUUID } from 'crypto';
import { ComposeService } from './ComposeService';
import { FileSystemService } from './FileSystemService';
import { buildEffectiveServiceModel } from './effectiveServiceModel';
import { getErrorMessage } from '../utils/errors';
export type ImageReferenceKind = 'moving_tag' | 'digest_pinned' | 'none';
const DIGEST_PIN_PATTERN = /@sha256:[a-f0-9]{64}$/i;
export function classifyReferenceKind(declaredImageRef: string | null): ImageReferenceKind {
if (!declaredImageRef) return 'none';
if (DIGEST_PIN_PATTERN.test(declaredImageRef)) return 'digest_pinned';
return 'moving_tag';
}
async function requireRenderableModel(nodeId: number, stackName: string) {
const model = await buildEffectiveServiceModel(nodeId, stackName);
if (!model.renderable) {
throw new Error(model.error || 'Effective Compose model failed to render');
}
return model;
}
export interface ComposeProjectContext {
readonly nodeId: number;
readonly stackName: string;
readonly stackDir: string;
backupSlotId: string | null;
toComposeArgs(action: string[]): Promise<string[]>;
validateForMutation(): Promise<void>;
backupFromContext(operation: 'update' | 'deployment'): Promise<string>;
restoreFromContext(): Promise<void>;
resolveServiceImageMap(): Promise<Map<string, string | null>>;
}
class AuthoredComposeProjectContext implements ComposeProjectContext {
backupSlotId: string | null = null;
constructor(
readonly nodeId: number,
readonly stackName: string,
readonly stackDir: string,
) {}
async toComposeArgs(action: string[]): Promise<string[]> {
return ComposeService.getInstance(this.nodeId).buildAuthoredComposeArgs(this.stackName, action);
}
async validateForMutation(): Promise<void> {
await ComposeService.getInstance(this.nodeId).validateStackForMutation(this.stackName);
await requireRenderableModel(this.nodeId, this.stackName);
}
async backupFromContext(_operation: 'update' | 'deployment'): Promise<string> {
await FileSystemService.getInstance(this.nodeId).backupStackFiles(this.stackName);
const slotId = randomUUID();
this.backupSlotId = slotId;
return slotId;
}
async restoreFromContext(): Promise<void> {
await FileSystemService.getInstance(this.nodeId).restoreStackFiles(this.stackName);
}
async resolveServiceImageMap(): Promise<Map<string, string | null>> {
const model = await requireRenderableModel(this.nodeId, this.stackName);
const map = new Map<string, string | null>();
for (const svc of model.services) {
map.set(svc.name, svc.declaredImage);
}
return map;
}
}
export async function resolveComposeProjectContext(
nodeId: number,
stackName: string,
): Promise<ComposeProjectContext> {
const stackDir = path.join(FileSystemService.getInstance(nodeId).getBaseDir(), stackName);
return new AuthoredComposeProjectContext(nodeId, stackName, stackDir);
}
export function describeContextError(error: unknown): string {
return getErrorMessage(error, 'Compose project context failed');
}
backend/src/services/updateGuard/readiness.ts
+2 -2
View File
@@ -304,11 +304,11 @@ export function buildRollbackItems(inputs: RollbackInputs, now: number): Rollbac
if (inputs.rollbackTarget === 'error') {
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The update preview is unavailable.' });
} else if (inputs.rollbackTarget.target && inputs.rollbackTarget.moving) {
items.push({ id: 'previous_images', state: 'not_covered', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag, so restoring the compose and env files does not revert the image: the local tag still resolves to the newer digest. Pin every image to an immutable version tag for a true image rollback.` });
items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Rollback target ${inputs.rollbackTarget.target}. This stack uses a moving image tag. Full-stack updates capture the running image ID before pull/build and retain it through the recovery window so an immediate restore can retarget the exact prior image, not only the tag string.` });
} else if (inputs.rollbackTarget.target) {
items.push({ id: 'previous_images', state: 'ready', label: 'Previous image tag', detail: `Known rollback target: ${inputs.rollbackTarget.target}. The compose file pins an immutable tag, so restoring files also restores the image.` });
} else {
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. A rollback restores compose and env files; a moving tag may keep the newer image.' });
items.push({ id: 'previous_images', state: 'unknown', label: 'Previous image tag', detail: 'The previous image tag could not be determined. When no prior image is recorded, a restore may fall back to compose and env files alone; a moving tag can still resolve to a newer digest.' });
}
if (inputs.lastDeployAt === 'error') {
backend/vitest.config.ts
+1 -1
View File
@@ -5,7 +5,7 @@ export default defineConfig({
environment: 'node',
// Only run TypeScript sources - exclude the compiled dist/ output.
include: ['src/__tests__/**/*.test.ts'],
exclude: ['dist/**', 'node_modules/**'],
exclude: ['dist/**', 'node_modules/**', 'src/__tests__/docker-integration/**'],
// Build the baseline DB (schema + migrations + admin seed) once; each
// test file's setupTestDb copies it instead of re-running migrations.
globalSetup: ['./src/__tests__/helpers/vitestGlobalSetup.ts'],
backend/vitest.docker-integration.config.ts
+16
View File
@@ -0,0 +1,16 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
environment: 'node',
include: ['src/__tests__/docker-integration/**/*.test.ts'],
exclude: ['dist/**', 'node_modules/**'],
globalSetup: ['./src/__tests__/helpers/vitestGlobalSetup.ts'],
pool: 'forks',
maxWorkers: 1,
minWorkers: 1,
testTimeout: 180_000,
hookTimeout: 180_000,
sequence: { concurrent: false },
},
});
docs/features/alerts-notifications.mdx
+4 -1
View File
@@ -123,11 +123,14 @@ Open **Settings · Notifications · Mute Rules** and click **+ Add mute rule**.
| **Severity** *(optional)* | One or more of info, warning, or error. Empty matches any severity. |
| **Apply to** | **Bell**, **External**, or **Both**. Bell skips the notification popover WebSocket push. External skips routing rules and global channels. |
| **Expiration** | Forever, 1 hour, 24 hours, or a custom date. Expired rules stop matching automatically. |
| **Weekly window (UTC)** *(optional)* | Recurring weekly maintenance window. Choose one or more start days and a UTC start/end time. The rule only mutes while the window is active. Same-day windows use an inclusive start and exclusive end. Cross-midnight windows (for example Sat 22:00 to Sun 02:00) list only the start day. Leave off for always-on mute (subject to expiration). |
| **Enabled** | Toggle the rule without deleting it. |
All non-empty matchers must match (AND). Suppressed alerts are still written to stack activity history; only delivery is affected. When a mute rule blocks delivery, the activity row shows a **Suppressed** badge with the matched rule name in a tooltip.
Rules you create on the control instance replicate to remote nodes so alerts emitted on a remote stack honor the same suppression. From the bell, admins can open a row menu and choose **Mute this category**, **Mute notifications like this**, or **Mute this stack** to create a quick rule with default **Both** targeting. The same presets are available from stack menus (sidebar, stack header, activity tab), fleet node cards, and label groups.
Rules you create on the control instance replicate to remote nodes so alerts emitted on a remote stack honor the same suppression. Replication is best-effort. Remotes that do not support weekly windows do not receive scheduled replicas as all-day mutes: Sencho skips the push and attempts to remove any prior replica. If a remote is unreachable, cleanup may stay pending until connectivity returns and you re-save or clear the rule. Before downgrading a node past a release that introduced weekly windows, clear every weekly window (or disable those rules) so an older binary does not treat them as always-on.
From the bell, admins can open a row menu and choose **Mute this category**, **Mute notifications like this**, or **Mute this stack** to create a quick rule with default **Both** targeting and no weekly window. The same presets are available from stack menus (sidebar, stack header, activity tab), fleet node cards, and label groups.
### Built-in bell quieting (not a mute rule)
docs/features/blueprint-model.mdx
+4 -4
View File
@@ -3,7 +3,7 @@ title: "Blueprints"
description: "Fleet-wide compose templates that Sencho keeps in sync across the nodes you choose, with stateless or stateful classification, drift detection, and three response modes."
---
A **Blueprint** bundles a `docker-compose.yml` with a node selector and a drift policy. Every minute Sencho compares the live state on each targeted node against the blueprint's desired revision, and either reports the drift, dispatches a notification, or auto-corrects, depending on the mode you picked. You declare a stack once; Sencho handles the distribution and the reconciliation loop.
A **Blueprint** bundles Compose YAML with a node selector and a drift policy. Every minute Sencho compares the live state on each targeted node against the blueprint's desired revision, and either reports the drift, dispatches a notification, or auto-corrects, depending on the mode you picked. You declare a stack once; Sencho handles the distribution and the reconciliation loop.
Blueprints live under **Fleet · Deployments**.
@@ -49,7 +49,7 @@ Drift detection runs on every tick for every Active deployment regardless of pol
|---|---|
| User role | **Admin** to create, edit, withdraw, accept, and apply. Operators and viewers can read the catalog and the detail sheet. Pinning requires admin. |
| Nodes | At least one node that the selector resolves to. Remote nodes need a healthy proxy connection; see [Multi-node management](/features/multi-node) and [Pilot Agent](/features/pilot-agent) for enrollment. |
| Compose YAML | Valid `docker-compose.yml`, 96 KiB or fewer. |
| Compose YAML | Valid Compose YAML, 96 KiB or fewer. |
| Blueprint name | 1 to 64 characters matching `^[a-z0-9][a-z0-9_-]*$`. The name doubles as the stack directory on every targeted node. Rename is blocked while any non-withdrawn deployment or guard exists. |
| Selector | A `labels` or `nodes` selector with up to 200 entries per side. An empty resolved set is allowed but produces no deployments. |
| Compose directory | Per-node compose directory must be writable. Remote nodes must accept the controlling instance's bearer token; this is the same channel the rest of the fleet management already uses. |
@@ -74,7 +74,7 @@ The first time you visit the tab, the catalog is empty and a three-step explaine
|---|---|
| **Name** | Used as the stack directory on every targeted node (`<COMPOSE_DIR>/<blueprint-name>/`). Lowercase letters, digits, hyphens, and underscores only. Rename is allowed only when every deployment is withdrawn (or there are none); live placements and guards block rename. |
| **Description** | Short prose for the catalog tile and the detail header. |
| **Compose** | Standard `docker-compose.yml`. The same file ships to every targeted node. Sencho parses it on save and classifies the blueprint as stateless, stateful, or unknown. The YAML must parse successfully and stay under 96 KiB. |
| **Compose** | Standard Compose YAML. The same content ships to every targeted node as `compose.yaml`. Sencho parses it on save and classifies the blueprint as stateless, stateful, or unknown. The YAML must parse successfully and stay under 96 KiB. |
| **Selector** | Either label expressions (any-of plus all-of) or a list of node IDs picked by hand. |
| **Drift policy** | Observe, Suggest, or Enforce. Drift detection always runs; only the response differs. |
| **Reconciler enabled** | Toggle the reconciliation loop without deleting the blueprint. The **Disable** button on the detail sheet flips this toggle off; **Enable** turns it back on. Useful when you want to pause auto-deploys without losing the configuration. |
@@ -321,7 +321,7 @@ A future Volume Migration feature will automate this with app-aware backup tooli
Sencho accepts valid YAML up to 96 KiB. Split very large compose files into smaller blueprints or move generated content out of the Blueprint. Do not paste secrets into the compose body; use environment files or [Fleet Secrets](/features/fleet-secrets) where appropriate.
</Accordion>
<Accordion title="A remote node is offline or disconnected during apply">
The row moves to **Failed** with the remote error. Reconnect the node (see [Pilot Agent](/features/pilot-agent) and [Multi-node management](/features/multi-node)), verify whether the stack directory contains `docker-compose.yml` and `.blueprint.json`, then click **Apply now**. If the directory exists without a matching marker, Sencho treats it as a name conflict until you rename or remove the remote stack manually.
The row moves to **Failed** with the remote error. Reconnect the node (see [Pilot Agent](/features/pilot-agent) and [Multi-node management](/features/multi-node)), verify whether the stack directory contains `compose.yaml` and `.blueprint.json`, then click **Apply now**. If the directory exists without a matching marker, Sencho treats it as a name conflict until you rename or remove the remote stack manually.
</Accordion>
<Accordion title="A Docker daemon or registry failure surfaced during apply">
The deployment row moves to **Failed** and records the Docker or registry error. Resolve the daemon, socket, registry credentials, rate limit, disk, or volume-permission issue on the affected node, then click **Apply now**. Watch that node's stack activity and security scan status after retry.
docs/reference/settings.mdx
+2 -1
View File
@@ -468,9 +468,10 @@ Create notification suppression rules that mute or drop matching alerts from the
| **Severity** | Info, warning, and/or error levels to suppress. |
| **Apply to** | Bell only, external channels only, or both. |
| **Expiration** | Forever, 1 hour, 24 hours, or a custom timestamp. |
| **Weekly window (UTC)** | Optional recurring weekly window. Off means no window (always eligible while enabled and unexpired). ACTIVE counts still reflect enabled and unexpired rules, not whether the window is currently open. |
| **Enabled** toggle | Disable a rule without deleting it. |
See [Mute Rules](/features/alerts-notifications#mute-rules) for the full walkthrough, compose-first shortcuts, and bell quick-mute.
See [Mute Rules](/features/alerts-notifications#mute-rules) for the full walkthrough, weekly-window semantics, best-effort fleet sync, and bell quick-mute.
---
frontend/package-lock.json
Generated
+480 -476
View File
File diff suppressed because it is too large Load Diff
frontend/package.json
+26 -26
View File
@@ -18,21 +18,21 @@
"dependencies": {
"@dagrejs/dagre": "^3.0.0",
"@monaco-editor/react": "^4.7.0",
"@radix-ui/react-alert-dialog": "^1.1.19",
"@radix-ui/react-checkbox": "^1.3.7",
"@radix-ui/react-context-menu": "^2.3.3",
"@radix-ui/react-dialog": "^1.1.19",
"@radix-ui/react-dropdown-menu": "^2.1.20",
"@radix-ui/react-hover-card": "^1.1.19",
"@radix-ui/react-label": "^2.1.11",
"@radix-ui/react-popover": "^1.1.19",
"@radix-ui/react-scroll-area": "^1.2.14",
"@radix-ui/react-select": "^2.3.3",
"@radix-ui/react-separator": "^1.1.11",
"@radix-ui/react-slider": "^1.4.3",
"@radix-ui/react-alert-dialog": "^1.1.20",
"@radix-ui/react-checkbox": "^1.3.8",
"@radix-ui/react-context-menu": "^2.3.4",
"@radix-ui/react-dialog": "^1.1.20",
"@radix-ui/react-dropdown-menu": "^2.1.21",
"@radix-ui/react-hover-card": "^1.1.20",
"@radix-ui/react-label": "^2.1.12",
"@radix-ui/react-popover": "^1.1.20",
"@radix-ui/react-scroll-area": "^1.2.15",
"@radix-ui/react-select": "^2.3.4",
"@radix-ui/react-separator": "^1.1.12",
"@radix-ui/react-slider": "^1.4.4",
"@radix-ui/react-slot": "^1.3.0",
"@radix-ui/react-tabs": "^1.1.17",
"@radix-ui/react-tooltip": "^1.2.12",
"@radix-ui/react-tabs": "^1.1.18",
"@radix-ui/react-tooltip": "^1.2.13",
"@xterm/addon-fit": "^0.11.0",
"@xterm/addon-search": "^0.16.0",
"@xterm/addon-serialize": "^0.14.0",
@@ -45,18 +45,18 @@
"date-fns": "^4.4.0",
"fflate": "^0.8.2",
"geist": "^1.7.2",
"lucide-react": "^1.24.0",
"lucide-react": "^1.25.0",
"monaco-editor": "^0.55.1",
"motion": "^12.42.2",
"qrcode.react": "^4.2.0",
"radix-ui": "^1.6.2",
"react": "^19.2.7",
"radix-ui": "^1.6.4",
"react": "^19.2.8",
"react-day-picker": "^10.0.1",
"react-dom": "^19.2.7",
"react-is": "^19.2.7",
"react-dom": "^19.2.8",
"react-is": "^19.2.8",
"react-markdown": "^10.1.0",
"react-use-measure": "^2.1.7",
"recharts": "^3.9.2",
"recharts": "^3.10.0",
"remark-gfm": "^4.0.1",
"tailwind-merge": "^3.6.0",
"tailwindcss-animate": "^1.0.7",
@@ -69,15 +69,15 @@
},
"devDependencies": {
"@eslint/js": "^10.0.1",
"@tailwindcss/vite": "^4.3.2",
"@tailwindcss/vite": "^4.3.3",
"@testing-library/dom": "^10.4.1",
"@testing-library/jest-dom": "^6.9.1",
"@testing-library/jest-dom": "^7.0.0",
"@testing-library/react": "^16.3.2",
"@testing-library/user-event": "^14.6.1",
"@types/node": "^26.1.1",
"@types/react": "^19.2.17",
"@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^6.0.3",
"@vitejs/plugin-react": "^6.0.4",
"eslint": "^10.7.0",
"eslint-plugin-react-hooks": "^7.1.1",
"eslint-plugin-react-refresh": "^0.5.3",
@@ -85,9 +85,9 @@
"jsdom": "^29.1.1",
"rollup-plugin-visualizer": "^7.0.1",
"tailwindcss": "^4.2.2",
"typescript": "^6.0.3",
"typescript-eslint": "^8.64.0",
"vite": "^8.1.4",
"typescript": "^6.0.2",
"typescript-eslint": "^8.65.0",
"vite": "^8.1.5",
"vitest": "^4.1.10"
}
}
frontend/src/components/EditorLayout.tsx
+14
View File
@@ -105,6 +105,8 @@ export default function EditorLayout() {
envFiles,
selectedEnvFile,
containers,
containersLoadStatus,
containersLoadError,
activeTab, setActiveTab,
logsMode, setLogsMode,
gitSourceOpen, setGitSourceOpen,
@@ -208,6 +210,10 @@ export default function EditorLayout() {
// useViewNavigationState needs onNavigateToDashboard -> resetEditorState
// but stackActions isn't created until after navState
const resetEditorStateRef = useRef<() => void>(() => {});
// Mobile state is declared after useStackActions; this ref is assigned once
// pendingDetailStack / mobileView exist so delete-of-open-stack can flip to
// the list surface without reordering the hook graph.
const onDeletedOpenStackRef = useRef<() => void>(() => {});
const navState = useViewNavigationState({
onNavigateToDashboard: () => resetEditorStateRef.current(),
@@ -282,6 +288,7 @@ export default function EditorLayout() {
return can('stack:edit', 'stack', stackName);
},
canOfferVolumeRemoval,
onDeletedOpenStack: () => onDeletedOpenStackRef.current(),
});
// Wire the ref now that stackActions is available
@@ -375,6 +382,10 @@ export default function EditorLayout() {
const [pendingDetailStack, setPendingDetailStack] = useState<string | null>(null);
const [pendingAnatomyTab, setPendingAnatomyTab] = useState<'networking' | 'doctor' | 'dossier' | 'drift' | undefined>();
const [fleetUpdatesIntent, setFleetUpdatesIntent] = useState<{ tab: 'nodes' | 'changelog' } | null>(null);
onDeletedOpenStackRef.current = () => {
setPendingDetailStack(null);
setMobileView('list');
};
const handleFleetUpdatesIntentConsumed = useCallback(() => setFleetUpdatesIntent(null), []);
@@ -607,6 +618,9 @@ export default function EditorLayout() {
stackName={stackName}
isDarkMode={isDarkMode}
containers={containers}
containersLoadStatus={containersLoadStatus}
containersLoadError={containersLoadError}
onRetryContainersLoad={() => { void stackActions.retryContainersLoad(); }}
containerStats={containerStats}
containerStatsError={containerStatsError}
content={content}
frontend/src/components/EditorLayout/EditorView.tsx
+12
View File
@@ -135,6 +135,9 @@ export interface EditorViewProps {
envFiles: string[];
selectedEnvFile: string;
isFileLoading: boolean;
containersLoadStatus?: 'idle' | 'loading' | 'success' | 'error';
containersLoadError?: string | null;
onRetryContainersLoad?: () => void;
backupInfo: { exists: boolean; timestamp: number | null };
gitSourcePendingMap: Record<string, boolean>;
notifications: NotificationItem[];
@@ -244,6 +247,9 @@ export function EditorView(props: EditorViewProps) {
envFiles,
selectedEnvFile,
isFileLoading,
containersLoadStatus = 'success',
containersLoadError = null,
onRetryContainersLoad,
backupInfo,
gitSourcePendingMap,
notifications,
@@ -475,6 +481,9 @@ export function EditorView(props: EditorViewProps) {
onRequestServiceUpdate={onRequestServiceUpdate}
containersExpanded={containersExpanded}
onToggleContainersExpand={toggleContainersExpand}
containersLoadStatus={containersLoadStatus}
containersLoadError={containersLoadError}
onRetryContainersLoad={onRetryContainersLoad}
key={`${activeNode?.id ?? 'local'}:${stackName}`}
/>
</ScrollArea>
@@ -494,6 +503,9 @@ export function EditorView(props: EditorViewProps) {
serviceUpdateStatuses={serviceUpdateStatuses}
serviceUpdateInProgress={serviceUpdateInProgress}
onRequestServiceUpdate={onRequestServiceUpdate}
containersLoadStatus={containersLoadStatus}
containersLoadError={containersLoadError}
onRetryContainersLoad={onRetryContainersLoad}
key={`${activeNode?.id ?? 'local'}:${stackName}`}
/>
</CardContent>
frontend/src/components/EditorLayout/MobileStackDetail.tsx
+3
View File
@@ -233,6 +233,9 @@ export function MobileStackDetail(props: EditorViewProps) {
serviceUpdateStatuses={serviceUpdateStatuses}
serviceUpdateInProgress={serviceUpdateInProgress}
onRequestServiceUpdate={onRequestServiceUpdate}
containersLoadStatus={props.containersLoadStatus}
containersLoadError={props.containersLoadError}
onRetryContainersLoad={props.onRetryContainersLoad}
key={`${activeNode?.id ?? 'local'}:${stackName}`}
/>
</div>
frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx
+60
View File
@@ -450,3 +450,63 @@ describe('declared-service headers (multi-service only)', () => {
expect(onToggle).toHaveBeenCalledTimes(1);
});
});
describe('containers load states', () => {
it('does not show empty copy while loading', () => {
render(
<ContainersHealth
safeContainers={[]}
containerStats={{}}
containerStatsError={null}
isAdmin
activeNode={LOCAL_NODE}
openLogViewer={vi.fn()}
openBashModal={vi.fn()}
serviceAction={vi.fn()}
containersLoadStatus="loading"
/>,
);
expect(screen.queryByText(/No containers running for this stack/i)).toBeNull();
expect(screen.queryByText(/No containers running for this service/i)).toBeNull();
});
it('shows confirmed empty only after success', () => {
render(
<ContainersHealth
safeContainers={[]}
containerStats={{}}
containerStatsError={null}
isAdmin
activeNode={LOCAL_NODE}
openLogViewer={vi.fn()}
openBashModal={vi.fn()}
serviceAction={vi.fn()}
containersLoadStatus="success"
/>,
);
expect(screen.getByText(/No containers running for this stack/i)).toBeInTheDocument();
});
it('shows error and invokes retry once', async () => {
const onRetry = vi.fn();
const user = userEvent.setup();
render(
<ContainersHealth
safeContainers={[]}
containerStats={{}}
containerStatsError={null}
isAdmin
activeNode={LOCAL_NODE}
openLogViewer={vi.fn()}
openBashModal={vi.fn()}
serviceAction={vi.fn()}
containersLoadStatus="error"
containersLoadError="Could not load containers."
onRetryContainersLoad={onRetry}
/>,
);
expect(screen.queryByText(/No containers running for this stack/i)).toBeNull();
await user.click(screen.getByRole('button', { name: /retry/i }));
expect(onRetry).toHaveBeenCalledTimes(1);
});
});
frontend/src/components/EditorLayout/editor-view-blocks.tsx
+39 -1
View File
@@ -21,9 +21,12 @@ import {
List,
Maximize2,
Minimize2,
AlertCircle,
RefreshCw
} from 'lucide-react';
import { useCallback, useEffect, useRef, useState } from 'react';
import { Button } from '../ui/button';
import { Skeleton } from '../ui/skeleton';
import { CardTitle } from '../ui/card';
import {
DropdownMenu,
@@ -317,6 +320,9 @@ export interface ContainersHealthProps {
onRequestServiceUpdate?: (serviceName: string, mode: 'update' | 'rebuild') => void;
containersExpanded?: boolean;
onToggleContainersExpand?: () => void;
containersLoadStatus?: 'idle' | 'loading' | 'success' | 'error';
containersLoadError?: string | null;
onRetryContainersLoad?: () => void;
}
// Per-container health strip: status badge, uptime, ports, and CPU/Mem/Net
@@ -336,6 +342,9 @@ export function ContainersHealth({
onRequestServiceUpdate,
containersExpanded,
onToggleContainersExpand,
containersLoadStatus = 'success',
containersLoadError = null,
onRetryContainersLoad,
}: ContainersHealthProps) {
// Multi-service only (§12): a single-service stack keeps the existing flat
// layout untouched, including its per-container Start/Stop/Restart kebab.
@@ -642,6 +651,35 @@ export function ContainersHealth({
);
};
const showConfirmedEmpty = containersLoadStatus === 'success' && safeContainers.length === 0;
if (containersLoadStatus === 'idle' || containersLoadStatus === 'loading') {
return (
<div className="space-y-2">
<Skeleton className="h-8 w-full" />
<Skeleton className="h-16 w-full" />
<Skeleton className="h-16 w-full" />
</div>
);
}
if (containersLoadStatus === 'error') {
return (
<div className="rounded-lg border border-card-border bg-card/40 p-4 text-center">
<AlertCircle className="mx-auto mb-2 h-8 w-8 text-muted-foreground" aria-hidden />
<p className="mb-3 text-sm text-muted-foreground">
{containersLoadError ?? 'Could not load containers.'}
</p>
{onRetryContainersLoad && (
<Button type="button" variant="outline" size="sm" onClick={onRetryContainersLoad}>
<RefreshCw className="h-4 w-4" />
Retry
</Button>
)}
</div>
);
}
return (
<div>
{containerStatsError && safeContainers.length > 0 && (
@@ -754,7 +792,7 @@ export function ContainersHealth({
);
})}
</div>
) : safeContainers.length === 0 ? (
) : showConfirmedEmpty ? (
<div className="text-muted-foreground text-sm">No containers running for this stack.</div>
) : (
<div className="flex flex-col gap-2">
frontend/src/components/EditorLayout/hooks/useEditorViewState.ts
+4
View File
@@ -31,6 +31,8 @@ export function useEditorViewState() {
const [envFiles, setEnvFiles] = useState<string[]>([]);
const [selectedEnvFile, setSelectedEnvFile] = useState<string>('');
const [containers, setContainers] = useState<ContainerInfo[]>([]);
const [containersLoadStatus, setContainersLoadStatus] = useState<'idle' | 'loading' | 'success' | 'error'>('idle');
const [containersLoadError, setContainersLoadError] = useState<string | null>(null);
// Declared-service facts for the loaded stack, from the effective Compose
// model. Empty for a single-service stack, an older node without the
// service-scoped-update capability, or a render failure; all three cases
@@ -67,6 +69,8 @@ export function useEditorViewState() {
envFiles, setEnvFiles,
selectedEnvFile, setSelectedEnvFile,
containers, setContainers,
containersLoadStatus, setContainersLoadStatus,
containersLoadError, setContainersLoadError,
effectiveServices, setEffectiveServices,
serviceUpdateInProgress, setServiceUpdateInProgress,
activeTab, setActiveTab,
frontend/src/components/EditorLayout/hooks/useStackActions.test.ts
+427 -3
View File
@@ -19,6 +19,7 @@ vi.mock('@/components/ui/toast-store', () => ({
}));
import { apiFetch } from '@/lib/api';
import { toast } from '@/components/ui/toast-store';
type EditorState = ReturnType<typeof useEditorViewState>;
type StackListState = ReturnType<typeof useStackListState>;
@@ -42,6 +43,11 @@ function makeEditorState(over: Partial<EditorState> = {}): EditorState {
setEditingCompose: vi.fn(),
setActiveTab: vi.fn(),
setContainers: vi.fn(),
containers: [],
containersLoadStatus: 'idle' as const,
containersLoadError: null as string | null,
setContainersLoadStatus: vi.fn(),
setContainersLoadError: vi.fn(),
setEnvFiles: vi.fn(),
setSelectedEnvFile: vi.fn(),
setEnvExists: vi.fn(),
@@ -101,6 +107,8 @@ function makeOverlay(over: Partial<OverlayState> = {}): OverlayState {
setPreDeployAdvisory: vi.fn(),
openSelfStackProtected: vi.fn(),
setDiffPreview: vi.fn(),
stackToDelete: null,
closeDeleteDialog: vi.fn(),
...over,
} as unknown as OverlayState;
}
@@ -115,17 +123,24 @@ function setup(over: {
editorState?: Partial<EditorState>;
overlay?: Partial<OverlayState>;
stackList?: Partial<StackListState>;
navState?: Partial<NavState>;
getLastDeployOutputLine?: (stackName: string) => string | undefined;
hasUpdateGuard?: boolean;
canEditStack?: (stackNameOrFilename: string) => boolean;
activeNode?: Parameters<typeof useStackActions>[0]['activeNode'];
setActiveNode?: Parameters<typeof useStackActions>[0]['setActiveNode'];
onDeletedOpenStack?: () => void;
} = {}) {
const editorState = makeEditorState(over.editorState);
const stackListState = makeStackListState(over.stackList);
const navState = { setActiveView: vi.fn() } as unknown as NavState;
const navState = {
activeView: 'editor',
setActiveView: vi.fn(),
...over.navState,
} as unknown as NavState;
const overlayState = makeOverlay(over.overlay);
const setActiveNode = over.setActiveNode ?? vi.fn();
const onDeletedOpenStack = over.onDeletedOpenStack ?? vi.fn();
const { result } = renderHook(() =>
useStackActions({
@@ -141,9 +156,10 @@ function setup(over: {
diffPreviewEnabled: false,
hasUpdateGuard: over.hasUpdateGuard ?? false,
canEditStack: over.canEditStack ?? (() => true),
onDeletedOpenStack,
}),
);
return { result, editorState, stackListState, overlayState, setActiveNode };
return { result, editorState, stackListState, overlayState, navState, setActiveNode, onDeletedOpenStack };
}
describe('useStackActions.saveFile', () => {
@@ -181,7 +197,7 @@ describe('useStackActions.saveFile', () => {
useStackActions({
editorState,
stackListState,
navState: { setActiveView: vi.fn() } as unknown as NavState,
navState: { activeView: 'editor', setActiveView: vi.fn() } as unknown as NavState,
overlayState: makeOverlay(),
activeNode: { id: 1, type: 'local' } as Parameters<typeof useStackActions>[0]['activeNode'],
setActiveNode: vi.fn(),
@@ -190,6 +206,7 @@ describe('useStackActions.saveFile', () => {
getLastDeployOutputLine: () => undefined,
diffPreviewEnabled: false,
canEditStack: () => true,
onDeletedOpenStack: vi.fn(),
}),
);
const ok = await result.current.saveFile();
@@ -314,6 +331,18 @@ describe('useStackActions policy-block dialog wiring', () => {
expect(overlayState.setPolicyBlock).not.toHaveBeenCalled();
});
it('toasts a deletion-in-progress message when the conflict action is delete', async () => {
const inProgress = JSON.stringify({
code: 'stack_op_in_progress',
inProgress: { action: 'delete', startedAt: Date.now(), user: 'admin' },
});
vi.mocked(apiFetch).mockResolvedValueOnce(new Response(inProgress, { status: 409 }));
const { result, overlayState } = setup();
await act(async () => { await result.current.updateStack(mouseEvent); });
expect(overlayState.setPolicyBlock).not.toHaveBeenCalled();
expect(toast.error).toHaveBeenCalledWith(expect.stringMatching(/already deleting/i));
});
it('opens the dialog with action "update" via the sidebar update entry point', async () => {
vi.mocked(apiFetch).mockResolvedValueOnce(new Response(JSON.stringify(policyPayload), { status: 409 }));
const { result, overlayState } = setup();
@@ -1194,3 +1223,398 @@ describe('useStackActions.openStackApp', () => {
expect(clickCount).toBe(0);
});
});
describe('container fetch contract', () => {
beforeEach(() => {
vi.mocked(apiFetch).mockReset();
});
it('soft refresh prior empty transitions to error instead of confirmed empty', async () => {
const setContainersLoadStatus = vi.fn();
const setContainersLoadError = vi.fn();
vi.mocked(apiFetch).mockResolvedValue(new Response('fail', { status: 500 }));
const { result } = setup({
editorState: {
containers: [],
containersLoadStatus: 'success',
containersLoadError: null,
setContainersLoadStatus,
setContainersLoadError,
} as never,
});
let ok = true;
await act(async () => {
ok = await result.current.refreshSelectedContainers('web', 'web.yml');
});
expect(ok).toBe(false);
expect(setContainersLoadStatus).toHaveBeenCalledWith('error');
expect(setContainersLoadError).toHaveBeenCalled();
});
it('soft refresh preserves prior non-empty containers on failure', async () => {
const setContainers = vi.fn();
const prior = [{ Id: 'abc', Names: ['/web'], State: 'running' }];
vi.mocked(apiFetch).mockResolvedValue(new Response('fail', { status: 500 }));
const { result } = setup({
editorState: {
containers: prior,
containersLoadStatus: 'success',
containersLoadError: null,
setContainers,
} as never,
});
await act(async () => {
await result.current.refreshSelectedContainers('web', 'web.yml');
});
expect(setContainers).not.toHaveBeenCalled();
});
it('malformed 200 is not treated as success empty in foreground retry', async () => {
const setContainersLoadStatus = vi.fn();
const setContainersLoadError = vi.fn();
vi.mocked(apiFetch).mockResolvedValue(
new Response(JSON.stringify({ not: 'an-array' }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}),
);
const { result } = setup({
editorState: {
containers: [],
setContainersLoadStatus,
setContainersLoadError,
setContainers: vi.fn(),
} as never,
});
await act(async () => {
await result.current.retryContainersLoad();
});
expect(setContainersLoadStatus).toHaveBeenCalledWith('error');
});
it('same-owner soft refreshes: older success does not overwrite newer', async () => {
const resolvers: Array<(r: Response) => void> = [];
vi.mocked(apiFetch).mockImplementation((endpoint: unknown) => {
if (typeof endpoint === 'string' && endpoint.includes('/containers')) {
return new Promise<Response>((resolve) => { resolvers.push(resolve); });
}
return Promise.resolve(new Response('[]', {
status: 200,
headers: { 'Content-Type': 'application/json' },
}));
});
const setContainers = vi.fn();
const setContainersLoadStatus = vi.fn();
const { result } = setup({
editorState: {
containers: [],
containersLoadStatus: 'success',
containersLoadError: null,
setContainers,
setContainersLoadStatus,
setContainersLoadError: vi.fn(),
} as never,
});
let olderPromise!: Promise<boolean>;
let newerPromise!: Promise<boolean>;
await act(async () => {
olderPromise = result.current.refreshSelectedContainers('web', 'web.yml');
});
await act(async () => {
newerPromise = result.current.refreshSelectedContainers('web', 'web.yml');
});
expect(resolvers).toHaveLength(2);
const older = [{ Id: 'old', Names: ['/old'], State: 'running' }];
const newer = [{ Id: 'new', Names: ['/new'], State: 'running' }];
const json = (body: unknown) => new Response(JSON.stringify(body), {
status: 200,
headers: { 'Content-Type': 'application/json' },
});
await act(async () => {
resolvers[1](json(newer));
await newerPromise;
});
expect(setContainers).toHaveBeenLastCalledWith(newer);
expect(setContainersLoadStatus).toHaveBeenCalledWith('success');
setContainers.mockClear();
setContainersLoadStatus.mockClear();
await act(async () => {
resolvers[0](json(older));
await olderPromise;
});
expect(setContainers).not.toHaveBeenCalled();
expect(setContainersLoadStatus).not.toHaveBeenCalled();
});
it('deferred response after stack switch does not apply setters', async () => {
let resolveContainers: ((r: Response) => void) | null = null;
vi.mocked(apiFetch).mockImplementation((endpoint: unknown) => {
if (typeof endpoint === 'string' && endpoint.includes('/containers')) {
return new Promise<Response>((resolve) => { resolveContainers = resolve; });
}
return Promise.resolve(new Response('[]', {
status: 200,
headers: { 'Content-Type': 'application/json' },
}));
});
const setContainers = vi.fn();
const setContainersLoadStatus = vi.fn();
const editorState = makeEditorState({
containers: [],
containersLoadStatus: 'success',
containersLoadError: null,
setContainers,
setContainersLoadStatus,
setContainersLoadError: vi.fn(),
});
const { result, rerender } = renderHook(
({ selectedFile }) =>
useStackActions({
editorState,
stackListState: makeStackListState({ selectedFile }),
navState: { setActiveView: vi.fn() } as unknown as NavState,
overlayState: makeOverlay(),
activeNode: { id: 1, type: 'local' } as Parameters<typeof useStackActions>[0]['activeNode'],
setActiveNode: vi.fn(),
nodes: [],
runWithLog,
getLastDeployOutputLine: () => undefined,
diffPreviewEnabled: false,
canEditStack: () => true,
onDeletedOpenStack: vi.fn(),
}),
{ initialProps: { selectedFile: 'web.yml' as string | null } },
);
let refreshPromise!: Promise<boolean>;
await act(async () => {
refreshPromise = result.current.refreshSelectedContainers('web', 'web.yml');
});
expect(resolveContainers).not.toBeNull();
rerender({ selectedFile: 'api.yml' });
await act(async () => { await Promise.resolve(); });
await act(async () => {
resolveContainers?.(new Response(JSON.stringify([{ Id: 'stale', Names: ['/web'], State: 'running' }]), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}));
await refreshPromise;
});
expect(setContainers).not.toHaveBeenCalled();
expect(setContainersLoadStatus).not.toHaveBeenCalledWith('success');
});
it('deferred response after node switch does not apply setters', async () => {
let resolveContainers: ((r: Response) => void) | null = null;
vi.mocked(apiFetch).mockImplementation((endpoint: unknown) => {
if (typeof endpoint === 'string' && endpoint.includes('/containers')) {
return new Promise<Response>((resolve) => { resolveContainers = resolve; });
}
return Promise.resolve(new Response('[]', {
status: 200,
headers: { 'Content-Type': 'application/json' },
}));
});
const setContainers = vi.fn();
const setContainersLoadStatus = vi.fn();
const editorState = makeEditorState({
containers: [],
containersLoadStatus: 'success',
containersLoadError: null,
setContainers,
setContainersLoadStatus,
setContainersLoadError: vi.fn(),
});
type NodeArg = Parameters<typeof useStackActions>[0]['activeNode'];
const { result, rerender } = renderHook(
({ activeNode }) =>
useStackActions({
editorState,
stackListState: makeStackListState({ selectedFile: 'web.yml' }),
navState: { setActiveView: vi.fn() } as unknown as NavState,
overlayState: makeOverlay(),
activeNode,
setActiveNode: vi.fn(),
nodes: [],
runWithLog,
getLastDeployOutputLine: () => undefined,
diffPreviewEnabled: false,
canEditStack: () => true,
onDeletedOpenStack: vi.fn(),
}),
{
initialProps: {
activeNode: { id: 1, type: 'local' } as NodeArg,
},
},
);
let refreshPromise!: Promise<boolean>;
await act(async () => {
refreshPromise = result.current.refreshSelectedContainers('web', 'web.yml');
});
rerender({ activeNode: { id: 2, type: 'remote', api_url: 'http://192.168.1.50:1852' } as NodeArg });
await act(async () => { await Promise.resolve(); });
await act(async () => {
resolveContainers?.(new Response(JSON.stringify([{ Id: 'stale', Names: ['/web'], State: 'running' }]), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}));
await refreshPromise;
});
expect(setContainers).not.toHaveBeenCalled();
expect(setContainersLoadStatus).not.toHaveBeenCalledWith('success');
});
});
describe('useStackActions.deleteStack', () => {
beforeEach(() => {
vi.mocked(apiFetch).mockReset();
});
it('leaves the editor for dashboard when deleting the open stack by filename', async () => {
vi.mocked(apiFetch).mockResolvedValue(new Response(null, { status: 200 }));
const { result, stackListState, overlayState, navState, onDeletedOpenStack } = setup({
overlay: { stackToDelete: 'web.yml' },
stackList: { selectedFile: 'web.yml', files: ['web.yml'] },
navState: { activeView: 'editor' },
});
await act(async () => {
await result.current.deleteStack(false);
});
expect(apiFetch).toHaveBeenCalledWith('/stacks/web.yml', { method: 'DELETE' });
expect(stackListState.setSelectedFile).toHaveBeenCalledWith(null);
expect(navState.setActiveView).toHaveBeenCalledWith('dashboard');
expect(navState.setActiveView).toHaveBeenCalledTimes(1);
expect(onDeletedOpenStack).toHaveBeenCalledTimes(1);
expect(overlayState.closeDeleteDialog).toHaveBeenCalled();
expect(stackListState.refreshStacks).toHaveBeenCalled();
});
it('clears isFileLoading on delete-leave so the URL writer is not blocked', async () => {
vi.mocked(apiFetch).mockResolvedValue(new Response(null, { status: 200 }));
const { result, editorState } = setup({
overlay: { stackToDelete: 'web.yml' },
stackList: { selectedFile: 'web.yml', files: ['web.yml'] },
editorState: { isFileLoading: true },
navState: { activeView: 'editor' },
});
await act(async () => {
await result.current.deleteStack(false);
});
expect(editorState.setIsFileLoading).toHaveBeenCalledWith(false);
});
it('leaves the editor when sidebar delete passes a basename', async () => {
vi.mocked(apiFetch).mockResolvedValue(new Response(null, { status: 200 }));
const { result, stackListState, navState, onDeletedOpenStack } = setup({
overlay: { stackToDelete: 'web' },
stackList: { selectedFile: 'web.yml', files: ['web.yml'] },
navState: { activeView: 'editor' },
});
await act(async () => {
await result.current.deleteStack(false);
});
expect(apiFetch).toHaveBeenCalledWith('/stacks/web', { method: 'DELETE' });
expect(stackListState.setSelectedFile).toHaveBeenCalledWith(null);
expect(navState.setActiveView).toHaveBeenCalledWith('dashboard');
expect(onDeletedOpenStack).toHaveBeenCalledTimes(1);
});
it('does not navigate when deleting a different stack', async () => {
vi.mocked(apiFetch).mockResolvedValue(new Response(null, { status: 200 }));
const { result, stackListState, navState, onDeletedOpenStack } = setup({
overlay: { stackToDelete: 'other.yml' },
stackList: {
selectedFile: 'web.yml',
files: ['web.yml', 'other.yml'],
},
navState: { activeView: 'editor' },
});
await act(async () => {
await result.current.deleteStack(false);
});
expect(stackListState.setSelectedFile).not.toHaveBeenCalledWith(null);
expect(navState.setActiveView).not.toHaveBeenCalled();
expect(onDeletedOpenStack).not.toHaveBeenCalled();
expect(stackListState.refreshStacks).toHaveBeenCalled();
});
it('clears selection without navigating when the matching stack is hidden behind another view', async () => {
vi.mocked(apiFetch).mockResolvedValue(new Response(null, { status: 200 }));
const { result, stackListState, navState, onDeletedOpenStack } = setup({
overlay: { stackToDelete: 'web.yml' },
stackList: { selectedFile: 'web.yml', files: ['web.yml'] },
navState: { activeView: 'resources' },
});
await act(async () => {
await result.current.deleteStack(false);
});
expect(stackListState.setSelectedFile).toHaveBeenCalledWith(null);
expect(navState.setActiveView).not.toHaveBeenCalled();
expect(onDeletedOpenStack).not.toHaveBeenCalled();
expect(stackListState.refreshStacks).toHaveBeenCalled();
});
it('does not reset or navigate on a non-OK delete response', async () => {
vi.mocked(apiFetch).mockResolvedValue(new Response('boom', { status: 500 }));
const { toast } = await import('@/components/ui/toast-store');
const { result, stackListState, overlayState, navState, onDeletedOpenStack } = setup({
overlay: { stackToDelete: 'web.yml' },
stackList: { selectedFile: 'web.yml', files: ['web.yml'] },
navState: { activeView: 'editor' },
});
await act(async () => {
await result.current.deleteStack(false);
});
expect(stackListState.setSelectedFile).not.toHaveBeenCalledWith(null);
expect(navState.setActiveView).not.toHaveBeenCalled();
expect(onDeletedOpenStack).not.toHaveBeenCalled();
expect(overlayState.closeDeleteDialog).not.toHaveBeenCalled();
expect(stackListState.refreshStacks).not.toHaveBeenCalled();
expect(toast.error).toHaveBeenCalled();
});
it('does not navigate on a self-stack-protected response', async () => {
vi.mocked(apiFetch).mockResolvedValue(
new Response(JSON.stringify({ code: 'self_stack_protected' }), { status: 409 }),
);
const { result, stackListState, overlayState, navState, onDeletedOpenStack } = setup({
overlay: { stackToDelete: 'web.yml' },
stackList: { selectedFile: 'web.yml', files: ['web.yml'] },
navState: { activeView: 'editor' },
});
await act(async () => {
await result.current.deleteStack(false);
});
expect(overlayState.openSelfStackProtected).toHaveBeenCalled();
expect(overlayState.closeDeleteDialog).toHaveBeenCalled();
expect(stackListState.setSelectedFile).not.toHaveBeenCalledWith(null);
expect(navState.setActiveView).not.toHaveBeenCalled();
expect(onDeletedOpenStack).not.toHaveBeenCalled();
expect(stackListState.refreshStacks).not.toHaveBeenCalled();
});
});
frontend/src/components/EditorLayout/hooks/useStackActions.ts
+226 -69
View File
@@ -6,6 +6,7 @@ import {
beginSpan,
endSpan,
flushPendingCommit,
markMilestone,
type PendingCommit,
type SpanHandle,
} from '@/lib/hydrationTiming';
@@ -22,7 +23,7 @@ import type { RunWithLogParams } from '@/context/DeployFeedbackContext';
import { parsePath } from '@/lib/router/senchoRoute';
import { resolveEnvFilePath } from '@/lib/router/envRoute';
import type { EditorTab, RouteStackLoadResult } from '@/lib/router/routeTypes';
import type { StackAction, RecoverableAction, FailureClassification } from '../EditorView';
import type { StackAction, RecoverableAction, FailureClassification, ContainerInfo } from '../EditorView';
import type { NotificationItem } from '../../dashboard/types';
import type { PolicyBlockPayload, PolicyBlockableAction } from '../../stack/PolicyBlockDialog';
import type {
@@ -107,7 +108,7 @@ const parseFailureClassification = (value: unknown): FailureClassification | und
return undefined;
};
type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update';
type StackOpAction = 'deploy' | 'down' | 'restart' | 'stop' | 'start' | 'update' | 'delete';
interface StackOpInProgressInfo {
action: StackOpAction;
@@ -122,6 +123,7 @@ const STACK_OP_PRESENT_PARTICIPLE: Record<StackOpAction, string> = {
stop: 'stopping',
start: 'starting',
update: 'updating',
delete: 'deleting',
};
const VALID_STACK_OP_ACTIONS: ReadonlySet<string> = new Set(
@@ -167,6 +169,13 @@ interface UseStackActionsOptions {
canEditStack: (stackNameOrFilename: string) => boolean;
/** Fail-closed: true only when active node meta explicitly lists stack-down-remove-volumes. */
canOfferVolumeRemoval?: boolean;
/**
* Mobile (and any shell-owned) cleanup after deleting the stack that is
* currently open in the editor. EditorLayout clears pending detail and
* flips to the stack list surface. Required: the sole production caller
* owns that state, and an optional callback would silently skip it.
*/
onDeletedOpenStack: () => void;
}
const isRecord = (value: unknown): value is Record<string, unknown> =>
@@ -389,6 +398,7 @@ export function useStackActions(options: UseStackActionsOptions) {
hasServiceScopedUpdate = false,
canEditStack,
canOfferVolumeRemoval = false,
onDeletedOpenStack,
} = options;
const pendingStackLoadRef = useRef<string | null>(null);
@@ -414,12 +424,27 @@ export function useStackActions(options: UseStackActionsOptions) {
// activeView unchanged) still re-run the commit effect.
const [detailVisibleEpoch, setDetailVisibleEpoch] = useState(0);
// Live ownership for container fetches: render-closure comparisons after an
// await can accept a response for a stack/node that is no longer active.
const selectedFileRef = useRef(stackListState.selectedFile);
const activeNodeIdRef = useRef(activeNode?.id);
const containersRef = useRef(editorState.containers);
// Same-owner arbitration: soft refresh, Retry, and detail load can overlap
// for one stack/node; only the newest generation may apply success or failure.
const containersFetchGenRef = useRef(0);
useEffect(() => {
selectedFileRef.current = stackListState.selectedFile;
activeNodeIdRef.current = activeNode?.id;
containersRef.current = editorState.containers;
});
useEffect(() => {
return () => {
if (checkUpdatesIntervalRef.current !== null) {
clearInterval(checkUpdatesIntervalRef.current);
}
loadFileAbortRef.current?.abort();
containersFetchGenRef.current += 1;
};
}, []);
@@ -492,6 +517,10 @@ export function useStackActions(options: UseStackActionsOptions) {
// previous node's data.
loadFileAbortRef.current?.abort();
loadFileAbortRef.current = null;
// loadFileCore's finally skips clearing loading when the signal is aborted,
// so clear it here. Otherwise useUrlSync's writer stays blocked after a
// delete-leave (or any other reset) that aborts a mid-flight load.
editorState.setIsFileLoading(false);
stackListState.setSelectedFile(null);
editorState.setContent('');
editorState.setOriginalContent('');
@@ -501,31 +530,174 @@ export function useStackActions(options: UseStackActionsOptions) {
editorState.setSelectedEnvFile('');
editorState.setEnvExists(false);
editorState.setContainers([]);
editorState.setContainersLoadStatus('idle');
editorState.setContainersLoadError(null);
containersFetchGenRef.current += 1;
editorState.setEffectiveServices([]);
editorState.setServiceUpdateInProgress(null);
editorState.setIsEditing(false);
};
type ContainersFetchMode = 'foreground' | 'soft';
type ContainersFetchResult =
| { ok: true; containers: ContainerInfo[] }
| { ok: false; reason: 'http' | 'malformed' | 'network' | 'aborted' | 'stale'; error?: string };
type ContainersFetchOwnership = {
signal?: AbortSignal;
attemptId?: string;
expectedFile: string;
expectedNodeId: number | undefined;
generation: number;
};
const ownershipStillValid = (
ownership: ContainersFetchOwnership,
): 'ok' | 'aborted' | 'stale' => {
if (ownership.signal?.aborted) return 'aborted';
if (selectedFileRef.current !== ownership.expectedFile) return 'stale';
if (activeNodeIdRef.current !== ownership.expectedNodeId) return 'stale';
if (containersFetchGenRef.current !== ownership.generation) return 'stale';
if (
ownership.attemptId !== undefined
&& detailAttemptRef.current !== ownership.attemptId
) {
return 'stale';
}
return 'ok';
};
const applyContainersFetchFailure = (mode: ContainersFetchMode, message: string) => {
if (mode === 'foreground') {
editorState.setContainers([]);
editorState.setContainersLoadStatus('error');
editorState.setContainersLoadError(message);
return;
}
// Soft: prior non-empty cards stay visible. Prior confirmed-empty becomes a
// recoverable error so soft failure never keeps "No containers running".
if (containersRef.current.length === 0) {
editorState.setContainersLoadStatus('error');
editorState.setContainersLoadError(message);
}
};
const fetchStackContainers = async (
stackFile: string,
mode: ContainersFetchMode,
ownership: Omit<ContainersFetchOwnership, 'generation'>,
): Promise<ContainersFetchResult> => {
const stackName = stackFile.replace(/\.(yml|yaml)$/, '');
const owned: ContainersFetchOwnership = {
...ownership,
generation: ++containersFetchGenRef.current,
};
let headersSpan: SpanHandle | null = null;
let bodySpan: SpanHandle | null = null;
if (mode === 'foreground') {
editorState.setContainersLoadStatus('loading');
editorState.setContainersLoadError(null);
}
try {
headersSpan = owned.attemptId
? beginSpan('fetch_headers', { attemptId: owned.attemptId })
: null;
const containersRes = await apiFetch(`/stacks/${stackName}/containers`, {
signal: owned.signal,
nodeId: owned.expectedNodeId ?? null,
});
const hopProxied = containersRes.headers.get('x-sencho-proxy') === '1';
if (headersSpan !== null) {
endSpan(headersSpan, { proxied: hopProxied, detail: { status: containersRes.status } });
headersSpan = null;
}
const afterHeaders = ownershipStillValid(owned);
if (afterHeaders !== 'ok') {
return { ok: false, reason: afterHeaders };
}
if (!containersRes.ok) {
const message = `Could not load containers (${containersRes.status}).`;
applyContainersFetchFailure(mode, message);
return { ok: false, reason: 'http', error: message };
}
bodySpan = owned.attemptId
? beginSpan('body_decode', { attemptId: owned.attemptId, proxied: hopProxied })
: null;
const conts: unknown = await containersRes.json();
if (bodySpan !== null) {
endSpan(bodySpan);
bodySpan = null;
}
const afterBody = ownershipStillValid(owned);
if (afterBody !== 'ok') {
return { ok: false, reason: afterBody };
}
if (!Array.isArray(conts)) {
const message = 'Container list response was invalid.';
applyContainersFetchFailure(mode, message);
return { ok: false, reason: 'malformed', error: message };
}
const list = conts as ContainerInfo[];
const dispatchSpan = owned.attemptId
? beginSpan('state_dispatch', { attemptId: owned.attemptId, proxied: hopProxied })
: null;
editorState.setContainers(list);
editorState.setContainersLoadStatus('success');
editorState.setContainersLoadError(null);
if (dispatchSpan !== null) endSpan(dispatchSpan);
return { ok: true, containers: list };
} catch (error) {
if (headersSpan !== null) endSpan(headersSpan, { outcome: 'error' });
if (bodySpan !== null) endSpan(bodySpan, { outcome: 'error' });
if (isAbortError(error) || owned.signal?.aborted) {
return { ok: false, reason: 'aborted' };
}
const afterCatch = ownershipStillValid(owned);
if (afterCatch !== 'ok') {
return { ok: false, reason: afterCatch };
}
console.error('Failed to load containers:', error);
const message = 'Could not load containers.';
applyContainersFetchFailure(mode, message);
return { ok: false, reason: 'network', error: message };
}
};
// Re-sync the open stack's container list. Used after both successful and
// failed/stalled operations so the detail never shows containers that no
// longer reflect reality. Returns true only when the live list was fetched;
// false on a non-applicable stack, a non-ok response, or a network error, so
// callers (e.g. the recovery panel's Refresh) can report the real outcome.
const refreshSelectedContainers = async (stackName: string, stackFile: string): Promise<boolean> => {
if (stackListState.selectedFile !== stackFile) return false;
try {
const res = await apiFetch(`/stacks/${stackName}/containers`);
if (!res.ok) return false;
const conts = await res.json();
editorState.setContainers(Array.isArray(conts) ? conts : []);
return true;
} catch {
// Non-critical when called from an action's failure path: refreshStacks(true)
// in the caller's finally still reconciles the sidebar status.
return false;
}
// stackName is kept for call-site clarity; the fetch derives the name from stackFile.
const refreshSelectedContainers = async (_stackName: string, stackFile: string): Promise<boolean> => {
if (selectedFileRef.current !== stackFile) return false;
const result = await fetchStackContainers(stackFile, 'soft', {
expectedFile: stackFile,
expectedNodeId: activeNodeIdRef.current,
});
return result.ok;
};
const retryContainersLoad = async () => {
const stackFile = selectedFileRef.current;
if (!stackFile) return;
await fetchStackContainers(stackFile, 'foreground', {
expectedFile: stackFile,
expectedNodeId: activeNodeIdRef.current,
});
};
const loadContainerState = (
filename: string,
signal?: AbortSignal,
attemptId?: string,
): Promise<ContainersFetchResult> =>
fetchStackContainers(filename, 'foreground', {
signal,
attemptId,
expectedFile: filename,
expectedNodeId: activeNodeIdRef.current,
});
// Stack operations whose failure produces a recovery panel. A failed
// stop/start/delete is not recoverable through retry/restart/rollback.
const RECOVERABLE_ACTIONS: readonly StackAction[] = ['deploy', 'update', 'restart', 'rollback'];
@@ -646,50 +818,6 @@ export function useStackActions(options: UseStackActionsOptions) {
}
};
const loadContainerState = async (
filename: string,
signal?: AbortSignal,
attemptId?: string,
proxied?: boolean,
): Promise<number> => {
let headersSpan: SpanHandle | null = null;
let bodySpan: SpanHandle | null = null;
try {
headersSpan = attemptId
? beginSpan('fetch_headers', { attemptId, proxied })
: null;
const containersRes = await apiFetch(`/stacks/${filename}/containers`, { signal });
const hopProxied = containersRes.headers.get('x-sencho-proxy') === '1' || proxied === true;
if (headersSpan !== null) {
endSpan(headersSpan, { proxied: hopProxied, detail: { status: containersRes.status } });
headersSpan = null;
}
if (signal?.aborted) return 0;
bodySpan = attemptId
? beginSpan('body_decode', { attemptId, proxied: hopProxied })
: null;
const conts = await containersRes.json();
if (bodySpan !== null) {
endSpan(bodySpan);
bodySpan = null;
}
const list = Array.isArray(conts) ? conts : [];
const dispatchSpan = attemptId
? beginSpan('state_dispatch', { attemptId, proxied: hopProxied })
: null;
editorState.setContainers(list);
if (dispatchSpan !== null) endSpan(dispatchSpan);
return list.length;
} catch (error) {
if (headersSpan !== null) endSpan(headersSpan, { outcome: 'error' });
if (bodySpan !== null) endSpan(bodySpan, { outcome: 'error' });
if (isAbortError(error)) return 0;
console.error('Failed to load containers:', error);
editorState.setContainers([]);
return 0;
}
};
const loadBackupState = async (filename: string, signal?: AbortSignal) => {
try {
const backupRes = await apiFetch(`/stacks/${filename}/backup`, { signal });
@@ -764,6 +892,15 @@ export function useStackActions(options: UseStackActionsOptions) {
editorState.setIsEditing(false);
editorState.setEditingCompose(false);
editorState.setActiveTab('compose');
// Clear prior stack health before the first request so compose/env hydration
// never shows another stack's containers or service grouping. Bump the
// containers fetch generation so an in-flight soft refresh cannot rewrite
// this cleared state before loadContainerState claims a newer generation.
editorState.setContainers([]);
editorState.setEffectiveServices([]);
editorState.setContainersLoadError(null);
editorState.setContainersLoadStatus('loading');
containersFetchGenRef.current += 1;
let headersSpan: SpanHandle | null = null;
let bodySpan: SpanHandle | null = null;
try {
@@ -791,13 +928,22 @@ export function useStackActions(options: UseStackActionsOptions) {
detailVisiblePendingRef.current = { attemptId, token: filename, proxied };
setDetailVisibleEpoch((n) => n + 1);
const envFiles = await loadEnvState(filename, signal);
const containerCount = await loadContainerState(filename, signal, attemptId, proxied);
const containersResult = await loadContainerState(filename, signal, attemptId);
if (!signal.aborted) {
detailContainersPendingRef.current = {
attemptId,
token: `${filename}:${containerCount}`,
proxied,
};
if (containersResult.ok) {
detailContainersPendingRef.current = {
attemptId,
token: `${filename}:${containersResult.containers.length}`,
proxied,
};
} else if (containersResult.reason !== 'aborted' && containersResult.reason !== 'stale') {
markMilestone('detail_containers_ready', {
attemptId,
outcome: 'error',
proxied,
detail: { reason: containersResult.reason },
});
}
}
await loadBackupState(filename, signal);
await loadEffectiveServicesState(filename, signal);
@@ -826,6 +972,8 @@ export function useStackActions(options: UseStackActionsOptions) {
editorState.setOriginalEnvContent('');
editorState.setEnvEtag(null);
editorState.setContainers([]);
editorState.setContainersLoadStatus('idle');
editorState.setContainersLoadError(null);
editorState.setEffectiveServices([]);
return { ok: false };
} finally {
@@ -1587,9 +1735,8 @@ export function useStackActions(options: UseStackActionsOptions) {
const label =
action === 'restart' ? 'restarted' : action === 'stop' ? 'stopped' : 'started';
toast.success(`Service "${serviceName}" ${label}`);
const cr = await apiFetch(`/stacks/${stackName}/containers`);
const conts = await cr.json();
editorState.setContainers(Array.isArray(conts) ? conts : []);
const selected = selectedFileRef.current;
if (selected) await refreshSelectedContainers(stackName, selected);
} catch (e) {
console.error(`Failed to ${action} service "${serviceName}":`, e);
toast.error((e as Error).message || `Failed to ${action} service "${serviceName}"`);
@@ -1766,8 +1913,17 @@ export function useStackActions(options: UseStackActionsOptions) {
}
toast.success('Stack deleted successfully!');
overlayState.closeDeleteDialog();
if (stackListState.selectedFile === stackToDelete) {
const selected = stackListState.selectedFile;
const stripExt = (name: string) => name.replace(/\.(yml|yaml)$/, '');
// Always clear a deleted selection, even when another top-level view is
// visible (Resources, Networking, etc.). Leaving that view is gated on
// the editor being the active surface below.
if (selected != null && stripExt(selected) === stripExt(deleteKey)) {
resetEditorState();
if (navState.activeView === 'editor') {
navState.setActiveView('dashboard');
onDeletedOpenStack();
}
}
await stackListState.refreshStacks();
} catch (error) {
@@ -2092,6 +2248,7 @@ export function useStackActions(options: UseStackActionsOptions) {
openStackApp,
resetEditorState,
refreshSelectedContainers,
retryContainersLoad,
refreshGitSourcePending,
loadFile,
loadFileForRoute,
frontend/src/components/EditorLayout/hooks/useStackListState.test.ts
+140
View File
@@ -0,0 +1,140 @@
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { renderHook, act } from '@testing-library/react';
const apiFetchMock = vi.fn();
vi.mock('@/lib/api', () => ({
apiFetch: (...args: unknown[]) => apiFetchMock(...args),
}));
vi.mock('@/components/ui/toast-store', () => ({
toast: { success: vi.fn(), error: vi.fn(), info: vi.fn() },
}));
const useNodesMock = vi.fn();
vi.mock('@/context/NodeContext', () => ({
useNodes: () => useNodesMock(),
}));
import { useStackListState } from './useStackListState';
function okJson(payload: unknown): Response {
return new Response(JSON.stringify(payload), {
status: 200,
headers: { 'Content-Type': 'application/json' },
});
}
function notFound(): Response {
return new Response('not found', { status: 404 });
}
beforeEach(() => {
apiFetchMock.mockReset();
useNodesMock.mockReset();
useNodesMock.mockReturnValue({
activeNode: { id: 1, name: 'Local', type: 'local' },
nodes: [{ id: 1, name: 'Local', type: 'local' }],
});
});
describe('useStackListState.refreshStacks failure classification', () => {
it('rejects a malformed (non-array) successful /stacks response as an error, not confirmed-empty', async () => {
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stacks') return Promise.resolve(okJson({}));
return Promise.resolve(notFound());
});
const { result } = renderHook(() => useStackListState());
await act(async () => {
await result.current.refreshStacks();
});
expect(result.current.stacksLoadStatus).toBe('error');
expect(result.current.files).toEqual([]);
});
it('surfaces a recoverable error on a background failure after the list was already confirmed empty', async () => {
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stacks') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') return Promise.resolve(okJson({}));
return Promise.resolve(notFound());
});
const { result } = renderHook(() => useStackListState());
await act(async () => {
await result.current.refreshStacks();
});
expect(result.current.stacksLoadStatus).toBe('success');
expect(result.current.files).toEqual([]);
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stacks') return Promise.resolve(new Response('fail', { status: 500 }));
return Promise.resolve(notFound());
});
await act(async () => {
await result.current.refreshStacks(true);
});
expect(result.current.stacksLoadStatus).toBe('error');
});
it('preserves a non-empty list on a background failure (soft-refresh semantics unchanged)', async () => {
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stacks') return Promise.resolve(okJson(['web.yml']));
if (endpoint === '/stacks/statuses') return Promise.resolve(okJson({ 'web.yml': { status: 'running' } }));
return Promise.resolve(notFound());
});
const { result } = renderHook(() => useStackListState());
await act(async () => {
await result.current.refreshStacks();
});
expect(result.current.stacksLoadStatus).toBe('success');
expect(result.current.files).toEqual(['web.yml']);
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stacks') return Promise.resolve(new Response('fail', { status: 500 }));
return Promise.resolve(notFound());
});
await act(async () => {
await result.current.refreshStacks(true);
});
expect(result.current.stacksLoadStatus).toBe('success');
expect(result.current.files).toEqual(['web.yml']);
expect(result.current.stacksLoadError).toBe('Could not load stacks (500)');
});
it('keeps a list that just loaded non-empty when the follow-up statuses fetch throws in the same background refresh', async () => {
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stacks') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') return Promise.resolve(okJson({}));
return Promise.resolve(notFound());
});
const { result } = renderHook(() => useStackListState());
await act(async () => {
await result.current.refreshStacks();
});
expect(result.current.files).toEqual([]);
// A background refresh discovers a real stack, but decoding the
// follow-up /stacks/statuses call throws. The just-committed non-empty
// list must survive: only the closure-stale `files` from before this
// call was empty, not the list this attempt just fetched.
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stacks') return Promise.resolve(okJson(['web.yml']));
if (endpoint === '/stacks/statuses') return Promise.reject(new Error('network error'));
return Promise.resolve(notFound());
});
await act(async () => {
await result.current.refreshStacks(true);
});
expect(result.current.files).toEqual(['web.yml']);
});
});
frontend/src/components/EditorLayout/hooks/useStackListState.ts
+31 -20
View File
@@ -234,6 +234,30 @@ export function useStackListState() {
setStacksLoadError(null);
}
// Tracks the most recently committed list for this attempt: `files` (the
// render-time closure) is stale once the list itself has just succeeded
// within this same call, e.g. the list decodes fine but the follow-up
// /stacks/statuses decode then throws. Seeded from `files` so a failure
// that happens before the list ever loads still consults the prior state.
let latestFileList = files;
// Soft (background) failure keeps a non-empty list visible, matching the
// soft-failure handling in applyContainersFetchFailure (useStackActions.ts).
// A list that was already confirmed empty must not stay masquerading as
// empty: it becomes a recoverable error instead, since a soft failure is
// otherwise indistinguishable from "still no stacks".
const applyStacksFailure = (message: string): string[] => {
if (background && hadSuccessfulListRef.current && latestFileList.length > 0) {
setStacksLoadError(message);
return latestFileList;
}
setFiles([]);
setFilesNodeId(fetchNodeId);
setStacksLoadStatus('error');
setStacksLoadError(message);
return [];
};
const headersSpan = beginSpan('fetch_headers', { attemptId, background });
let bodySpan: SpanHandle | null = null;
try {
@@ -242,22 +266,17 @@ export function useStackListState() {
endSpan(headersSpan, { proxied, detail: { status: res.status } });
if (stale()) { abortAttempt(attemptId); return []; }
if (!res.ok) {
const message = `Could not load stacks (${res.status})`;
if (background && hadSuccessfulListRef.current) {
setStacksLoadError(message);
return files;
}
setFiles([]);
setFilesNodeId(fetchNodeId);
setStacksLoadStatus('error');
setStacksLoadError(message);
return [];
return applyStacksFailure(`Could not load stacks (${res.status})`);
}
bodySpan = beginSpan('body_decode', { attemptId, background, proxied });
const data = await res.json();
endSpan(bodySpan);
bodySpan = null;
const fileList: string[] = Array.isArray(data) ? data : [];
if (!Array.isArray(data)) {
return applyStacksFailure('Stack list response was invalid.');
}
const fileList: string[] = data;
latestFileList = fileList;
const listDispatch = beginSpan('state_dispatch', { attemptId, background, proxied });
setFiles(fileList);
setFilesNodeId(fetchNodeId);
@@ -344,15 +363,7 @@ export function useStackListState() {
if (listSucceeded && !background) {
markMilestone('list_hydrated', { attemptId, outcome: 'error', proxied });
}
if (background && hadSuccessfulListRef.current) {
setStacksLoadError(message);
return files;
}
setFiles([]);
setFilesNodeId(fetchNodeId);
setStacksLoadStatus('error');
setStacksLoadError(message);
return [];
return applyStacksFailure(message);
} finally {
setIsLoading(false);
}
frontend/src/components/EditorLayout/hooks/useUrlSync.test.ts
+70
View File
@@ -554,6 +554,76 @@ describe('useUrlSync', () => {
pushSpy.mockRestore();
});
it('writes dashboard URL after leaving a deleted editor stack', () => {
window.history.replaceState({ senchoIdx: 0 }, '', '/nodes/local/stacks/radarr');
const pushSpy = vi.spyOn(window.history, 'pushState');
const { rerender } = renderHook(
(props) => useUrlSync(props),
{
initialProps: makeOpts({
activeView: 'editor',
selectedFile: 'radarr',
files: ['radarr'],
}),
},
);
act(() => {
rerender(makeOpts({
activeView: 'dashboard',
selectedFile: null,
files: [],
}));
});
const paths = [
...pushSpy.mock.calls.map((call) => String(call[2] ?? '')),
window.location.pathname,
];
expect(paths.some((p) => p === '/nodes/local/dashboard')).toBe(true);
expect(window.location.pathname).not.toContain('/stacks/radarr');
pushSpy.mockRestore();
});
it('writes mobile stack-list URL after leaving a deleted editor stack', () => {
window.history.replaceState({ senchoIdx: 0 }, '', '/nodes/local/stacks/radarr');
const pushSpy = vi.spyOn(window.history, 'pushState');
const { rerender } = renderHook(
(props) => useUrlSync(props),
{
initialProps: makeOpts({
activeView: 'editor',
selectedFile: 'radarr',
files: ['radarr'],
isMobile: true,
mobileSurface: 'detail',
}),
},
);
act(() => {
rerender(makeOpts({
activeView: 'dashboard',
selectedFile: null,
files: [],
isMobile: true,
mobileSurface: 'list',
}));
});
const paths = [
...pushSpy.mock.calls.map((call) => String(call[2] ?? '')),
window.location.pathname,
];
expect(paths.some((p) => p === '/nodes/local/stacks')).toBe(true);
expect(window.location.pathname).not.toContain('/stacks/radarr');
pushSpy.mockRestore();
});
it('opens Monaco when hydrating /compose deep link', async () => {
const loadFileForRoute = vi.fn().mockResolvedValue({ ok: true, envFiles: [] });
const applyEditorRouteState = vi.fn();
frontend/src/components/EditorLayout/operation-phase.ts
+1 -1
View File
@@ -38,7 +38,7 @@ export function classifyOperationPhase(rows: ParsedLogRow[], action: ActionVerb)
if (stage === 'BUILD') {
return 'Building images';
}
if (message.includes('Backup created for atomic') || message.includes('Cleaning up existing containers')) {
if (message.includes('Backup created for atomic') || message.includes('Validating stack for update') || message.includes('Capturing rollback generation')) {
return 'Preparing';
}
}
frontend/src/components/HomeDashboard.tsx
+3
View File
@@ -48,6 +48,9 @@ export default function HomeDashboard({ onNavigateToStack, onOpenSettingsSection
<StackHealthTable
stackStatuses={data.stackStatuses}
stackStatusesLoadStatus={data.stackStatusesLoadStatus}
stackStatusesLoadError={data.stackStatusesLoadError}
onRetryStackStatuses={data.retryStackStatuses}
metrics={data.metrics}
stackCpuSeries={data.stackCpuSeries}
onNavigateToStack={onNavigateToStack ?? NOOP}
frontend/src/components/blueprints/RolloutPreviewDialog.tsx
+2
View File
@@ -78,6 +78,8 @@ export function RolloutPreviewDialog({
const { failed = 0, pending = 0 } = result.outcomeSummary ?? {};
if (failed > 0) {
toast.warning(result.message || 'Rollout confirmed with node failures');
} else if (result.effectiveApproval !== 'approved') {
toast.warning(result.message || 'Confirmed snapshot finished; approval is no longer current');
} else if (pending > 0) {
toast.info(result.message || 'Rollout confirmed; some actions are still in progress');
} else {
frontend/src/components/dashboard/StackHealthTable.tsx
+39 -2
View File
@@ -1,9 +1,10 @@
import { useEffect, useMemo, useState } from 'react';
import { Button } from '@/components/ui/button';
import { Sparkline } from '@/components/ui/sparkline';
import { ArrowUp, ArrowDown, ChevronLeft, ChevronRight, Layers } from 'lucide-react';
import { AlertCircle, ArrowUp, ArrowDown, ChevronLeft, ChevronRight, Layers, RefreshCw } from 'lucide-react';
import { cn } from '@/lib/utils';
import type { StackStatusEntry, MetricPoint, StackCpuSeries } from './types';
import { Skeleton } from '@/components/ui/skeleton';
import type { StackStatusEntry, MetricPoint, StackCpuSeries, StackStatusesLoadStatus } from './types';
import type { StackUpdateInfo } from '@/types/imageUpdates';
import { aggregateCurrentUsage } from './aggregateCurrentUsage';
import { classifyRow, type RowState } from './classifyRow';
@@ -11,6 +12,9 @@ import { updateAvailableBadge, updateAvailableLabel } from '@/lib/updateAvailabl
interface StackHealthTableProps {
stackStatuses: Record<string, StackStatusEntry>;
stackStatusesLoadStatus: StackStatusesLoadStatus;
stackStatusesLoadError: string | null;
onRetryStackStatuses?: () => void;
metrics: MetricPoint[];
stackCpuSeries: Record<string, StackCpuSeries>;
onNavigateToStack: (stackFile: string) => void;
@@ -84,6 +88,9 @@ const sparkStroke: Record<RowState, string> = {
export function StackHealthTable({
stackStatuses,
stackStatusesLoadStatus,
stackStatusesLoadError,
onRetryStackStatuses,
metrics,
stackCpuSeries,
onNavigateToStack,
@@ -173,6 +180,36 @@ export function StackHealthTable({
const stackCount = Object.keys(stackStatuses).length;
if (stackStatusesLoadStatus === 'idle' || stackStatusesLoadStatus === 'loading') {
return (
<div className="rounded-lg border border-card-border border-t-card-border-top bg-card shadow-card-bevel p-5 space-y-3">
<Skeleton className="h-6 w-40" />
<Skeleton className="h-10 w-full" />
<Skeleton className="h-10 w-full" />
<Skeleton className="h-10 w-full" />
</div>
);
}
if (stackStatusesLoadStatus === 'error') {
return (
<div className="rounded-lg border border-card-border border-t-card-border-top bg-card shadow-card-bevel py-10">
<div className="flex flex-col items-center justify-center gap-3 text-stat-subtitle">
<AlertCircle className="h-8 w-8 text-stat-icon" strokeWidth={1.5} aria-hidden />
<p className="text-sm text-center px-4">
{stackStatusesLoadError ?? 'Could not load stack health.'}
</p>
{onRetryStackStatuses && (
<Button type="button" variant="outline" size="sm" onClick={onRetryStackStatuses}>
<RefreshCw className="w-4 h-4" />
Retry
</Button>
)}
</div>
</div>
);
}
if (stackCount === 0) {
return (
<div className="rounded-lg border border-card-border border-t-card-border-top bg-card shadow-card-bevel py-10">
frontend/src/components/dashboard/__tests__/StackHealthTable.loadStates.test.tsx
+52
View File
@@ -0,0 +1,52 @@
import { describe, it, expect, vi } from 'vitest';
import { render, screen } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { StackHealthTable } from '../StackHealthTable';
describe('StackHealthTable load states', () => {
it('does not show empty copy while loading', () => {
render(
<StackHealthTable
stackStatuses={{}}
stackStatusesLoadStatus="loading"
stackStatusesLoadError={null}
metrics={[]}
stackCpuSeries={{}}
onNavigateToStack={vi.fn()}
/>,
);
expect(screen.queryByText(/No stacks found/i)).toBeNull();
});
it('shows empty copy only after success', () => {
render(
<StackHealthTable
stackStatuses={{}}
stackStatusesLoadStatus="success"
stackStatusesLoadError={null}
metrics={[]}
stackCpuSeries={{}}
onNavigateToStack={vi.fn()}
/>,
);
expect(screen.getByText(/No stacks found/i)).toBeInTheDocument();
});
it('shows retry on error', async () => {
const onRetry = vi.fn();
const user = userEvent.setup();
render(
<StackHealthTable
stackStatuses={{}}
stackStatusesLoadStatus="error"
stackStatusesLoadError="Could not load stack health."
onRetryStackStatuses={onRetry}
metrics={[]}
stackCpuSeries={{}}
onNavigateToStack={vi.fn()}
/>,
);
await user.click(screen.getByRole('button', { name: /retry/i }));
expect(onRetry).toHaveBeenCalledTimes(1);
});
});
frontend/src/components/dashboard/__tests__/useDashboardData.test.tsx
+278 -7
View File
@@ -14,17 +14,16 @@ vi.mock('@/context/NodeContext', () => ({
useNodes: () => useNodesMock(),
}));
// `visibilityInterval` from the live utils library uses
// `document.visibilityState`, which jsdom treats as `prerender` until a
// listener is attached. The polling tests below assert mount-time fetches and
// the debounced refetch path; the long-running interval ticks themselves are
// covered by the existing useNextAutoUpdateRun suite. Replace with a no-op
// cleanup so the hook does not retain a real timer across tests.
// Statuses soft-poll uses visibilityInterval (setInterval). Use a real timer so
// tests can advance past the 10s cadence; skip document.visibility wiring.
vi.mock('@/lib/utils', async () => {
const actual = await vi.importActual<typeof import('@/lib/utils')>('@/lib/utils');
return {
...actual,
visibilityInterval: () => () => {},
visibilityInterval: (fn: () => void, ms: number) => {
const id = setInterval(fn, ms);
return () => clearInterval(id);
},
};
});
@@ -112,3 +111,275 @@ describe('useDashboardData state-invalidate handling', () => {
expect(apiFetchMock).not.toHaveBeenCalled();
});
});
describe('useDashboardData stackStatuses load states', () => {
it('reaches success with an empty map without treating deferral as empty UI state', async () => {
let resolveStatuses: ((r: Response) => void) | null = null;
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
if (endpoint === '/system/stats') return Promise.resolve(okJson(SYS_PAYLOAD));
if (endpoint === '/metrics/historical') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') {
return new Promise<Response>((resolve) => { resolveStatuses = resolve; });
}
return Promise.resolve(okJson(null));
});
const { result } = renderHook(() => useDashboardData());
await act(async () => { await Promise.resolve(); });
expect(result.current.stackStatusesLoadStatus).toBe('loading');
await act(async () => {
resolveStatuses?.(okJson({}));
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatusesLoadStatus).toBe('success');
expect(result.current.stackStatuses).toEqual({});
});
it('surfaces error on failed statuses fetch and recovers on retry', async () => {
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
if (endpoint === '/system/stats') return Promise.resolve(okJson(SYS_PAYLOAD));
if (endpoint === '/metrics/historical') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') {
return Promise.resolve(new Response('nope', { status: 500 }));
}
return Promise.resolve(okJson(null));
});
const { result } = renderHook(() => useDashboardData());
await act(async () => { await Promise.resolve(); await Promise.resolve(); });
expect(result.current.stackStatusesLoadStatus).toBe('error');
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
if (endpoint === '/system/stats') return Promise.resolve(okJson(SYS_PAYLOAD));
if (endpoint === '/metrics/historical') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') return Promise.resolve(okJson({}));
return Promise.resolve(okJson(null));
});
await act(async () => {
result.current.retryStackStatuses();
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatusesLoadStatus).toBe('success');
});
it('ignores an older soft success after a newer foreground retry', async () => {
const resolvers: Array<(r: Response) => void> = [];
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
if (endpoint === '/system/stats') return Promise.resolve(okJson(SYS_PAYLOAD));
if (endpoint === '/metrics/historical') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') {
return new Promise<Response>((resolve) => { resolvers.push(resolve); });
}
return Promise.resolve(okJson(null));
});
const { result } = renderHook(() => useDashboardData());
await act(async () => { await Promise.resolve(); });
expect(resolvers).toHaveLength(1);
await act(async () => {
resolvers[0](okJson({}));
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatusesLoadStatus).toBe('success');
act(() => { fireInvalidate({ scope: 'container' }); });
await act(async () => { vi.advanceTimersByTime(300); });
expect(resolvers).toHaveLength(2);
await act(async () => {
result.current.retryStackStatuses();
await Promise.resolve();
});
expect(resolvers).toHaveLength(3);
const softMap = { 'old.yml': { status: 'exited' as const } };
const retryMap = { 'web.yml': { status: 'running' as const } };
await act(async () => {
resolvers[1](okJson(softMap));
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatuses).toEqual({});
await act(async () => {
resolvers[2](okJson(retryMap));
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatusesLoadStatus).toBe('success');
expect(result.current.stackStatuses).toEqual(retryMap);
});
it('ignores an older soft failure after a newer foreground retry success', async () => {
const resolvers: Array<(r: Response) => void> = [];
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
if (endpoint === '/system/stats') return Promise.resolve(okJson(SYS_PAYLOAD));
if (endpoint === '/metrics/historical') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') {
return new Promise<Response>((resolve) => { resolvers.push(resolve); });
}
return Promise.resolve(okJson(null));
});
const { result } = renderHook(() => useDashboardData());
await act(async () => { await Promise.resolve(); });
expect(resolvers).toHaveLength(1);
await act(async () => {
resolvers[0](okJson({}));
await Promise.resolve();
await Promise.resolve();
});
act(() => { fireInvalidate({ scope: 'container' }); });
await act(async () => { vi.advanceTimersByTime(300); });
expect(resolvers).toHaveLength(2);
await act(async () => {
result.current.retryStackStatuses();
await Promise.resolve();
});
expect(resolvers).toHaveLength(3);
const retryMap = { 'web.yml': { status: 'running' as const } };
await act(async () => {
resolvers[2](okJson(retryMap));
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatuses).toEqual(retryMap);
await act(async () => {
resolvers[1](new Response('nope', { status: 500 }));
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatusesLoadStatus).toBe('success');
expect(result.current.stackStatuses).toEqual(retryMap);
});
it('lets a slow foreground statuses response commit after soft poll and invalidate ticks', async () => {
const resolvers: Array<(r: Response) => void> = [];
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
if (endpoint === '/system/stats') return Promise.resolve(okJson(SYS_PAYLOAD));
if (endpoint === '/metrics/historical') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') {
return new Promise<Response>((resolve) => { resolvers.push(resolve); });
}
return Promise.resolve(okJson(null));
});
const { result } = renderHook(() => useDashboardData());
await act(async () => { await Promise.resolve(); });
expect(resolvers).toHaveLength(1);
expect(result.current.stackStatusesLoadStatus).toBe('loading');
act(() => { fireInvalidate({ scope: 'container' }); });
await act(async () => { vi.advanceTimersByTime(300); });
expect(resolvers).toHaveLength(1);
await act(async () => { vi.advanceTimersByTime(10000); });
expect(resolvers).toHaveLength(1);
await act(async () => { vi.advanceTimersByTime(10000); });
expect(resolvers).toHaveLength(1);
const settled = { 'web.yml': { status: 'running' as const } };
await act(async () => {
resolvers[0](okJson(settled));
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatusesLoadStatus).toBe('success');
expect(result.current.stackStatuses).toEqual(settled);
});
it('turns a soft poll failure into a recoverable error when the prior success was empty', async () => {
const resolvers: Array<(r: Response) => void> = [];
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
if (endpoint === '/system/stats') return Promise.resolve(okJson(SYS_PAYLOAD));
if (endpoint === '/metrics/historical') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') {
return new Promise<Response>((resolve) => { resolvers.push(resolve); });
}
return Promise.resolve(okJson(null));
});
const { result } = renderHook(() => useDashboardData());
await act(async () => { await Promise.resolve(); });
expect(resolvers).toHaveLength(1);
// Foreground load settles on confirmed-empty.
await act(async () => {
resolvers[0](okJson({}));
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatusesLoadStatus).toBe('success');
expect(result.current.stackStatuses).toEqual({});
// A subsequent soft poll fails: this must not stay a silent empty state.
act(() => { fireInvalidate({ scope: 'container' }); });
await act(async () => { vi.advanceTimersByTime(300); });
expect(resolvers).toHaveLength(2);
await act(async () => {
resolvers[1](new Response('nope', { status: 500 }));
await Promise.resolve();
await Promise.resolve();
});
expect(result.current.stackStatusesLoadStatus).toBe('error');
});
it('drops a malformed per-stack entry instead of crashing, keeping the rest of a valid response', async () => {
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
if (endpoint === '/system/stats') return Promise.resolve(okJson(SYS_PAYLOAD));
if (endpoint === '/metrics/historical') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') {
return Promise.resolve(okJson({
'web.yml': { status: 'running' },
'broken.yml': null,
'also-broken.yml': 'running',
}));
}
return Promise.resolve(okJson(null));
});
const { result } = renderHook(() => useDashboardData());
await act(async () => { await Promise.resolve(); await Promise.resolve(); });
expect(result.current.stackStatusesLoadStatus).toBe('success');
expect(result.current.stackStatuses).toEqual({ 'web.yml': { status: 'running' } });
});
it('treats a non-empty response where every entry is malformed as an error, not confirmed-empty', async () => {
apiFetchMock.mockImplementation((endpoint: string) => {
if (endpoint === '/stats') return Promise.resolve(okJson(STATS_PAYLOAD));
if (endpoint === '/system/stats') return Promise.resolve(okJson(SYS_PAYLOAD));
if (endpoint === '/metrics/historical') return Promise.resolve(okJson([]));
if (endpoint === '/stacks/statuses') {
return Promise.resolve(okJson({ 'a.yml': null, 'b.yml': 'running' }));
}
return Promise.resolve(okJson(null));
});
const { result } = renderHook(() => useDashboardData());
await act(async () => { await Promise.resolve(); await Promise.resolve(); });
expect(result.current.stackStatusesLoadStatus).toBe('error');
expect(result.current.stackStatuses).toEqual({});
});
});
frontend/src/components/dashboard/types.ts
+5
View File
@@ -96,11 +96,16 @@ export interface StackCpuSeries {
latestValue: number;
}
export type StackStatusesLoadStatus = 'idle' | 'loading' | 'success' | 'error';
export interface DashboardData {
stats: Stats;
systemStats: SystemStats | null;
metrics: MetricPoint[];
stackStatuses: Record<string, StackStatusEntry>;
stackStatusesLoadStatus: StackStatusesLoadStatus;
stackStatusesLoadError: string | null;
retryStackStatuses: () => void;
lastSyncAt: number | null;
nodeCount: number;
stackCpuSeries: Record<string, StackCpuSeries>;
frontend/src/components/dashboard/useDashboardData.ts
+170 -13
View File
@@ -9,6 +9,7 @@ import type {
StackStatusEntry,
DashboardData,
StackCpuSeries,
StackStatusesLoadStatus,
} from './types';
const DEFAULT_STATS: Stats = { active: 0, managed: 0, unmanaged: 0, exited: 0, total: 0 };
@@ -90,6 +91,20 @@ export function buildNetHistory(
// is chosen so a single transient hiccup does not trip the indicator.
const METRICS_STALE_THRESHOLD = 3;
const VALID_STACK_STATUS_VALUES = new Set(['running', 'exited', 'unknown', 'partial']);
// A malformed per-stack entry (null, a bare string, or an object missing
// `status`) must never reach the table renderer, which indexes straight into
// `entry.status` and other fields without a null check.
function isValidStatusEntry(value: unknown): value is StackStatusEntry {
return (
!!value
&& typeof value === 'object'
&& !Array.isArray(value)
&& VALID_STACK_STATUS_VALUES.has((value as { status?: unknown }).status as string)
);
}
export function useDashboardData(): DashboardData {
const { activeNode, nodes } = useNodes();
const nodeId = activeNode?.id;
@@ -98,6 +113,8 @@ export function useDashboardData(): DashboardData {
const [systemStats, setSystemStats] = useState<SystemStats | null>(null);
const [metrics, setMetrics] = useState<MetricPoint[]>([]);
const [stackStatuses, setStackStatuses] = useState<Record<string, StackStatusEntry>>({});
const [stackStatusesLoadStatus, setStackStatusesLoadStatus] = useState<StackStatusesLoadStatus>('idle');
const [stackStatusesLoadError, setStackStatusesLoadError] = useState<string | null>(null);
const [lastSyncAt, setLastSyncAt] = useState<number | null>(null);
const [metricsStale, setMetricsStale] = useState(false);
@@ -106,6 +123,27 @@ export function useDashboardData(): DashboardData {
const nodeIdRef = useRef(nodeId);
useEffect(() => { nodeIdRef.current = nodeId; }, [nodeId]);
// Whether the last committed success held a non-empty map. Soft poll failures
// keep the prior map only in that case; a confirmed-empty fleet must surface a
// recoverable error instead. Set from each committed success and reset on node
// change, so commitStackStatusesFailure (a useCallback that does not depend on
// stackStatuses) can read it without the map identity.
const hadNonEmptyStatusesRef = useRef(false);
// Latest-request arbitration for /stacks/statuses: polling, invalidation,
// mount, and Retry can overlap; only the current generation may commit.
const stackStatusesFetchGenRef = useRef(0);
// Soft poll/invalidation must not start while any statuses request is in
// flight. Fixed-interval ticks would otherwise bump generation forever and
// starve a slow foreground hydration. Foreground (mount/retry/node change)
// always starts and supersedes obsolete work.
const stackStatusesInFlightRef = useRef(false);
useEffect(() => {
hadNonEmptyStatusesRef.current = false;
}, [nodeId]);
useEffect(() => () => {
stackStatusesFetchGenRef.current += 1;
}, []);
// Consecutive failure counters per live-metrics endpoint. Either reaching
// METRICS_STALE_THRESHOLD trips the metricsStale indicator; the first
// successful response on the failing endpoint clears its own counter and,
@@ -190,19 +228,136 @@ export function useDashboardData(): DashboardData {
return cleanup;
}, [nodeId, fetchJson]);
// Stack statuses: 10s polling, resets on node change
// Stack statuses: 10s polling, resets on node change. Foreground / retry
// expose loading and recoverable error; soft poll failures after success keep
// the prior map so the dashboard never flashes a false empty state.
const isCurrentStatusesFetch = useCallback((
currentNodeId: number | undefined,
generation: number,
) => (
nodeIdRef.current === currentNodeId
&& stackStatusesFetchGenRef.current === generation
), []);
const commitStackStatusesSuccess = useCallback((
currentNodeId: number | undefined,
generation: number,
data: Record<string, StackStatusEntry>,
) => {
if (!isCurrentStatusesFetch(currentNodeId, generation)) return;
setStackStatuses(data);
setStackStatusesLoadStatus('success');
setStackStatusesLoadError(null);
hadNonEmptyStatusesRef.current = Object.keys(data).length > 0;
}, [isCurrentStatusesFetch]);
const commitStackStatusesFailure = useCallback((
currentNodeId: number | undefined,
generation: number,
mode: 'foreground' | 'soft',
failureMessage: string,
) => {
if (!isCurrentStatusesFetch(currentNodeId, generation)) return;
// Soft: prior non-empty rows stay visible on a transient failure. Prior
// confirmed-empty becomes a recoverable error so a soft failure can never
// look identical to "no stacks".
if (mode === 'soft' && hadNonEmptyStatusesRef.current) return;
setStackStatusesLoadStatus('error');
setStackStatusesLoadError(failureMessage);
}, [isCurrentStatusesFetch]);
const fetchStackStatuses = useCallback(async (
currentNodeId: number | undefined,
mode: 'foreground' | 'soft',
) => {
if (nodeIdRef.current !== currentNodeId) return;
if (mode === 'soft' && stackStatusesInFlightRef.current) return;
const generation = ++stackStatusesFetchGenRef.current;
stackStatusesInFlightRef.current = true;
if (mode === 'foreground') {
setStackStatusesLoadStatus('loading');
setStackStatusesLoadError(null);
}
try {
const res = await apiFetch('/stacks/statuses');
if (!isCurrentStatusesFetch(currentNodeId, generation)) return;
if (!res.ok) {
commitStackStatusesFailure(
currentNodeId,
generation,
mode,
`Could not load stack health (${res.status}).`,
);
return;
}
const body: unknown = await res.json();
if (!isCurrentStatusesFetch(currentNodeId, generation)) return;
if (body && typeof body === 'object' && !Array.isArray(body)) {
// Drop any entry isValidStatusEntry rejects rather than trusting the
// whole map: one bad entry must not crash or misrepresent the rest of
// a valid response.
const rawEntries = Object.entries(body as Record<string, unknown>);
const sanitized: Record<string, StackStatusEntry> = {};
for (const [file, entry] of rawEntries) {
if (isValidStatusEntry(entry)) {
sanitized[file] = entry;
} else {
console.error('[Dashboard] Dropped malformed stack status entry:', file, entry);
}
}
// A non-empty map where every entry failed validation is a malformed
// response, not a confirmed-empty fleet: committing it as success
// would be indistinguishable from a genuine empty fleet.
if (rawEntries.length > 0 && Object.keys(sanitized).length === 0) {
commitStackStatusesFailure(
currentNodeId,
generation,
mode,
'Stack health response was invalid.',
);
return;
}
commitStackStatusesSuccess(currentNodeId, generation, sanitized);
return;
}
commitStackStatusesFailure(
currentNodeId,
generation,
mode,
'Stack health response was invalid.',
);
} catch {
if (!isCurrentStatusesFetch(currentNodeId, generation)) return;
commitStackStatusesFailure(
currentNodeId,
generation,
mode,
'Could not load stack health.',
);
} finally {
// Only the latest generation clears the gate. A superseded request that
// finishes later must not reopen soft polling while a newer fetch is live.
if (stackStatusesFetchGenRef.current === generation) {
stackStatusesInFlightRef.current = false;
}
}
}, [commitStackStatusesSuccess, commitStackStatusesFailure, isCurrentStatusesFetch]);
const retryStackStatuses = useCallback(() => {
void fetchStackStatuses(nodeIdRef.current, 'foreground');
}, [fetchStackStatuses]);
useEffect(() => {
setStackStatuses({}); // eslint-disable-line react-hooks/set-state-in-effect
setStackStatusesLoadStatus('loading');
setStackStatusesLoadError(null);
const currentNodeId = nodeId;
const fetchStatuses = async () => {
if (nodeIdRef.current !== currentNodeId) return;
const data = await fetchJson<Record<string, StackStatusEntry>>('/stacks/statuses');
if (data && nodeIdRef.current === currentNodeId) setStackStatuses(data);
};
fetchStatuses();
const cleanup = visibilityInterval(fetchStatuses, 10000);
void fetchStackStatuses(currentNodeId, 'foreground');
const cleanup = visibilityInterval(() => {
void fetchStackStatuses(currentNodeId, 'soft');
}, 10000);
return cleanup;
}, [nodeId, fetchJson]);
}, [nodeId, fetchStackStatuses]);
// React to live `state-invalidate` signals from /ws/notifications: when a
// Docker container event fires (start/stop/die/restart/health), the layout
@@ -219,10 +374,9 @@ export function useDashboardData(): DashboardData {
let invalidateTimer: ReturnType<typeof setTimeout> | null = null;
const refresh = async () => {
if (!active || nodeIdRef.current !== currentNodeId) return;
const [statsData, sysData, statusesData] = await Promise.all([
const [statsData, sysData] = await Promise.all([
fetchJson<Stats>('/stats'),
fetchJson<SystemStats>('/system/stats'),
fetchJson<Record<string, StackStatusEntry>>('/stacks/statuses'),
]);
// Re-check after the await: an unmount or node switch may have
// happened while the fetches were in flight, in which case the
@@ -233,7 +387,7 @@ export function useDashboardData(): DashboardData {
setLastSyncAt(Date.now());
}
if (sysData) setSystemStats(sysData);
if (statusesData) setStackStatuses(statusesData);
await fetchStackStatuses(currentNodeId, 'soft');
};
const onInvalidate = () => {
if (!active || nodeIdRef.current !== currentNodeId) return;
@@ -249,7 +403,7 @@ export function useDashboardData(): DashboardData {
window.removeEventListener('sencho:state-invalidate', onInvalidate);
if (invalidateTimer) clearTimeout(invalidateTimer);
};
}, [nodeId, fetchJson]);
}, [nodeId, fetchJson, fetchStackStatuses]);
const stackCpuSeries = useMemo<Record<string, StackCpuSeries>>(() => {
if (metrics.length === 0) return {};
@@ -337,6 +491,9 @@ export function useDashboardData(): DashboardData {
systemStats,
metrics,
stackStatuses,
stackStatusesLoadStatus,
stackStatusesLoadError,
retryStackStatuses,
lastSyncAt,
nodeCount: nodes.length,
stackCpuSeries,
frontend/src/components/mobile/MobileDashboard.tsx
+58 -32
View File
@@ -141,6 +141,63 @@ export function MobileDashboard({ notifications, headerActions, onNavigateToStac
? `avg ${cpuAvg.toFixed(0)}% last 10m · peak ${cpuPeak.toFixed(0)}%${cpuPeakLabel ? ` @ ${cpuPeakLabel}` : ''}`
: 'collecting metrics…';
let stackHealthBody: ReactNode;
if (data.stackStatusesLoadStatus === 'idle' || data.stackStatusesLoadStatus === 'loading') {
stackHealthBody = (
<p className="px-1 py-4 font-mono text-[12px] text-stat-subtitle">Loading stacks…</p>
);
} else if (data.stackStatusesLoadStatus === 'error') {
stackHealthBody = (
<div className="flex flex-col items-start gap-2 px-1 py-4">
<p className="font-mono text-[12px] text-stat-subtitle">
{data.stackStatusesLoadError ?? 'Could not load stack health.'}
</p>
<button
type="button"
onClick={data.retryStackStatuses}
className="font-mono text-[12px] text-brand underline-offset-2 hover:underline"
>
Retry
</button>
</div>
);
} else if (visibleRows.length === 0) {
stackHealthBody = (
<p className="px-1 py-4 font-mono text-[12px] text-stat-subtitle">No stacks yet.</p>
);
} else {
stackHealthBody = (
<div className="flex flex-col gap-px">
{visibleRows.map(row => (
<button
key={row.file}
type="button"
onClick={() => onNavigateToStack(row.file)}
className={`flex min-h-11 items-center gap-2.5 rounded-[7px] px-2.5 py-[9px] text-left ${ROW_TINT[row.state]}`}
>
<StateDot tone={ROW_TONE[row.state]} size={7} glow={row.state !== 'healthy'} />
<span className="min-w-0 flex-1">
<span className="block truncate font-mono text-[13px] text-stat-value">{row.name}</span>
<span className="block truncate font-mono text-[10px] text-stat-icon">{activeNodeName}</span>
</span>
<span className="block h-[18px] w-[60px] shrink-0">
{row.points.length > 1 ? (
<MSparkline values={row.points} height={18} color={ROW_STROKE[row.state]} peak={false} />
) : (
<span className="block h-full w-full border-b border-dashed border-hairline" />
)}
</span>
<span
className={`w-[34px] shrink-0 text-right font-mono tabular-nums text-[12px] ${row.cpu >= 80 ? 'text-warning' : 'text-stat-subtitle'}`}
>
{`${row.cpu.toFixed(0)}%`}
</span>
</button>
))}
</div>
);
}
return (
<div className="flex h-full min-h-0 flex-col">
<Masthead
@@ -202,38 +259,7 @@ export function MobileDashboard({ notifications, headerActions, onNavigateToStac
<SectionHead right={<button type="button" onClick={onViewAllStacks} className="text-brand">view all →</button>}>
stack health
</SectionHead>
{visibleRows.length === 0 ? (
<p className="px-1 py-4 font-mono text-[12px] text-stat-subtitle">No stacks yet.</p>
) : (
<div className="flex flex-col gap-px">
{visibleRows.map(row => (
<button
key={row.file}
type="button"
onClick={() => onNavigateToStack(row.file)}
className={`flex min-h-11 items-center gap-2.5 rounded-[7px] px-2.5 py-[9px] text-left ${ROW_TINT[row.state]}`}
>
<StateDot tone={ROW_TONE[row.state]} size={7} glow={row.state !== 'healthy'} />
<span className="min-w-0 flex-1">
<span className="block truncate font-mono text-[13px] text-stat-value">{row.name}</span>
<span className="block truncate font-mono text-[10px] text-stat-icon">{activeNodeName}</span>
</span>
<span className="block h-[18px] w-[60px] shrink-0">
{row.points.length > 1 ? (
<MSparkline values={row.points} height={18} color={ROW_STROKE[row.state]} peak={false} />
) : (
<span className="block h-full w-full border-b border-dashed border-hairline" />
)}
</span>
<span
className={`w-[34px] shrink-0 text-right font-mono tabular-nums text-[12px] ${row.cpu >= 80 ? 'text-warning' : 'text-stat-subtitle'}`}
>
{`${row.cpu.toFixed(0)}%`}
</span>
</button>
))}
</div>
)}
{stackHealthBody}
</div>
</div>
</div>
frontend/src/components/settings/NotificationSuppressionSection.tsx
+179
View File
@@ -40,10 +40,42 @@ interface NotificationSuppressionRule {
applies_to: AppliesTo;
enabled: boolean;
expires_at: number | null;
schedule: MuteRuleSchedule | null;
scheduleInvalid: boolean;
created_at: number;
updated_at: number;
}
type MuteRuleSchedule = {
days: number[];
start_minute: number;
end_minute: number;
tz: 'UTC';
};
const DAY_LABELS = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'] as const;
function minuteToTimeInput(minute: number): string {
const h = Math.floor(minute / 60);
const m = minute % 60;
return `${String(h).padStart(2, '0')}:${String(m).padStart(2, '0')}`;
}
function timeInputToMinute(value: string): number | null {
const match = /^(\d{2}):(\d{2})(?::\d{2})?$/.exec(value);
if (!match) return null;
const h = Number(match[1]);
const m = Number(match[2]);
if (h > 23 || m > 59) return null;
return h * 60 + m;
}
function formatScheduleSummary(schedule: MuteRuleSchedule | null): string | null {
if (!schedule) return null;
const days = schedule.days.map((d) => DAY_LABELS[d] ?? String(d)).join(', ');
return `UTC ${days} ${minuteToTimeInput(schedule.start_minute)}-${minuteToTimeInput(schedule.end_minute)}`;
}
const LEVEL_LABELS: Record<NotificationLevel, string> = {
info: 'Info',
warning: 'Warning',
@@ -127,6 +159,15 @@ export function NotificationSuppressionSection({
const [formEnabled, setFormEnabled] = useState(true);
const [formExpirationPreset, setFormExpirationPreset] = useState<ExpirationPreset>('forever');
const [formCustomExpiry, setFormCustomExpiry] = useState('');
const [formScheduleEnabled, setFormScheduleEnabled] = useState(false);
const [formScheduleDays, setFormScheduleDays] = useState<number[]>([]);
const [formScheduleStart, setFormScheduleStart] = useState('02:00');
const [formScheduleEnd, setFormScheduleEnd] = useState('06:00');
// True when editing a rule whose stored schedule could not be read. Its window
// must not be silently cleared: the operator has to touch the controls first.
const [formScheduleInvalid, setFormScheduleInvalid] = useState(false);
const [formScheduleTouched, setFormScheduleTouched] = useState(false);
const markScheduleTouched = () => setFormScheduleTouched(true);
const fetchRules = useCallback(async () => {
try {
@@ -176,6 +217,12 @@ export function NotificationSuppressionSection({
});
setFormExpirationPreset('forever');
setFormCustomExpiry('');
setFormScheduleEnabled(false);
setFormScheduleDays([]);
setFormScheduleStart('02:00');
setFormScheduleEnd('06:00');
setFormScheduleInvalid(false);
setFormScheduleTouched(false);
setEditingId(null);
setShowForm(true);
onPrefillConsumed?.();
@@ -192,6 +239,12 @@ export function NotificationSuppressionSection({
setFormEnabled(true);
setFormExpirationPreset('forever');
setFormCustomExpiry('');
setFormScheduleEnabled(false);
setFormScheduleDays([]);
setFormScheduleStart('02:00');
setFormScheduleEnd('06:00');
setFormScheduleInvalid(false);
setFormScheduleTouched(false);
setEditingId(null);
setShowForm(false);
};
@@ -209,11 +262,30 @@ export function NotificationSuppressionSection({
setFormEnabled(rule.enabled);
setFormExpirationPreset(preset);
setFormCustomExpiry(customMs != null ? new Date(customMs).toISOString().slice(0, 16) : '');
if (rule.schedule) {
setFormScheduleEnabled(true);
setFormScheduleDays([...rule.schedule.days]);
setFormScheduleStart(minuteToTimeInput(rule.schedule.start_minute));
setFormScheduleEnd(minuteToTimeInput(rule.schedule.end_minute));
} else {
setFormScheduleEnabled(false);
setFormScheduleDays([]);
setFormScheduleStart('02:00');
setFormScheduleEnd('06:00');
}
setFormScheduleInvalid(rule.scheduleInvalid);
setFormScheduleTouched(false);
setShowForm(true);
};
const handleSave = async () => {
if (!formName.trim()) { toast.error('Name is required.'); return; }
if (formScheduleInvalid && !formScheduleTouched) {
toast.error(
"This rule's stored weekly window could not be read. Turn the weekly window on to set a new schedule, or toggle it on then off to confirm clearing it.",
);
return;
}
const preparedPatterns = patternChipsRef.current?.prepareSave();
if (!preparedPatterns?.ok) {
toast.error('Fix invalid stack patterns before saving.');
@@ -225,6 +297,30 @@ export function NotificationSuppressionSection({
return;
}
let schedule: MuteRuleSchedule | null = null;
if (formScheduleEnabled) {
if (formScheduleDays.length === 0) {
toast.error('Select at least one day for the weekly window.');
return;
}
const startMinute = timeInputToMinute(formScheduleStart);
const endMinute = timeInputToMinute(formScheduleEnd);
if (startMinute == null || endMinute == null) {
toast.error('Enter valid UTC start and end times.');
return;
}
if (startMinute === endMinute) {
toast.error('Weekly window start and end must differ.');
return;
}
schedule = {
days: [...new Set(formScheduleDays)].sort((a, b) => a - b),
start_minute: startMinute,
end_minute: endMinute,
tz: 'UTC',
};
}
setSaving(true);
try {
const body = {
@@ -237,6 +333,7 @@ export function NotificationSuppressionSection({
applies_to: formAppliesTo,
enabled: formEnabled,
expires_at: expirationFromPreset(formExpirationPreset, customMs),
schedule,
};
const url = editingId
@@ -483,6 +580,77 @@ export function NotificationSuppressionSection({
)}
</div>
<div className="space-y-2">
<div className="flex items-center gap-2">
<TogglePill
checked={formScheduleEnabled}
onChange={(v) => { setFormScheduleEnabled(v); markScheduleTouched(); }}
id="mute-rule-schedule"
/>
<Label htmlFor="mute-rule-schedule" className="mb-0">Weekly window (UTC)</Label>
</div>
{formScheduleInvalid && !formScheduleTouched && (
<p className="text-xs text-destructive">
Stored weekly window could not be read and was not applied (alerts have been
delivering normally). Configure a new window above, or toggle it on then off to
confirm clearing it.
</p>
)}
{formScheduleEnabled && (
<div className="space-y-2 rounded-md border border-border/60 p-3">
<div className="flex flex-wrap gap-1.5">
{DAY_LABELS.map((label, day) => {
const selected = formScheduleDays.includes(day);
return (
<Button
key={label}
type="button"
size="sm"
variant={selected ? 'default' : 'outline'}
className="h-7 px-2 text-xs"
aria-pressed={selected}
aria-label={label}
onClick={() => {
setFormScheduleDays((prev) =>
selected
? prev.filter((d) => d !== day)
: [...prev, day].sort((a, b) => a - b),
);
markScheduleTouched();
}}
>
{label}
</Button>
);
})}
</div>
<div className="grid grid-cols-2 gap-2">
<div className="space-y-1">
<Label htmlFor="mute-schedule-start" className="text-xs text-muted-foreground">Start (UTC)</Label>
<Input
id="mute-schedule-start"
type="time"
value={formScheduleStart}
onChange={(e) => { setFormScheduleStart(e.target.value); markScheduleTouched(); }}
/>
</div>
<div className="space-y-1">
<Label htmlFor="mute-schedule-end" className="text-xs text-muted-foreground">End (UTC)</Label>
<Input
id="mute-schedule-end"
type="time"
value={formScheduleEnd}
onChange={(e) => { setFormScheduleEnd(e.target.value); markScheduleTouched(); }}
/>
</div>
</div>
<p className="text-xs text-muted-foreground">
Outside this window the rule does not mute. Cross-midnight windows use the start day only.
</p>
</div>
)}
</div>
<div className="flex items-center gap-2">
<TogglePill checked={formEnabled} onChange={setFormEnabled} id="mute-rule-enabled" />
<span className="text-sm text-stat-value select-none">
@@ -531,6 +699,11 @@ export function NotificationSuppressionSection({
{rule.expires_at != null && rule.expires_at <= Date.now() && (
<Badge variant="secondary" className="text-[10px] shrink-0 text-muted-foreground">Expired</Badge>
)}
{rule.scheduleInvalid && (
<Badge variant="destructive" className="text-[10px] shrink-0" title="Stored weekly window could not be read; alerts deliver normally until repaired in Edit">
Invalid schedule
</Badge>
)}
</div>
<div className="flex items-center gap-1 shrink-0">
<TogglePill checked={rule.enabled} onChange={() => handleToggleEnabled(rule)} className="scale-75" />
@@ -553,6 +726,12 @@ export function NotificationSuppressionSection({
)}
<span className="text-muted-foreground/50">|</span>
<span className="tabular-nums">Expires: {formatExpiry(rule.expires_at)}</span>
{formatScheduleSummary(rule.schedule) && (
<>
<span className="text-muted-foreground/50">|</span>
<span className="tabular-nums">{formatScheduleSummary(rule.schedule)}</span>
</>
)}
</div>
</div>
))}
frontend/src/components/settings/__tests__/NotificationSuppressionSection.test.tsx
+277 -13
View File
@@ -1,8 +1,8 @@
/**
* NotificationSuppressionSection stack pattern chips.
* NotificationSuppressionSection stack pattern chips and weekly schedule UI.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { render, screen, waitFor } from '@testing-library/react';
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() }));
@@ -38,10 +38,67 @@ vi.mock('@/lib/muteRules', () => ({
}));
import { apiFetch } from '@/lib/api';
import { toast } from '@/components/ui/toast-store';
import { NotificationSuppressionSection } from '../NotificationSuppressionSection';
const mockedFetch = apiFetch as unknown as ReturnType<typeof vi.fn>;
const scheduledRule = {
id: 42,
name: 'Weekend mute',
node_id: null,
stack_patterns: ['prod-*'],
label_ids: null,
categories: null,
levels: null,
applies_to: 'both',
enabled: true,
expires_at: null,
schedule: {
days: [6],
start_minute: 120,
end_minute: 360,
tz: 'UTC',
},
scheduleInvalid: false,
created_at: 1,
updated_at: 1,
};
const corruptScheduleRule = {
id: 43,
name: 'Corrupt window',
node_id: null,
stack_patterns: [],
label_ids: null,
categories: null,
levels: null,
applies_to: 'both',
enabled: true,
expires_at: null,
schedule: null,
scheduleInvalid: true,
created_at: 1,
updated_at: 1,
};
function mockListRules(rules: unknown[] = []) {
mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => {
if (url === '/notification-suppression-rules' && !opts?.method) {
return { ok: true, json: async () => rules };
}
if (url === '/stacks') return { ok: true, json: async () => ['staging'] };
if (url === '/labels') return { ok: true, json: async () => [] };
if (url === '/notification-suppression-rules' && opts?.method === 'POST') {
return { ok: true, json: async () => ({ id: 1 }) };
}
if (typeof url === 'string' && url.startsWith('/notification-suppression-rules/') && opts?.method === 'PUT') {
return { ok: true, json: async () => ({ id: 42 }) };
}
return { ok: true, json: async () => ([]) };
});
}
async function openMuteForm() {
render(<NotificationSuppressionSection />);
await waitFor(() => expect(screen.getByRole('button', { name: /Add mute rule|Add rule/i })).toBeInTheDocument());
@@ -49,20 +106,16 @@ async function openMuteForm() {
await waitFor(() => expect(screen.getByRole('dialog', { name: /New mute rule/i })).toBeInTheDocument());
}
async function enableWeeklyWindow() {
await userEvent.click(screen.getByLabelText(/Weekly window \(UTC\)/i));
await waitFor(() => expect(screen.getByRole('button', { name: 'Sat' })).toBeInTheDocument());
}
describe('NotificationSuppressionSection', () => {
beforeEach(() => {
mockedFetch.mockReset();
mockedFetch.mockImplementation(async (url: string, opts?: { method?: string }) => {
if (url === '/notification-suppression-rules' && !opts?.method) {
return { ok: true, json: async () => [] };
}
if (url === '/stacks') return { ok: true, json: async () => ['staging'] };
if (url === '/labels') return { ok: true, json: async () => [] };
if (url === '/notification-suppression-rules' && opts?.method === 'POST') {
return { ok: true, json: async () => ({ id: 1 }) };
}
return { ok: true, json: async () => ([]) };
});
vi.mocked(toast.error).mockClear();
mockListRules([]);
});
it('posts normalized stack patterns and null levels', async () => {
@@ -129,4 +182,215 @@ describe('NotificationSuppressionSection', () => {
expect(posts).toHaveLength(0);
});
});
it('posts a weekly UTC schedule with accessible day and time controls', async () => {
await openMuteForm();
await userEvent.type(screen.getByPlaceholderText(/Mute staging/i), 'Sat window');
await userEvent.type(screen.getByPlaceholderText(/Type a pattern/i), 'prod-*{Enter}');
await enableWeeklyWindow();
const sat = screen.getByRole('button', { name: 'Sat' });
expect(sat).toHaveAttribute('aria-pressed', 'false');
await userEvent.click(sat);
expect(sat).toHaveAttribute('aria-pressed', 'true');
expect(screen.getByLabelText('Start (UTC)')).toHaveAttribute('id', 'mute-schedule-start');
expect(screen.getByLabelText('End (UTC)')).toHaveAttribute('id', 'mute-schedule-end');
fireEvent.change(screen.getByLabelText('Start (UTC)'), { target: { value: '02:00' } });
fireEvent.change(screen.getByLabelText('End (UTC)'), { target: { value: '06:00' } });
await userEvent.click(screen.getByRole('button', { name: /Create|Update/i }));
await waitFor(() => {
const post = mockedFetch.mock.calls.find(
([url, opts]) => url === '/notification-suppression-rules' && (opts as { method?: string })?.method === 'POST',
);
expect(post).toBeTruthy();
const body = JSON.parse((post![1] as { body: string }).body);
expect(body.schedule).toEqual({
days: [6],
start_minute: 120,
end_minute: 360,
tz: 'UTC',
});
});
});
it('blocks scheduled create when no weekday is selected', async () => {
await openMuteForm();
await userEvent.type(screen.getByPlaceholderText(/Mute staging/i), 'No days');
await userEvent.type(screen.getByPlaceholderText(/Type a pattern/i), 'prod-*{Enter}');
await enableWeeklyWindow();
await userEvent.click(screen.getByRole('button', { name: /Create|Update/i }));
await waitFor(() => {
expect(toast.error).toHaveBeenCalledWith('Select at least one day for the weekly window.');
});
const posts = mockedFetch.mock.calls.filter(
([url, opts]) => url === '/notification-suppression-rules' && (opts as { method?: string })?.method === 'POST',
);
expect(posts).toHaveLength(0);
});
it('hydrates edit form from an existing schedule and keeps it on PUT', async () => {
mockListRules([scheduledRule]);
render(<NotificationSuppressionSection />);
await waitFor(() => expect(screen.getByText('Weekend mute')).toBeInTheDocument());
expect(screen.getByText(/UTC Sat 02:00-06:00/)).toBeInTheDocument();
await userEvent.click(screen.getByTitle('Edit'));
await waitFor(() => expect(screen.getByRole('dialog', { name: /Edit mute rule/i })).toBeInTheDocument());
expect(screen.getByLabelText(/Weekly window \(UTC\)/i)).toHaveAttribute('aria-checked', 'true');
expect(screen.getByRole('button', { name: 'Sat' })).toHaveAttribute('aria-pressed', 'true');
expect(screen.getByLabelText('Start (UTC)')).toHaveValue('02:00');
expect(screen.getByLabelText('End (UTC)')).toHaveValue('06:00');
await userEvent.click(screen.getByRole('button', { name: /Create|Update/i }));
await waitFor(() => {
const put = mockedFetch.mock.calls.find(
([url, opts]) =>
url === '/notification-suppression-rules/42' && (opts as { method?: string })?.method === 'PUT',
);
expect(put).toBeTruthy();
const body = JSON.parse((put![1] as { body: string }).body);
expect(body.schedule).toEqual({
days: [6],
start_minute: 120,
end_minute: 360,
tz: 'UTC',
});
});
});
it('clears schedule to null when weekly window is turned off on edit', async () => {
mockListRules([scheduledRule]);
render(<NotificationSuppressionSection />);
await waitFor(() => expect(screen.getByText('Weekend mute')).toBeInTheDocument());
await userEvent.click(screen.getByTitle('Edit'));
await waitFor(() => expect(screen.getByRole('dialog', { name: /Edit mute rule/i })).toBeInTheDocument());
await userEvent.click(screen.getByLabelText(/Weekly window \(UTC\)/i));
await waitFor(() => {
expect(screen.queryByRole('button', { name: 'Sat' })).not.toBeInTheDocument();
});
await userEvent.click(screen.getByRole('button', { name: /Create|Update/i }));
await waitFor(() => {
const put = mockedFetch.mock.calls.find(
([url, opts]) =>
url === '/notification-suppression-rules/42' && (opts as { method?: string })?.method === 'PUT',
);
expect(put).toBeTruthy();
const body = JSON.parse((put![1] as { body: string }).body);
expect(body.schedule).toBeNull();
});
});
it('marks a corrupt stored schedule as invalid instead of an ordinary unscheduled rule', async () => {
mockListRules([corruptScheduleRule]);
render(<NotificationSuppressionSection />);
await waitFor(() => expect(screen.getByText('Corrupt window')).toBeInTheDocument());
expect(screen.getByText('Invalid schedule')).toBeInTheDocument();
});
it('blocks saving a corrupt-schedule rule until the operator touches the weekly window', async () => {
mockListRules([corruptScheduleRule]);
render(<NotificationSuppressionSection />);
await waitFor(() => expect(screen.getByText('Corrupt window')).toBeInTheDocument());
await userEvent.click(screen.getByTitle('Edit'));
await waitFor(() => expect(screen.getByRole('dialog', { name: /Edit mute rule/i })).toBeInTheDocument());
expect(screen.getByLabelText(/Weekly window \(UTC\)/i)).toHaveAttribute('aria-checked', 'false');
await userEvent.click(screen.getByRole('button', { name: /Create|Update/i }));
await waitFor(() => {
expect(toast.error).toHaveBeenCalledWith(
expect.stringContaining('could not be read'),
);
});
const puts = mockedFetch.mock.calls.filter(
([url, opts]) => url === '/notification-suppression-rules/43' && (opts as { method?: string })?.method === 'PUT',
);
expect(puts).toHaveLength(0);
});
it('allows saving a corrupt-schedule rule once the operator explicitly clears it', async () => {
mockListRules([corruptScheduleRule]);
render(<NotificationSuppressionSection />);
await waitFor(() => expect(screen.getByText('Corrupt window')).toBeInTheDocument());
await userEvent.click(screen.getByTitle('Edit'));
await waitFor(() => expect(screen.getByRole('dialog', { name: /Edit mute rule/i })).toBeInTheDocument());
// Explicit acknowledgement: turn the window on, then off again, to intentionally clear it.
await userEvent.click(screen.getByLabelText(/Weekly window \(UTC\)/i));
await userEvent.click(screen.getByLabelText(/Weekly window \(UTC\)/i));
await userEvent.click(screen.getByRole('button', { name: /Create|Update/i }));
await waitFor(() => {
const put = mockedFetch.mock.calls.find(
([url, opts]) =>
url === '/notification-suppression-rules/43' && (opts as { method?: string })?.method === 'PUT',
);
expect(put).toBeTruthy();
const body = JSON.parse((put![1] as { body: string }).body);
expect(body.schedule).toBeNull();
});
});
it('allows saving a corrupt-schedule rule once the operator configures a new valid schedule', async () => {
mockListRules([corruptScheduleRule]);
render(<NotificationSuppressionSection />);
await waitFor(() => expect(screen.getByText('Corrupt window')).toBeInTheDocument());
await userEvent.click(screen.getByTitle('Edit'));
await waitFor(() => expect(screen.getByRole('dialog', { name: /Edit mute rule/i })).toBeInTheDocument());
await enableWeeklyWindow();
await userEvent.click(screen.getByRole('button', { name: 'Sat' }));
fireEvent.change(screen.getByLabelText('Start (UTC)'), { target: { value: '02:00' } });
fireEvent.change(screen.getByLabelText('End (UTC)'), { target: { value: '06:00' } });
await userEvent.click(screen.getByRole('button', { name: /Create|Update/i }));
await waitFor(() => {
const put = mockedFetch.mock.calls.find(
([url, opts]) =>
url === '/notification-suppression-rules/43' && (opts as { method?: string })?.method === 'PUT',
);
expect(put).toBeTruthy();
const body = JSON.parse((put![1] as { body: string }).body);
expect(body.schedule).toEqual({
days: [6],
start_minute: 120,
end_minute: 360,
tz: 'UTC',
});
});
});
it('does not carry the invalid-schedule save gate over to a later edit of a different, valid rule', async () => {
mockListRules([corruptScheduleRule, scheduledRule]);
render(<NotificationSuppressionSection />);
await waitFor(() => expect(screen.getByText('Corrupt window')).toBeInTheDocument());
const editButtons = screen.getAllByTitle('Edit');
await userEvent.click(editButtons[0]);
await waitFor(() => expect(screen.getByRole('dialog', { name: /Edit mute rule/i })).toBeInTheDocument());
await userEvent.click(screen.getByRole('button', { name: 'Cancel' }));
await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument());
await userEvent.click(screen.getAllByTitle('Edit')[1]);
await waitFor(() => expect(screen.getByRole('dialog', { name: /Edit mute rule/i })).toBeInTheDocument());
await userEvent.click(screen.getByRole('button', { name: /Create|Update/i }));
await waitFor(() => {
const put = mockedFetch.mock.calls.find(
([url, opts]) =>
url === '/notification-suppression-rules/42' && (opts as { method?: string })?.method === 'PUT',
);
expect(put).toBeTruthy();
});
expect(toast.error).not.toHaveBeenCalledWith(expect.stringContaining('could not be read'));
});
});
frontend/src/components/sidebar/StackList.tsx
+11 -5
View File
@@ -18,6 +18,7 @@ import type { StacksLoadStatus } from '@/components/EditorLayout/hooks/useStackL
import { Button } from '@/components/ui/button';
import { useLabelMuteActions } from '@/hooks/useMuteRuleActions';
import { LabelGroupMuteKebab } from '@/components/mute/MuteMenuItems';
import { isStacksListLoading } from './stacksLoadUi';
interface RemoteNodeResult {
nodeId: number;
@@ -169,7 +170,7 @@ export function StackList(props: StackListProps & StackListBulkProps) {
useStackKeyboardShortcuts(selectedFile, buildMenuCtx);
if (isLoading) {
if (isStacksListLoading(isLoading, stacksLoadStatus)) {
return (
<div className="space-y-2 px-2 mt-2">
<Skeleton className="h-8 w-full" />
@@ -196,10 +197,15 @@ export function StackList(props: StackListProps & StackListBulkProps) {
);
}
// First-run prompt only when the node has no stacks at all: no search text and
// no active filter chip, so a filter that happens to match nothing does not
// masquerade as an empty fleet.
if (files.length === 0 && !searchQuery.trim() && filterChip === 'all') {
// First-run prompt only after a successful list load with zero stacks: no
// search text and no active filter chip, so a filter that matches nothing is
// not mistaken for an empty fleet, and idle/loading never looks like first-run.
if (
stacksLoadStatus === 'success'
&& files.length === 0
&& !searchQuery.trim()
&& filterChip === 'all'
) {
return (
<DiscoveryEmptyState
onOpenAdopt={onOpenAdopt}
frontend/src/components/sidebar/StackSidebar.tsx
+6 -1
View File
@@ -11,6 +11,7 @@ import { StackList, type StackListProps } from './StackList';
import type { FilterChip } from './sidebar-types';
import type { BulkAction } from '@/hooks/useBulkStackActions';
import type { SidebarActivitySummary } from './useSidebarActivitySummary';
import { isStacksListSettled } from './stacksLoadUi';
export interface StackSidebarProps {
isDarkMode: boolean;
@@ -99,7 +100,11 @@ export function StackSidebar(props: StackSidebarProps) {
/>
)}
<ScrollArea block className="flex-1 px-2 pb-2">
<div data-stacks-loaded={list.isLoading ? 'false' : 'true'}>
<div
data-stacks-loaded={
isStacksListSettled(list.isLoading, list.stacksLoadStatus) ? 'true' : 'false'
}
>
<StackList {...list} bulkMode={bulkMode} selectedFiles={selectedFiles} onToggleSelect={onToggleSelect} />
</div>
</ScrollArea>
frontend/src/components/sidebar/__tests__/StackList.test.tsx
+106
View File
@@ -0,0 +1,106 @@
import type React from 'react';
import { describe, it, expect, vi } from 'vitest';
import { render, screen } from '@testing-library/react';
import { StackList } from '../StackList';
import { isStacksListSettled, isStacksListLoading } from '../stacksLoadUi';
type StackListRenderProps = React.ComponentProps<typeof StackList>;
vi.mock('../DiscoveryEmptyState', () => ({
DiscoveryEmptyState: () => <div data-testid="discovery-empty">No compose projects yet</div>,
}));
vi.mock('@/hooks/useStackKeyboardShortcuts', () => ({
useStackKeyboardShortcuts: () => {},
}));
vi.mock('@/hooks/useMuteRuleActions', () => ({
useLabelMuteActions: () => ({ canMute: false }),
}));
function baseProps(over: Partial<StackListRenderProps> = {}): StackListRenderProps {
return {
files: [],
isLoading: false,
selectedFile: null,
searchQuery: '',
stackLabelMap: {},
stackStatuses: {},
stackCounts: {},
stackUpdates: {},
gitSourcePendingMap: {},
pinnedFiles: [],
isCollapsed: () => false,
toggleCollapse: () => {},
isBusy: () => false,
getDisplayName: (f) => f,
onSelectFile: () => {},
buildMenuCtx: () => ({}) as never,
remoteResults: [],
remoteLoading: false,
remoteFailedNodes: [],
onSelectRemoteFile: () => {},
filterChip: 'all',
stacksLoadStatus: 'idle',
stacksLoadError: null,
bulkMode: false,
selectedFiles: new Set<string>(),
onToggleSelect: () => {},
...over,
};
}
describe('stacksLoadUi helpers', () => {
it('treats success while isLoading as unsettled', () => {
expect(isStacksListSettled(true, 'success')).toBe(false);
expect(isStacksListLoading(true, 'success')).toBe(true);
});
it('settles only when not loading and status is success or error', () => {
expect(isStacksListSettled(false, 'success')).toBe(true);
expect(isStacksListSettled(false, 'error')).toBe(true);
expect(isStacksListSettled(false, 'idle')).toBe(false);
expect(isStacksListSettled(false, 'loading')).toBe(false);
});
});
describe('StackList load gating', () => {
it('does not mount discovery empty while idle', () => {
render(<StackList {...baseProps({ stacksLoadStatus: 'idle', isLoading: false, files: [] })} />);
expect(screen.queryByTestId('discovery-empty')).toBeNull();
});
it('does not mount discovery empty while loading even if status is already success', () => {
render(
<StackList
{...baseProps({ stacksLoadStatus: 'success', isLoading: true, files: [] })}
/>,
);
expect(screen.queryByTestId('discovery-empty')).toBeNull();
});
it('mounts discovery empty only after successful empty load', () => {
render(
<StackList
{...baseProps({ stacksLoadStatus: 'success', isLoading: false, files: [] })}
/>,
);
expect(screen.getByTestId('discovery-empty')).toBeInTheDocument();
});
it('shows retry on error with empty files', () => {
render(
<StackList
{...baseProps({
stacksLoadStatus: 'error',
stacksLoadError: 'Could not load stacks for this node.',
isLoading: false,
files: [],
onRetryStacksLoad: vi.fn(),
})}
/>,
);
expect(screen.getByRole('button', { name: /retry/i })).toBeInTheDocument();
expect(screen.queryByTestId('discovery-empty')).toBeNull();
});
});
frontend/src/components/sidebar/stacksLoadUi.ts
+17
View File
@@ -0,0 +1,17 @@
import type { StacksLoadStatus } from '@/components/EditorLayout/hooks/useStackListState';
/** True when the sidebar stack list request has finished for the active node. */
export function isStacksListSettled(
isLoading: boolean,
stacksLoadStatus: StacksLoadStatus | undefined,
): boolean {
return !isLoading && (stacksLoadStatus === 'success' || stacksLoadStatus === 'error');
}
/** True while the sidebar should show the stack-list skeleton. */
export function isStacksListLoading(
isLoading: boolean,
stacksLoadStatus: StacksLoadStatus | undefined,
): boolean {
return isLoading || stacksLoadStatus === 'idle' || stacksLoadStatus === 'loading' || stacksLoadStatus == null;
}

Some files were not shown because too many files have changed in this diff Show More