diff --git a/.release-please-manifest.json b/.release-please-manifest.json index efb7bd94..6b5b821b 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.95.0" + ".": "0.96.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 21e7af70..ba5f2423 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,67 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.96.0](https://github.com/Studio-Saelix/sencho/compare/v0.95.0...v0.96.0) (2026-07-26) + + +### Added + +* add Admiral Hardened Build channel and business assurance surfaces ([#1629](https://github.com/Studio-Saelix/sencho/issues/1629)) ([381ed2a](https://github.com/Studio-Saelix/sencho/commit/381ed2a91fa7ff5743a365511d9f5cd6bff141a9)) +* add Apprise as a fourth notification channel ([#1644](https://github.com/Studio-Saelix/sencho/issues/1644)) ([83b3d93](https://github.com/Studio-Saelix/sencho/commit/83b3d932e5a9664264b4b5ac40d601925654e4b1)) +* add configurable notification dispatch retries ([#1655](https://github.com/Studio-Saelix/sencho/issues/1655)) ([090a0d7](https://github.com/Studio-Saelix/sencho/commit/090a0d73acbad1a85db1603372d79b65e044392d)) +* add developer-mode startup and stack hydration timing ([#1619](https://github.com/Studio-Saelix/sencho/issues/1619)) ([b70a529](https://github.com/Studio-Saelix/sencho/commit/b70a529656a232df0012c56b3280aed9aa49c4fa)) +* add service-scoped Compose update and restore ([#1648](https://github.com/Studio-Saelix/sencho/issues/1648)) ([63213c0](https://github.com/Studio-Saelix/sencho/commit/63213c096026551f9fb3b10ada625eb2be8ff35f)) +* add service-scoped stack alert rules ([#1681](https://github.com/Studio-Saelix/sencho/issues/1681)) ([85842cc](https://github.com/Studio-Saelix/sencho/commit/85842cc54788b453071890fb57764ab608de78a0)) +* **blueprints:** require confirmed rollout preview before reconcile ([#1649](https://github.com/Studio-Saelix/sencho/issues/1649)) ([d94e586](https://github.com/Studio-Saelix/sencho/commit/d94e586af3011733af5f2d9c991c745385b7b29e)) +* graduate Host Console to Community admins ([#1669](https://github.com/Studio-Saelix/sencho/issues/1669)) ([dd54a2e](https://github.com/Studio-Saelix/sencho/commit/dd54a2e483f4432c59f602ea1129a47ec87c1ec8)) +* guide missing external network creation during deploy ([#1645](https://github.com/Studio-Saelix/sencho/issues/1645)) ([35bb744](https://github.com/Studio-Saelix/sencho/commit/35bb74425be6d906d13dc9a2fd67f7c8acd5b534)) +* node-scoped Networking operator page ([#1603](https://github.com/Studio-Saelix/sencho/issues/1603)) ([8980910](https://github.com/Studio-Saelix/sencho/commit/8980910153c1c6264af35076dbeb64fca0f3301e)) +* stack glob patterns and route severity levels ([#1651](https://github.com/Studio-Saelix/sencho/issues/1651)) ([972f2b9](https://github.com/Studio-Saelix/sencho/commit/972f2b9483432648689ba01aa7a2d64bb0eed917)) +* surface stack Monitor from header and service cards ([#1693](https://github.com/Studio-Saelix/sencho/issues/1693)) ([524cc56](https://github.com/Studio-Saelix/sencho/commit/524cc56d2ff43d1d38b737c0a8751509e72d29bc)) +* **ui:** add Classic, Smart, and Compact desktop navigation styles ([#1642](https://github.com/Studio-Saelix/sencho/issues/1642)) ([25586fc](https://github.com/Studio-Saelix/sencho/commit/25586fc8ab8a8fdd8d2ef7b23d57b41be6c11d83)) +* weekly UTC maintenance windows for mute rules ([#1661](https://github.com/Studio-Saelix/sencho/issues/1661)) ([a3edee5](https://github.com/Studio-Saelix/sencho/commit/a3edee5e6a6c6319bb292f2e37d96aa3260acf3f)) + + +### Fixed + +* assorted UI/UX polish fixes ([#1670](https://github.com/Studio-Saelix/sencho/issues/1670)) ([cfb42af](https://github.com/Studio-Saelix/sencho/commit/cfb42af4e098ce8375529eff9c78c33b17c0e9fa)) +* **blueprints:** fail closed on marker ownership for apply and withdraw ([#1694](https://github.com/Studio-Saelix/sencho/issues/1694)) ([17a8dc8](https://github.com/Studio-Saelix/sencho/commit/17a8dc8a94b18b396492735de8cd89bc2cf45c97)) +* **blueprints:** gate confirmed apply on live intent fingerprint ([#1663](https://github.com/Studio-Saelix/sencho/issues/1663)) ([155db30](https://github.com/Studio-Saelix/sencho/commit/155db30554e566fcb1a449451fc79b6e6bb4e3d9)) +* **blueprints:** write compose.yaml so first-time apply is not shadowed ([#1668](https://github.com/Studio-Saelix/sencho/issues/1668)) ([e15b9d1](https://github.com/Studio-Saelix/sencho/commit/e15b9d1244f1e59f7140aaa2139fe34a16f681bb)) +* bump brace-expansion override to 5.0.8 to clear high-severity audit ([#1702](https://github.com/Studio-Saelix/sencho/issues/1702)) ([d0a4f1e](https://github.com/Studio-Saelix/sencho/commit/d0a4f1ead9652667580319e1c83f31baea675060)) +* dedupe healthcheck alerts and share crash rate limits ([#1690](https://github.com/Studio-Saelix/sencho/issues/1690)) ([ec0f59a](https://github.com/Studio-Saelix/sencho/commit/ec0f59a85e3a54f29670d2424e4e04d21b07d6c4)) +* **docs:** remove duplicate license url key in openapi.yaml ([#1628](https://github.com/Studio-Saelix/sencho/issues/1628)) ([8ca8eba](https://github.com/Studio-Saelix/sencho/commit/8ca8ebaa2468ad6df79bd0be87a94dba6c4faf15)) +* **fleet:** expose Community cordon on NodeCard ([#1646](https://github.com/Studio-Saelix/sencho/issues/1646)) ([674220b](https://github.com/Studio-Saelix/sencho/commit/674220b9deaee8f4e8a58626fdf1d0679a7c669b)) +* **fleet:** isolate corrupt snapshot file decrypt failures ([#1650](https://github.com/Studio-Saelix/sencho/issues/1650)) ([3b02795](https://github.com/Studio-Saelix/sencho/commit/3b027957c4369a4d8c0ea5513b9e631b1bf2508b)) +* **fleet:** refresh prune reclaimable estimate after successful run ([#1675](https://github.com/Studio-Saelix/sencho/issues/1675)) ([ed5ca9c](https://github.com/Studio-Saelix/sencho/commit/ed5ca9c4f690d07dc1ccf25c3a8557e57780410d)) +* **fleet:** verify update status before removing readiness cards ([#1697](https://github.com/Studio-Saelix/sencho/issues/1697)) ([719180f](https://github.com/Studio-Saelix/sencho/commit/719180f156c49bea48ad2b319d81cb9823cfb9fa)) +* **image-updates:** match any local RepoDigest against the remote tag ([#1695](https://github.com/Studio-Saelix/sencho/issues/1695)) ([6688da9](https://github.com/Studio-Saelix/sencho/commit/6688da97b1e3b107b379539354ceddc56b549b7d)) +* **image-updates:** normalize docker.io host aliases to the registry API host ([#1706](https://github.com/Studio-Saelix/sencho/issues/1706)) ([bb7c76b](https://github.com/Studio-Saelix/sencho/commit/bb7c76ba4664abcdfafcdc5816e35073076c66d7)) +* **image-updates:** treat multi-arch child digests as up to date ([#1641](https://github.com/Studio-Saelix/sencho/issues/1641)) ([66ec4eb](https://github.com/Studio-Saelix/sencho/commit/66ec4ebdd2c6c67576f6e4c328bd7fe87c28285c)) +* keep running containers until stack pull/build succeeds ([#1657](https://github.com/Studio-Saelix/sencho/issues/1657)) ([3f1f15a](https://github.com/Studio-Saelix/sencho/commit/3f1f15a6f433eb137206833550c2eb0a0134fbf0)) +* leave editor after deleting the open stack ([#1665](https://github.com/Studio-Saelix/sencho/issues/1665)) ([55fa29f](https://github.com/Studio-Saelix/sencho/commit/55fa29f565473865f4c262f2b8516bc1899faa82)) +* **notifications:** align Sencho update alerts with Fleet cache ([#1620](https://github.com/Studio-Saelix/sencho/issues/1620)) ([4079cb9](https://github.com/Studio-Saelix/sencho/commit/4079cb919817509e17ba4ed9f13889be892a09e0)) +* **notifications:** neutralize satellite-local node names in alert bodies ([#1640](https://github.com/Studio-Saelix/sencho/issues/1640)) ([d8e4ede](https://github.com/Studio-Saelix/sencho/commit/d8e4ede94f1fcacc631e70eaca5ec3a8540951e5)) +* **notifications:** stop embedding Local in janitor alerts ([#1631](https://github.com/Studio-Saelix/sencho/issues/1631)) ([678c198](https://github.com/Studio-Saelix/sencho/commit/678c198faadc065c081013590dd810dfbcf35242)) +* **notifications:** version mute replica retractions for soft-cleanup restore ([#1703](https://github.com/Studio-Saelix/sencho/issues/1703)) ([9859ce6](https://github.com/Studio-Saelix/sencho/commit/9859ce60b813fd3fc5247234a1f21ac7259eb9e3)) +* pin brace-expansion to a patched version in both packages ([#1658](https://github.com/Studio-Saelix/sencho/issues/1658)) ([8598390](https://github.com/Studio-Saelix/sencho/commit/859839082c89f912b346b11a42acfc0576268314)) +* pin postcss override to clear backend audit vulnerability ([#1701](https://github.com/Studio-Saelix/sencho/issues/1701)) ([e33eda3](https://github.com/Studio-Saelix/sencho/commit/e33eda3c38423b047f1c870ea7a452cf11b1741f)) +* prevent false empty states during stack hydration ([#1659](https://github.com/Studio-Saelix/sencho/issues/1659)) ([b06dfd7](https://github.com/Studio-Saelix/sencho/commit/b06dfd71754242e6a09104aa9b61f38ba1bcbc7c)) +* purge deleted-stack notifications from panel and ticker ([#1674](https://github.com/Studio-Saelix/sencho/issues/1674)) ([698b7d0](https://github.com/Studio-Saelix/sencho/commit/698b7d0713919b9f91a63e05d84b44057b7f1d12)) +* **rate-limit:** verify node_proxy JWT before skipping limiters ([#1647](https://github.com/Studio-Saelix/sencho/issues/1647)) ([31d4e46](https://github.com/Studio-Saelix/sencho/commit/31d4e4669b335ce270cece716045b34072385e7b)) +* **rbac:** cover stack assignment cleanup on blueprint withdraw ([#1664](https://github.com/Studio-Saelix/sencho/issues/1664)) ([6484c79](https://github.com/Studio-Saelix/sencho/commit/6484c79015b5da7321379a76bc34d43478eb7dd0)) +* recognize clean one-shot completions in health gate and drift ([#1691](https://github.com/Studio-Saelix/sencho/issues/1691)) ([79914fe](https://github.com/Studio-Saelix/sencho/commit/79914fe7503923edebe64e9f006b4fbe5b7d189f)) +* reconcile sticky update indicators with Anatomy preview ([#1698](https://github.com/Studio-Saelix/sencho/issues/1698)) ([0daddfd](https://github.com/Studio-Saelix/sencho/commit/0daddfde00dfe37a8e0622f639d2a84125c031f3)) +* **security:** bump grpc to 1.82.1 in docker CLI and compose builds ([#1678](https://github.com/Studio-Saelix/sencho/issues/1678)) ([ce4eeb6](https://github.com/Studio-Saelix/sencho/commit/ce4eeb6b933b299a880dd566d40148c970a78422)) +* **settings:** toast and block save when node settings fail to load ([#1654](https://github.com/Studio-Saelix/sencho/issues/1654)) ([ad00517](https://github.com/Studio-Saelix/sencho/commit/ad00517a0ea5c9ab1a2b992e0cad622b0f2d4503)) +* **sso:** correct settings description to drop unsupported SAML claim ([#1662](https://github.com/Studio-Saelix/sencho/issues/1662)) ([a4d9fa7](https://github.com/Studio-Saelix/sencho/commit/a4d9fa79a1d6043708fb3f951b58337b2c0abfaa)) +* **stacks:** fail closed when compose ps errors during update orphan classify ([#1708](https://github.com/Studio-Saelix/sencho/issues/1708)) ([6887670](https://github.com/Studio-Saelix/sencho/commit/688767092a3b4be5a11709fac72ffcd7ffadb299)) +* **ui:** default Reduced Motion on Calm and quiet decorative rails ([#1622](https://github.com/Studio-Saelix/sencho/issues/1622)) ([c170c3f](https://github.com/Studio-Saelix/sencho/commit/c170c3f30ca0673df1e5e395dcd82dd1baed33b6)) +* **ui:** hide log service chips on single-service stacks ([#1689](https://github.com/Studio-Saelix/sencho/issues/1689)) ([a89498a](https://github.com/Studio-Saelix/sencho/commit/a89498ae5b4bb8b88178fe8088e1acb46e3821b9)) +* **ui:** lower-overhead Reduced effects for constrained GPUs ([#1616](https://github.com/Studio-Saelix/sencho/issues/1616)) ([c01f247](https://github.com/Studio-Saelix/sencho/commit/c01f2479b94b2214a949ae736a3c5141cc52b5be)) +* **ui:** use Docker health status terms on container cards ([#1696](https://github.com/Studio-Saelix/sencho/issues/1696)) ([4d2270a](https://github.com/Studio-Saelix/sencho/commit/4d2270a097b797c632353efaf513b67ccd5f57a6)), closes [#1677](https://github.com/Studio-Saelix/sencho/issues/1677), thanks @Crosis47 +* unlock Community deploy policy hard-blocking ([#1643](https://github.com/Studio-Saelix/sencho/issues/1643)) ([b91025d](https://github.com/Studio-Saelix/sencho/commit/b91025dc8b5d476dd295f12695f4225a88081b90)) + ## [0.95.0](https://github.com/Studio-Saelix/sencho/compare/v0.94.1...v0.95.0) (2026-07-12) diff --git a/Dockerfile b/Dockerfile index 4c3bdf04..e0e13789 100644 --- a/Dockerfile +++ b/Dockerfile @@ -102,14 +102,16 @@ RUN if [ "$TARGETARCH" = "$BUILDARCH" ]; then \ # The fetch pulls only the v29.4.1 commit, minimising transfer size. # docker/cli uses CalVer and ships vendor.mod instead of go.mod to avoid SemVer # compliance requirements. We copy vendor.mod -> go.mod, drop the committed vendor -# tree, bump golang.org/x/net to v0.55.0 and google.golang.org/grpc to v1.82.1, -# and build with -mod=mod so the patched modules are resolved from the module -# proxy. x/net v0.53.0 is flagged for six HIGH advisories (CVE-2026-25680, -# -25681, -27136, -39821, -42502, -42506; x/net/html parsing and x/net/idna). -# grpc v1.80.0 is flagged for GHSA-hrxh-6v49-42gf (xDS RBAC / HTTP/2). Removing -# vendor/ keeps -mod=mod from reading the stale copy, and avoids `go mod tidy` -# (which does not run cleanly against docker/cli's vendor.mod manifest). This -# stage now fetches modules at build time rather than building fully offline. +# tree, bump golang.org/x/net to v0.55.0, golang.org/x/text to v0.39.0, and +# google.golang.org/grpc to v1.82.1, and build with -mod=mod so the patched +# modules are resolved from the module proxy. x/net v0.53.0 is flagged for six +# HIGH advisories (CVE-2026-25680, -25681, -27136, -39821, -42502, -42506; +# x/net/html parsing and x/net/idna). x/text v0.37.0 is flagged for +# CVE-2026-56852 (norm.Iter infinite loop on crafted input). grpc v1.80.0 is +# flagged for GHSA-hrxh-6v49-42gf (xDS RBAC / HTTP/2). Removing vendor/ keeps +# -mod=mod from reading the stale copy, and avoids `go mod tidy` (which does +# not run cleanly against docker/cli's vendor.mod manifest). This stage now +# fetches modules at build time rather than building fully offline. # Base image pinned by digest so the Go toolchain that compiles the static # Docker CLI binary cannot change without an explicit Dependabot bump. FROM --platform=$BUILDPLATFORM golang:1.27rc2-alpine@sha256:dcbb18cc5fa1082364dc6aa95224b6b55429d09cbb9631a053d8064c1c367300 AS cli-builder @@ -134,6 +136,7 @@ RUN mkdir -p /build RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \ rm -rf vendor && \ go get golang.org/x/net@v0.55.0 \ + golang.org/x/text@v0.39.0 \ google.golang.org/grpc@v1.82.1 && \ CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ -mod=mod \ @@ -172,9 +175,11 @@ RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \ # so this is defense-in-depth rather than a live exposure. # # The same go get also bumps google.golang.org/grpc from v1.80.0 to v1.82.1 to -# clear GHSA-hrxh-6v49-42gf (xDS RBAC fail-open and HTTP/2 transport issues). -# Base image pinned by digest (same image as cli-builder above) so both -# source builds share an identical, immutable Go toolchain. +# clear GHSA-hrxh-6v49-42gf (xDS RBAC fail-open and HTTP/2 transport issues), +# and golang.org/x/text from v0.38.0 to v0.39.0 to clear CVE-2026-56852 +# (norm.Iter infinite loop on crafted input). Base image pinned by digest +# (same image as cli-builder above) so both source builds share an +# identical, immutable Go toolchain. FROM --platform=$BUILDPLATFORM golang:1.27rc2-alpine@sha256:dcbb18cc5fa1082364dc6aa95224b6b55429d09cbb9631a053d8064c1c367300 AS compose-builder ARG TARGETARCH @@ -212,7 +217,8 @@ RUN --mount=type=cache,id=go-mod,sharing=locked,target=/go/pkg/mod \ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc@v1.43.0 \ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp@v1.43.0 \ github.com/containerd/containerd/v2@v2.2.5 \ - google.golang.org/grpc@v1.82.1 && \ + google.golang.org/grpc@v1.82.1 \ + golang.org/x/text@v0.39.0 && \ go mod tidy # Build target is ./cmd (the package main with plugin.Run), per docker/compose's diff --git a/README.md b/README.md index 6953500f..88bbf980 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ ## What Sencho is -Sencho is a Docker Compose control plane for DevOps engineers, platform teams, and system administrators who run services on Compose and need a real operational surface: a graphical interface that does not give up file-on-disk workflows, and the ability to manage more than one machine without SSH gymnastics or a VPN. +Sencho is a Docker Compose control plane for DevOps engineers, platform teams, system administrators and homelab users who run services on Compose and need a real operational surface: a graphical interface that does not give up file-on-disk workflows, and the ability to manage more than one machine without SSH gymnastics or a VPN. It runs as a single container on your hardware and provides a UI for common Compose operations: deploying, editing files, watching logs, restarting containers, browsing volumes, and recovering from failures. Your compose files stay on the host filesystem and remain the source of truth. diff --git a/backend/package-lock.json b/backend/package-lock.json index 756d7fa9..7cc65f43 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -2490,16 +2490,16 @@ "optional": true }, "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", + "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/braces": { diff --git a/backend/package.json b/backend/package.json index 467de07d..15e76215 100644 --- a/backend/package.json +++ b/backend/package.json @@ -18,6 +18,7 @@ "create-emergency-admin": "node dist/cli/createEmergencyAdmin.js", "clear-sessions": "node dist/cli/clearSessions.js", "disable-sso": "node dist/cli/disableSso.js", + "enable-local-login": "node dist/cli/enableLocalLogin.js", "diagnostics": "node dist/cli/diagnostics.js", "validate-db": "node dist/cli/validateDb.js", "backup-data": "node dist/cli/backupData.js" @@ -30,7 +31,7 @@ "node": ">=26.0.0" }, "overrides": { - "brace-expansion": "^5.0.7", + "brace-expansion": "^5.0.8", "postcss": "^8.5.23" }, "devDependencies": { diff --git a/backend/src/__tests__/auth-mode.test.ts b/backend/src/__tests__/auth-mode.test.ts new file mode 100644 index 00000000..0851f1a4 --- /dev/null +++ b/backend/src/__tests__/auth-mode.test.ts @@ -0,0 +1,214 @@ +/** + * Authentication mode (SSO-only) route and activation safeguards. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { + setupTestDb, + cleanupTestDb, + loginAsTestAdmin, + TEST_USERNAME, + TEST_PASSWORD, +} from './helpers/setupTestDb'; +import { setAuthenticationMode } from '../helpers/authenticationMode'; + +let tmpDir: string; +let app: import('express').Express; +let adminCookie: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let LicenseService: typeof import('../services/LicenseService').LicenseService; +let SSOService: typeof import('../services/SSOService').SSOService; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ LicenseService } = await import('../services/LicenseService')); + ({ SSOService } = await import('../services/SSOService')); + adminCookie = await loginAsTestAdmin(app); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +function markAdminAsSso(): void { + const db = DatabaseService.getInstance(); + db.getDb() + .prepare("UPDATE users SET auth_provider = 'oidc_custom', provider_id = 'sso-admin-1' WHERE username = ?") + .run(TEST_USERNAME); +} + +function markAdminAsLocal(): void { + const db = DatabaseService.getInstance(); + db.getDb() + .prepare("UPDATE users SET auth_provider = 'local', provider_id = NULL WHERE username = ?") + .run(TEST_USERNAME); +} + +function enableGithubProvider(): void { + DatabaseService.getInstance().upsertSSOConfig( + 'oidc_github', + true, + JSON.stringify({ + provider: 'oidc_github', + enabled: true, + displayName: 'GitHub', + oidcClientId: 'test-client', + }), + ); +} + +beforeEach(() => { + setAuthenticationMode('local_and_sso'); + markAdminAsLocal(); + const db = DatabaseService.getInstance(); + for (const cfg of db.getSSOConfigs()) { + db.upsertSSOConfig(cfg.provider, false, cfg.config_json); + } + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); +}); + +describe('GET /api/sso/auth-mode', () => { + it('returns the current mode for an admin', async () => { + const res = await request(app).get('/api/sso/auth-mode').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.authenticationMode).toBe('local_and_sso'); + expect(res.body.localLoginEnabled).toBe(true); + }); +}); + +describe('PUT /api/sso/auth-mode', () => { + it('rejects PUT sso_only without confirm: true', async () => { + markAdminAsSso(); + enableGithubProvider(); + vi.spyOn(SSOService.getInstance(), 'testOidcDiscovery').mockResolvedValue({ success: true }); + + const missing = await request(app) + .put('/api/sso/auth-mode') + .set('Cookie', adminCookie) + .send({ mode: 'sso_only' }); + expect(missing.status).toBe(400); + expect(missing.body.error).toMatch(/confirm/i); + + const falsy = await request(app) + .put('/api/sso/auth-mode') + .set('Cookie', adminCookie) + .send({ mode: 'sso_only', confirm: false }); + expect(falsy.status).toBe(400); + expect(falsy.body.error).toMatch(/confirm/i); + }); + + it('allows Community admin to enable sso_only', async () => { + markAdminAsSso(); + enableGithubProvider(); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + vi.spyOn(SSOService.getInstance(), 'testOidcDiscovery').mockResolvedValue({ success: true }); + + const res = await request(app) + .put('/api/sso/auth-mode') + .set('Cookie', adminCookie) + .send({ mode: 'sso_only', confirm: true }); + expect(res.status).toBe(200); + expect(res.body.authenticationMode).toBe('sso_only'); + expect(res.body.localLoginEnabled).toBe(false); + }); + + it('rejects local-only admin entering sso_only', async () => { + enableGithubProvider(); + vi.spyOn(SSOService.getInstance(), 'testOidcDiscovery').mockResolvedValue({ success: true }); + + const res = await request(app) + .put('/api/sso/auth-mode') + .set('Cookie', adminCookie) + .send({ mode: 'sso_only', confirm: true }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/Sign in with SSO/i); + }); + + it('rejects sso_only when no provider is enabled', async () => { + markAdminAsSso(); + const res = await request(app) + .put('/api/sso/auth-mode') + .set('Cookie', adminCookie) + .send({ mode: 'sso_only', confirm: true }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/at least one SSO provider/i); + }); + + it('enables sso_only for an SSO admin when a provider test passes', async () => { + markAdminAsSso(); + enableGithubProvider(); + vi.spyOn(SSOService.getInstance(), 'testOidcDiscovery').mockResolvedValue({ success: true }); + + const res = await request(app) + .put('/api/sso/auth-mode') + .set('Cookie', adminCookie) + .send({ mode: 'sso_only', confirm: true }); + expect(res.status).toBe(200); + expect(res.body.authenticationMode).toBe('sso_only'); + expect(res.body.localLoginEnabled).toBe(false); + }); + + it('lets a Community admin revert to local_and_sso (not paid-gated)', async () => { + setAuthenticationMode('sso_only'); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + + const res = await request(app) + .put('/api/sso/auth-mode') + .set('Cookie', adminCookie) + .send({ mode: 'local_and_sso' }); + expect(res.status).toBe(200); + expect(res.body.authenticationMode).toBe('local_and_sso'); + expect(res.body.localLoginEnabled).toBe(true); + }); +}); + +describe('Last-provider guard while sso_only', () => { + it('rejects disabling the last enabled provider', async () => { + markAdminAsSso(); + enableGithubProvider(); + setAuthenticationMode('sso_only'); + + const res = await request(app) + .put('/api/sso/config/oidc_github') + .set('Cookie', adminCookie) + .send({ + provider: 'oidc_github', + enabled: false, + displayName: 'GitHub', + oidcClientId: 'test-client', + }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/last SSO provider/i); + expect(DatabaseService.getInstance().getEnabledSSOConfigs()).toHaveLength(1); + }); + + it('rejects deleting the last enabled provider', async () => { + markAdminAsSso(); + enableGithubProvider(); + setAuthenticationMode('sso_only'); + + const res = await request(app) + .delete('/api/sso/config/oidc_github') + .set('Cookie', adminCookie); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/last SSO provider/i); + }); +}); + +describe('Password change under sso_only', () => { + it('still allows an authenticated password change', async () => { + setAuthenticationMode('local_and_sso'); + const freshCookie = await loginAsTestAdmin(app); + setAuthenticationMode('sso_only'); + + const res = await request(app) + .put('/api/auth/password') + .set('Cookie', freshCookie) + .send({ oldPassword: TEST_PASSWORD, newPassword: TEST_PASSWORD }); + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + }); +}); diff --git a/backend/src/__tests__/auth.test.ts b/backend/src/__tests__/auth.test.ts index 0672ea22..952bb49a 100644 --- a/backend/src/__tests__/auth.test.ts +++ b/backend/src/__tests__/auth.test.ts @@ -51,6 +51,54 @@ describe('POST /api/auth/login', () => { const res = await request(app).post('/api/auth/login').send({}); expect(res.status).toBe(400); }); + + it('returns 403 when authentication_mode is sso_only', async () => { + const { setAuthenticationMode } = await import('../helpers/authenticationMode'); + setAuthenticationMode('sso_only'); + try { + const res = await request(app) + .post('/api/auth/login') + .send({ username: TEST_USERNAME, password: TEST_PASSWORD }); + expect(res.status).toBe(403); + expect(res.body.error).toMatch(/Local password authentication is disabled/i); + } finally { + setAuthenticationMode('local_and_sso'); + } + }); +}); + +describe('GET /api/auth/status', () => { + it('reports localLoginEnabled true by default', async () => { + const res = await request(app).get('/api/auth/status'); + expect(res.status).toBe(200); + expect(res.body.localLoginEnabled).toBe(true); + expect(res.body.authenticationMode).toBe('local_and_sso'); + }); + + it('reports localLoginEnabled false when sso_only', async () => { + const { setAuthenticationMode } = await import('../helpers/authenticationMode'); + setAuthenticationMode('sso_only'); + try { + const res = await request(app).get('/api/auth/status'); + expect(res.status).toBe(200); + expect(res.body.localLoginEnabled).toBe(false); + expect(res.body.authenticationMode).toBe('sso_only'); + } finally { + setAuthenticationMode('local_and_sso'); + } + }); + + it('defaults localLoginEnabled to true when the setting key is missing', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + db.getDb().prepare('DELETE FROM global_settings WHERE key = ?').run('authentication_mode'); + // Bust the settings cache so the next read rebuilds without the deleted key. + const cpu = db.getDb().prepare('SELECT value FROM global_settings WHERE key = ?').get('host_cpu_limit') as { value: string }; + db.updateGlobalSetting('host_cpu_limit', cpu.value); + const res = await request(app).get('/api/auth/status'); + expect(res.status).toBe(200); + expect(res.body.localLoginEnabled).toBe(true); + }); }); // ─── Auth middleware ────────────────────────────────────────────────────────── diff --git a/backend/src/__tests__/auto-update-digest-gate.test.ts b/backend/src/__tests__/auto-update-digest-gate.test.ts new file mode 100644 index 00000000..45a6f7dd --- /dev/null +++ b/backend/src/__tests__/auto-update-digest-gate.test.ts @@ -0,0 +1,38 @@ +import { describe, it, expect } from 'vitest'; +import { + createAutoUpdateDigestGateState, + messageWhenDigestApplyBlockedByCheckErrors, + recordAutoUpdateImageCheck, +} from '../helpers/autoUpdateDigestGate'; + +describe('autoUpdateDigestGate', () => { + it('blocks digest apply when sibling check errors exist', () => { + const state = createAutoUpdateDigestGateState(); + recordAutoUpdateImageCheck(state, 'nginx:latest', { + hasUpdate: true, + digestUpdate: true, + tagUpdate: false, + }); + recordAutoUpdateImageCheck(state, 'redis:latest', { + hasUpdate: false, + digestUpdate: false, + tagUpdate: false, + checkStatus: 'failed', + error: 'registry timeout', + }); + + const msg = messageWhenDigestApplyBlockedByCheckErrors('web', state); + expect(msg).toContain('image check(s) failed'); + expect(msg).toContain('registry timeout'); + }); + + it('does not block when digest update exists without check errors', () => { + const state = createAutoUpdateDigestGateState(); + recordAutoUpdateImageCheck(state, 'nginx:latest', { + hasUpdate: true, + digestUpdate: true, + tagUpdate: false, + }); + expect(messageWhenDigestApplyBlockedByCheckErrors('web', state)).toBeNull(); + }); +}); diff --git a/backend/src/__tests__/cache-service.test.ts b/backend/src/__tests__/cache-service.test.ts index 24725c5d..6af1b8b3 100644 --- a/backend/src/__tests__/cache-service.test.ts +++ b/backend/src/__tests__/cache-service.test.ts @@ -298,6 +298,26 @@ describe('CacheService', () => { it('is a no-op for missing keys', () => { expect(() => cache.invalidate('ns:missing')).not.toThrow(); }); + + it('prevents an in-flight fetch started before invalidate from committing afterward', async () => { + let resolveFetch!: (value: string) => void; + const staleFetcher = vi.fn(() => new Promise((resolve) => { + resolveFetch = resolve; + })); + + const stalePromise = cache.getOrFetch('fleet-updates', 60_000, staleFetcher); + cache.invalidate('fleet-updates'); + + const freshFetcher = vi.fn().mockResolvedValue('fresh'); + const fresh = await cache.getOrFetch('fleet-updates', 60_000, freshFetcher); + expect(fresh).toBe('fresh'); + expect(freshFetcher).toHaveBeenCalledTimes(1); + + resolveFetch('stale'); + await expect(stalePromise).resolves.toBe('stale'); + // Stale writer must not overwrite the post-invalidate entry. + expect(cache.get('fleet-updates')).toBe('fresh'); + }); }); describe('invalidateNamespace', () => { diff --git a/backend/src/__tests__/compose-doctor-service.test.ts b/backend/src/__tests__/compose-doctor-service.test.ts index 1d8cf331..a0b46e6d 100644 --- a/backend/src/__tests__/compose-doctor-service.test.ts +++ b/backend/src/__tests__/compose-doctor-service.test.ts @@ -28,6 +28,9 @@ function stubDocker( rendered: object | null, stderr = '', snapshot: { containers: unknown[]; networks: unknown[]; volumes: unknown[] } | 'reject' = { containers: [], networks: [], volumes: [] }, + inspectImage: ReturnType = vi.fn().mockRejectedValue( + Object.assign(new Error('No such image'), { statusCode: 404 }), + ), ) { vi.spyOn(ComposeService, 'getInstance').mockReturnValue({ renderConfig: vi.fn().mockResolvedValue({ @@ -41,6 +44,13 @@ function stubDocker( getDependencySnapshot: snapshot === 'reject' ? vi.fn().mockRejectedValue(new Error('docker down')) : vi.fn().mockResolvedValue(snapshot), + getDocker: vi.fn(() => ({ + listContainers: vi.fn().mockResolvedValue([]), + getContainer: vi.fn(() => ({ + inspect: vi.fn().mockResolvedValue({ Config: {} }), + })), + })), + inspectImage, } as unknown as DockerController); } @@ -85,7 +95,7 @@ describe('runPreflight', () => { expect(report.renderable).toBe(true); expect(report.status).toBe('high'); // env-unset + 0.0.0.0 exposure are high expect(report.highestSeverity).toBe('high'); - expect(report.findings.map(f => f.ruleId)).toEqual(expect.arrayContaining(['env-literal-dollar', 'port-exposed-all-interfaces', 'image-latest', 'no-healthcheck'])); + expect(report.findings.map(f => f.ruleId)).toEqual(expect.arrayContaining(['env-literal-dollar', 'port-exposed-all-interfaces', 'image-latest', 'healthcheck-unverifiable'])); expect(report.ranBy).toBe('tester'); expect(report.sourceHash).toBeTruthy(); @@ -173,6 +183,30 @@ describe('runPreflight', () => { expect(doctor().getLatest(nodeId, STACK).status).toBe('pass'); }); + it('treats inherited healthcheck as a note that does not block All Clear', async () => { + const model = { + name: STACK, + services: { web: { image: 'nginx:1.27', restart: 'always' } }, + networks: {}, + volumes: {}, + }; + stubDocker( + model, + '', + { containers: [], networks: [], volumes: [] }, + vi.fn().mockResolvedValue({ + inspect: { Config: { Healthcheck: { Test: ['CMD', 'true'] } } }, + history: [], + }), + ); + + const report = await doctor().runPreflight(nodeId, STACK, 'tester'); + expect(report.findings.some(f => f.ruleId === 'healthcheck-inherited')).toBe(true); + expect(report.activeCount).toBe(0); + expect(report.activeStatus).toBe('pass'); + expect(report.status).toBe('pass'); + }); + it('returns an unrenderable report and never stores raw stderr', async () => { stubDocker(null, `bad yaml near ${SECRET}`); // stderr can echo arbitrary file content const report = await doctor().runPreflight(nodeId, STACK, null); diff --git a/backend/src/__tests__/compose-network-inspector.test.ts b/backend/src/__tests__/compose-network-inspector.test.ts index 035d1599..702a9328 100644 --- a/backend/src/__tests__/compose-network-inspector.test.ts +++ b/backend/src/__tests__/compose-network-inspector.test.ts @@ -14,10 +14,15 @@ import { import { assembleStackNetworkFacts } from '../services/network/composeNetworkInspector'; function effSvc(over: Partial = {}): EffService { + const hasHealthcheck = over.hasHealthcheck ?? true; + const composeHealthcheck = over.composeHealthcheck ?? (hasHealthcheck ? 'active' : 'absent'); return { name: 'web', image: 'nginx:1.27', ports: [], binds: [], namedVolumes: [], storageMounts: [], - privileged: false, hasHealthcheck: true, restart: 'unless-stopped', envKeys: [], - networks: [], extraHosts: [], labelKeys: [], ...over, + privileged: false, restart: 'unless-stopped', envKeys: [], + networks: [], extraHosts: [], labelKeys: [], + ...over, + hasHealthcheck, + composeHealthcheck, }; } diff --git a/backend/src/__tests__/dashboard-routes.test.ts b/backend/src/__tests__/dashboard-routes.test.ts index ba4d0758..27c329ef 100644 --- a/backend/src/__tests__/dashboard-routes.test.ts +++ b/backend/src/__tests__/dashboard-routes.test.ts @@ -6,11 +6,15 @@ * - GET /api/dashboard/configuration returns the documented shape and * applies tier-correct `locked` flags for the Community and paid * personas (toggled via LicenseService spies). + * - Alert-rule counts are scoped to stacks present on the active node + * (dashboard and fleet local-node row agree on exact cardinality). * - GET /api/dashboard/stack-restarts clamps the `days` query parameter * to [1, 30] and falls back to 7 for invalid inputs. * - Neither endpoint leaks secret material (agent URLs, tokens) in the * response payload. */ +import fs from 'fs'; +import path from 'path'; import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; import request from 'supertest'; import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; @@ -113,7 +117,7 @@ describe('GET /api/dashboard/configuration', () => { last_error: null, prune_targets: null, target_services: null, - prune_label_filter: null, + prune_label_filter: null, selector_type: null, selector_value: null, }; const idA = db.createScheduledTask({ ...baseTask, @@ -192,6 +196,57 @@ describe('GET /api/dashboard/configuration', () => { db.getDb().prepare('DELETE FROM agents WHERE url = ?').run(SECRET_URL); } }); + + it('scopes alertRules to stacks on the active node for dashboard and fleet local row', async () => { + const db = DatabaseService.getInstance(); + const composeDir = process.env.COMPOSE_DIR as string; + const stackName = 'cfg-alert-scope'; + const stackDir = path.join(composeDir, stackName); + const alertIds: number[] = []; + + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync( + path.join(stackDir, 'compose.yaml'), + 'services:\n web:\n image: nginx:latest\n', + ); + + const baseAlert = { + service_name: null as string | null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 15, + last_fired_at: 0, + }; + + try { + // Two rules on a discovered stack (exact rule count, not unique stacks) + // plus one orphaned rule that must not inflate the scoped count. + alertIds.push(db.addStackAlert({ ...baseAlert, stack_name: stackName, threshold: 80 }).id!); + alertIds.push(db.addStackAlert({ ...baseAlert, stack_name: stackName, threshold: 90 }).id!); + alertIds.push(db.addStackAlert({ ...baseAlert, stack_name: 'cfg-alert-orphan', threshold: 70 }).id!); + + const dash = await request(app).get('/api/dashboard/configuration').set('Cookie', adminCookie); + expect(dash.status).toBe(200); + expect(dash.body.notifications.alertRules).toBe(2); + + const fleet = await request(app).get('/api/fleet/configuration').set('Cookie', adminCookie); + expect(fleet.status).toBe(200); + expect(Array.isArray(fleet.body)).toBe(true); + const localRow = fleet.body.find( + (row: { type: string; configuration: { notifications: { alertRules: number } } | null }) => + row.type === 'local' && row.configuration != null, + ); + expect(localRow).toBeDefined(); + expect(localRow.configuration.notifications.alertRules).toBe(2); + } finally { + for (const id of alertIds) { + db.deleteStackAlert(id); + } + fs.rmSync(stackDir, { recursive: true, force: true }); + } + }); }); describe('GET /api/dashboard/stack-restarts', () => { diff --git a/backend/src/__tests__/database-cascade.test.ts b/backend/src/__tests__/database-cascade.test.ts index 14c5c12a..2d5c7f62 100644 --- a/backend/src/__tests__/database-cascade.test.ts +++ b/backend/src/__tests__/database-cascade.test.ts @@ -41,6 +41,8 @@ const SCHEMA = ` prune_targets TEXT, target_services TEXT, prune_label_filter TEXT, + selector_type TEXT, + selector_value TEXT, FOREIGN KEY(node_id) REFERENCES nodes(id) ); diff --git a/backend/src/__tests__/database-stack-update-status.test.ts b/backend/src/__tests__/database-stack-update-status.test.ts index ce74e2bb..e7c22633 100644 --- a/backend/src/__tests__/database-stack-update-status.test.ts +++ b/backend/src/__tests__/database-stack-update-status.test.ts @@ -46,10 +46,50 @@ describe('stack_update_status tri-state accessors', () => { expect(db().getStackUpdateDetail(NODE).web.checkStatus).toBe('ok'); }); - it('keeps getStackUpdateStatus a boolean map for the fleet contract', () => { + it('keeps getStackUpdateStatus a raw boolean map ignoring check_status', () => { db().upsertStackUpdateStatus(NODE, 'web', true, 1000, 'ok', null); db().upsertStackUpdateStatus(NODE, 'api', false, 1000, 'failed', 'boom'); - expect(db().getStackUpdateStatus(NODE)).toEqual({ web: true, api: false }); + db().upsertStackUpdateStatus(NODE, 'sticky', true, 1000, 'partial', 'half'); + expect(db().getStackUpdateStatus(NODE)).toEqual({ web: true, api: false, sticky: true }); + }); + + it('projects confirmed updates only via getConfirmedStackUpdateStatus', () => { + db().upsertStackUpdateStatus(NODE, 'web', true, 1000, 'ok', null); + db().upsertStackUpdateStatus(NODE, 'sticky', true, 1000, 'partial', 'half'); + db().upsertStackUpdateStatus(NODE, 'failed', true, 1000, 'failed', 'boom'); + db().upsertStackUpdateStatus(NODE, 'clean', false, 1000, 'ok', null); + expect(db().getConfirmedStackUpdateStatus(NODE)).toEqual({ + web: true, + sticky: false, + failed: false, + clean: false, + }); + }); + + it('clearStackUpdateStatus returns deleted row count and removes services_json', () => { + db().upsertStackUpdateStatus(NODE, 'web', true, 1000, 'partial', 'half', [ + { service: 'web', image: 'web:1', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ]); + expect(db().clearStackUpdateStatus(NODE, 'web')).toBe(1); + expect(db().getStackUpdateDetail(NODE).web).toBeUndefined(); + expect(db().clearStackUpdateStatus(NODE, 'web')).toBe(0); + }); + + it('clearAllStackUpdateStatus deletes only the given node rows', () => { + const other = NODE + 1; + db().upsertStackUpdateStatus(NODE, 'web', true, 1000, 'ok', null); + db().upsertStackUpdateStatus(NODE, 'api', true, 1000, 'ok', null); + db().upsertStackUpdateStatus(other, 'web', true, 1000, 'ok', null); + expect(db().clearAllStackUpdateStatus(NODE)).toBe(2); + expect(db().getStackUpdateDetail(NODE)).toEqual({}); + expect(db().getStackUpdateDetail(other).web).toBeDefined(); + }); + + it('getNodeUpdateSummary counts only confirmed updates', () => { + db().upsertStackUpdateStatus(NODE, 'web', true, 1000, 'ok', null); + db().upsertStackUpdateStatus(NODE, 'sticky', true, 1000, 'partial', 'half'); + const summary = db().getNodeUpdateSummary().find((r) => r.node_id === NODE); + expect(summary?.stacks_with_updates).toBe(1); }); it('recordStackCheckFailure preserves a prior has_update while marking failed', () => { diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index e55e7082..67591035 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -2,11 +2,11 @@ * Unit tests for DockerController — validateApiData, state-safe container ops, * disk usage, classified resources, orphan detection, and error paths. */ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; // ── Hoisted mocks ────────────────────────────────────────────────────── -const { mockDocker } = vi.hoisted(() => { +const { mockDocker, composeDirRef, mockExecFileAsync } = vi.hoisted(() => { const mockDocker = { df: vi.fn(), listImages: vi.fn().mockResolvedValue([]), @@ -23,7 +23,11 @@ const { mockDocker } = vi.hoisted(() => { pruneVolumes: vi.fn().mockResolvedValue({ SpaceReclaimed: 0 }), createNetwork: vi.fn(), }; - return { mockDocker }; + return { + mockDocker, + composeDirRef: { current: '/test/compose' }, + mockExecFileAsync: vi.fn(), + }; }); vi.mock('../services/NodeRegistry', () => ({ @@ -31,7 +35,16 @@ vi.mock('../services/NodeRegistry', () => ({ getInstance: () => ({ getDocker: () => mockDocker, getDefaultNodeId: () => 1, - getComposeDir: () => '/test/compose', + getComposeDir: () => composeDirRef.current, + }), + }, +})); + +vi.mock('../services/DatabaseService', () => ({ + DatabaseService: { + getInstance: () => ({ + getGitSource: () => undefined, + getStackProjectEnvFiles: () => [], }), }, })); @@ -42,14 +55,19 @@ vi.mock('child_process', () => ({ execFile: vi.fn(), })); vi.mock('util', () => ({ - promisify: () => vi.fn(), + promisify: () => mockExecFileAsync, })); import DockerController, { selectMainWebPort, parseExitCode, isContainerFailed } from '../services/DockerController'; import { CacheService } from '../services/CacheService'; +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; beforeEach(() => { vi.clearAllMocks(); + composeDirRef.current = '/test/compose'; + mockExecFileAsync.mockResolvedValue({ stdout: '', stderr: '' }); }); // ── validateApiData ──────────────────────────────────────────────────── @@ -1694,13 +1712,23 @@ describe('DockerController - getBulkStackStatuses partial status', () => { // ── getLegacyOrphanContainersByStack (#1565) ─────────────────────────── +type OrphanDcSpies = { + fetchComposePsContainers: (...a: unknown[]) => Promise; + smartFallback: (...a: unknown[]) => Promise; +}; + +function spyOrphanDc( + dc: ReturnType, + method: keyof OrphanDcSpies, +) { + return vi.spyOn(dc as unknown as OrphanDcSpies, method); +} + describe('DockerController - getLegacyOrphanContainersByStack', () => { it('returns [] when compose ps already manages containers', async () => { const dc = DockerController.getInstance(1); - const fetchSpy = vi.spyOn(dc as unknown as { fetchComposePsContainers: (...a: unknown[]) => Promise }, 'fetchComposePsContainers') - .mockResolvedValue([{ ID: 'managed-c1', Name: 'web-1' }]); - const fallbackSpy = vi.spyOn(dc as unknown as { smartFallback: (...a: unknown[]) => Promise }, 'smartFallback') - .mockResolvedValue([{ Id: 'legacy-c1' }]); + const fetchSpy = spyOrphanDc(dc, 'fetchComposePsContainers').mockResolvedValue([{ ID: 'managed-c1', Name: 'web-1' }]); + const fallbackSpy = spyOrphanDc(dc, 'smartFallback').mockResolvedValue([{ Id: 'legacy-c1' }]); await expect(dc.getLegacyOrphanContainersByStack('my-stack')).resolves.toEqual([]); expect(fallbackSpy).not.toHaveBeenCalled(); @@ -1710,45 +1738,174 @@ describe('DockerController - getLegacyOrphanContainersByStack', () => { it('returns legacy orphan IDs when compose ps is empty', async () => { const dc = DockerController.getInstance(1); - const fetchSpy = vi.spyOn(dc as unknown as { fetchComposePsContainers: (...a: unknown[]) => Promise }, 'fetchComposePsContainers') - .mockResolvedValue([]); - const fallbackSpy = vi.spyOn(dc as unknown as { smartFallback: (...a: unknown[]) => Promise }, 'smartFallback') - .mockResolvedValue([{ Id: 'legacy-c1' }, { Id: '' }, {}]); + const fetchSpy = spyOrphanDc(dc, 'fetchComposePsContainers').mockResolvedValue([]); + const fallbackSpy = spyOrphanDc(dc, 'smartFallback').mockResolvedValue([{ Id: 'legacy-c1' }, { Id: '' }, {}]); await expect(dc.getLegacyOrphanContainersByStack('my-stack')).resolves.toEqual([{ Id: 'legacy-c1' }]); fetchSpy.mockRestore(); fallbackSpy.mockRestore(); }); - it('falls back to legacy orphan lookup when compose ps throws', async () => { + it('returns [] when compose ps throws, even if fallback would match containers', async () => { const dc = DockerController.getInstance(1); - const fetchSpy = vi.spyOn(dc as unknown as { fetchComposePsContainers: (...a: unknown[]) => Promise }, 'fetchComposePsContainers') - .mockRejectedValue(new Error('compose ps failed')); - const fallbackSpy = vi.spyOn(dc as unknown as { smartFallback: (...a: unknown[]) => Promise }, 'smartFallback') - .mockResolvedValue([{ Id: 'legacy-c2' }]); + const fetchSpy = spyOrphanDc(dc, 'fetchComposePsContainers').mockRejectedValue(new Error('compose ps failed')); + const fallbackSpy = spyOrphanDc(dc, 'smartFallback').mockResolvedValue([{ Id: 'running-compose-c2' }]); - await expect(dc.getLegacyOrphanContainersByStack('my-stack')).resolves.toEqual([{ Id: 'legacy-c2' }]); + await expect(dc.getLegacyOrphanContainersByStack('my-stack')).resolves.toEqual([]); + // Must not consult name-matching fallback: those IDs may be healthy Compose runtimes. + expect(fallbackSpy).not.toHaveBeenCalled(); fetchSpy.mockRestore(); fallbackSpy.mockRestore(); }); + + it('returns [] and skips fallback when compose ps stdout is unparseable', async () => { + mockExecFileAsync.mockResolvedValue({ + stdout: 'this is not json at all just random text { broken [', + stderr: '', + }); + const dc = DockerController.getInstance(1); + const fallbackSpy = spyOrphanDc(dc, 'smartFallback').mockResolvedValue([{ Id: 'running-c1' }]); + + await expect(dc.getLegacyOrphanContainersByStack('my-stack')).resolves.toEqual([]); + expect(fallbackSpy).not.toHaveBeenCalled(); + fallbackSpy.mockRestore(); + }); }); describe('DockerController - classifyLegacyOrphansForUpdate', () => { - it('returns none when compose ps already manages containers', async () => { - // Reuse the same mocks as getLegacyOrphanContainersByStack tests in this file. + async function classifyDc() { const { default: DockerController } = await import('../services/DockerController'); - const dc = DockerController.getInstance(1); - vi.spyOn(dc as any, 'fetchComposePsContainers').mockResolvedValue([{ ID: 'c1' }]); + return DockerController.getInstance(1); + } + + it('returns none when compose ps already manages containers', async () => { + const dc = await classifyDc(); + spyOrphanDc(dc, 'fetchComposePsContainers').mockResolvedValue([{ ID: 'c1' }]); await expect(dc.classifyLegacyOrphansForUpdate('my-stack')).resolves.toEqual({ status: 'none' }); }); - it('returns classification_failed when compose ps and fallback both fail', async () => { - const { default: DockerController } = await import('../services/DockerController'); - const dc = DockerController.getInstance(1); - vi.spyOn(dc as any, 'fetchComposePsContainers').mockRejectedValue(new Error('compose boom')); - vi.spyOn(dc as any, 'smartFallback').mockRejectedValue(new Error('fallback boom')); + it('returns orphans when compose ps is empty and fallback finds legacy containers', async () => { + const dc = await classifyDc(); + spyOrphanDc(dc, 'fetchComposePsContainers').mockResolvedValue([]); + spyOrphanDc(dc, 'smartFallback').mockResolvedValue([{ Id: 'legacy-c1' }, { Id: 'legacy-c2' }]); + await expect(dc.classifyLegacyOrphansForUpdate('my-stack')).resolves.toEqual({ + status: 'orphans', + ids: ['legacy-c1', 'legacy-c2'], + }); + }); + + it('returns classification_failed when compose ps throws, even if fallback would match containers', async () => { + const dc = await classifyDc(); + spyOrphanDc(dc, 'fetchComposePsContainers').mockRejectedValue(new Error('compose boom')); + const fallbackSpy = spyOrphanDc(dc, 'smartFallback').mockResolvedValue([{ Id: 'running-compose-c1' }]); + const result = await dc.classifyLegacyOrphansForUpdate('my-stack'); + expect(result).toEqual({ status: 'classification_failed', error: 'compose boom' }); + // Must not consult name-matching fallback: those IDs may be healthy Compose runtimes. + expect(fallbackSpy).not.toHaveBeenCalled(); + }); + + it('returns classification_failed when compose ps is empty and fallback throws', async () => { + const dc = await classifyDc(); + spyOrphanDc(dc, 'fetchComposePsContainers').mockResolvedValue([]); + spyOrphanDc(dc, 'smartFallback').mockRejectedValue(new Error('fallback boom')); + const result = await dc.classifyLegacyOrphansForUpdate('my-stack'); + expect(result).toEqual({ status: 'classification_failed', error: 'fallback boom' }); + }); + + it('returns classification_failed when compose ps stdout is unparseable', async () => { + mockExecFileAsync.mockResolvedValue({ + stdout: 'this is not json at all just random text { broken [', + stderr: '', + }); + const dc = await classifyDc(); + const fallbackSpy = spyOrphanDc(dc, 'smartFallback').mockResolvedValue([{ Id: 'running-c1' }]); const result = await dc.classifyLegacyOrphansForUpdate('my-stack'); expect(result.status).toBe('classification_failed'); + expect(String((result as { error?: string }).error ?? '')).toMatch(/unparseable JSON/i); + expect(fallbackSpy).not.toHaveBeenCalled(); + }); +}); + +describe('DockerController - smartFallback stack-dir evidence', () => { + const composeYaml = `services:\n web:\n image: nginx:alpine\n`; + let stackDir: string; + let tmpRoot: string; + + function runningContainer(id: string, name: string, labels: Record = {}) { + return { Id: id, Names: [name], Labels: labels, State: 'running', Status: 'Up', Ports: [] }; + } + + async function orphansWithEmptyPs() { + const dc = DockerController.getInstance(1); + spyOrphanDc(dc, 'fetchComposePsContainers').mockResolvedValue([]); + return dc.getLegacyOrphanContainersByStack('my-stack'); + } + + beforeEach(async () => { + tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'sencho-orphan-')); + composeDirRef.current = tmpRoot; + stackDir = path.join(tmpRoot, 'my-stack'); + await fs.mkdir(stackDir); + await fs.writeFile(path.join(stackDir, 'docker-compose.yml'), composeYaml); + }); + + afterEach(async () => { + composeDirRef.current = '/test/compose'; + await fs.rm(tmpRoot, { recursive: true, force: true }); + }); + + it('excludes name-matched containers without stack-dir evidence', async () => { + mockDocker.listContainers.mockResolvedValue([ + runningContainer('unrelated-web', '/web'), + ]); + await expect(orphansWithEmptyPs()).resolves.toEqual([]); + }); + + it('includes name-matched containers whose working_dir equals the stack dir', async () => { + mockDocker.listContainers.mockResolvedValue([ + runningContainer('legacy-web', '/web', { + 'com.docker.compose.project.working_dir': stackDir, + }), + ]); + await expect(orphansWithEmptyPs()).resolves.toEqual([{ Id: 'legacy-web' }]); + }); + + it('matches working_dir case-insensitively on win32', async () => { + if (process.platform !== 'win32') return; + mockDocker.listContainers.mockResolvedValue([ + runningContainer('legacy-web-case', '/web', { + 'com.docker.compose.project.working_dir': stackDir.toUpperCase(), + }), + ]); + await expect(orphansWithEmptyPs()).resolves.toEqual([{ Id: 'legacy-web-case' }]); + }); + + it('excludes name-matched containers whose working_dir points elsewhere', async () => { + mockDocker.listContainers.mockResolvedValue([ + runningContainer('other-stack-web', '/web', { + 'com.docker.compose.project.working_dir': path.join(tmpRoot, 'other-stack'), + }), + ]); + await expect(orphansWithEmptyPs()).resolves.toEqual([]); + }); + + it('includes name-matched containers whose config_files path is under the stack dir', async () => { + mockDocker.listContainers.mockResolvedValue([ + runningContainer('legacy-cfg', '/my-stack-web-1', { + 'com.docker.compose.project.config_files': path.join(stackDir, 'docker-compose.yml'), + }), + ]); + await expect(orphansWithEmptyPs()).resolves.toEqual([{ Id: 'legacy-cfg' }]); + }); + + it('surfaces listContainers failure as classification_failed on update', async () => { + mockDocker.listContainers.mockRejectedValue(new Error('daemon unavailable')); + + const { default: DockerController } = await import('../services/DockerController'); + const dc = DockerController.getInstance(1); + spyOrphanDc(dc, 'fetchComposePsContainers').mockResolvedValue([]); + const result = await dc.classifyLegacyOrphansForUpdate('my-stack'); + expect(result).toEqual({ status: 'classification_failed', error: 'daemon unavailable' }); }); }); diff --git a/backend/src/__tests__/drift-detection.test.ts b/backend/src/__tests__/drift-detection.test.ts index efb4694a..c5198e68 100644 --- a/backend/src/__tests__/drift-detection.test.ts +++ b/backend/src/__tests__/drift-detection.test.ts @@ -43,10 +43,15 @@ function container(p: Partial & { id: string }): Dependency const findingKinds = (r: { findings: { kind: string }[] }): string[] => r.findings.map((f) => f.kind).sort(); function effSvc(over: Partial = {}): EffService { + const hasHealthcheck = over.hasHealthcheck ?? true; + const composeHealthcheck = over.composeHealthcheck ?? (hasHealthcheck ? 'active' : 'absent'); return { name: 'web', image: 'nginx:1.27', ports: [], binds: [], namedVolumes: [], storageMounts: [], - privileged: false, hasHealthcheck: true, restart: 'unless-stopped', envKeys: [], - networks: [], extraHosts: [], labelKeys: [], ...over, + privileged: false, restart: 'unless-stopped', envKeys: [], + networks: [], extraHosts: [], labelKeys: [], + ...over, + hasHealthcheck, + composeHealthcheck, }; } diff --git a/backend/src/__tests__/effective-service-model.test.ts b/backend/src/__tests__/effective-service-model.test.ts index e7ffccfb..ca52c9ff 100644 --- a/backend/src/__tests__/effective-service-model.test.ts +++ b/backend/src/__tests__/effective-service-model.test.ts @@ -79,6 +79,15 @@ describe('buildEffectiveServiceModel', () => { expect(result.services[0].hasHealthcheck).toBe(false); }); + it('treats test NONE as a disabled healthcheck', async () => { + stubRender(JSON.stringify({ + services: { web: { image: 'a', healthcheck: { test: ['NONE'] } } }, + })); + const result = await buildEffectiveServiceModel(1, 'mystack'); + if (!result.renderable) throw new Error('expected renderable'); + expect(result.services[0].hasHealthcheck).toBe(false); + }); + it('parses depends_on given in the short list form', async () => { stubRender(JSON.stringify({ services: { web: { image: 'a', depends_on: ['db', 'cache'] } }, diff --git a/backend/src/__tests__/exposure.test.ts b/backend/src/__tests__/exposure.test.ts index 132e1406..7333b3fe 100644 --- a/backend/src/__tests__/exposure.test.ts +++ b/backend/src/__tests__/exposure.test.ts @@ -14,6 +14,7 @@ function svc(overrides: Record) { networkMode: undefined as string | undefined, restart: undefined as string | undefined, hasHealthcheck: false, + composeHealthcheck: 'absent' as const, envKeys: [], networks: [], extraHosts: [], diff --git a/backend/src/__tests__/fleet-update-cache.test.ts b/backend/src/__tests__/fleet-update-cache.test.ts new file mode 100644 index 00000000..312bde25 --- /dev/null +++ b/backend/src/__tests__/fleet-update-cache.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { CacheService } from '../services/CacheService'; +import { + FLEET_UPDATE_CACHE_KEY, + invalidateFleetUpdateCache, + isFullStackUpdatePath, + isUpdatePreviewPath, +} from '../helpers/fleetUpdateCache'; + +describe('isFullStackUpdatePath', () => { + it('matches full-stack update paths after the /api mount strip', () => { + expect(isFullStackUpdatePath('/stacks/paperless/update')).toBe(true); + expect(isFullStackUpdatePath('/stacks/paperless/update?nodeId=2')).toBe(true); + }); + + it('rejects service-scoped update and restore paths', () => { + expect(isFullStackUpdatePath('/stacks/paperless/services/redis/update')).toBe(false); + expect(isFullStackUpdatePath('/stacks/paperless/services/redis/restore')).toBe(false); + expect(isFullStackUpdatePath('/stacks/paperless/deploy')).toBe(false); + }); +}); + +describe('isUpdatePreviewPath', () => { + it('matches update-preview paths after the /api mount strip', () => { + expect(isUpdatePreviewPath('/stacks/paperless/update-preview')).toBe(true); + expect(isUpdatePreviewPath('/stacks/paperless/update-preview?nodeId=2')).toBe(true); + }); + + it('rejects full-stack update and other stack paths', () => { + expect(isUpdatePreviewPath('/stacks/paperless/update')).toBe(false); + expect(isUpdatePreviewPath('/stacks/paperless/services/redis/update')).toBe(false); + }); +}); + +describe('invalidateFleetUpdateCache', () => { + beforeEach(() => { + CacheService.getInstance().flush(); + }); + + afterEach(() => { + CacheService.getInstance().flush(); + }); + + it('drops the shared fleet-updates key', async () => { + const cache = CacheService.getInstance(); + await cache.getOrFetch(FLEET_UPDATE_CACHE_KEY, 60_000, async () => ({ '1': { web: true } })); + invalidateFleetUpdateCache(); + expect(cache.get(FLEET_UPDATE_CACHE_KEY)).toBeUndefined(); + }); +}); diff --git a/backend/src/__tests__/fleet-update-hardening.test.ts b/backend/src/__tests__/fleet-update-hardening.test.ts index 13b11a7f..834d5454 100644 --- a/backend/src/__tests__/fleet-update-hardening.test.ts +++ b/backend/src/__tests__/fleet-update-hardening.test.ts @@ -73,6 +73,7 @@ function setTracker(over: Partial { FleetUpdateTrackerService.getInstance().set(proxyNodeId, { status: 'failed', startedAt: Date.now(), previousVersion: null, previousProcessStart: null, wasOffline: false, resolvedAt: Date.now(), error: 'boom', + operationKind: 'update', }); const second = await request(app) @@ -413,3 +415,170 @@ describe('GET /api/fleet/update-status/release-notes', () => { expect(res.body.htmlUrl).toBeNull(); }); }); + +describe('compose reapply status and concurrency', () => { + it('exposes canReapplyCompose for local when SelfUpdateService is available', async () => { + vi.spyOn(SelfUpdateService.getInstance(), 'isAvailable').mockReturnValue(true); + mockCompareTargetFetch(); + const res = await request(app).get('/api/fleet/update-status').set('Authorization', adminAuth); + expect(res.status).toBe(200); + const local = res.body.nodes.find((n: { type: string }) => n.type === 'local'); + expect(local.canReapplyCompose).toBe(true); + }); + + it('sets canReapplyCompose false for a remote without self-update capability', async () => { + mockMeta(ONLINE({ capabilities: ['stacks'] })); + mockCompareTargetFetch(); + const res = await request(app).get('/api/fleet/update-status').set('Authorization', adminAuth); + expect(res.status).toBe(200); + const remote = res.body.nodes.find((n: { nodeId: number }) => n.nodeId === proxyNodeId); + expect(remote.canReapplyCompose).toBe(false); + }); + + it('resolves a reapply tracker via startedAt change without requiring a version bump', async () => { + setTracker({ + operationKind: 'reapply_configuration', + previousVersion: '0.83.0', + previousProcessStart: 1, + startedAt: Date.now() - RECENT_MS, + }); + mockMeta(ONLINE({ version: '0.83.0', startedAt: 2 })); + mockCompareTargetFetch(); + expect(await getStatus()).toBe('completed'); + }); + + it('does not complete a reapply tracker via signal 4 when version is already current', async () => { + setTracker({ + operationKind: 'reapply_configuration', + previousVersion: '0.99.0', + previousProcessStart: 1, + startedAt: Date.now() - 20_000, + }); + // Node already at compare target; signal 4 would false-complete an update, + // but must not for reapply while startedAt is unchanged. + mockMeta(ONLINE({ version: '0.99.0', startedAt: 1 })); + mockCompareTargetFetch(); + expect(await getStatus()).toBe('updating'); + }); + + it('returns 409 when reapply is requested while an update tracker is in flight', async () => { + setTracker({ operationKind: 'update' }); + mockTarget(); + const res = await request(app) + .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`) + .set('Authorization', adminAuth); + expect(res.status).toBe(409); + expect(res.body?.error).toMatch(/already in progress/i); + }); + + it('dispatches remote reapply to /api/system/reapply-compose without updateBlocked gating', async () => { + mockTarget(); + mockMeta(ONLINE({ updateBlocked: true, imagePinKind: 'digest', imageChannel: 'community' })); + let reapplyUrl: string | null = null; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + try { + if (new URL(url).hostname === 'api.github.com') { + return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 }); + } + } catch { /* fall through */ } + reapplyUrl = url; + return new Response(JSON.stringify({ message: 'ok' }), { status: 202 }); + }); + + const res = await request(app) + .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`) + .set('Authorization', adminAuth); + + expect(res.status).toBe(202); + expect(reapplyUrl).toContain('/api/system/reapply-compose'); + expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)?.operationKind) + .toBe('reapply_configuration'); + }); + + it('reserves the tracker before remote dispatch so a concurrent reapply gets 409 without overwriting success', async () => { + mockTarget(); + // Hold meta so the first request sits in the dispatch set before the + // pollable tracker exists; the second must still 409 on that lock. + let releaseMeta!: (value: RemoteMeta) => void; + const metaHeld = new Promise((resolve) => { releaseMeta = resolve; }); + vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockImplementation(async () => metaHeld); + + let releaseRemote!: (value: Response) => void; + const remoteHeld = new Promise((resolve) => { releaseRemote = resolve; }); + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + try { + if (new URL(url).hostname === 'api.github.com') { + return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 }); + } + } catch { /* fall through */ } + if (url.includes('/api/system/reapply-compose')) { + return remoteHeld; + } + return new Response('{}', { status: 200 }); + }); + + // Supertest is lazy until the thenable is consumed; start the request now. + const firstPromise = request(app) + .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`) + .set('Authorization', adminAuth) + .then((res) => res); + + await vi.waitFor(() => { + expect(NodeRegistry.getInstance().fetchMetaForNode).toHaveBeenCalled(); + }); + + const secondDuringMeta = await request(app) + .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`) + .set('Authorization', adminAuth); + expect(secondDuringMeta.status).toBe(409); + expect(secondDuringMeta.body?.error).toMatch(/already in progress/i); + + releaseMeta(ONLINE()); + + await vi.waitFor(() => { + expect(FleetUpdateTrackerService.getInstance().get(proxyNodeId)?.status).toBe('updating'); + }); + + const secondDuringPost = await request(app) + .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`) + .set('Authorization', adminAuth); + expect(secondDuringPost.status).toBe(409); + + releaseRemote(new Response(JSON.stringify({ message: 'ok' }), { status: 202 })); + const first = await firstPromise; + expect(first.status).toBe(202); + const tracker = FleetUpdateTrackerService.getInstance().get(proxyNodeId); + expect(tracker?.status).toBe('updating'); + expect(tracker?.operationKind).toBe('reapply_configuration'); + expect(tracker?.previousVersion).toBe('0.83.0'); + expect(tracker?.previousProcessStart).toBe(1); + expect(tracker?.error).toBeUndefined(); + }); + + it('marks a reserved remote reapply as failed when the peer rejects, without leaving a false updating row for a second request', async () => { + mockTarget(); + mockMeta(ONLINE()); + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + try { + if (new URL(url).hostname === 'api.github.com') { + return new Response(JSON.stringify({ tag_name: 'v0.99.0' }), { status: 200 }); + } + } catch { /* fall through */ } + return new Response(JSON.stringify({ + error: 'An image operation is already in progress.', + code: 'IMAGE_OPERATION_IN_FLIGHT', + }), { status: 409 }); + }); + + const res = await request(app) + .post(`/api/fleet/nodes/${proxyNodeId}/reapply-compose`) + .set('Authorization', adminAuth); + expect(res.status).toBe(502); + const tracker = FleetUpdateTrackerService.getInstance().get(proxyNodeId); + expect(tracker?.status).toBe('failed'); + expect(tracker?.error).toMatch(/already in progress/i); + }); +}); diff --git a/backend/src/__tests__/fleet-update-tracker-service.test.ts b/backend/src/__tests__/fleet-update-tracker-service.test.ts index c596f727..2c6a6645 100644 --- a/backend/src/__tests__/fleet-update-tracker-service.test.ts +++ b/backend/src/__tests__/fleet-update-tracker-service.test.ts @@ -14,6 +14,7 @@ function mk(over: Partial): UpdateTracker { previousVersion: null, previousProcessStart: null, wasOffline: false, + operationKind: 'update', ...over, }; } diff --git a/backend/src/__tests__/healthcheck-evidence-collector.test.ts b/backend/src/__tests__/healthcheck-evidence-collector.test.ts new file mode 100644 index 00000000..2fd0e002 --- /dev/null +++ b/backend/src/__tests__/healthcheck-evidence-collector.test.ts @@ -0,0 +1,292 @@ +/** + * Unit tests for collectServiceHealthcheckEvidence. Mocks Docker list/inspect + * and image inspect; asserts structural evidence states and that Test command + * text never appears in returned evidence. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { collectServiceHealthcheckEvidence } from '../services/healthcheck/collectServiceHealthcheckEvidence'; +import type { EffectiveModel, EffService } from '../services/preflight/effectiveModel'; +import DockerController from '../services/DockerController'; + +function svc(over: Partial = {}): EffService { + const hasHealthcheck = over.hasHealthcheck ?? false; + const composeHealthcheck = over.composeHealthcheck ?? (hasHealthcheck ? 'active' : 'absent'); + return { + name: 'web', image: 'nginx:1.27', ports: [], binds: [], namedVolumes: [], storageMounts: [], + privileged: false, restart: 'unless-stopped', envKeys: [], + networks: [], extraHosts: [], labelKeys: [], + ...over, + hasHealthcheck, + composeHealthcheck, + }; +} + +function model(services: EffService[]): EffectiveModel { + return { projectName: 'proj', services, networks: {}, volumes: {} }; +} + +function mockDocker(opts: { + list?: unknown[]; + inspectById?: Record; + image?: { Config?: { Healthcheck?: { Test?: unknown } } } | 'missing' | 'error'; +}) { + const getContainer = vi.fn((id: string) => ({ + inspect: vi.fn(async () => { + const hit = opts.inspectById?.[id]; + if (!hit) { + const err = Object.assign(new Error('not found'), { statusCode: 404 }); + throw err; + } + return hit; + }), + })); + const listContainers = vi.fn(async () => opts.list ?? []); + const getDocker = vi.fn(() => ({ listContainers, getContainer })); + const inspectImage = vi.fn(async () => { + if (opts.image === 'missing' || opts.image === 'error') { + throw Object.assign(new Error('No such image'), { statusCode: 404 }); + } + return { inspect: opts.image ?? { Config: {} }, history: [] }; + }); + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getDocker, + inspectImage, + } as unknown as DockerController); + return { listContainers, getContainer, inspectImage }; +} + +describe('collectServiceHealthcheckEvidence', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('returns compose-declared for an active Compose healthcheck without Docker calls', async () => { + const { listContainers } = mockDocker({}); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ hasHealthcheck: true, composeHealthcheck: 'active' })]), true, + ); + expect(evidence.web).toEqual({ state: 'compose-declared', origin: 'compose', consistentReplicas: null }); + expect(listContainers).not.toHaveBeenCalled(); + }); + + it('returns explicitly-disabled for Compose disablement', async () => { + mockDocker({}); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'disabled' })]), true, + ); + expect(evidence.web.state).toBe('explicitly-disabled'); + }); + + it('lists containers by Compose projectName, not the stack directory name', async () => { + const { listContainers } = mockDocker({ + list: [{ + Id: 'c1', + Labels: { 'com.docker.compose.service': 'web' }, + Image: 'nginx:1.27', + }], + inspectById: { + c1: { Config: { Image: 'nginx:1.27', Healthcheck: { Test: ['CMD', 'true'] } } }, + }, + }); + const m: EffectiveModel = { + projectName: 'qa-hc-1713', + services: [svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })], + networks: {}, + volumes: {}, + }; + const evidence = await collectServiceHealthcheckEvidence(1, 'qa-healthcheck', m, true); + expect(listContainers).toHaveBeenCalledWith({ + all: true, + filters: { label: ['com.docker.compose.project=qa-hc-1713'] }, + }); + expect(evidence.web.state).toBe('runtime-inherited'); + }); + + it('recognizes runtime-inherited healthchecks and never returns Test text', async () => { + mockDocker({ + list: [{ + Id: 'c1', + Names: ['/proj-web-1'], + Labels: { 'com.docker.compose.service': 'web' }, + Image: 'nginx:1.27', + }], + inspectById: { + c1: { + Config: { + Image: 'nginx:1.27', + Healthcheck: { Test: ['CMD-SHELL', 'curl -f http://x/?token=secret-token || exit 1'] }, + }, + }, + }, + }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web).toEqual({ + state: 'runtime-inherited', + origin: 'runtime', + consistentReplicas: true, + }); + expect(JSON.stringify(evidence)).not.toContain('secret-token'); + expect(JSON.stringify(evidence)).not.toContain('CMD-SHELL'); + }); + + it('matches containers via composeServiceMatch when the service label is absent', async () => { + mockDocker({ + list: [{ Id: 'c1', Names: ['/web'], Image: 'nginx:1.27' }], + inspectById: { + c1: { Config: { Image: 'nginx:1.27', Healthcheck: { Test: ['CMD', 'true'] } } }, + }, + }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ name: 'web', composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web.state).toBe('runtime-inherited'); + }); + + it('ignores stale image replicas and uses local-image evidence', async () => { + mockDocker({ + list: [{ + Id: 'c1', + Labels: { 'com.docker.compose.service': 'web' }, + Image: 'nginx:old', + }], + inspectById: { + c1: { Config: { Image: 'nginx:old', Healthcheck: { Test: ['CMD', 'true'] } } }, + }, + image: { Config: { Healthcheck: { Test: ['CMD', 'true'] } } }, + }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web.state).toBe('local-image-inherited'); + expect(evidence.web.origin).toBe('local-image'); + }); + + it('reports inconsistent-replicas when coverage mixes', async () => { + mockDocker({ + list: [ + { Id: 'c1', Labels: { 'com.docker.compose.service': 'web' }, Image: 'nginx:1.27' }, + { Id: 'c2', Labels: { 'com.docker.compose.service': 'web' }, Image: 'nginx:1.27' }, + ], + inspectById: { + c1: { Config: { Image: 'nginx:1.27', Healthcheck: { Test: ['CMD', 'true'] } } }, + c2: { Config: { Image: 'nginx:1.27', Healthcheck: { Test: ['NONE'] } } }, + }, + }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web.state).toBe('inconsistent-replicas'); + }); + + it('reports unverifiable when Docker is unavailable', async () => { + mockDocker({}); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent' })]), false, + ); + expect(evidence.web.state).toBe('unverifiable'); + }); + + it('reports unverifiable for a missing local image', async () => { + mockDocker({ list: [], image: 'missing' }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web.state).toBe('unverifiable'); + }); + + it('reports unverifiable for build-only services with no image', async () => { + mockDocker({ list: [] }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: undefined })]), true, + ); + expect(evidence.web.state).toBe('unverifiable'); + }); + + it('reports absent when local image has no healthcheck', async () => { + mockDocker({ list: [], image: { Config: {} } }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web).toEqual({ + state: 'absent', + origin: 'local-image', + consistentReplicas: null, + }); + }); + + it('reports absent when matching replicas lack a healthcheck even if the local image has one', async () => { + mockDocker({ + list: [{ + Id: 'c1', + Labels: { 'com.docker.compose.service': 'web' }, + Image: 'nginx:1.27', + }], + inspectById: { + c1: { Config: { Image: 'nginx:1.27', Healthcheck: { Test: ['NONE'] } } }, + }, + image: { Config: { Healthcheck: { Test: ['CMD', 'true'] } } }, + }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web).toEqual({ + state: 'absent', + origin: 'runtime', + consistentReplicas: true, + }); + }); + + it('reports unverifiable when some replica inspects fail amid otherwise full coverage', async () => { + mockDocker({ + list: [ + { Id: 'c1', Labels: { 'com.docker.compose.service': 'web' }, Image: 'nginx:1.27' }, + { Id: 'gone', Labels: { 'com.docker.compose.service': 'web' }, Image: 'nginx:1.27' }, + ], + inspectById: { + c1: { Config: { Image: 'nginx:1.27', Healthcheck: { Test: ['CMD', 'true'] } } }, + }, + }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web.state).toBe('unverifiable'); + expect(evidence.web.origin).toBe('runtime'); + }); + + it('treats a disappearing container as inspect failure and falls through', async () => { + mockDocker({ + list: [{ Id: 'gone', Labels: { 'com.docker.compose.service': 'web' }, Image: 'nginx:1.27' }], + inspectById: {}, + image: { Config: { Healthcheck: { Test: ['CMD', 'true'] } } }, + }); + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web.state).toBe('local-image-inherited'); + }); +}); + +describe('collectServiceHealthcheckEvidence list failure', () => { + beforeEach(() => { + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getDocker: () => ({ + listContainers: vi.fn(async () => { throw new Error('daemon down'); }), + getContainer: vi.fn(), + }), + inspectImage: vi.fn(async () => ({ + inspect: { Config: { Healthcheck: { Test: ['CMD', 'true'] } } }, + history: [], + })), + } as unknown as DockerController); + }); + afterEach(() => vi.restoreAllMocks()); + + it('falls through to local-image evidence when the container list fails', async () => { + const evidence = await collectServiceHealthcheckEvidence( + 1, 'proj', model([svc({ composeHealthcheck: 'absent', image: 'nginx:1.27' })]), true, + ); + expect(evidence.web.state).toBe('local-image-inherited'); + }); +}); diff --git a/backend/src/__tests__/healthcheck-presence.test.ts b/backend/src/__tests__/healthcheck-presence.test.ts new file mode 100644 index 00000000..21010e72 --- /dev/null +++ b/backend/src/__tests__/healthcheck-presence.test.ts @@ -0,0 +1,51 @@ +import { describe, it, expect } from 'vitest'; +import { + classifyComposeHealthcheck, + isComposeHealthcheckActive, + isDockerHealthcheckActive, +} from '../helpers/healthcheckPresence'; + +describe('classifyComposeHealthcheck', () => { + it('treats a missing or non-object healthcheck as absent', () => { + expect(classifyComposeHealthcheck(undefined)).toBe('absent'); + expect(classifyComposeHealthcheck(null)).toBe('absent'); + expect(classifyComposeHealthcheck('CMD')).toBe('absent'); + }); + + it('treats disable: true as disabled', () => { + expect(classifyComposeHealthcheck({ disable: true })).toBe('disabled'); + expect(isComposeHealthcheckActive({ disable: true })).toBe(false); + }); + + it('treats test NONE forms as disabled', () => { + expect(classifyComposeHealthcheck({ test: 'NONE' })).toBe('disabled'); + expect(classifyComposeHealthcheck({ test: ['NONE'] })).toBe('disabled'); + expect(classifyComposeHealthcheck({ test: ['none'] })).toBe('disabled'); + }); + + it('treats an active test as active', () => { + expect(classifyComposeHealthcheck({ test: ['CMD', 'true'] })).toBe('active'); + expect(isComposeHealthcheckActive({ test: ['CMD', 'curl', '-f', 'http://localhost'] })).toBe(true); + }); + + it('treats empty or timing-only healthcheck objects as absent, not active', () => { + expect(classifyComposeHealthcheck({})).toBe('absent'); + expect(classifyComposeHealthcheck({ disable: false })).toBe('absent'); + expect(classifyComposeHealthcheck({ interval: '30s', timeout: '3s' })).toBe('absent'); + expect(isComposeHealthcheckActive({ disable: false })).toBe(false); + }); +}); + +describe('isDockerHealthcheckActive', () => { + it('is false for missing, empty, and NONE', () => { + expect(isDockerHealthcheckActive(undefined)).toBe(false); + expect(isDockerHealthcheckActive([])).toBe(false); + expect(isDockerHealthcheckActive(['NONE'])).toBe(false); + expect(isDockerHealthcheckActive('NONE')).toBe(false); + }); + + it('is true for a real Test array without exposing its contents', () => { + const secretTest = ['CMD-SHELL', 'curl -f http://x/?token=secret-token || exit 1']; + expect(isDockerHealthcheckActive(secretTest)).toBe(true); + }); +}); diff --git a/backend/src/__tests__/image-operation-service.test.ts b/backend/src/__tests__/image-operation-service.test.ts index 02847750..f91c2d82 100644 --- a/backend/src/__tests__/image-operation-service.test.ts +++ b/backend/src/__tests__/image-operation-service.test.ts @@ -270,4 +270,46 @@ describe('ImageOperationService', () => { expect(changed).not.toBe(baseline); }); + + it('executeClaimedComposeReapply transitions to recreating and watches helper exit before trigger', async () => { + const callOrder: string[] = []; + vi.spyOn(SelfUpdateService.getInstance(), 'getResolvedComposeImageForUpdate').mockResolvedValue(null); + vi.spyOn(SelfUpdateService.getInstance(), 'getComposeServiceName').mockReturnValue('sencho'); + vi.spyOn(SelfUpdateService.getInstance(), 'onceHelperExit').mockImplementation(() => { + callOrder.push('watch'); + }); + vi.spyOn(SelfUpdateService.getInstance(), 'triggerComposeReapply').mockImplementation(async () => { + callOrder.push('trigger'); + }); + vi.spyOn(SelfUpdateService.getInstance(), 'getLastError').mockReturnValue(null); + + const service = ImageOperationService.getInstance(); + const claim = await service.claimComposeReapply(); + expect(claim).toEqual({ ok: true }); + const result = await service.executeClaimedComposeReapply(); + const current = await service.getCurrentOperation(); + + expect(result).toEqual({ ok: true }); + expect(current?.kind).toBe('compose_reapply'); + expect(current?.state).toBe('recreating'); + expect(callOrder).toEqual(['watch', 'trigger']); + }); + + it('reconcileOnStartup resolves compose_reapply via marker-only success without pin match', async () => { + const service = ImageOperationService.getInstance(); + vi.spyOn(SelfUpdateService.getInstance(), 'getResolvedComposeImageForUpdate').mockResolvedValue(null); + vi.spyOn(SelfUpdateService.getInstance(), 'getComposeServiceName').mockReturnValue('sencho'); + const claim = await service.claimComposeReapply(); + expect(claim).toEqual({ ok: true }); + const current = await service.getCurrentOperation(); + expect(current?.kind).toBe('compose_reapply'); + expect(current?.targetImageRef).toBeNull(); + + const markerPath = path.join(tmpDir, `image-op-success-${current!.operationId}.json`); + await fs.writeFile(markerPath, JSON.stringify({ ok: true, operationId: current!.operationId }), 'utf8'); + + await service.reconcileOnStartup(); + const resolved = await service.getCurrentOperation(); + expect(resolved?.state).toBe('succeeded'); + }); }); diff --git a/backend/src/__tests__/image-update-detect-cross-surface.test.ts b/backend/src/__tests__/image-update-detect-cross-surface.test.ts new file mode 100644 index 00000000..f59fd014 --- /dev/null +++ b/backend/src/__tests__/image-update-detect-cross-surface.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, vi } from 'vitest'; +import { detectImageUpdate } from '../services/imageUpdateDetect'; +import { computeImagePreview } from '../services/UpdatePreviewService'; +import type { DigestComparisonResult } from '../services/registry-api'; + +const PLATFORM = { os: 'linux', architecture: 'amd64' }; +const LOCAL_DIGEST = `sha256:${'a'.repeat(64)}`; +const CREDENTIALS = { username: 'u', password: 'p' }; +const IMAGE = 'nginx:1.2.3'; + +/** + * COR-1 regression: persisted sidebar status (via detectImageUpdate / + * checkImage) and Fleet/Anatomy preview (computeImagePreview) must agree when + * the declared-tag digest matches but a higher semantic tag exists. + */ +describe('cross-surface update detection (digest match + higher semver)', () => { + async function runDetectionAndPreview( + comparison: DigestComparisonResult, + tags: string[], + localDigests: string[] = [LOCAL_DIGEST], + ) { + const compareDigest = vi.fn().mockResolvedValue(comparison); + const listRegistryTagsResult = vi.fn().mockResolvedValue({ ok: true, tags }); + + const detection = await detectImageUpdate({ + localDigests, + platform: PLATFORM, + registry: 'registry-1.docker.io', + repo: 'library/nginx', + tag: '1.2.3', + credentials: CREDENTIALS, + deps: { compareDigest, listRegistryTagsResult }, + }); + + const preview = await computeImagePreview('app', IMAGE, { + getCredentials: vi.fn().mockResolvedValue(CREDENTIALS), + getLocalDigest: vi.fn().mockResolvedValue({ digests: localDigests, platform: PLATFORM, emptyReason: null }), + compareDigest, + listRegistryTagsResult, + }); + + return { detection, preview, compareDigest }; + } + + it('shared detector and preview both report hasUpdate for app:1.2.3 when 1.2.4 exists', async () => { + const { detection, preview } = await runDetectionAndPreview( + { kind: 'match' }, + ['1.2.3', '1.2.4'], + ); + + expect(detection.hasUpdate).toBe(true); + expect(detection.nextTag).toBe('1.2.4'); + expect(detection.digestUpdate).toBe(false); + expect(preview.has_update).toBe(true); + expect(preview.next_tag).toBe('1.2.4'); + expect(preview.has_update).toBe(detection.hasUpdate); + }); + + it('shared detector and preview both clear when digest matches and no higher tag exists', async () => { + const { detection, preview } = await runDetectionAndPreview({ kind: 'match' }, ['1.2.3']); + + expect(detection.hasUpdate).toBe(false); + expect(preview.has_update).toBe(false); + expect(preview.has_update).toBe(detection.hasUpdate); + }); + + it('shared detector and preview both report hasUpdate when digest errors but 1.2.4 exists', async () => { + const { detection, preview } = await runDetectionAndPreview( + { kind: 'error', reason: 'Registry unreachable' }, + ['1.2.3', '1.2.4'], + ); + + expect(detection.hasUpdate).toBe(true); + expect(detection.nextTag).toBe('1.2.4'); + expect(detection.digestUpdate).toBe(false); + expect(detection.digestError).toBe('Registry unreachable'); + expect(preview.has_update).toBe(true); + expect(preview.next_tag).toBe('1.2.4'); + expect(preview.has_update).toBe(detection.hasUpdate); + }); + + it('shared detector and preview both report hasUpdate with no local digests when 1.2.4 exists', async () => { + const { detection, preview, compareDigest } = await runDetectionAndPreview( + { kind: 'update' }, + ['1.2.3', '1.2.4'], + [], + ); + + expect(compareDigest).not.toHaveBeenCalled(); + expect(detection.hasUpdate).toBe(true); + expect(detection.digestUpdate).toBe(false); + expect(detection.nextTag).toBe('1.2.4'); + expect(preview.has_update).toBe(true); + expect(preview.next_tag).toBe('1.2.4'); + expect(preview.has_update).toBe(detection.hasUpdate); + }); +}); diff --git a/backend/src/__tests__/image-update-service-reduction.test.ts b/backend/src/__tests__/image-update-service-reduction.test.ts index 75287f94..e951709e 100644 --- a/backend/src/__tests__/image-update-service-reduction.test.ts +++ b/backend/src/__tests__/image-update-service-reduction.test.ts @@ -15,13 +15,13 @@ import { import type { StackServiceStatus } from '../services/DatabaseService'; function ok(hasUpdate: boolean): ImageCheckResult { - return { hasUpdate }; + return { hasUpdate, checkStatus: 'ok', digestUpdate: hasUpdate, tagUpdate: false }; } function errored(message: string): ImageCheckResult { - return { hasUpdate: false, error: message }; + return { hasUpdate: false, checkStatus: 'failed', error: message }; } function notCheckable(): ImageCheckResult { - return { hasUpdate: false, notCheckable: true }; + return { hasUpdate: false, checkStatus: 'not_checkable', notCheckable: true }; } describe('reduceServiceStatus', () => { diff --git a/backend/src/__tests__/image-update-service.test.ts b/backend/src/__tests__/image-update-service.test.ts index 3645ba22..c6b9dc15 100644 --- a/backend/src/__tests__/image-update-service.test.ts +++ b/backend/src/__tests__/image-update-service.test.ts @@ -9,9 +9,10 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; const { mockGetAuthForRegistry, mockGetStackUpdateStatus, mockUpsertStackUpdateStatus, mockClearStackUpdateStatus, + mockClearAllStackUpdateStatus, mockUpdateGlobalSetting, mockRecordStackCheckFailure, mockGetStackServicesJson, mockGetSystemState, mockSetSystemState, mockAddNotificationHistory, - mockDispatchAlert, + mockDispatchAlert, mockBroadcastEvent, mockGetStacks, mockGetStackContent, mockGetEnvContent, mockEnvExists, mockGetAllContainers, mockGetGlobalSettings, mockInspect, mockBuildEffectiveServiceModel, @@ -20,12 +21,15 @@ const { mockGetStackUpdateStatus: vi.fn().mockReturnValue({}), mockUpsertStackUpdateStatus: vi.fn(), mockClearStackUpdateStatus: vi.fn(), + mockClearAllStackUpdateStatus: vi.fn().mockReturnValue(0), + mockUpdateGlobalSetting: vi.fn(), mockRecordStackCheckFailure: vi.fn(), mockGetStackServicesJson: vi.fn().mockReturnValue([]), mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled mockSetSystemState: vi.fn(), mockAddNotificationHistory: vi.fn(), mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), + mockBroadcastEvent: vi.fn(), mockGetStacks: vi.fn().mockResolvedValue([]), mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockRejectedValue(new Error('no env')), @@ -53,12 +57,14 @@ vi.mock('../services/DatabaseService', () => ({ DatabaseService: { getInstance: () => ({ getGlobalSettings: mockGetGlobalSettings, + updateGlobalSetting: mockUpdateGlobalSetting, getNodes: () => [], getGitSource: () => undefined, getStackProjectEnvFiles: () => [], upsertStackUpdateStatus: mockUpsertStackUpdateStatus, getStackUpdateStatus: mockGetStackUpdateStatus, clearStackUpdateStatus: mockClearStackUpdateStatus, + clearAllStackUpdateStatus: mockClearAllStackUpdateStatus, recordStackCheckFailure: mockRecordStackCheckFailure, getStackServicesJson: mockGetStackServicesJson, getSystemState: mockGetSystemState, @@ -76,10 +82,15 @@ vi.mock('../services/NotificationService', () => ({ NotificationService: { getInstance: () => ({ dispatchAlert: mockDispatchAlert, + broadcastEvent: mockBroadcastEvent, }), }, })); +vi.mock('../helpers/fleetUpdateCache', () => ({ + invalidateFleetUpdateCache: vi.fn(), +})); + vi.mock('../services/FileSystemService', () => ({ FileSystemService: { getInstance: () => ({ @@ -110,11 +121,21 @@ vi.mock('../services/NodeRegistry', () => ({ })); // compareLocalToRemoteTag is module-scoped inside checkImage; mock it to drive the -// comparison outcome while keeping the real parseImageRef / selectLocalRepoDigest. -const { mockCompareLocalToRemoteTag } = vi.hoisted(() => ({ mockCompareLocalToRemoteTag: vi.fn() })); +// comparison outcome while keeping the real parseImageRef / selectLocalRepoDigests. +const { mockCompareLocalToRemoteTag, mockListRegistryTagsResult } = vi.hoisted(() => ({ + mockCompareLocalToRemoteTag: vi.fn(), + // Detection now checks tags alongside digests; default to an empty, successful + // list so digest-focused tests are not accidentally driven by a real network + // call finding a genuine newer tag for whatever image ref they pass. + mockListRegistryTagsResult: vi.fn().mockResolvedValue({ ok: true, tags: [] }), +})); vi.mock('../services/registry-api', async (importOriginal) => { const actual = await importOriginal(); - return { ...actual, compareLocalToRemoteTag: mockCompareLocalToRemoteTag }; + return { + ...actual, + compareLocalToRemoteTag: mockCompareLocalToRemoteTag, + listRegistryTagsResult: mockListRegistryTagsResult, + }; }); // ── Re-export internal helpers via the module ───────────────────────── @@ -158,7 +179,7 @@ describe('ImageUpdateService - image ref parsing (via checkImage)', () => { it('marks sha256-only refs not-checkable (no tag to track)', async () => { const docker = makeMockDocker(); const result = await service.checkImage(docker, 'sha256:abc123'); - expect(result).toEqual({ hasUpdate: false, notCheckable: true }); + expect(result).toEqual({ hasUpdate: false, checkStatus: 'not_checkable', notCheckable: true }); }); it('returns error when local image inspect fails', async () => { @@ -195,7 +216,7 @@ describe('ImageUpdateService - image ref parsing (via checkImage)', () => { // Empty RepoDigests means locally built / not registry-backed. const docker = makeMockDocker([]); const result = await service.checkImage(docker, 'nginx:latest'); - expect(result).toEqual({ hasUpdate: false, notCheckable: true }); + expect(result).toEqual({ hasUpdate: false, checkStatus: 'not_checkable', notCheckable: true }); }); it('errors when RepoDigests are present but none resolves a digest', async () => { @@ -207,6 +228,17 @@ describe('ImageUpdateService - image ref parsing (via checkImage)', () => { expect(result.notCheckable).toBeUndefined(); expect(result.error).toContain('Could not resolve a local registry digest'); }); + + it('errors (not a comparison) when the sole valid RepoDigest belongs to an unrelated repository', async () => { + // A well-formed digest is present, but it names a different repo (e.g. + // left over from a retag): comparing it against this ref's registry state + // would risk a false update against unrelated content. + const docker = makeMockDocker([`ghcr.io/other/image@sha256:${'b'.repeat(64)}`]); + const result = await service.checkImage(docker, 'nginx:latest'); + expect(result.hasUpdate).toBe(false); + expect(result.notCheckable).toBeUndefined(); + expect(result.error).toContain('Could not resolve a local registry digest'); + }); }); // ── checkImage surfaces the comparison resolver's outcome ────────────── @@ -218,6 +250,7 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco beforeEach(() => { vi.clearAllMocks(); + mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: [] }); (ImageUpdateService as any).instance = undefined; service = ImageUpdateService.getInstance(); }); @@ -234,29 +267,39 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco }), } as any); + const dockerWithNginxSemver = (repoDigests: string[] = [`registry-1.docker.io/library/nginx@${LOCAL_DIGEST}`]) => ({ + getDocker: () => ({ + getImage: () => ({ inspect: vi.fn().mockResolvedValue({ + RepoDigests: repoDigests, + Os: 'linux', + Architecture: 'amd64', + }) }), + }), + } as any); + it('surfaces the specific failure reason (not a generic "unreachable") as the check error', async () => { mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'error', reason: 'Authentication failed for ghcr.io/linuxserver/radarr:latest' }); const result = await service.checkImage(dockerWithLocalDigest(LOCAL_DIGEST), 'ghcr.io/linuxserver/radarr:latest'); - expect(result).toEqual({ hasUpdate: false, error: 'Authentication failed for ghcr.io/linuxserver/radarr:latest' }); + expect(result).toMatchObject({ hasUpdate: false, checkStatus: 'failed', error: 'Authentication failed for ghcr.io/linuxserver/radarr:latest' }); }); it('reports an update when the comparison resolver classifies the remote as an update', async () => { mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'update' }); const result = await service.checkImage(dockerWithLocalDigest(LOCAL_DIGEST), 'ghcr.io/linuxserver/radarr:latest'); - expect(result).toEqual({ hasUpdate: true }); + expect(result).toMatchObject({ hasUpdate: true, digestUpdate: true, checkStatus: 'ok' }); }); it('reports no update when the comparison resolver classifies the remote as a match', async () => { mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'match' }); const result = await service.checkImage(dockerWithLocalDigest(LOCAL_DIGEST), 'ghcr.io/linuxserver/radarr:latest'); - expect(result).toEqual({ hasUpdate: false }); + expect(result).toMatchObject({ hasUpdate: false, digestUpdate: false, checkStatus: 'ok' }); }); it('passes the local digest, platform, and parsed ref through to the comparison resolver', async () => { mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'match' }); await service.checkImage(dockerWithLocalDigest(LOCAL_DIGEST), 'ghcr.io/linuxserver/radarr:latest'); expect(mockCompareLocalToRemoteTag).toHaveBeenCalledWith( - LOCAL_DIGEST, + [LOCAL_DIGEST], 'ghcr.io', 'linuxserver/radarr', 'latest', @@ -264,6 +307,50 @@ describe('ImageUpdateService - checkImage surfaces the comparison resolver outco null, ); }); + + it('reports an update when the declared tag digest matches but a higher semver tag exists', async () => { + mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'match' }); + mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: ['1.2.3', '1.2.4'] }); + const result = await service.checkImage(dockerWithNginxSemver(), 'nginx:1.2.3'); + expect(result).toMatchObject({ hasUpdate: true, digestUpdate: false, tagUpdate: true, checkStatus: 'ok' }); + }); + + it('reports an update when digest comparison errors but a higher semver tag exists', async () => { + mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'error', reason: 'Registry unreachable' }); + mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: ['1.2.3', '1.2.4'] }); + const result = await service.checkImage(dockerWithNginxSemver(), 'nginx:1.2.3'); + expect(result).toMatchObject({ hasUpdate: true, digestUpdate: false, tagUpdate: true, checkStatus: 'ok' }); + }); + + it('forwards every matching RepoDigest (stale index ahead of current) to the comparison resolver', async () => { + const STALE = `sha256:${'f'.repeat(64)}`; + const CURRENT = `sha256:${'e'.repeat(64)}`; + const docker = { + getDocker: () => ({ + getImage: () => ({ + inspect: vi.fn().mockResolvedValue({ + RepoDigests: [ + `redis@${STALE}`, + `redis@${CURRENT}`, + ], + Os: 'linux', + Architecture: 'amd64', + }), + }), + }), + } as any; + mockCompareLocalToRemoteTag.mockResolvedValue({ kind: 'match' }); + const result = await service.checkImage(docker, 'redis:8.8.0'); + expect(result).toEqual({ hasUpdate: false, digestUpdate: false, tagUpdate: false, checkStatus: 'ok' }); + expect(mockCompareLocalToRemoteTag).toHaveBeenCalledWith( + [STALE, CURRENT], + 'registry-1.docker.io', + 'library/redis', + '8.8.0', + { os: 'linux', architecture: 'amd64' }, + null, + ); + }); }); // ── Multi-arch digest comparison persistence (end-to-end via checkNode) ─ @@ -288,6 +375,7 @@ services: beforeEach(() => { vi.clearAllMocks(); + mockListRegistryTagsResult.mockResolvedValue({ ok: true, tags: [] }); (ImageUpdateService as any).instance = undefined; mockGetSystemState.mockReturnValue('1'); mockGetStacks.mockResolvedValue(['stackA']); @@ -1132,6 +1220,62 @@ describe('ImageUpdateService - configurable interval & status', () => { service.stop(); checkSpy.mockRestore(); }); + + it('start() while checks disabled arms no timer and reports enabled false', () => { + vi.useFakeTimers(); + mockGetGlobalSettings.mockReturnValue({ image_update_checks_enabled: '0', image_update_check_interval_minutes: '60' }); + const service = ImageUpdateService.getInstance(); + const checkSpy = vi.spyOn(service as any, 'check').mockResolvedValue(undefined); + service.start(); + const status = service.getStatus(); + expect(status.enabled).toBe(false); + expect(status.nextCheckAt).toBeNull(); + expect(status.checking).toBe(false); + expect(vi.getTimerCount()).toBe(0); + vi.advanceTimersByTime(10 * 60 * 1000); + expect(checkSpy).not.toHaveBeenCalled(); + checkSpy.mockRestore(); + }); + + it('treats a missing checks-enabled key as enabled', () => { + mockGetGlobalSettings.mockReturnValue({}); + const service = ImageUpdateService.getInstance(); + expect(ImageUpdateService.isChecksEnabled()).toBe(true); + expect(service.getStatus().enabled).toBe(true); + }); + + it('applyChecksEnabled(false) stops polling, clears local findings, and broadcasts invalidate', () => { + vi.useFakeTimers(); + mockGetGlobalSettings.mockReturnValue({ image_update_check_interval_minutes: '60' }); + const service = ImageUpdateService.getInstance(); + service.start(); + expect(service.getStatus().nextCheckAt).not.toBeNull(); + + mockGetGlobalSettings.mockReturnValue({ image_update_checks_enabled: '0', image_update_check_interval_minutes: '60' }); + mockUpdateGlobalSetting.mockImplementation((key: string, value: string) => { + if (key === 'image_update_checks_enabled') { + mockGetGlobalSettings.mockReturnValue({ image_update_checks_enabled: value, image_update_check_interval_minutes: '60' }); + } + }); + + const status = service.applyChecksEnabled(false); + expect(status.enabled).toBe(false); + expect(status.nextCheckAt).toBeNull(); + expect(mockUpdateGlobalSetting).toHaveBeenCalledWith('image_update_checks_enabled', '0'); + expect(mockClearAllStackUpdateStatus).toHaveBeenCalledWith(1); + expect(mockBroadcastEvent).toHaveBeenCalledWith(expect.objectContaining({ + type: 'state-invalidate', + scope: 'image-updates', + nodeId: 1, + })); + expect(vi.getTimerCount()).toBe(0); + }); + + it('triggerManualRefresh returns false when checks are disabled', () => { + mockGetGlobalSettings.mockReturnValue({ image_update_checks_enabled: '0' }); + const service = ImageUpdateService.getInstance(); + expect(service.triggerManualRefresh()).toBe(false); + }); }); // ── Stale stack pruning ───────────────────────────────────────────────── @@ -1495,6 +1639,7 @@ services: mockGetSystemState.mockReturnValue('1'); mockGetAllContainers.mockResolvedValue([]); mockEnvExists.mockResolvedValue(false); + mockGetGlobalSettings.mockReturnValue({ developer_mode: '0' }); }); it('reduces per-service status through the effective model and persists services_json with a generation', async () => { @@ -1616,7 +1761,26 @@ services: }); describe('recheckStack', () => { - it('persists a fresh per-service reduction and returns no warning on success', async () => { + it('skips registry probes and DB writes when checks are disabled', async () => { + mockGetGlobalSettings.mockReturnValueOnce({ image_update_checks_enabled: '0' }); + const service = ImageUpdateService.getInstance(); + (service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: true }); + const genBefore = service.peekStackWriteGeneration(1, 'stackA'); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ outcome: 'cleared', warning: null }); + expect(service.peekStackWriteGeneration(1, 'stackA')).toBe(genBefore); + expect(mockBuildEffectiveServiceModel).not.toHaveBeenCalled(); + expect(mockGetAllContainers).not.toHaveBeenCalled(); + expect((service as any).checkImage).not.toHaveBeenCalled(); + expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockRecordStackCheckFailure).not.toHaveBeenCalled(); + expect(mockClearStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockClearAllStackUpdateStatus).not.toHaveBeenCalled(); + }); + + it('returns still_present when a checkable service still has an update', async () => { mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: true, services: [specFor('web', 'web:latest'), specFor('worker', 'worker:latest')], @@ -1629,7 +1793,10 @@ services: const result = await service.recheckStack(1, 'stackA'); - expect(result).toEqual({ warning: null }); + expect(result).toEqual({ + outcome: 'still_present', + warning: 'The update command completed, but Sencho still detects an available image update.', + }); expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith( 1, 'stackA', true, expect.any(Number), 'ok', null, [ @@ -1640,13 +1807,103 @@ services: ); }); - it('returns a warning and leaves the prior row untouched when the model cannot render', async () => { + it('returns cleared when every checkable service is up to date', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest')], + }); + mockGetAllContainers.mockResolvedValue([ + { Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } }, + ]); + const service = ImageUpdateService.getInstance(); + (service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false }); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ outcome: 'cleared', warning: null }); + expect(mockUpsertStackUpdateStatus).toHaveBeenCalledWith( + 1, 'stackA', false, expect.any(Number), 'ok', null, + expect.any(Array), + expect.any(Number), + ); + }); + + it('returns verification_failed and leaves the prior row untouched when the model cannot render', async () => { mockBuildEffectiveServiceModel.mockResolvedValueOnce({ renderable: false, code: 'effective_model_render_failed', error: 'no model in test' }); const service = ImageUpdateService.getInstance(); const result = await service.recheckStack(1, 'stackA'); - expect(result).toEqual({ warning: 'no model in test' }); + expect(result).toEqual({ outcome: 'verification_failed', warning: 'no model in test' }); + expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockRecordStackCheckFailure).not.toHaveBeenCalled(); + }); + + it('returns verification_incomplete and preserves prior hasUpdate on a fully failed check', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest')], + }); + mockGetAllContainers.mockResolvedValue([ + { Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } }, + ]); + mockGetStackServicesJson.mockReturnValueOnce([ + { service: 'web', image: 'web:latest', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ]); + const service = ImageUpdateService.getInstance(); + (service as any).checkImage = vi.fn().mockResolvedValue({ + hasUpdate: false, + error: 'registry timeout', + }); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ + outcome: 'verification_incomplete', + warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.', + }); + expect(mockRecordStackCheckFailure).toHaveBeenCalled(); + expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); + }); + + it('returns verification_incomplete when the write lock discards a stale commit', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest')], + }); + mockGetAllContainers.mockResolvedValue([ + { Id: 'c1', Image: 'web:latest', Labels: { 'com.docker.compose.project': 'stackA', 'com.docker.compose.service': 'web' } }, + ]); + const service = ImageUpdateService.getInstance(); + (service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false }); + (service as any).withStackWriteLock = vi.fn().mockResolvedValue(false); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ + outcome: 'verification_incomplete', + warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.', + }); + expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockRecordStackCheckFailure).not.toHaveBeenCalled(); + }); + + it('returns verification_incomplete when container listing fails', async () => { + mockBuildEffectiveServiceModel.mockResolvedValueOnce({ + renderable: true, + services: [specFor('web', 'web:latest')], + }); + mockGetAllContainers.mockRejectedValueOnce(new Error('docker socket down')); + const service = ImageUpdateService.getInstance(); + (service as any).checkImage = vi.fn().mockResolvedValue({ hasUpdate: false }); + + const result = await service.recheckStack(1, 'stackA'); + + expect(result).toEqual({ + outcome: 'verification_incomplete', + warning: 'The update command completed, but Sencho could not fully verify whether an image update remains.', + }); + expect((service as any).checkImage).not.toHaveBeenCalled(); expect(mockUpsertStackUpdateStatus).not.toHaveBeenCalled(); }); }); diff --git a/backend/src/__tests__/image-updates-routes.test.ts b/backend/src/__tests__/image-updates-routes.test.ts index 3d07a7b0..cf4019fe 100644 --- a/backend/src/__tests__/image-updates-routes.test.ts +++ b/backend/src/__tests__/image-updates-routes.test.ts @@ -44,6 +44,18 @@ describe('GET /api/image-updates', () => { expect(res.status).toBe(200); expect(res.body).toBeInstanceOf(Object); }); + + it('excludes partial and failed retained rows from the confirmed boolean map', async () => { + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + DatabaseService.getInstance().upsertStackUpdateStatus(nodeId, 'ok-stack', true, 1000, 'ok', null); + DatabaseService.getInstance().upsertStackUpdateStatus(nodeId, 'partial-stack', true, 1000, 'partial', 'half'); + DatabaseService.getInstance().upsertStackUpdateStatus(nodeId, 'failed-stack', true, 1000, 'failed', 'boom'); + const res = await request(app).get('/api/image-updates').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body['ok-stack']).toBe(true); + expect(res.body['partial-stack']).toBe(false); + expect(res.body['failed-stack']).toBe(false); + }); }); describe('GET /api/image-updates/detail', () => { @@ -103,6 +115,65 @@ describe('GET /api/image-updates/status', () => { expect(typeof res.body.manualCooldownRemainingMs).toBe('number'); expect('lastCheckedAt' in res.body).toBe(true); expect('nextCheckAt' in res.body).toBe(true); + expect(res.body.enabled).toBe(true); + }); +}); + +describe('PUT /api/image-updates/enabled', () => { + it('rejects unauthenticated requests with 401', async () => { + const res = await request(app).put('/api/image-updates/enabled').send({ enabled: false }); + expect(res.status).toBe(401); + }); + + it('rejects non-admin users with 403', async () => { + const res = await request(app).put('/api/image-updates/enabled').set('Cookie', viewerCookie).send({ enabled: false }); + expect(res.status).toBe(403); + }); + + it('disables checks, clears local findings, and returns enabled false', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'pending-stack', true, Date.now(), 'ok', null); + expect(Object.keys(db.getStackUpdateDetail(nodeId)).length).toBeGreaterThan(0); + + const res = await request(app).put('/api/image-updates/enabled').set('Cookie', adminCookie).send({ enabled: false }); + expect(res.status).toBe(200); + expect(res.body.enabled).toBe(false); + expect(res.body.nextCheckAt).toBeNull(); + expect(db.getGlobalSettings().image_update_checks_enabled).toBe('0'); + expect(db.getStackUpdateDetail(nodeId)).toEqual({}); + }); + + it('re-enables checks and returns enabled true', async () => { + DatabaseService.getInstance().updateGlobalSetting('image_update_checks_enabled', '0'); + const res = await request(app).put('/api/image-updates/enabled').set('Cookie', adminCookie).send({ enabled: true }); + expect(res.status).toBe(200); + expect(res.body.enabled).toBe(true); + expect(DatabaseService.getInstance().getGlobalSettings().image_update_checks_enabled).toBe('1'); + }); +}); + +describe('POST /api/image-updates/refresh when disabled', () => { + it('returns 409 with enabled false instead of rate-limit 429', async () => { + DatabaseService.getInstance().updateGlobalSetting('image_update_checks_enabled', '0'); + const res = await request(app).post('/api/image-updates/refresh').set('Cookie', adminCookie); + expect(res.status).toBe(409); + expect(res.body.enabled).toBe(false); + expect(res.body.error).toMatch(/disabled/i); + DatabaseService.getInstance().updateGlobalSetting('image_update_checks_enabled', '1'); + }); +}); + +describe('POST /api/image-updates/fleet/refresh when disabled', () => { + it('lists the local node in disabled rather than triggered or rateLimited', async () => { + DatabaseService.getInstance().updateGlobalSetting('image_update_checks_enabled', '0'); + const localId = DatabaseService.getInstance().getDefaultNode()!.id!; + const res = await request(app).post('/api/image-updates/fleet/refresh').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.disabled).toContain(localId); + expect(res.body.triggered).not.toContain(localId); + expect(res.body.rateLimited).not.toContain(localId); + DatabaseService.getInstance().updateGlobalSetting('image_update_checks_enabled', '1'); }); }); @@ -341,6 +412,35 @@ describe('POST /api/auto-update/execute', () => { expect(res.body.error).toMatch(/Missing "target"/); }); + it('rejects an empty targets array with 400', async () => { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ targets: [] }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/non-empty array/); + }); + + it('rejects invalid names in targets with 400', async () => { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ targets: ['ok-stack', '../bad'] }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/Invalid stack name/); + }); + + it('accepts targets[] and returns a per-stack summary string', async () => { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ targets: ['missing-a', 'missing-b'] }); + expect(res.status).toBe(200); + expect(typeof res.body.result).toBe('string'); + expect(res.body.result).toMatch(/missing-a/); + expect(res.body.result).toMatch(/missing-b/); + }); + it('rejects invalid stack name with 400', async () => { const res = await request(app) .post('/api/auto-update/execute') @@ -374,7 +474,7 @@ describe('POST /api/auto-update/execute', () => { const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') - .mockResolvedValue({ hasUpdate: true } as never); + .mockResolvedValue({ hasUpdate: true, digestUpdate: true } as never); const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); const gateSpy = vi.spyOn(PolicyEnforcement, 'enforcePolicyPreDeploy').mockResolvedValue({ ok: false, @@ -416,13 +516,22 @@ describe('POST /api/auto-update/execute', () => { const { ComposeService } = await import('../services/ComposeService'); const { HealthGateService } = await import('../services/HealthGateService'); const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + const callOrder: string[] = []; const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') - .mockResolvedValue({ hasUpdate: true } as never); + .mockResolvedValue({ hasUpdate: true, digestUpdate: true } as never); const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); - const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-au'); + const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack') + .mockImplementation(async () => { + callOrder.push('recheckStack'); + return { outcome: 'cleared', warning: null } as never; + }); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockImplementation(() => { + callOrder.push('beginStack'); + return 'gate-au'; + }); try { const res = await request(app) .post('/api/auto-update/execute') @@ -430,12 +539,113 @@ describe('POST /api/auto-update/execute', () => { .send({ target: 'auto-upd-gate' }); expect(res.status).toBe(200); expect(updateSpy).toHaveBeenCalledWith('auto-upd-gate', undefined, true); + expect(recheckSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate'); expect(beginSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate', 'update', `auto-update:${TEST_USERNAME}`); + expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); } finally { containersSpy.mockRestore(); checkSpy.mockRestore(); updateSpy.mockRestore(); + recheckSpy.mockRestore(); beginSpy.mockRestore(); } }); + + it('skips Compose apply for tag-only availability without clearing status', async () => { + const DockerController = (await import('../services/DockerController')).default; + const { ImageUpdateService } = await import('../services/ImageUpdateService'); + const { ComposeService } = await import('../services/ComposeService'); + const { DatabaseService } = await import('../services/DatabaseService'); + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + + const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') + .mockResolvedValue([{ Id: 'c1', Image: 'nginx:1.2.3' }] as never); + const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') + .mockResolvedValue({ hasUpdate: true, digestUpdate: false, tagUpdate: true } as never); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack'); + const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus'); + try { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ target: 'auto-upd-tag-only' }); + expect(res.status).toBe(200); + expect(res.body.result).toContain('Compose pin unchanged'); + expect(updateSpy).not.toHaveBeenCalled(); + expect(recheckSpy).not.toHaveBeenCalled(); + expect(clearSpy).not.toHaveBeenCalledWith(nodeId, 'auto-upd-tag-only'); + } finally { + containersSpy.mockRestore(); + checkSpy.mockRestore(); + updateSpy.mockRestore(); + recheckSpy.mockRestore(); + clearSpy.mockRestore(); + } + }); + + it('skips digest apply when a sibling image check failed', async () => { + const DockerController = (await import('../services/DockerController')).default; + const { ImageUpdateService } = await import('../services/ImageUpdateService'); + const { ComposeService } = await import('../services/ComposeService'); + + const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') + .mockResolvedValue([ + { Id: 'c1', Image: 'nginx:latest' }, + { Id: 'c2', Image: 'redis:latest' }, + ] as never); + const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') + .mockResolvedValueOnce({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never) + .mockResolvedValueOnce({ hasUpdate: false, error: 'registry timeout', checkStatus: 'failed' } as never); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack'); + try { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ target: 'auto-upd-check-err' }); + expect(res.status).toBe(200); + expect(res.body.result).toContain('image check(s) failed'); + expect(updateSpy).not.toHaveBeenCalled(); + expect(recheckSpy).not.toHaveBeenCalled(); + } finally { + containersSpy.mockRestore(); + checkSpy.mockRestore(); + updateSpy.mockRestore(); + recheckSpy.mockRestore(); + } + }); + + it('still applies when checkImage reports same-tag digestUpdate', async () => { + const DockerController = (await import('../services/DockerController')).default; + const { ImageUpdateService } = await import('../services/ImageUpdateService'); + const { ComposeService } = await import('../services/ComposeService'); + const { DatabaseService } = await import('../services/DatabaseService'); + const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; + + const containersSpy = vi.spyOn(DockerController.prototype, 'getContainersByStack') + .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); + const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') + .mockResolvedValue({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack') + .mockResolvedValue({ outcome: 'still_present', warning: null } as never); + const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus'); + try { + const res = await request(app) + .post('/api/auto-update/execute') + .set('Cookie', adminCookie) + .send({ target: 'auto-upd-digest' }); + expect(res.status).toBe(200); + expect(updateSpy).toHaveBeenCalledWith('auto-upd-digest', undefined, true); + expect(recheckSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-digest'); + expect(clearSpy).not.toHaveBeenCalledWith(nodeId, 'auto-upd-digest'); + } finally { + containersSpy.mockRestore(); + checkSpy.mockRestore(); + updateSpy.mockRestore(); + recheckSpy.mockRestore(); + clearSpy.mockRestore(); + } + }); }); diff --git a/backend/src/__tests__/labels.test.ts b/backend/src/__tests__/labels.test.ts index 28604ae2..3b941a9a 100644 --- a/backend/src/__tests__/labels.test.ts +++ b/backend/src/__tests__/labels.test.ts @@ -66,6 +66,8 @@ const SCHEMA = ` prune_targets TEXT, target_services TEXT, prune_label_filter TEXT, + selector_type TEXT, + selector_value TEXT, FOREIGN KEY(node_id) REFERENCES nodes(id) ); diff --git a/backend/src/__tests__/missing-external-networks-route.test.ts b/backend/src/__tests__/missing-external-networks-route.test.ts index 4b2e7105..52759650 100644 --- a/backend/src/__tests__/missing-external-networks-route.test.ts +++ b/backend/src/__tests__/missing-external-networks-route.test.ts @@ -96,6 +96,7 @@ describe('deploy provenance trust boundary', () => { const res = { status: vi.fn().mockReturnThis(), json: vi.fn(() => resolve()), + cookie: vi.fn(), } as unknown as import('express').Response; void Promise.resolve(authMiddleware(req, res, () => { nextCalled = true; diff --git a/backend/src/__tests__/notification-suppression-routes-api.test.ts b/backend/src/__tests__/notification-suppression-routes-api.test.ts index ef6c54ce..4cd78162 100644 --- a/backend/src/__tests__/notification-suppression-routes-api.test.ts +++ b/backend/src/__tests__/notification-suppression-routes-api.test.ts @@ -506,7 +506,7 @@ describe('Notification suppression - CRUD', () => { DatabaseService.getInstance().deleteNotificationSuppressionRule(940005); }); - it('replica does not resurrect a rule after it was deleted, even with a newer updated_at', async () => { + it('omitted-body replica DELETE is permanent; delayed POST cannot resurrect', async () => { const jwt = await import('jsonwebtoken'); const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); @@ -538,9 +538,8 @@ describe('Notification suppression - CRUD', () => { expect(del.status).toBe(200); expect(DatabaseService.getInstance().getNotificationSuppressionRule(950006)).toBeUndefined(); - // A delayed POST arrives after the DELETE, reordered by the network. Even - // though its updated_at is newer than anything the sender ever sent before - // the delete, the delete is authoritative: this id must stay gone. + // Omitted DELETE body (old hub) fails closed as permanent. A delayed POST + // with any updated_at must stay blocked. const delayed = await request(app) .post('/api/notification-suppression-rules/replica') .set('Authorization', `Bearer ${token}`) @@ -612,21 +611,479 @@ describe('Notification suppression - CRUD', () => { .set('Authorization', `Bearer ${token}`) .send({ rule: replicaRule({ updated_at: 1000 }) }); expect(first.status).toBe(200); + expect(first.body.outcome).toBe('applied'); expect(DatabaseService.getInstance().getNotificationSuppressionRule(970008)?.schedule).not.toBeNull(); - // This is the worst case the fix protects: a capability-cleanup DELETE - // retracts an all-day/scheduled mute from a node that stopped supporting - // it. A delayed re-push of the scheduled rule must not undo that cleanup. + // Capability-cleanup DELETE is recoverable at the pushed version. A delayed + // re-push at the same or older watermark must not undo cleanup. const del = await request(app) .delete('/api/notification-suppression-rules/replica/970008') - .set('Authorization', `Bearer ${token}`); + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'recoverable', source_updated_at: 1000 }); expect(del.status).toBe(200); + expect(del.body.outcome).toBe('applied'); const delayed = await request(app) .post('/api/notification-suppression-rules/replica') .set('Authorization', `Bearer ${token}`) - .send({ rule: replicaRule({ updated_at: 2000 }) }); + .send({ rule: replicaRule({ updated_at: 1000 }) }); expect(delayed.status).toBe(200); + expect(delayed.body.outcome).toBe('ignored_recoverable_watermark'); expect(DatabaseService.getInstance().getNotificationSuppressionRule(970008)).toBeUndefined(); + expect(DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(970008)?.kind).toBe( + 'recoverable', + ); + }); + + it('recoverable soft-cleanup allows recreate when hub re-save is newer', async () => { + const jwt = await import('jsonwebtoken'); + const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); + const replicaRule = (overrides: Record) => ({ + id: 980009, + name: 'replica-soft-cleanup-resave', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + schedule: { days: [2], start_minute: 60, end_minute: 120, tz: 'UTC' }, + created_at: 1, + ...overrides, + }); + + // Receiver clock skew must not affect ordering. Sign the JWT under the same + // mocked clock so exp verification stays valid. + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(9_000_000_000_000); + const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1h' }); + try { + const first = await request(app) + .post('/api/notification-suppression-rules/replica') + .set('Authorization', `Bearer ${token}`) + .send({ rule: replicaRule({ updated_at: 1000 }) }); + expect(first.status).toBe(200); + expect(first.body.outcome).toBe('applied'); + + const del = await request(app) + .delete('/api/notification-suppression-rules/replica/980009') + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'recoverable', source_updated_at: 1000 }); + expect(del.status).toBe(200); + expect(del.body.outcome).toBe('applied'); + expect(DatabaseService.getInstance().getNotificationSuppressionRule(980009)).toBeUndefined(); + + const tie = await request(app) + .post('/api/notification-suppression-rules/replica') + .set('Authorization', `Bearer ${token}`) + .send({ rule: replicaRule({ updated_at: 1000 }) }); + expect(tie.status).toBe(200); + expect(tie.body.outcome).toBe('ignored_recoverable_watermark'); + expect(DatabaseService.getInstance().getNotificationSuppressionRule(980009)).toBeUndefined(); + + const resave = await request(app) + .post('/api/notification-suppression-rules/replica') + .set('Authorization', `Bearer ${token}`) + .send({ rule: replicaRule({ name: 'replica-soft-cleanup-resave-v2', updated_at: 2000 }) }); + expect(resave.status).toBe(200); + expect(resave.body.outcome).toBe('applied'); + const restored = DatabaseService.getInstance().getNotificationSuppressionRule(980009); + expect(restored?.name).toBe('replica-soft-cleanup-resave-v2'); + expect(restored?.updated_at).toBe(2000); + expect(DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(980009)).toBeUndefined(); + } finally { + nowSpy.mockRestore(); + } + }); + + it('permanent DELETE blocks any later POST regardless of updated_at', async () => { + const jwt = await import('jsonwebtoken'); + const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); + const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + + const del = await request(app) + .delete('/api/notification-suppression-rules/replica/990010') + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'permanent', source_updated_at: 50 }); + expect(del.status).toBe(200); + expect(del.body.outcome).toBe('applied'); + + const post = await request(app) + .post('/api/notification-suppression-rules/replica') + .set('Authorization', `Bearer ${token}`) + .send({ + rule: { + id: 990010, + name: 'should-not-return', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + created_at: 1, + updated_at: Number.MAX_SAFE_INTEGER, + }, + }); + expect(post.status).toBe(200); + expect(post.body.outcome).toBe('ignored_permanent_tombstone'); + expect(DatabaseService.getInstance().getNotificationSuppressionRule(990010)).toBeUndefined(); + expect(DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(990010)?.kind).toBe( + 'permanent', + ); + }); + + it('stale recoverable DELETE does not remove a newer stored row', async () => { + const jwt = await import('jsonwebtoken'); + const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); + const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + const replicaRule = (overrides: Record) => ({ + id: 991011, + name: 'v200', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + created_at: 1, + ...overrides, + }); + + const post = await request(app) + .post('/api/notification-suppression-rules/replica') + .set('Authorization', `Bearer ${token}`) + .send({ rule: replicaRule({ updated_at: 200 }) }); + expect(post.status).toBe(200); + + const del = await request(app) + .delete('/api/notification-suppression-rules/replica/991011') + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'recoverable', source_updated_at: 100 }); + expect(del.status).toBe(200); + expect(del.body.outcome).toBe('ignored_stale'); + expect(DatabaseService.getInstance().getNotificationSuppressionRule(991011)?.updated_at).toBe(200); + expect(DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(991011)).toBeUndefined(); + }); + + it('recoverable DELETE at exact stored version deletes and tombstones', async () => { + const jwt = await import('jsonwebtoken'); + const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); + const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + + await request(app) + .post('/api/notification-suppression-rules/replica') + .set('Authorization', `Bearer ${token}`) + .send({ + rule: { + id: 992012, + name: 'exact-tie-delete', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + created_at: 1, + updated_at: 150, + }, + }); + + const del = await request(app) + .delete('/api/notification-suppression-rules/replica/992012') + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'recoverable', source_updated_at: 150 }); + expect(del.status).toBe(200); + expect(DatabaseService.getInstance().getNotificationSuppressionRule(992012)).toBeUndefined(); + const tomb = DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(992012); + expect(tomb?.kind).toBe('recoverable'); + expect(tomb?.source_updated_at).toBe(150); + }); + + it('reordered recoverable tombstones keep the max watermark; permanent wins', async () => { + const jwt = await import('jsonwebtoken'); + const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); + const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + + await request(app) + .delete('/api/notification-suppression-rules/replica/993013') + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'recoverable', source_updated_at: 200 }); + await request(app) + .delete('/api/notification-suppression-rules/replica/993013') + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'recoverable', source_updated_at: 100 }); + expect( + DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(993013)?.source_updated_at, + ).toBe(200); + + await request(app) + .delete('/api/notification-suppression-rules/replica/993013') + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'permanent', source_updated_at: 50 }); + const tomb = DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(993013); + expect(tomb?.kind).toBe('permanent'); + expect(tomb?.source_updated_at).toBe(200); + + // Later recoverable cannot weaken permanent. + await request(app) + .delete('/api/notification-suppression-rules/replica/993013') + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'recoverable', source_updated_at: 999 }); + expect(DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(993013)?.kind).toBe( + 'permanent', + ); + }); + + it('partial or invalid DELETE body returns 400 without mutation', async () => { + const jwt = await import('jsonwebtoken'); + const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); + const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + + await request(app) + .post('/api/notification-suppression-rules/replica') + .set('Authorization', `Bearer ${token}`) + .send({ + rule: { + id: 994014, + name: 'keep-me', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + created_at: 1, + updated_at: 10, + }, + }); + + const cases: object[] = [ + { kind: 'recoverable' }, + { source_updated_at: 1 }, + { kind: 'nope', source_updated_at: 1 }, + { kind: 'recoverable', source_updated_at: 1.5 }, + { kind: 'recoverable', source_updated_at: -1 }, + { kind: 'recoverable', source_updated_at: '1' }, + { kind: 'recoverable', source_updated_at: null }, + { kind: 'recoverable', source_updated_at: Number.MAX_SAFE_INTEGER + 1 }, + ]; + for (const body of cases) { + const res = await request(app) + .delete('/api/notification-suppression-rules/replica/994014') + .set('Authorization', `Bearer ${token}`) + .send(body); + expect(res.status).toBe(400); + } + expect(DatabaseService.getInstance().getNotificationSuppressionRule(994014)?.name).toBe('keep-me'); + expect(DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(994014)).toBeUndefined(); + }); + + it('replica POST rejects missing or invalid created_at and updated_at', async () => { + const jwt = await import('jsonwebtoken'); + const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); + const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + const base = { + id: 995015, + name: 'bad-ts', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + }; + + for (const rule of [ + { ...base, updated_at: 1 }, + { ...base, created_at: 1 }, + { ...base, created_at: -1, updated_at: 1 }, + { ...base, created_at: 1, updated_at: 1.5 }, + { ...base, created_at: 1, updated_at: '1' }, + ]) { + const res = await request(app) + .post('/api/notification-suppression-rules/replica') + .set('Authorization', `Bearer ${token}`) + .send({ rule }); + expect(res.status).toBe(400); + } + expect(DatabaseService.getInstance().getNotificationSuppressionRule(995015)).toBeUndefined(); + }); + + it('permanent DELETE removes a newer stored row', async () => { + const jwt = await import('jsonwebtoken'); + const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); + const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + + await request(app) + .post('/api/notification-suppression-rules/replica') + .set('Authorization', `Bearer ${token}`) + .send({ + rule: { + id: 999019, + name: 'newer-row', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + created_at: 1, + updated_at: 200, + }, + }); + + const del = await request(app) + .delete('/api/notification-suppression-rules/replica/999019') + .set('Authorization', `Bearer ${token}`) + .send({ kind: 'permanent', source_updated_at: 50 }); + expect(del.status).toBe(200); + expect(DatabaseService.getInstance().getNotificationSuppressionRule(999019)).toBeUndefined(); + expect(DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(999019)?.kind).toBe( + 'permanent', + ); + }); + + it('empty JSON DELETE body fails closed as permanent', async () => { + const jwt = await import('jsonwebtoken'); + const { TEST_JWT_SECRET } = await import('./helpers/testConstants'); + const token = jwt.default.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + + const del = await request(app) + .delete('/api/notification-suppression-rules/replica/999020') + .set('Authorization', `Bearer ${token}`) + .set('Content-Type', 'application/json') + .send({}); + expect(del.status).toBe(200); + expect(DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(999020)?.kind).toBe( + 'permanent', + ); + expect( + DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(999020)?.source_updated_at, + ).toBe(0); + }); + + it('one-arg deleteNotificationSuppressionRule defaults to permanent', () => { + DatabaseService.getInstance().upsertNotificationSuppressionRuleReplica({ + id: 996016, + name: 'one-arg', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + schedule: null, + scheduleInvalid: false, + created_at: 1, + updated_at: 5, + }); + DatabaseService.getInstance().deleteNotificationSuppressionRule(996016); + expect(DatabaseService.getInstance().getNotificationSuppressionRule(996016)).toBeUndefined(); + expect(DatabaseService.getInstance().getNotificationSuppressionRuleTombstone(996016)?.kind).toBe( + 'permanent', + ); + }); + + it('delete rolls back when tombstone upsert fails', () => { + const db = DatabaseService.getInstance(); + db.upsertNotificationSuppressionRuleReplica({ + id: 997017, + name: 'atomic-del', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + schedule: null, + scheduleInvalid: false, + created_at: 1, + updated_at: 7, + }); + const raw = db.getDb(); + const orig = raw.prepare.bind(raw); + const spy = vi.spyOn(raw, 'prepare').mockImplementation(((sql: string) => { + if ( + typeof sql === 'string' && + sql.includes('INSERT INTO notification_suppression_rule_tombstones') + ) { + throw new Error('forced tombstone upsert failure'); + } + return orig(sql); + }) as typeof raw.prepare); + + expect(() => + db.deleteNotificationSuppressionRule(997017, { + kind: 'recoverable', + source_updated_at: 7, + }), + ).toThrow(/forced tombstone upsert failure/); + spy.mockRestore(); + + expect(db.getNotificationSuppressionRule(997017)?.name).toBe('atomic-del'); + expect(db.getNotificationSuppressionRuleTombstone(997017)).toBeUndefined(); + }); + + it('recoverable recreate rolls back when insert fails after tombstone clear attempt', () => { + const db = DatabaseService.getInstance(); + db.deleteNotificationSuppressionRule(998018, { + kind: 'recoverable', + source_updated_at: 10, + }); + expect(db.getNotificationSuppressionRuleTombstone(998018)?.kind).toBe('recoverable'); + + const raw = db.getDb(); + const orig = raw.prepare.bind(raw); + const spy = vi.spyOn(raw, 'prepare').mockImplementation(((sql: string) => { + if ( + typeof sql === 'string' && + sql.includes('INSERT INTO notification_suppression_rules') + ) { + throw new Error('forced replica insert failure'); + } + return orig(sql); + }) as typeof raw.prepare); + + expect(() => + db.upsertNotificationSuppressionRuleReplica({ + id: 998018, + name: 'recreate-fail', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + schedule: null, + scheduleInvalid: false, + created_at: 1, + updated_at: 20, + }), + ).toThrow(/forced replica insert failure/); + spy.mockRestore(); + + expect(db.getNotificationSuppressionRule(998018)).toBeUndefined(); + expect(db.getNotificationSuppressionRuleTombstone(998018)?.kind).toBe('recoverable'); + expect(db.getNotificationSuppressionRuleTombstone(998018)?.source_updated_at).toBe(10); }); }); diff --git a/backend/src/__tests__/notification-suppression-sync.test.ts b/backend/src/__tests__/notification-suppression-sync.test.ts index 3a401972..d46ae8d9 100644 --- a/backend/src/__tests__/notification-suppression-sync.test.ts +++ b/backend/src/__tests__/notification-suppression-sync.test.ts @@ -1,5 +1,6 @@ /** - * Fleet sync for suppression rules: node_id normalize, capability gate, stale DELETE. + * Fleet sync for suppression rules: node_id normalize, capability gate, stale DELETE, + * durable pending retractions, permanent fan-out to all remotes. */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; @@ -8,12 +9,18 @@ const mockGetProxyTarget = vi.fn(); const mockGetNodes = vi.fn(); const mockGetNode = vi.fn(); const mockRemoteAdvertises = vi.fn(); +const mockUpsertPending = vi.fn(); +const mockDeletePending = vi.fn(); +const mockListPending = vi.fn(); vi.mock('../services/DatabaseService', () => ({ DatabaseService: { getInstance: () => ({ getNodes: mockGetNodes, getNode: mockGetNode, + upsertNotificationSuppressionPendingRetraction: mockUpsertPending, + deleteNotificationSuppressionPendingRetraction: mockDeletePending, + listNotificationSuppressionPendingRetractions: mockListPending, }), }, })); @@ -46,6 +53,7 @@ import { syncSuppressionRuleToFleet, syncSuppressionRuleUpdateToFleet, replicationTargetIds, + flushPendingSuppressionRetractions, } from '../helpers/notificationSuppressionSync'; import type { NotificationSuppressionRule } from '../services/DatabaseService'; @@ -72,6 +80,21 @@ function makeRule(overrides: Partial = {}): Notific const remoteA = { id: 10, name: 'remote-a', type: 'remote' as const }; const remoteB = { id: 11, name: 'remote-b', type: 'remote' as const }; +const RETRACTION_CAP = 'notification-suppression-replica-retraction'; + +function metaWith(...caps: string[]) { + return { capabilities: caps, online: true }; +} + +function okDeleteResponse(outcome = 'applied') { + return { + ok: true, + status: 200, + text: async () => '', + json: async () => ({ success: true, outcome }), + }; +} + describe('notificationSuppressionSync', () => { beforeEach(() => { vi.clearAllMocks(); @@ -83,8 +106,10 @@ describe('notificationSuppressionSync', () => { apiUrl: `http://node-${id}.example:1852`, apiToken: 'tok', })); - mockFetch.mockResolvedValue({ ok: true, status: 200, text: async () => '' }); + mockFetch.mockResolvedValue(okDeleteResponse()); mockRemoteAdvertises.mockResolvedValue(true); + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP, 'notification-suppression-schedule')); + mockListPending.mockReturnValue([]); }); afterEach(() => { @@ -100,6 +125,7 @@ describe('notificationSuppressionSync', () => { syncSuppressionRuleToFleet(makeRule({ schedule: null, node_id: 10 })); await vi.waitFor(() => expect(mockFetch).toHaveBeenCalled()); expect(mockRemoteAdvertises).not.toHaveBeenCalled(); + expect(mockFetchMeta).not.toHaveBeenCalled(); const body = JSON.parse((mockFetch.mock.calls[0][1] as { body: string }).body); expect(body.rule.node_id).toBeNull(); expect(body.rule.schedule).toBeNull(); @@ -121,44 +147,99 @@ describe('notificationSuppressionSync', () => { expect(body.rule.schedule.days).toEqual([6]); }); - it('probe false + DELETE success: no POST; cleanup logged as removed', async () => { + it('schedule unsupported + retraction supported: recoverable DELETE, no POST', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); const error = vi.spyOn(console, 'error').mockImplementation(() => {}); mockRemoteAdvertises.mockResolvedValue(false); - mockFetch.mockResolvedValue({ ok: true, status: 200, text: async () => '' }); + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); syncSuppressionRuleToFleet(makeRule({ node_id: 10, + updated_at: 555, schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' }, })); await vi.waitFor(() => expect(mockFetch).toHaveBeenCalled()); expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true); - expect(warn.mock.calls.some((c) => String(c[0]).includes('replica was removed'))).toBe(true); + expect(JSON.parse((mockFetch.mock.calls[0][1] as { body: string }).body)).toEqual({ + kind: 'recoverable', + source_updated_at: 555, + }); + expect(warn.mock.calls.some((c) => String(c[0]).includes('recoverable DELETE applied'))).toBe(true); expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(false); + expect(mockUpsertPending).not.toHaveBeenCalled(); + expect(mockDeletePending).toHaveBeenCalledWith(42, 10); }); - it('probe false + no proxy target: no successful-cleanup claim', async () => { + it('schedule unsupported + retraction unsupported: no DELETE; queues pending', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + mockRemoteAdvertises.mockResolvedValue(false); + mockFetchMeta.mockResolvedValue(metaWith('notification-suppression')); + + syncSuppressionRuleToFleet(makeRule({ + node_id: 10, + updated_at: 555, + schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' }, + })); + await vi.waitFor(() => expect(mockUpsertPending).toHaveBeenCalled()); + + expect(mockFetch).not.toHaveBeenCalled(); + expect(warn.mock.calls.some((c) => String(c[0]).includes('queued pending retraction'))).toBe(true); + expect(mockUpsertPending).toHaveBeenCalledWith( + expect.objectContaining({ + rule_id: 42, + node_id: 10, + kind: 'recoverable', + source_updated_at: 555, + }), + ); + }); + + it('schedule unsupported + probe unreachable (offline meta): no DELETE; queues pending', async () => { + mockRemoteAdvertises.mockResolvedValue(false); + mockFetchMeta.mockResolvedValue({ capabilities: [], online: false }); + + syncSuppressionRuleToFleet(makeRule({ + node_id: 10, + schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' }, + })); + await vi.waitFor(() => expect(mockUpsertPending).toHaveBeenCalled()); + expect(mockFetch).not.toHaveBeenCalled(); + expect(mockUpsertPending.mock.calls[0][0].last_error).toMatch(/unreachable/); + }); + + it('schedule unsupported + probe unreachable (throw): no DELETE; queues pending', async () => { + mockRemoteAdvertises.mockResolvedValue(false); + mockFetchMeta.mockRejectedValue(new Error('timeout')); + + syncSuppressionRuleToFleet(makeRule({ + node_id: 10, + schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' }, + })); + await vi.waitFor(() => expect(mockUpsertPending).toHaveBeenCalled()); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('schedule unsupported + no proxy after supported probe: queues pending, no POST', async () => { const error = vi.spyOn(console, 'error').mockImplementation(() => {}); mockRemoteAdvertises.mockResolvedValue(false); + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); mockGetProxyTarget.mockReturnValue(null); syncSuppressionRuleToFleet(makeRule({ node_id: 10, schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' }, })); - await vi.waitFor(() => expect(error).toHaveBeenCalled()); + await vi.waitFor(() => expect(mockUpsertPending).toHaveBeenCalled()); expect(mockFetch).not.toHaveBeenCalled(); - expect(warn.mock.calls.some((c) => String(c[0]).includes('replica was removed'))).toBe(false); expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(true); }); - it('scheduleInvalid: DELETE success, no POST', async () => { + it('scheduleInvalid: DELETE success when retraction supported, no POST', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); const error = vi.spyOn(console, 'error').mockImplementation(() => {}); - mockFetch.mockResolvedValue({ ok: true, status: 200, text: async () => '' }); + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); syncSuppressionRuleToFleet(makeRule({ node_id: 10, @@ -166,31 +247,44 @@ describe('notificationSuppressionSync', () => { scheduleInvalid: true, })); await vi.waitFor(() => { - expect(warn.mock.calls.some((c) => String(c[0]).includes('replica removed'))).toBe(true); + expect(warn.mock.calls.some((c) => String(c[0]).includes('recoverable DELETE applied'))).toBe(true); }); expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true); expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(false); }); - it('scheduleInvalid: DELETE 404 counts as cleanup success, no POST', async () => { - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - mockFetch.mockResolvedValue({ ok: false, status: 404, text: async () => 'gone' }); - - syncSuppressionRuleToFleet(makeRule({ - node_id: 10, - schedule: null, - scheduleInvalid: true, - })); - await vi.waitFor(() => { - expect(warn.mock.calls.some((c) => String(c[0]).includes('replica removed'))).toBe(true); + it('scheduleInvalid: opaque DELETE 404 queues pending (not treated as success)', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); + mockFetch.mockResolvedValue({ + ok: false, + status: 404, + text: async () => 'Not Found', + json: async () => { + throw new Error('no json'); + }, }); - expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true); + + syncSuppressionRuleToFleet(makeRule({ + node_id: 10, + schedule: null, + scheduleInvalid: true, + })); + await vi.waitFor(() => expect(mockUpsertPending).toHaveBeenCalled()); + expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(true); + expect(mockDeletePending).not.toHaveBeenCalled(); }); - it('scheduleInvalid: DELETE failure logs pending cleanup, no POST', async () => { + it('scheduleInvalid: DELETE failure queues pending, no POST', async () => { const error = vi.spyOn(console, 'error').mockImplementation(() => {}); - mockFetch.mockResolvedValue({ ok: false, status: 503, text: async () => 'down' }); + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); + mockFetch.mockResolvedValue({ + ok: false, + status: 503, + text: async () => 'down', + json: async () => ({}), + }); syncSuppressionRuleToFleet(makeRule({ node_id: 10, @@ -199,28 +293,32 @@ describe('notificationSuppressionSync', () => { })); await vi.waitFor(() => expect(error).toHaveBeenCalled()); expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true); + expect(mockUpsertPending).toHaveBeenCalled(); expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(true); }); - it('scheduleInvalid: no proxy target logs pending, no POST', async () => { - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const error = vi.spyOn(console, 'error').mockImplementation(() => {}); - mockGetProxyTarget.mockReturnValue(null); + it('scheduleInvalid without retraction capability: no DELETE; queues pending', async () => { + mockFetchMeta.mockResolvedValue(metaWith()); syncSuppressionRuleToFleet(makeRule({ node_id: 10, + updated_at: 777, schedule: null, scheduleInvalid: true, })); - await vi.waitFor(() => expect(error).toHaveBeenCalled()); + await vi.waitFor(() => expect(mockUpsertPending).toHaveBeenCalled()); expect(mockFetch).not.toHaveBeenCalled(); - expect(warn.mock.calls.some((c) => String(c[0]).includes('replica removed'))).toBe(false); - expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(true); + expect(mockUpsertPending).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'recoverable', + source_updated_at: 777, + }), + ); }); - it('unscheduled-to-scheduled on unsupported target attempts DELETE', async () => { + it('unscheduled-to-scheduled on unsupported schedule target attempts recoverable path', async () => { mockRemoteAdvertises.mockResolvedValue(false); - mockFetch.mockResolvedValue({ ok: true, status: 200, text: async () => '' }); + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); const previous = makeRule({ node_id: 10, schedule: null }); const updated = makeRule({ node_id: 10, @@ -231,22 +329,6 @@ describe('notificationSuppressionSync', () => { expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true); }); - it('probe false + DELETE failure: no POST; logs cleanup pending', async () => { - const error = vi.spyOn(console, 'error').mockImplementation(() => {}); - mockRemoteAdvertises.mockResolvedValue(false); - mockFetch.mockResolvedValue({ ok: false, status: 503, text: async () => 'down' }); - - syncSuppressionRuleToFleet(makeRule({ - node_id: 10, - schedule: { days: [1], start_minute: 0, end_minute: 60, tz: 'UTC' }, - })); - await vi.waitFor(() => expect(error).toHaveBeenCalled()); - - expect(mockFetch.mock.calls.every((c) => (c[1] as { method: string }).method === 'DELETE')).toBe(true); - expect(error.mock.calls.some((c) => String(c[0]).includes('cleanup pending'))).toBe(true); - expect(error.mock.calls.some((c) => String(c[0]).includes('rule 42'))).toBe(true); - }); - it('scheduled-to-unscheduled POST refresh does not require capability', async () => { mockRemoteAdvertises.mockResolvedValue(false); const previous = makeRule({ @@ -261,7 +343,8 @@ describe('notificationSuppressionSync', () => { expect(body.rule.schedule).toBeNull(); }); - it('stale targets receive DELETE on scope change', async () => { + it('stale targets receive recoverable DELETE when retraction supported', async () => { + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); const previous = makeRule({ node_id: null, schedule: null }); const updated = makeRule({ node_id: 10, schedule: null }); syncSuppressionRuleUpdateToFleet(previous, updated); @@ -272,4 +355,147 @@ describe('notificationSuppressionSync', () => { const posts = mockFetch.mock.calls.filter((c) => (c[1] as { method: string }).method === 'POST'); expect(posts.some((c) => String(c[0]).includes('node-10'))).toBe(true); }); + + it('stale-target without retraction capability: no DELETE; queues pending', async () => { + mockFetchMeta.mockResolvedValue(metaWith()); + const previous = makeRule({ id: 42, node_id: null, schedule: null, updated_at: 10 }); + const updated = makeRule({ id: 42, node_id: 10, schedule: null, updated_at: 99 }); + syncSuppressionRuleUpdateToFleet(previous, updated); + await vi.waitFor(() => expect(mockUpsertPending).toHaveBeenCalled()); + + const deletes = mockFetch.mock.calls.filter((c) => (c[1] as { method: string }).method === 'DELETE'); + expect(deletes).toHaveLength(0); + expect(mockUpsertPending).toHaveBeenCalledWith( + expect.objectContaining({ + rule_id: 42, + node_id: 11, + kind: 'recoverable', + source_updated_at: 99, + }), + ); + }); + + it('stale-target DELETE sends recoverable watermark from updated rule', async () => { + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); + const previous = makeRule({ id: 42, node_id: null, schedule: null, updated_at: 10 }); + const updated = makeRule({ id: 42, node_id: 10, schedule: null, updated_at: 99 }); + syncSuppressionRuleUpdateToFleet(previous, updated); + await vi.waitFor(() => expect(mockFetch.mock.calls.length).toBeGreaterThanOrEqual(2)); + const deletes = mockFetch.mock.calls.filter((c) => (c[1] as { method: string }).method === 'DELETE'); + const stale = deletes.find((c) => String(c[0]).includes('node-11')); + expect(stale).toBeTruthy(); + expect(JSON.parse((stale![1] as { body: string }).body)).toEqual({ + kind: 'recoverable', + source_updated_at: 99, + }); + }); + + it('authoritative fleet delete fans permanent retraction to all remotes', async () => { + const { deleteSuppressionRuleFromFleet } = await import('../helpers/notificationSuppressionSync'); + deleteSuppressionRuleFromFleet(makeRule({ node_id: 10, updated_at: 321 })); + await vi.waitFor(() => expect(mockFetch.mock.calls.length).toBe(2)); + + for (const call of mockFetch.mock.calls) { + const [, init] = call as [string, { method: string; body: string }]; + expect(init.method).toBe('DELETE'); + expect(JSON.parse(init.body)).toEqual({ kind: 'permanent', source_updated_at: 321 }); + } + expect(mockFetch.mock.calls.some((c) => String(c[0]).includes('node-10'))).toBe(true); + expect(mockFetch.mock.calls.some((c) => String(c[0]).includes('node-11'))).toBe(true); + }); + + it('authoritative delete transport failure queues pending permanent row', async () => { + const { deleteSuppressionRuleFromFleet } = await import('../helpers/notificationSuppressionSync'); + mockFetch.mockResolvedValue({ + ok: false, + status: 503, + text: async () => 'down', + json: async () => ({}), + }); + deleteSuppressionRuleFromFleet(makeRule({ node_id: 10, updated_at: 321 })); + await vi.waitFor(() => expect(mockUpsertPending).toHaveBeenCalled()); + expect(mockUpsertPending).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'permanent', + source_updated_at: 321, + }), + ); + }); + + it('ignored_stale DELETE keeps pending and does not clear', async () => { + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + text: async () => '', + json: async () => ({ success: true, outcome: 'ignored_stale' }), + }); + + syncSuppressionRuleToFleet(makeRule({ + node_id: 10, + schedule: null, + scheduleInvalid: true, + })); + await vi.waitFor(() => expect(mockUpsertPending).toHaveBeenCalled()); + expect(mockDeletePending).not.toHaveBeenCalled(); + expect(mockUpsertPending).toHaveBeenCalledWith( + expect.objectContaining({ + last_error: expect.stringMatching(/ignored_stale/), + }), + ); + }); + + it('flushPendingSuppressionRetractions retries recoverable only when supported', async () => { + mockListPending.mockReturnValue([ + { + rule_id: 7, + node_id: 10, + kind: 'recoverable', + source_updated_at: 50, + created_at: 1, + updated_at: 2, + attempts: 1, + last_error: 'earlier', + }, + ]); + mockFetchMeta.mockResolvedValue(metaWith()); + + await flushPendingSuppressionRetractions(10); + expect(mockFetch).not.toHaveBeenCalled(); + expect(mockUpsertPending).toHaveBeenCalled(); + + mockUpsertPending.mockClear(); + mockFetchMeta.mockResolvedValue(metaWith(RETRACTION_CAP)); + await flushPendingSuppressionRetractions(10); + await vi.waitFor(() => expect(mockFetch).toHaveBeenCalled()); + expect(JSON.parse((mockFetch.mock.calls[0][1] as { body: string }).body)).toEqual({ + kind: 'recoverable', + source_updated_at: 50, + }); + expect(mockDeletePending).toHaveBeenCalledWith(7, 10); + }); + + it('flushPendingSuppressionRetractions sends permanent without retraction capability', async () => { + mockListPending.mockReturnValue([ + { + rule_id: 8, + node_id: 11, + kind: 'permanent', + source_updated_at: 90, + created_at: 1, + updated_at: 2, + attempts: 2, + last_error: 'offline', + }, + ]); + mockFetchMeta.mockResolvedValue(metaWith()); + + await flushPendingSuppressionRetractions(11); + await vi.waitFor(() => expect(mockFetch).toHaveBeenCalled()); + expect(JSON.parse((mockFetch.mock.calls[0][1] as { body: string }).body)).toEqual({ + kind: 'permanent', + source_updated_at: 90, + }); + expect(mockDeletePending).toHaveBeenCalledWith(8, 11); + }); }); diff --git a/backend/src/__tests__/notification-suppression-tombstone-migration.test.ts b/backend/src/__tests__/notification-suppression-tombstone-migration.test.ts new file mode 100644 index 00000000..eb39ab8e --- /dev/null +++ b/backend/src/__tests__/notification-suppression-tombstone-migration.test.ts @@ -0,0 +1,159 @@ +/** + * Additive kind + source_updated_at on notification_suppression_rule_tombstones. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import Database from 'better-sqlite3'; +import { DatabaseService } from '../services/DatabaseService'; + +function resetDatabaseSingleton(): void { + const holder = DatabaseService as unknown as { instance?: DatabaseService }; + const existing = holder.instance; + if (existing) { + try { + existing.getDb().close(); + } catch { + // already closed + } + holder.instance = undefined; + } +} + +describe('notification suppression tombstone migration', () => { + let scratchDir: string | null = null; + let prevDataDir: string | undefined; + + afterEach(() => { + resetDatabaseSingleton(); + if (prevDataDir === undefined) { + delete process.env.DATA_DIR; + } else { + process.env.DATA_DIR = prevDataDir; + } + if (scratchDir) { + try { + fs.rmSync(scratchDir, { recursive: true, force: true }); + } catch { + // best-effort + } + scratchDir = null; + } + }); + + it('adds kind and source_updated_at; legacy rows stay permanent', { timeout: 60_000 }, () => { + scratchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-supp-tomb-mig-')); + const dbPath = path.join(scratchDir, 'sencho.db'); + const seed = new Database(dbPath); + try { + seed.exec(` + CREATE TABLE notification_suppression_rules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + node_id INTEGER NULL, + stack_patterns TEXT NOT NULL, + label_ids TEXT NULL, + categories TEXT NULL, + levels TEXT NULL, + applies_to TEXT NOT NULL, + enabled INTEGER DEFAULT 1, + expires_at INTEGER NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE notification_suppression_rule_tombstones ( + id INTEGER PRIMARY KEY, + deleted_at INTEGER NOT NULL + ); + INSERT INTO notification_suppression_rule_tombstones (id, deleted_at) + VALUES (42, 1700000000000); + `); + } finally { + seed.close(); + } + + prevDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = scratchDir; + resetDatabaseSingleton(); + const db = DatabaseService.getInstance(); + + const cols = db.getDb().prepare('PRAGMA table_info(notification_suppression_rule_tombstones)').all() as Array<{ + name: string; + }>; + const names = cols.map((c) => c.name); + expect(names).toContain('kind'); + expect(names).toContain('source_updated_at'); + + const tomb = db.getNotificationSuppressionRuleTombstone(42); + expect(tomb?.kind).toBe('permanent'); + expect(tomb?.source_updated_at).toBe(1700000000000); + + // Legacy permanent cannot be cleared by a newer hub POST. + db.upsertNotificationSuppressionRuleReplica({ + id: 42, + name: 'should-stay-gone', + applies_to: 'both', + stack_patterns: [], + node_id: null, + label_ids: null, + categories: null, + levels: null, + enabled: true, + expires_at: null, + schedule: null, + scheduleInvalid: false, + created_at: 1, + updated_at: 9_999_999_999_999, + }); + expect(db.getNotificationSuppressionRule(42)).toBeUndefined(); + expect(db.getNotificationSuppressionRuleTombstone(42)?.kind).toBe('permanent'); + }); + + it('creates pending retractions table and merges permanent over recoverable', { timeout: 60_000 }, () => { + scratchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sencho-supp-pending-')); + prevDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = scratchDir; + resetDatabaseSingleton(); + const db = DatabaseService.getInstance(); + + const cols = db.getDb().prepare('PRAGMA table_info(notification_suppression_pending_retractions)').all() as Array<{ + name: string; + }>; + expect(cols.map((c) => c.name)).toEqual( + expect.arrayContaining([ + 'rule_id', + 'node_id', + 'kind', + 'source_updated_at', + 'attempts', + 'last_error', + ]), + ); + + db.upsertNotificationSuppressionPendingRetraction({ + rule_id: 7, + node_id: 3, + kind: 'recoverable', + source_updated_at: 100, + last_error: 'unsupported', + }); + db.upsertNotificationSuppressionPendingRetraction({ + rule_id: 7, + node_id: 3, + kind: 'permanent', + source_updated_at: 50, + last_error: 'offline', + }); + + const rows = db.listNotificationSuppressionPendingRetractions(3); + expect(rows).toHaveLength(1); + expect(rows[0].kind).toBe('permanent'); + expect(rows[0].source_updated_at).toBe(100); + expect(rows[0].attempts).toBe(2); + expect(rows[0].last_error).toBe('offline'); + + db.deleteNotificationSuppressionPendingRetraction(7, 3); + expect(db.listNotificationSuppressionPendingRetractions(3)).toHaveLength(0); + }); +}); diff --git a/backend/src/__tests__/preflight-effective-model.test.ts b/backend/src/__tests__/preflight-effective-model.test.ts index 4bd7f69a..8709faa7 100644 --- a/backend/src/__tests__/preflight-effective-model.test.ts +++ b/backend/src/__tests__/preflight-effective-model.test.ts @@ -157,6 +157,24 @@ describe('parseEffectiveModel', () => { it('treats a disabled healthcheck as none', () => { const m = parseEffectiveModel({ services: { web: { healthcheck: { disable: true } } } }, 'fallback'); expect(m.services[0].hasHealthcheck).toBe(false); + expect(m.services[0].composeHealthcheck).toBe('disabled'); + }); + + it('treats test NONE as a disabled healthcheck', () => { + const m = parseEffectiveModel({ services: { web: { healthcheck: { test: ['NONE'] } } } }, 'fallback'); + expect(m.services[0].hasHealthcheck).toBe(false); + expect(m.services[0].composeHealthcheck).toBe('disabled'); + }); + + it('treats empty or disable:false healthcheck objects as absent', () => { + expect(parseEffectiveModel({ services: { web: { healthcheck: {} } } }, 'fallback').services[0]).toMatchObject({ + hasHealthcheck: false, + composeHealthcheck: 'absent', + }); + expect(parseEffectiveModel({ services: { web: { healthcheck: { disable: false } } } }, 'fallback').services[0]).toMatchObject({ + hasHealthcheck: false, + composeHealthcheck: 'absent', + }); }); it('falls back to the provided project name and yields an empty model for garbage', () => { diff --git a/backend/src/__tests__/preflight-route.test.ts b/backend/src/__tests__/preflight-route.test.ts index cbb2580b..cf91e360 100644 --- a/backend/src/__tests__/preflight-route.test.ts +++ b/backend/src/__tests__/preflight-route.test.ts @@ -11,6 +11,7 @@ import jwt from 'jsonwebtoken'; import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; import DockerController from '../services/DockerController'; import { ComposeService } from '../services/ComposeService'; +import { isPreflightNoteFinding } from '../services/preflight/rules'; let tmpDir: string; let app: import('express').Express; @@ -111,7 +112,10 @@ describe('preflight acknowledgement routes', () => { const acked = get.body.findings.find((f: { ruleId: string; service?: string }) => f.ruleId === target.ruleId && f.service === target.service); expect(acked?.acknowledged).toBe(true); - expect(get.body.activeCount).toBe(get.body.findings.length - get.body.acknowledgedCount); + expect(get.body.activeCount).toBe( + get.body.findings.filter((f: { ruleId: string; acknowledged?: boolean }) => + !f.acknowledged && !isPreflightNoteFinding(f.ruleId)).length, + ); }); it('DELETE clears an acknowledgement', async () => { @@ -140,4 +144,14 @@ describe('preflight acknowledgement routes', () => { .send({ ruleId: 'port-exposed-all-interfaces', expiryMode: 'until_image_change' }); expect(res.status).toBe(400); }); + + it('rejects acknowledging informational note findings', async () => { + await request(app).post(`/api/stacks/${STACK}/preflight/run`).set('Authorization', authHeader); + const res = await request(app) + .post(`/api/stacks/${STACK}/preflight/acknowledgements`) + .set('Authorization', authHeader) + .send({ ruleId: 'healthcheck-inherited', service: 'web', expiryMode: 'forever' }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/notes cannot be acknowledged/i); + }); }); diff --git a/backend/src/__tests__/preflight-rules.test.ts b/backend/src/__tests__/preflight-rules.test.ts index b6ffb191..a7a50e23 100644 --- a/backend/src/__tests__/preflight-rules.test.ts +++ b/backend/src/__tests__/preflight-rules.test.ts @@ -11,10 +11,15 @@ import type { EffService, EffectiveModel } from '../services/preflight/effective import type { PreflightContext, PreflightFinding } from '../services/preflight/types'; function svc(over: Partial = {}): EffService { + const hasHealthcheck = over.hasHealthcheck ?? true; + const composeHealthcheck = over.composeHealthcheck ?? (hasHealthcheck ? 'active' : 'absent'); return { name: 'web', image: 'nginx:1.27', ports: [], binds: [], namedVolumes: [], storageMounts: [], - privileged: false, hasHealthcheck: true, restart: 'unless-stopped', envKeys: [], - networks: [], extraHosts: [], labelKeys: [], ...over, + privileged: false, restart: 'unless-stopped', envKeys: [], + networks: [], extraHosts: [], labelKeys: [], + ...over, + hasHealthcheck, + composeHealthcheck, }; } @@ -33,7 +38,9 @@ function ctx(over: Partial = {}): PreflightContext { existingContainers: [], nodeStateAvailable: true, bindChecks: [], stackIntent: null, serviceIntents: {}, accessUrlPorts: new Set(), hasAccessUrls: false, exposureAvailable: true, - isSelfStack: false, ...over, + isSelfStack: false, + healthchecks: {}, + ...over, }; } @@ -217,10 +224,56 @@ describe('hygiene rules', () => { expect(restartFindings[0].remediation).toMatch(/one-shot|init jobs/i); expect(restartFindings[0].remediation).toMatch(/restart: "no"/); expect(restartFindings[0].remediation).toMatch(/unless-stopped/); - expect(ids(runRules(ctx({ model: bare })), 'no-healthcheck')).toHaveLength(1); + expect(ids(runRules(ctx({ + model: bare, + healthchecks: { web: { state: 'absent', origin: 'local-image', consistentReplicas: null } }, + })), 'no-healthcheck')).toHaveLength(1); const withDeployRestart = model([svc({ restart: undefined, deploy: { restart_policy: { condition: 'any' } }})]); expect(ids(runRules(ctx({ model: withDeployRestart })), 'no-restart-policy')).toHaveLength(0); }); + + it('emits the healthcheck evidence rule family', () => { + const bare = model([svc({ hasHealthcheck: false })]); + expect(ids(runRules(ctx({ + model: bare, + healthchecks: { web: { state: 'explicitly-disabled', origin: 'compose', consistentReplicas: null } }, + })), 'healthcheck-disabled')).toHaveLength(1); + expect(ids(runRules(ctx({ + model: bare, + healthchecks: { web: { state: 'runtime-inherited', origin: 'runtime', consistentReplicas: true } }, + })), 'healthcheck-inherited')[0]).toMatchObject({ + severity: 'info', + title: 'Healthcheck inherited from image', + }); + expect(ids(runRules(ctx({ + model: bare, + healthchecks: { web: { state: 'local-image-inherited', origin: 'local-image', consistentReplicas: null } }, + })), 'healthcheck-inherited')[0].remediation).toMatch(/Optionally declare/); + expect(ids(runRules(ctx({ + model: bare, + healthchecks: { web: { state: 'unverifiable', origin: 'none', consistentReplicas: null } }, + })), 'healthcheck-unverifiable')[0].severity).toBe('info'); + expect(ids(runRules(ctx({ + model: bare, + healthchecks: { web: { state: 'inconsistent-replicas', origin: 'runtime', consistentReplicas: false } }, + })), 'healthcheck-inconsistent')).toHaveLength(1); + expect(ids(runRules(ctx({ + model: bare, + healthchecks: { web: { state: 'compose-declared', origin: 'compose', consistentReplicas: null } }, + })), 'no-healthcheck')).toHaveLength(0); + }); + + it('never embeds healthcheck Test command text in findings', () => { + const bare = model([svc({ hasHealthcheck: false })]); + const findings = runRules(ctx({ + model: bare, + healthchecks: { web: { state: 'runtime-inherited', origin: 'runtime', consistentReplicas: true } }, + })); + const blob = findings.map(f => `${f.title}\n${f.message}\n${f.remediation ?? ''}`).join('\n'); + expect(blob).not.toMatch(/\bCMD\b/); + expect(blob).not.toMatch(/CMD-SHELL/); + expect(blob).not.toContain('secret-token'); + }); it('flags swarm-only deploy fields but not honored ones', () => { expect(ids(runRules(ctx({ model: model([svc({ deploy: { placement: {} }})]) })), 'deploy-swarm-only')).toHaveLength(1); expect(ids(runRules(ctx({ model: model([svc({ deploy: { replicas: 3 }})]) })), 'deploy-swarm-only')).toHaveLength(0); @@ -491,7 +544,9 @@ describe('rule registry completeness', () => { const EXPECTED_RULE_IDS = [ 'render-failed', 'env-unset', 'env-literal-dollar', 'env-file-missing', 'port-conflict-node', 'port-conflict-internal', 'port-exposed-all-interfaces', 'bind-path-missing', 'bind-path-permission', 'docker-socket-mount', 'privileged', 'network-mode-host', - 'uid-gid-risk', 'image-latest', 'no-restart-policy', 'no-healthcheck', 'deploy-swarm-only', + 'uid-gid-risk', 'image-latest', 'no-restart-policy', 'no-healthcheck', + 'healthcheck-disabled', 'healthcheck-inherited', 'healthcheck-unverifiable', 'healthcheck-inconsistent', + 'deploy-swarm-only', 'node-state-unavailable', 'external-network-missing', 'external-volume-missing', 'new-network', 'new-volume', 'anonymous-volume', 'container-name-internal-dup', 'container-name-collision', diff --git a/backend/src/__tests__/recovery-cli.test.ts b/backend/src/__tests__/recovery-cli.test.ts index 3f05e375..ce27b5b0 100644 --- a/backend/src/__tests__/recovery-cli.test.ts +++ b/backend/src/__tests__/recovery-cli.test.ts @@ -15,6 +15,7 @@ let resetPassword: typeof import('../cli/resetPassword').resetPassword; let createEmergencyAdmin: typeof import('../cli/createEmergencyAdmin').createEmergencyAdmin; let clearSessions: typeof import('../cli/clearSessions').clearSessions; let disableSso: typeof import('../cli/disableSso').disableSso; +let enableLocalLogin: typeof import('../cli/enableLocalLogin').enableLocalLogin; let validateDb: typeof import('../cli/validateDb').validateDb; let backupData: typeof import('../cli/backupData').backupData; @@ -25,6 +26,7 @@ beforeAll(async () => { ({ createEmergencyAdmin } = await import('../cli/createEmergencyAdmin')); ({ clearSessions } = await import('../cli/clearSessions')); ({ disableSso } = await import('../cli/disableSso')); + ({ enableLocalLogin } = await import('../cli/enableLocalLogin')); ({ validateDb } = await import('../cli/validateDb')); ({ backupData } = await import('../cli/backupData')); }); @@ -126,6 +128,59 @@ describe('disableSso', () => { expect(result.ok).toBe(true); expect(db.getEnabledSSOConfigs()).toHaveLength(0); }); + + it('rejects disabling the last provider while sso_only', () => { + const db = DatabaseService.getInstance(); + db.updateGlobalSetting('authentication_mode', 'sso_only'); + db.upsertSSOConfig('oidc_custom', true, '{"clientId":"abc"}'); + for (const cfg of db.getEnabledSSOConfigs()) { + if (cfg.provider !== 'oidc_custom') { + db.upsertSSOConfig(cfg.provider, false, cfg.config_json); + } + } + const result = disableSso('oidc_custom'); + expect(result.ok).toBe(false); + expect(result.message).toMatch(/last SSO provider/i); + expect(db.getEnabledSSOConfigs()).toHaveLength(1); + expect(db.getGlobalSettings().authentication_mode).toBe('sso_only'); + }); + + it('preserves sso_only when disabling one of several providers', () => { + const db = DatabaseService.getInstance(); + db.updateGlobalSetting('authentication_mode', 'sso_only'); + for (const cfg of db.getEnabledSSOConfigs()) { + db.upsertSSOConfig(cfg.provider, false, cfg.config_json); + } + db.upsertSSOConfig('oidc_google', true, '{"clientId":"g"}'); + db.upsertSSOConfig('oidc_github', true, '{"clientId":"h"}'); + const result = disableSso('oidc_google'); + expect(result.ok).toBe(true); + expect(result.message).toMatch(/remains SSO only/i); + expect(db.getGlobalSettings().authentication_mode).toBe('sso_only'); + const remaining = db.getEnabledSSOConfigs().map(c => c.provider).sort(); + expect(remaining).toEqual(['oidc_github']); + }); + + it('restores local_and_sso before disabling all providers under sso_only', () => { + const db = DatabaseService.getInstance(); + db.updateGlobalSetting('authentication_mode', 'sso_only'); + db.upsertSSOConfig('oidc_custom', true, '{"clientId":"abc"}'); + const result = disableSso(); + expect(result.ok).toBe(true); + expect(db.getGlobalSettings().authentication_mode).toBe('local_and_sso'); + expect(db.getEnabledSSOConfigs()).toHaveLength(0); + }); +}); + +describe('enableLocalLogin', () => { + it('sets authentication_mode to local_and_sso', () => { + const db = DatabaseService.getInstance(); + db.updateGlobalSetting('authentication_mode', 'sso_only'); + const result = enableLocalLogin(); + expect(result.ok).toBe(true); + expect(result.message).toMatch(/Restart Sencho/i); + expect(db.getGlobalSettings().authentication_mode).toBe('local_and_sso'); + }); }); describe('backupData', () => { diff --git a/backend/src/__tests__/registry-api.test.ts b/backend/src/__tests__/registry-api.test.ts index ff03dae6..4e93cab1 100644 --- a/backend/src/__tests__/registry-api.test.ts +++ b/backend/src/__tests__/registry-api.test.ts @@ -58,9 +58,11 @@ import { getRemoteDigest, getRemoteDigestResult, getAuthToken, + listRegistryTags, listRegistryTagsResult, parseImageRef, selectLocalRepoDigest, + selectLocalRepoDigests, compareLocalToRemoteTag, MANIFEST_CLASSIFICATION_CACHE_TTL_MS, MANIFEST_INDEX_DESCRIPTOR_CAP, @@ -113,6 +115,34 @@ describe('repoDigestMatchesRef', () => { }); }); +describe('parseImageRef', () => { + // docker.io / index.docker.io / registry-1.docker.io are the same registry, but only + // the literal 'registry-1.docker.io' is recognized elsewhere (getAuthToken, the + // library/ auto-prefix in parseImageRef). An unnormalized 'docker.io' or 'index.docker.io' + // leaks through into request URLs and hits the marketing domain instead of the registry API. + // Each alias is listed with and without an explicit library/ namespace to pin both paths. + it.each([ + 'docker.io/library/traefik:latest', + 'docker.io/traefik:latest', + 'index.docker.io/library/traefik:latest', + 'index.docker.io/traefik:latest', + ])('normalizes %s to the registry API host and the library/ namespace', (ref) => { + expect(parseImageRef(ref)).toEqual({ + registry: 'registry-1.docker.io', + repo: 'library/traefik', + tag: 'latest', + }); + }); + + it('leaves a bare official image name unchanged (regression guard)', () => { + expect(parseImageRef('traefik:latest')).toEqual({ + registry: 'registry-1.docker.io', + repo: 'library/traefik', + tag: 'latest', + }); + }); +}); + describe('getRemoteDigest HEAD-first lookup', () => { beforeEach(() => { calls.length = 0; @@ -358,6 +388,31 @@ describe('listRegistryTagsResult', () => { }); }); +describe('listRegistryTags (compatibility wrapper)', () => { + beforeEach(() => { + calls.length = 0; + }); + + it('lists tags for a public Docker Hub repository with no credentials configured', async () => { + route = (url) => tokenOk(url) ?? ( + url.includes('/tags/list') + ? { statusCode: 200, headers: {}, body: JSON.stringify({ tags: ['8.7.0', '8.8.0'] }) } + : { statusCode: 500, headers: {} } + ); + await expect(listRegistryTags('registry-1.docker.io', 'library/redis', null)).resolves.toEqual(['8.7.0', '8.8.0']); + }); + + it('still returns empty for a registry that actually requires credentials the caller does not have', async () => { + const CHALLENGE = 'Bearer realm="https://ghcr.io/token",service="ghcr.io",scope="repository:user/image:pull"'; + route = (url): FakeResp => { + if (url === 'https://ghcr.io/v2/') return { statusCode: 401, headers: { 'www-authenticate': CHALLENGE } }; + if (url.startsWith('https://ghcr.io/token')) return { statusCode: 401, headers: {} }; + return { statusCode: 500, headers: {} }; + }; + await expect(listRegistryTags('ghcr.io', 'private/app', undefined)).resolves.toEqual([]); + }); +}); + // ─── selectLocalRepoDigest ─────────────────────────────────────────────── describe('selectLocalRepoDigest', () => { @@ -374,9 +429,9 @@ describe('selectLocalRepoDigest', () => { expect(selectLocalRepoDigest(repoDigests, parsed('nginx:latest'))).toBe(DIGEST_A); }); - it('falls back to the sole valid entry when nothing matches the ref', () => { + it('returns null when the sole valid entry belongs to an unrelated repository, rather than guessing', () => { const repoDigests = [`ghcr.io/other/image@${DIGEST_A}`]; - expect(selectLocalRepoDigest(repoDigests, parsed('nginx:latest'))).toBe(DIGEST_A); + expect(selectLocalRepoDigest(repoDigests, parsed('nginx:latest'))).toBeNull(); }); it('returns null when multiple valid entries exist and none matches the ref', () => { @@ -407,6 +462,64 @@ describe('selectLocalRepoDigest', () => { }); }); +// ─── selectLocalRepoDigests ────────────────────────────────────────────── + +describe('selectLocalRepoDigests', () => { + const parsed = (ref: string) => { + const p = parseImageRef(ref); + if (!p) throw new Error(`unparseable ${ref}`); + return p; + }; + const DIGEST_A = `sha256:${'a'.repeat(64)}`; + const DIGEST_B = `sha256:${'b'.repeat(64)}`; + const DIGEST_C = `sha256:${'c'.repeat(64)}`; + + it('returns every matching entry in first-seen order', () => { + const repoDigests = [`redis@${DIGEST_A}`, `nginx@${DIGEST_B}`, `redis@${DIGEST_C}`]; + expect(selectLocalRepoDigests(repoDigests, parsed('redis:8.8.0'))).toEqual([DIGEST_A, DIGEST_C]); + }); + + it('deduplicates matching digests while preserving first-seen order', () => { + const repoDigests = [`redis@${DIGEST_A}`, `redis@${DIGEST_B}`, `redis@${DIGEST_A}`]; + expect(selectLocalRepoDigests(repoDigests, parsed('redis:latest'))).toEqual([DIGEST_A, DIGEST_B]); + }); + + it('deduplicates case-insensitively on hex digits', () => { + const upper = `sha256:${'A'.repeat(64)}`; + const lower = `sha256:${'a'.repeat(64)}`; + expect(selectLocalRepoDigests([`nginx@${upper}`, `nginx@${lower}`], parsed('nginx:latest'))).toEqual([upper]); + }); + + it('filters malformed entries and keeps valid matches', () => { + const repoDigests = ['redis@sha256:tooshort', `redis@${DIGEST_A}`, 'redis:latest', `redis@${DIGEST_B}`]; + expect(selectLocalRepoDigests(repoDigests, parsed('redis:latest'))).toEqual([DIGEST_A, DIGEST_B]); + }); + + it('returns empty when the sole valid entry belongs to an unrelated repository, rather than guessing', () => { + // A legitimate retag can leave a lone RepoDigest from a different repo, but + // comparing it against this ref's registry state risks a false update + // against a registry that has nothing to do with the declared image. + expect(selectLocalRepoDigests([`ghcr.io/other/image@${DIGEST_A}`], parsed('nginx:latest'))).toEqual([]); + }); + + it('returns empty when multiple valid entries exist and none matches the ref', () => { + expect(selectLocalRepoDigests([`redis@${DIGEST_A}`, `postgres@${DIGEST_B}`], parsed('nginx:latest'))).toEqual([]); + }); + + it('returns empty for an empty list', () => { + expect(selectLocalRepoDigests([], parsed('nginx:latest'))).toEqual([]); + }); + + it('returns every matching RepoDigest for the image ref', () => { + const digests = selectLocalRepoDigests([ + `nginx@${DIGEST_A}`, + `nginx@${DIGEST_B}`, + `redis@sha256:${'c'.repeat(64)}`, + ], parsed('nginx:latest')); + expect(digests).toEqual([DIGEST_A, DIGEST_B]); + }); +}); + // ─── compareLocalToRemoteTag ───────────────────────────────────────────── // // Reproduces and fixes the false-positive multi-arch update: a local @@ -499,7 +612,7 @@ describe('compareLocalToRemoteTag', () => { ? { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST, 'content-type': INDEX_CONTENT_TYPE } } : { statusCode: 500, headers: {} } ); - const result = await compareLocalToRemoteTag(INDEX_DIGEST, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([INDEX_DIGEST], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'match' }); expect(calls.filter((c) => c.url.includes('/manifests/'))).toHaveLength(1); }); @@ -516,7 +629,7 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'match' }); expect(calls.filter((c) => c.url.includes('/manifests/'))).toEqual([ { url: MANIFEST_URL_TAG, method: 'HEAD' }, @@ -524,6 +637,215 @@ describe('compareLocalToRemoteTag', () => { ]); }); + // #1684: Docker can list a stale index digest ahead of the current one. + const STALE_INDEX = `sha256:${'f'.repeat(64)}`; + + it('matches when a later candidate equals the primary even if the first candidate is a stale index', async () => { + route = (url, method) => tokenOk(url) ?? ( + method === 'HEAD' + ? { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST, 'content-type': INDEX_CONTENT_TYPE } } + : { statusCode: 500, headers: {} } + ); + const result = await compareLocalToRemoteTag([STALE_INDEX, INDEX_DIGEST], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'match' }); + expect(calls.filter((c) => c.url.includes('/manifests/'))).toHaveLength(1); + }); + + it('matches when the current primary is first among multiple candidates', async () => { + route = (url, method) => tokenOk(url) ?? ( + method === 'HEAD' + ? { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST, 'content-type': INDEX_CONTENT_TYPE } } + : { statusCode: 500, headers: {} } + ); + const result = await compareLocalToRemoteTag([INDEX_DIGEST, STALE_INDEX], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'match' }); + }); + + it('expands the index once when a later candidate matches a platform child', async () => { + route = (url, method) => { + const token = tokenOk(url); + if (token) return token; + if (url === MANIFEST_URL_TAG && method === 'HEAD') { + return { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST, 'content-type': INDEX_CONTENT_TYPE } }; + } + if (url === manifestDigestUrl(INDEX_DIGEST) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST }, body: STANDARD_INDEX_BODY }; + } + return { statusCode: 500, headers: {} }; + }; + const result = await compareLocalToRemoteTag([STALE_INDEX, CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'match' }); + expect(calls.filter((c) => c.url.includes('/manifests/'))).toEqual([ + { url: MANIFEST_URL_TAG, method: 'HEAD' }, + { url: manifestDigestUrl(INDEX_DIGEST), method: 'GET' }, + ]); + }); + + it('reports update when every candidate is stale after a complete index classification', async () => { + route = (url, method) => { + const token = tokenOk(url); + if (token) return token; + if (url === MANIFEST_URL_TAG && method === 'HEAD') { + return { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST, 'content-type': INDEX_CONTENT_TYPE } }; + } + if (url === manifestDigestUrl(INDEX_DIGEST) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST }, body: STANDARD_INDEX_BODY }; + } + return { statusCode: 500, headers: {} }; + }; + const result = await compareLocalToRemoteTag([STALE_INDEX], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'update' }); + }); + + it('errors (not update) when the remote index has no descriptor at all for the local platform', async () => { + // STANDARD_INDEX_BODY only carries linux/amd64 and linux/arm64 children. + // A windows/amd64 node cannot pull anything from this index; that is not + // the same fact as "a newer build is available". + route = (url, method) => { + const token = tokenOk(url); + if (token) return token; + if (url === MANIFEST_URL_TAG && method === 'HEAD') { + return { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST, 'content-type': INDEX_CONTENT_TYPE } }; + } + if (url === manifestDigestUrl(INDEX_DIGEST) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST }, body: STANDARD_INDEX_BODY }; + } + return { statusCode: 500, headers: {} }; + }; + const result = await compareLocalToRemoteTag([STALE_INDEX], REGISTRY, REPO, TAG, { os: 'windows', architecture: 'amd64' }); + expect(result).toEqual({ kind: 'error', reason: expect.stringContaining('no windows/amd64 variant') }); + }); + + it('errors (not update) for an empty index with no manifests, rather than a speculative update', async () => { + const emptyIndexBody = indexBody([]); + const emptyIndexDigest = contentDigest(emptyIndexBody); + route = (url, method) => { + const token = tokenOk(url); + if (token) return token; + if (url === MANIFEST_URL_TAG && method === 'HEAD') { + return { statusCode: 200, headers: { 'docker-content-digest': emptyIndexDigest, 'content-type': INDEX_CONTENT_TYPE } }; + } + if (url === manifestDigestUrl(emptyIndexDigest) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': emptyIndexDigest }, body: emptyIndexBody }; + } + return { statusCode: 500, headers: {} }; + }; + const result = await compareLocalToRemoteTag([STALE_INDEX], REGISTRY, REPO, TAG, AMD64); + expect(result.kind).toBe('error'); + }); + + it('errors (not update) for an index whose only entries are filtered out (attestation-only), not just a literally empty one', async () => { + // Exercises the filtering path (parseIndexBody's attestation-manifest and + // unknown/unknown continues), distinct from manifests:[] never entering + // the per-entry loop at all. + const filteredIndexBody = indexBody([ + { + digest: CHILD_AMD64, os: 'unknown', architecture: 'unknown', + annotations: { 'vnd.docker.reference.type': 'attestation-manifest', 'vnd.docker.reference.digest': CHILD_ARM64 }, + }, + ]); + const filteredIndexDigest = contentDigest(filteredIndexBody); + route = (url, method) => { + const token = tokenOk(url); + if (token) return token; + if (url === MANIFEST_URL_TAG && method === 'HEAD') { + return { statusCode: 200, headers: { 'docker-content-digest': filteredIndexDigest, 'content-type': INDEX_CONTENT_TYPE } }; + } + if (url === manifestDigestUrl(filteredIndexDigest) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': filteredIndexDigest }, body: filteredIndexBody }; + } + return { statusCode: 500, headers: {} }; + }; + const result = await compareLocalToRemoteTag([STALE_INDEX], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'error', reason: expect.stringContaining('no linux/amd64 variant') }); + }); + + it('matches (not errors) a platform-less runnable descriptor even though no platform-labeled descriptor exists', async () => { + // OCI allows a runnable descriptor to omit platform; parseIndexBody routes + // it to exactDigests. The index has real, pullable content, so this must + // not be confused with the "nothing to pull" case. + const platformlessDigest = `sha256:${'7'.repeat(64)}`; + const noPlatformBody = JSON.stringify({ + schemaVersion: 2, + mediaType: INDEX_CONTENT_TYPE, + manifests: [{ digest: platformlessDigest, mediaType: 'application/vnd.oci.image.manifest.v1+json' }], + }); + const noPlatformDigest = contentDigest(noPlatformBody); + route = (url, method) => { + const token = tokenOk(url); + if (token) return token; + if (url === MANIFEST_URL_TAG && method === 'HEAD') { + return { statusCode: 200, headers: { 'docker-content-digest': noPlatformDigest, 'content-type': INDEX_CONTENT_TYPE } }; + } + if (url === manifestDigestUrl(noPlatformDigest) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': noPlatformDigest }, body: noPlatformBody }; + } + return { statusCode: 500, headers: {} }; + }; + const result = await compareLocalToRemoteTag([platformlessDigest], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'match' }); + }); + + it('errors (not update) for a mixed index: a platform-less leaf cannot be attributed when another descriptor is explicitly labeled for a different platform', async () => { + // Regression: an index with one arm64-labeled descriptor and one + // unlabeled leaf must not let the unlabeled leaf stand in as amd64 + // content just because SOME descriptor in the index is unlabeled. + const unrelatedLeaf = `sha256:${'6'.repeat(64)}`; + const mixedIndexBody = JSON.stringify({ + schemaVersion: 2, + mediaType: INDEX_CONTENT_TYPE, + manifests: [ + { digest: CHILD_ARM64, mediaType: 'application/vnd.oci.image.manifest.v1+json', platform: { os: 'linux', architecture: 'arm64' } }, + { digest: unrelatedLeaf, mediaType: 'application/vnd.oci.image.manifest.v1+json' }, + ], + }); + const mixedIndexDigest = contentDigest(mixedIndexBody); + route = (url, method) => { + const token = tokenOk(url); + if (token) return token; + if (url === MANIFEST_URL_TAG && method === 'HEAD') { + return { statusCode: 200, headers: { 'docker-content-digest': mixedIndexDigest, 'content-type': INDEX_CONTENT_TYPE } }; + } + if (url === manifestDigestUrl(mixedIndexDigest) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': mixedIndexDigest }, body: mixedIndexBody }; + } + return { statusCode: 500, headers: {} }; + }; + // A local candidate that matches neither the arm64 descriptor nor the + // unlabeled leaf: with no confirmed amd64 variant, this must fail closed. + const result = await compareLocalToRemoteTag([STALE_INDEX], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'error', reason: expect.stringContaining('no confirmed linux/amd64 variant') }); + }); + + it('errors (not update) when a nested index also has no descriptor for the local platform after full expansion', async () => { + const nestedIndexBody = indexBody([{ digest: CHILD_ARM64, os: 'linux', architecture: 'arm64' }]); + const nestedIndexDigest = contentDigest(nestedIndexBody); + const outerIndexBody = JSON.stringify({ + schemaVersion: 2, + mediaType: INDEX_CONTENT_TYPE, + manifests: [{ digest: nestedIndexDigest, mediaType: INDEX_CONTENT_TYPE }], + }); + const outerIndexDigest = contentDigest(outerIndexBody); + route = (url, method) => { + const token = tokenOk(url); + if (token) return token; + if (url === MANIFEST_URL_TAG && method === 'HEAD') { + return { statusCode: 200, headers: { 'docker-content-digest': outerIndexDigest, 'content-type': INDEX_CONTENT_TYPE } }; + } + if (url === manifestDigestUrl(outerIndexDigest) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': outerIndexDigest }, body: outerIndexBody }; + } + if (url === manifestDigestUrl(nestedIndexDigest) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': nestedIndexDigest }, body: nestedIndexBody }; + } + return { statusCode: 500, headers: {} }; + }; + // The outer index only nests an arm64-only child index; an amd64 node has + // no descriptor anywhere in the fully-expanded tree. + const result = await compareLocalToRemoteTag([STALE_INDEX], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'error', reason: expect.stringContaining('no linux/amd64 variant') }); + }); + it('never re-fetches the mutable tag: the expansion GET targets the primary digest from HEAD, not a second tag lookup', async () => { const DIVERGED_DIGEST = `sha256:${'e'.repeat(64)}`; let tagCallCount = 0; @@ -544,7 +866,7 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'match' }); expect(tagCallCount).toBe(1); }); @@ -555,7 +877,7 @@ describe('compareLocalToRemoteTag', () => { ? { statusCode: 200, headers: { 'docker-content-digest': SINGLE_DIGEST, 'content-type': 'application/vnd.docker.distribution.manifest.v2+json' } } : { statusCode: 500, headers: {} } ); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'update' }); expect(calls.filter((c) => c.url.includes('/manifests/'))).toHaveLength(1); }); @@ -570,7 +892,7 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(CHILD_ARM64, REGISTRY, REPO, TAG, ARM64); + const result = await compareLocalToRemoteTag([CHILD_ARM64], REGISTRY, REPO, TAG, ARM64); expect(result).toEqual({ kind: 'match' }); expect(calls.filter((c) => c.url.includes('/manifests/'))).toEqual([ { url: MANIFEST_URL_TAG, method: 'HEAD' }, @@ -590,7 +912,7 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result.kind).toBe('error'); expect(calls.filter((c) => c.url === MANIFEST_URL_TAG)).toHaveLength(1); }); @@ -609,9 +931,9 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const first = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const first = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(first.kind).toBe('error'); - const second = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const second = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(second.kind).toBe('error'); expect(digestGetCount).toBe(2); }); @@ -635,12 +957,12 @@ describe('compareLocalToRemoteTag', () => { return { statusCode: 500, headers: {} }; }; - const first = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const first = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(first).toEqual({ kind: 'match' }); await vi.advanceTimersByTimeAsync(MANIFEST_CLASSIFICATION_CACHE_TTL_MS + 1000); - const second = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const second = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(second).toEqual({ kind: 'match' }); expect(digestGetCount).toBe(2); }); @@ -659,8 +981,8 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); - await compareLocalToRemoteTag(CHILD_ARM64, REGISTRY, REPO, TAG, ARM64); + await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); + await compareLocalToRemoteTag([CHILD_ARM64], REGISTRY, REPO, TAG, ARM64); expect(digestGetCount).toBe(1); }); @@ -679,8 +1001,8 @@ describe('compareLocalToRemoteTag', () => { return { statusCode: 500, headers: {} }; }; const [a, b] = await Promise.all([ - compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64), - compareLocalToRemoteTag(CHILD_ARM64, REGISTRY, REPO, TAG, ARM64), + compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64), + compareLocalToRemoteTag([CHILD_ARM64], REGISTRY, REPO, TAG, ARM64), ]); expect(a).toEqual({ kind: 'match' }); expect(b).toEqual({ kind: 'match' }); @@ -688,7 +1010,13 @@ describe('compareLocalToRemoteTag', () => { }); it('misses the cache when the primary digest changes (new manifest, new immutable key)', async () => { - const INDEX_BODY_2 = indexBody([{ digest: CHILD_AMD64, os: 'linux', architecture: 'amd64' }]); + // Carries a genuinely different arm64 child (not CHILD_ARM64) so the second + // lookup is a real same-platform mismatch, not an absent-platform index. + const NEW_CHILD_ARM64 = `sha256:${'9'.repeat(64)}`; + const INDEX_BODY_2 = indexBody([ + { digest: CHILD_AMD64, os: 'linux', architecture: 'amd64' }, + { digest: NEW_CHILD_ARM64, os: 'linux', architecture: 'arm64' }, + ]); const INDEX_DIGEST_2 = contentDigest(INDEX_BODY_2); let headDigest = INDEX_DIGEST; let digestGetCount = 0; @@ -708,11 +1036,11 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const first = await compareLocalToRemoteTag(CHILD_ARM64, REGISTRY, REPO, TAG, ARM64); + const first = await compareLocalToRemoteTag([CHILD_ARM64], REGISTRY, REPO, TAG, ARM64); expect(first).toEqual({ kind: 'match' }); headDigest = INDEX_DIGEST_2; - const second = await compareLocalToRemoteTag(CHILD_ARM64, REGISTRY, REPO, TAG, ARM64); + const second = await compareLocalToRemoteTag([CHILD_ARM64], REGISTRY, REPO, TAG, ARM64); expect(second).toEqual({ kind: 'update' }); expect(digestGetCount).toBe(2); }); @@ -734,11 +1062,11 @@ describe('compareLocalToRemoteTag', () => { }; route = routeFor(REPO); - await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(digestGetCount).toBe(1); route = routeFor('otherorg/otherapp'); - await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, 'otherorg/otherapp', TAG, AMD64); + await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, 'otherorg/otherapp', TAG, AMD64); expect(digestGetCount).toBe(2); }); @@ -761,7 +1089,7 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(VARIANT_V7, REGISTRY, REPO, TAG, { os: 'linux', architecture: 'arm' }); + const result = await compareLocalToRemoteTag([VARIANT_V7], REGISTRY, REPO, TAG, { os: 'linux', architecture: 'arm' }); expect(result).toEqual({ kind: 'match' }); }); @@ -797,12 +1125,12 @@ describe('compareLocalToRemoteTag', () => { // A local digest equal to the filtered-out annotated-attestation entry must // never match, since that descriptor is dropped before the membership check. - const filtered = await compareLocalToRemoteTag(ATTESTATION_ANNOTATED, REGISTRY, REPO, TAG, AMD64); + const filtered = await compareLocalToRemoteTag([ATTESTATION_ANNOTATED], REGISTRY, REPO, TAG, AMD64); expect(filtered).toEqual({ kind: 'update' }); // The real platform descriptor still matches normally (cache hit reuses the // same parsed classification from the previous call). - const realMatch = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const realMatch = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(realMatch).toEqual({ kind: 'match' }); }); @@ -825,7 +1153,7 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result.kind).toBe('error'); }); @@ -843,7 +1171,7 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result.kind).toBe('error'); }); @@ -860,7 +1188,7 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'error', reason: expect.stringContaining('mismatched digest') }); }); @@ -879,7 +1207,7 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'error', reason: expect.stringContaining('does not match the requested digest') }); }); @@ -895,17 +1223,53 @@ describe('compareLocalToRemoteTag', () => { } return { statusCode: 500, headers: {} }; }; - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, { os: '', architecture: '' }); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, { os: '', architecture: '' }); expect(result.kind).toBe('error'); }); it('rejects a truncated local digest as an error, never as a speculative update', async () => { route = () => ({ statusCode: 500, headers: {} }); // must never be reached - const result = await compareLocalToRemoteTag('sha256:tooshort', REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag(['sha256:tooshort'], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'error', reason: 'Local digest is malformed or truncated' }); expect(calls).toHaveLength(0); }); + it('rejects an empty candidate list as an error without contacting the registry', async () => { + route = () => ({ statusCode: 500, headers: {} }); + const result = await compareLocalToRemoteTag([], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'error', reason: 'Local digest is malformed or truncated' }); + expect(calls).toHaveLength(0); + }); + + it('rejects an all-malformed candidate list as an error without contacting the registry', async () => { + route = () => ({ statusCode: 500, headers: {} }); + const result = await compareLocalToRemoteTag(['sha256:short', 'not-a-digest'], REGISTRY, REPO, TAG, AMD64); + expect(result).toEqual({ kind: 'error', reason: 'Local digest is malformed or truncated' }); + expect(calls).toHaveLength(0); + }); + + it('errors on unknown platform for a multi-candidate index mismatch (fail closed)', async () => { + route = (url, method) => { + const token = tokenOk(url); + if (token) return token; + if (url === MANIFEST_URL_TAG && method === 'HEAD') { + return { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST, 'content-type': INDEX_CONTENT_TYPE } }; + } + if (url === manifestDigestUrl(INDEX_DIGEST) && method === 'GET') { + return { statusCode: 200, headers: { 'docker-content-digest': INDEX_DIGEST }, body: STANDARD_INDEX_BODY }; + } + return { statusCode: 500, headers: {} }; + }; + const result = await compareLocalToRemoteTag( + [STALE_INDEX, `sha256:${'e'.repeat(64)}`], + REGISTRY, + REPO, + TAG, + { os: '', architecture: '' }, + ); + expect(result.kind).toBe('error'); + }); + it('degrades to an uncached comparison (not an error) when the classification cache is at capacity', async () => { const cache = CacheService.getInstance(); for (let i = 0; i < 1000; i++) cache.set(`filler:${i}`, { kind: 'single' as const }, 3_600_000); @@ -925,8 +1289,8 @@ describe('compareLocalToRemoteTag', () => { return { statusCode: 500, headers: {} }; }; - const first = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); - const second = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const first = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); + const second = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(first).toEqual({ kind: 'match' }); expect(second).toEqual({ kind: 'match' }); @@ -964,7 +1328,7 @@ describe('compareLocalToRemoteTag', () => { }, }); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'match' }); }); @@ -992,7 +1356,7 @@ describe('compareLocalToRemoteTag', () => { [nestedDigest]: nestedBody, }); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'match' }); expect(calls.filter((c) => c.method === 'GET' && c.url.includes('/manifests/'))).toHaveLength(2); }); @@ -1018,7 +1382,7 @@ describe('compareLocalToRemoteTag', () => { [nestedDigest]: { statusCode: 404, headers: {} }, }); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result.kind).toBe('error'); }); @@ -1038,7 +1402,7 @@ describe('compareLocalToRemoteTag', () => { const primary = contentDigest(body); route = routePrimaryDigest(primary, { [primary]: body }); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'match' }); }); @@ -1065,7 +1429,7 @@ describe('compareLocalToRemoteTag', () => { route = routePrimaryDigest(outerDigest, digests); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result.kind).toBe('error'); if (result.kind === 'error') { expect(result.reason).toMatch(/depth/i); @@ -1087,7 +1451,7 @@ describe('compareLocalToRemoteTag', () => { const primary = contentDigest(body); route = routePrimaryDigest(primary, { [primary]: body }); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result.kind).toBe('error'); }); @@ -1111,7 +1475,7 @@ describe('compareLocalToRemoteTag', () => { [nestedDigest]: nestedBody, }); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result).toEqual({ kind: 'match' }); }); @@ -1131,7 +1495,7 @@ describe('compareLocalToRemoteTag', () => { const outerDigest = contentDigest(outerBody); route = routePrimaryDigest(outerDigest, { [outerDigest]: outerBody }); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result.kind).toBe('error'); expect(calls.some((c) => c.url.includes('../') || c.url.includes('/evil'))).toBe(false); }); @@ -1147,7 +1511,7 @@ describe('compareLocalToRemoteTag', () => { const primary = contentDigest(body); route = routePrimaryDigest(primary, { [primary]: body }); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result.kind).toBe('error'); }); @@ -1161,7 +1525,7 @@ describe('compareLocalToRemoteTag', () => { const primary = contentDigest(body); route = routePrimaryDigest(primary, { [primary]: body }); - const result = await compareLocalToRemoteTag(CHILD_AMD64, REGISTRY, REPO, TAG, AMD64); + const result = await compareLocalToRemoteTag([CHILD_AMD64], REGISTRY, REPO, TAG, AMD64); expect(result.kind).toBe('error'); }); }); diff --git a/backend/src/__tests__/scheduled-tasks-routes.test.ts b/backend/src/__tests__/scheduled-tasks-routes.test.ts index 39d0271b..24e92161 100644 --- a/backend/src/__tests__/scheduled-tasks-routes.test.ts +++ b/backend/src/__tests__/scheduled-tasks-routes.test.ts @@ -78,7 +78,7 @@ describe('GET /api/scheduled-tasks', () => { last_error: null, prune_targets: null, target_services: null, - prune_label_filter: null, + prune_label_filter: null, selector_type: null, selector_value: null, }); const res = await request(app).get('/api/scheduled-tasks?window_hours=48').set('Cookie', adminCookie); @@ -108,7 +108,7 @@ describe('GET /api/scheduled-tasks', () => { last_error: null, prune_targets: null, target_services: null, - prune_label_filter: null, + prune_label_filter: null, selector_type: null, selector_value: null, }); db.createScheduledTask({ name: 'daily-snapshot', @@ -127,7 +127,7 @@ describe('GET /api/scheduled-tasks', () => { last_error: null, prune_targets: null, target_services: null, - prune_label_filter: null, + prune_label_filter: null, selector_type: null, selector_value: null, }); db.createScheduledTask({ name: 'system-prune', @@ -146,7 +146,7 @@ describe('GET /api/scheduled-tasks', () => { last_error: null, prune_targets: JSON.stringify(['images']), target_services: null, - prune_label_filter: null, + prune_label_filter: null, selector_type: null, selector_value: null, }); const res = await request(app).get('/api/scheduled-tasks').set('Cookie', adminCookie); @@ -440,7 +440,7 @@ describe('GET /api/scheduled-tasks/:id', () => { name: 't', target_type: 'stack', target_id: 's', node_id: 1, action: 'update', cron_expression: '0 3 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: null, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, }); }); @@ -468,7 +468,7 @@ describe('PATCH /api/scheduled-tasks/:id/toggle', () => { name: 't', target_type: 'stack', target_id: 's', node_id: 1, action: 'update', cron_expression: '0 3 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: now + 1000, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, }); const off = await request(app).patch(`/api/scheduled-tasks/${id}/toggle`).set('Cookie', adminCookie); @@ -490,7 +490,7 @@ describe('PATCH /api/scheduled-tasks/:id/toggle', () => { name: 'once', target_type: 'stack', target_id: 's', node_id: 1, action: 'auto_backup', cron_expression: '0 23 1 7 *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: pinned, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 1, run_at: pinned, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, delete_after_run: 1, run_at: pinned, }); const off = await request(app).patch(`/api/scheduled-tasks/${id}/toggle`).set('Cookie', adminCookie); @@ -515,7 +515,7 @@ describe('DELETE /api/scheduled-tasks/:id', () => { name: 't', target_type: 'stack', target_id: 's', node_id: 1, action: 'update', cron_expression: '0 3 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: null, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, }); const del = await request(app).delete(`/api/scheduled-tasks/${id}`).set('Cookie', adminCookie); @@ -534,7 +534,7 @@ describe('GET /api/scheduled-tasks/:id/runs', () => { name: 't', target_type: 'stack', target_id: 's', node_id: 1, action: 'update', cron_expression: '0 3 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: null, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, }); const res = await request(app).get(`/api/scheduled-tasks/${id}/runs`).set('Cookie', adminCookie); @@ -712,7 +712,7 @@ describe('PUT /api/scheduled-tasks/:id - delete_after_run', () => { name: 't', target_type: 'stack', target_id: 's', node_id: 1, action: 'auto_backup', cron_expression: '0 3 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: null, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 0, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, delete_after_run: 0, }); const res = await request(app) @@ -736,7 +736,7 @@ describe('PUT /api/scheduled-tasks/:id - delete_after_run', () => { name: 'once', target_type: 'stack', target_id: 's', node_id: 1, action: 'auto_backup', cron_expression: '0 23 1 7 *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: now + 1000, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 1, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, delete_after_run: 1, }); const runAt = new Date(new Date().getFullYear() + 1, 6, 1, 23, 0, 0, 0).getTime(); @@ -757,7 +757,7 @@ describe('PUT /api/scheduled-tasks/:id - delete_after_run', () => { name: 'once', target_type: 'stack', target_id: 's', node_id: 1, action: 'auto_backup', cron_expression: '0 23 1 7 *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: pinned, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 1, run_at: pinned, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, delete_after_run: 1, run_at: pinned, }); // Editing to a recurring daily schedule sends run_at: null (the frontend @@ -780,7 +780,7 @@ describe('PUT /api/scheduled-tasks/:id - delete_after_run', () => { name: 'once', target_type: 'stack', target_id: 's', node_id: 1, action: 'auto_backup', cron_expression: '0 23 1 7 *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: pinned, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 1, run_at: pinned, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, delete_after_run: 1, run_at: pinned, }); // Disabling via Save: next_run_at clears, but the pinned instant is retained @@ -800,7 +800,7 @@ describe('PUT /api/scheduled-tasks/:id - delete_after_run', () => { name: 'once', target_type: 'stack', target_id: 's', node_id: 1, action: 'auto_backup', cron_expression: '0 23 1 7 *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: now + 1000, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 1, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, delete_after_run: 1, }); const res = await request(app) @@ -821,7 +821,7 @@ describe('PUT /api/scheduled-tasks/:id - stack target validation', () => { name: 't', target_type: 'stack', target_id: 's', node_id: 1, action: 'update', cron_expression: '0 3 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: null, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 0, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, delete_after_run: 0, }); }); @@ -862,7 +862,7 @@ describe('PUT /api/scheduled-tasks/:id - stack target validation', () => { name: 'prune', target_type: 'system', target_id: null, node_id: 1, action: 'prune', cron_expression: '0 4 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: null, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, }); const res = await request(app) @@ -881,7 +881,7 @@ describe('PUT /api/scheduled-tasks/:id - stack target validation', () => { name: 'prune', target_type: 'system', target_id: null, node_id: 1, action: 'prune', cron_expression: '0 4 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: null, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, }); const remoteNodeId = db.addNode({ name: 'remote-prune-update-node', type: 'remote', api_url: 'http://remote.local:1852', @@ -937,7 +937,7 @@ describe('PUT /api/scheduled-tasks/:id - stack target validation', () => { last_error: null, prune_targets: null, target_services: JSON.stringify(['api']), - prune_label_filter: null, + prune_label_filter: null, selector_type: null, selector_value: null, delete_after_run: 0, }); @@ -966,7 +966,7 @@ describe('PUT /api/scheduled-tasks/:id - cron validation', () => { name: 't', target_type: 'stack', target_id: 's', node_id: 1, action: 'update', cron_expression: '0 3 * * *', enabled: 1, created_by: 'admin', created_at: now, updated_at: now, last_run_at: null, next_run_at: null, last_status: null, last_error: null, - prune_targets: null, target_services: null, prune_label_filter: null, delete_after_run: 0, + prune_targets: null, target_services: null, prune_label_filter: null, selector_type: null, selector_value: null, delete_after_run: 0, }); }); @@ -1087,3 +1087,83 @@ describe('scheduled-tasks state-invalidate broadcast', () => { } }); }); + +describe('POST/PUT /api/scheduled-tasks - stack-label selector', () => { + const labelPayload = { + name: 'label-update', + target_type: 'fleet', + target_id: null, + node_id: null, + action: 'update', + cron_expression: '0 3 * * *', + enabled: true, + selector_type: 'stack-label', + selector_value: 'production', + }; + + it('creates a fleet-wide label update with node_id null', async () => { + const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send(labelPayload); + expect(res.status).toBe(201); + expect(res.body.selector_type).toBe('stack-label'); + expect(res.body.selector_value).toBe('production'); + expect(res.body.node_id).toBeNull(); + expect(res.body.action).toBe('update'); + expect(res.body.target_type).toBe('fleet'); + }); + + it('persists selector_value changes through PUT (column map)', async () => { + const create = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send(labelPayload); + expect(create.status).toBe(201); + const id = create.body.id as number; + + const put = await request(app) + .put(`/api/scheduled-tasks/${id}`) + .set('Cookie', adminCookie) + .send({ selector_type: 'stack-label', selector_value: 'staging' }); + expect(put.status).toBe(200); + expect(put.body.selector_value).toBe('staging'); + + const get = await request(app).get(`/api/scheduled-tasks/${id}`).set('Cookie', adminCookie); + expect(get.status).toBe(200); + expect(get.body.selector_value).toBe('staging'); + }); + + it('still requires node_id for non-selector fleet updates', async () => { + const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({ + name: 'fleet-no-node', + target_type: 'fleet', + action: 'update', + cron_expression: '0 3 * * *', + node_id: null, + }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('Fleet update requires node_id.'); + }); + + it('rejects selector fields on unsupported actions', async () => { + const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({ + name: 'restart-with-selector', + target_type: 'stack', + target_id: 'my-stack', + node_id: 1, + action: 'restart', + cron_expression: '0 3 * * *', + selector_type: 'stack-label', + selector_value: 'prod', + }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/selector fields can only be used with update action on fleet target/); + }); + + it('creates a node-scoped label update when node_id is set', async () => { + const res = await request(app).post('/api/scheduled-tasks').set('Cookie', adminCookie).send({ + ...labelPayload, + name: 'label-update-node', + node_id: 1, + selector_value: 'Databases', + }); + expect(res.status).toBe(201); + expect(res.body.node_id).toBe(1); + expect(res.body.selector_value).toBe('Databases'); + }); +}); diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index f629f717..17541bbf 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -18,8 +18,8 @@ const { mockStartContainer, mockStopContainer, mockPruneSystem, mockUpdateStack, mockGetStacks, mockGetStackContent, mockGetEnvContent, - mockCheckImage, - mockDispatchAlert, + mockCheckImage, mockRecheckStack, + mockDispatchAlert, mockBroadcastEvent, mockGetProxyTarget, mockIsTrivyAvailable, mockScanAllNodeImages, @@ -59,7 +59,9 @@ const { mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockResolvedValue(''), mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }), + mockRecheckStack: vi.fn().mockResolvedValue({ outcome: 'cleared', warning: null }), mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), + mockBroadcastEvent: vi.fn(), mockGetProxyTarget: vi.fn().mockReturnValue(null), mockIsTrivyAvailable: vi.fn().mockReturnValue(true), mockScanAllNodeImages: vi.fn().mockResolvedValue({ @@ -165,16 +167,22 @@ vi.mock('../services/FileSystemService', () => ({ vi.mock('../services/ImageUpdateService', () => ({ ImageUpdateService: { + // Default on so existing executeUpdate tests keep prior behavior. + isChecksEnabled: () => true, getInstance: () => ({ checkImage: mockCheckImage, + recheckStack: mockRecheckStack, }), }, + UPDATE_VERIFICATION_INCOMPLETE_WARNING: + 'The update command completed, but Sencho could not fully verify whether an image update remains.', })); vi.mock('../services/NotificationService', () => ({ NotificationService: { getInstance: () => ({ dispatchAlert: mockDispatchAlert, + broadcastEvent: mockBroadcastEvent, }), }, })); @@ -789,13 +797,53 @@ describe('SchedulerService - executeUpdate', () => { mockGetContainersByStack.mockResolvedValue([ { Id: 'c1', Image: 'nginx:latest' }, ]); - mockCheckImage.mockResolvedValue({ hasUpdate: true }); // Update available + mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: true }); // Update available const svc = SchedulerService.getInstance(); await svc.triggerTask(80); expect(mockUpdateStack).toHaveBeenCalledWith('web-app', undefined, true); - expect(mockClearStackUpdateStatus).toHaveBeenCalledWith(1, 'web-app'); + expect(mockRecheckStack).toHaveBeenCalledWith(1, 'web-app'); + expect(mockClearStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockBroadcastEvent).toHaveBeenCalledWith(expect.objectContaining({ + type: 'state-invalidate', + scope: 'image-updates', + nodeId: 1, + stackName: 'web-app', + })); + }); + + it('blocks scheduled Compose apply when a sibling image check failed', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 186, + name: 'update-check-errors', + action: 'update', + cron_expression: '0 4 * * *', + enabled: true, + target_id: 'web-app', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([ + { Id: 'c1', Image: 'nginx:latest' }, + { Id: 'c2', Image: 'redis:latest' }, + ]); + mockCheckImage + .mockResolvedValueOnce({ hasUpdate: true, digestUpdate: true, tagUpdate: false }) + .mockResolvedValueOnce({ hasUpdate: false, error: 'registry timeout', checkStatus: 'failed' }); + + await SchedulerService.getInstance().triggerTask(186); + + expect(mockUpdateStack).not.toHaveBeenCalled(); + expect(mockRecheckStack).not.toHaveBeenCalled(); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ + status: 'success', + output: expect.stringContaining('image check(s) failed'), + }), + ); }); it('runs a scheduled update on the community tier (no paid gate)', async () => { @@ -814,7 +862,7 @@ describe('SchedulerService - executeUpdate', () => { last_status: null, }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]); - mockCheckImage.mockResolvedValue({ hasUpdate: true }); + mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: true }); const svc = SchedulerService.getInstance(); await svc.triggerTask(82); @@ -828,7 +876,15 @@ describe('SchedulerService - executeUpdate', () => { it('begins a health gate after a scheduled update succeeds', async () => { const { HealthGateService } = await import('../services/HealthGateService'); - const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-1'); + const callOrder: string[] = []; + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockImplementation(() => { + callOrder.push('beginStack'); + return 'gate-1'; + }); + mockRecheckStack.mockImplementation(async () => { + callOrder.push('recheckStack'); + return { outcome: 'cleared', warning: null }; + }); try { mockGetScheduledTask.mockReturnValue({ id: 83, @@ -842,11 +898,13 @@ describe('SchedulerService - executeUpdate', () => { last_status: null, }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]); - mockCheckImage.mockResolvedValue({ hasUpdate: true }); + mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: true }); await SchedulerService.getInstance().triggerTask(83); expect(beginSpy).toHaveBeenCalledWith(1, 'web-app', 'update', 'system:scheduler'); + expect(mockRecheckStack).toHaveBeenCalledWith(1, 'web-app'); + expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); } finally { beginSpy.mockRestore(); } @@ -875,6 +933,45 @@ describe('SchedulerService - executeUpdate', () => { expect(mockUpdateStack).not.toHaveBeenCalled(); }); + + it('skips Compose apply for tag-only availability (pinned semver cannot be rewritten)', async () => { + mockGetScheduledTask.mockReturnValue({ + id: 185, + name: 'update-tag-only', + action: 'update', + cron_expression: '0 4 * * *', + enabled: true, + target_id: 'web-app', + node_id: 1, + created_by: 'admin', + last_status: null, + }); + mockGetContainersByStack.mockResolvedValue([ + { Id: 'c1', Image: 'nginx:1.2.3' }, + ]); + // Higher tag is visible (hasUpdate) but not actionable via Compose pull. + mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: false, tagUpdate: true }); + + const svc = SchedulerService.getInstance(); + await svc.triggerTask(84); + + expect(mockUpdateStack).not.toHaveBeenCalled(); + expect(mockClearStackUpdateStatus).not.toHaveBeenCalled(); + expect(mockDispatchAlert).not.toHaveBeenCalledWith( + 'info', + 'image_update_applied', + expect.any(String), + expect.anything(), + ); + expect(mockUpdateScheduledTaskRun).toHaveBeenCalledWith( + 1, + expect.objectContaining({ + status: 'success', + output: expect.stringContaining('Compose pin unchanged'), + }), + ); + }); + it('handles wildcard target (*) by updating all stacks', async () => { mockGetScheduledTask.mockReturnValue({ id: 82, @@ -891,7 +988,7 @@ describe('SchedulerService - executeUpdate', () => { mockGetContainersByStack.mockResolvedValue([ { Id: 'c1', Image: 'nginx:latest' }, ]); - mockCheckImage.mockResolvedValue({ hasUpdate: true }); + mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: true }); const svc = SchedulerService.getInstance(); await svc.triggerTask(82); @@ -1035,7 +1132,7 @@ describe('SchedulerService - executeUpdate', () => { last_status: null, }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:1.14' }]); - mockCheckImage.mockResolvedValue({ hasUpdate: true }); + mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: true }); mockEnforcePolicyPreDeploy.mockResolvedValue({ ok: false, bypassed: false, @@ -1114,7 +1211,7 @@ describe('SchedulerService - executeUpdate', () => { }); mockGetStacks.mockResolvedValue(['app1', 'app2', 'app3']); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }]); - mockCheckImage.mockResolvedValue({ hasUpdate: true }); + mockCheckImage.mockResolvedValue({ hasUpdate: true, digestUpdate: true }); const svc = SchedulerService.getInstance(); await svc.triggerTask(87); @@ -1778,7 +1875,7 @@ function makeLifecycleTask(action: ScheduledTask['action'], overrides: Partial { expect(args).not.toContain('/opt/sencho:/opt/sencho:rw'); }); }); + +describe('buildComposeConfigValidateArgs', () => { + it('runs compose config in a throwaway helper with the working dir mounted read-only', () => { + const args = buildComposeConfigValidateArgs({ + workingDir: '/opt/sencho', + imageName: 'saelix/sencho:1.0.0', + configFiles: 'docker-compose.yml,/opt/sencho/override.yml', + hostBindMounts: [{ source: '/etc/sencho', destination: '/etc/sencho' }], + }); + expect(args).toContain('/opt/sencho:/opt/sencho:ro'); + expect(args).toContain('/var/run/docker.sock:/var/run/docker.sock'); + expect(args).toContain('/etc/sencho:/etc/sencho:ro'); + const cmd = args[args.length - 1]; + expect(cmd).toContain('docker compose'); + expect(cmd).toContain('config'); + expect(cmd).toContain(shQuote('docker-compose.yml')); + expect(cmd).toContain(shQuote('/opt/sencho/override.yml')); + }); +}); diff --git a/backend/src/__tests__/session-remember-refresh.test.ts b/backend/src/__tests__/session-remember-refresh.test.ts new file mode 100644 index 00000000..991c7f43 --- /dev/null +++ b/backend/src/__tests__/session-remember-refresh.test.ts @@ -0,0 +1,260 @@ +/** + * Tests for the two pieces added to fix sessions expiring out from under + * active users: sliding-refresh (authMiddleware silently reissues a + * near-expiry session cookie) and "stay signed in" (a longer-lived session + * chosen at login, carried through MFA and password-change reissues). + */ +import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { OTP } from 'otplib'; +import { + setupTestDb, + cleanupTestDb, + loginAsTestAdmin, + seedMfaUser, + TEST_USERNAME, + TEST_PASSWORD, + TEST_JWT_SECRET, +} from './helpers/setupTestDb'; + +// Match the server-side otplib configuration so test-generated OTPs are +// accepted by the verify path (see MfaService and __tests__/mfa.test.ts). +const authenticator = new OTP({ strategy: 'totp' }); +const TOTP_PARAMS = { algorithm: 'sha1' as const, digits: 6, period: 30 }; + +let tmpDir: string; +let app: import('express').Express; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ app } = await import('../index')); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +/** All Set-Cookie entries in a response for one cookie name. */ +function cookieEntries(setCookieHeader: string | string[] | undefined, name: string): string[] { + const cookies = Array.isArray(setCookieHeader) ? setCookieHeader : setCookieHeader ? [setCookieHeader] : []; + return cookies.filter((c) => c.startsWith(`${name}=`)); +} + +/** All sencho_token Set-Cookie entries in a response (should never be more than one). */ +function sessionCookieEntries(setCookieHeader: string | string[] | undefined): string[] { + return cookieEntries(setCookieHeader, 'sencho_token'); +} + +/** Extract the sencho_token cookie's raw JWT value from a Set-Cookie header. */ +function extractSessionToken(setCookieHeader: string | string[] | undefined): string | undefined { + const match = sessionCookieEntries(setCookieHeader)[0]; + return match?.split(';')[0].split('=')[1]; +} + +describe('sliding session refresh', () => { + afterEach(() => { + DatabaseService.getInstance().updateGlobalSetting('session_sliding_refresh', '1'); + }); + + it('refreshes a session nearing expiry', async () => { + const nearExpiryToken = jwt.sign({ username: TEST_USERNAME, remember: false }, TEST_JWT_SECRET, { expiresIn: '30s' }); + const res = await request(app) + .get('/api/auth/check') + .set('Authorization', `Bearer ${nearExpiryToken}`); + + expect(res.status).toBe(200); + const refreshed = extractSessionToken(res.headers['set-cookie']); + expect(refreshed).toBeDefined(); + const decoded = jwt.verify(refreshed!, TEST_JWT_SECRET) as { exp: number; remember?: boolean }; + // Refreshed back to a full 24h session, not just extended by seconds. + expect(decoded.exp * 1000 - Date.now()).toBeGreaterThan(23 * 60 * 60 * 1000); + expect(decoded.remember).toBe(false); + }); + + it('does not refresh a session with plenty of life left', async () => { + const freshCookie = await loginAsTestAdmin(app); + const res = await request(app) + .get('/api/auth/check') + .set('Cookie', freshCookie); + + expect(res.status).toBe(200); + expect(res.headers['set-cookie']).toBeUndefined(); + }); + + it('reissues a "stay signed in" session back to a 30-day TTL, not 24h', async () => { + const nearExpiryRememberToken = jwt.sign({ username: TEST_USERNAME, remember: true }, TEST_JWT_SECRET, { expiresIn: '30s' }); + const res = await request(app) + .get('/api/auth/check') + .set('Authorization', `Bearer ${nearExpiryRememberToken}`); + + const refreshed = extractSessionToken(res.headers['set-cookie']); + expect(refreshed).toBeDefined(); + const decoded = jwt.verify(refreshed!, TEST_JWT_SECRET) as { exp: number; remember?: boolean }; + expect(decoded.remember).toBe(true); + expect(decoded.exp * 1000 - Date.now()).toBeGreaterThan(29 * 24 * 60 * 60 * 1000); + }); + + it('does not refresh when session_sliding_refresh is disabled', async () => { + DatabaseService.getInstance().updateGlobalSetting('session_sliding_refresh', '0'); + const nearExpiryToken = jwt.sign({ username: TEST_USERNAME, remember: false }, TEST_JWT_SECRET, { expiresIn: '30s' }); + const res = await request(app) + .get('/api/auth/check') + .set('Authorization', `Bearer ${nearExpiryToken}`); + + expect(res.status).toBe(200); + expect(res.headers['set-cookie']).toBeUndefined(); + }); + + // The refresh sits after the token-version check in authMiddleware, which is + // what makes it safe: a stale token cannot ride the refresh back to life. If + // that ordering ever moved, these would be the tests to catch it. + it('rejects (does not refresh) a near-expiry token with a stale token_version', async () => { + const user = DatabaseService.getInstance().getUserByUsername(TEST_USERNAME)!; + const staleToken = jwt.sign( + { username: TEST_USERNAME, remember: false, tv: user.token_version - 1 }, + TEST_JWT_SECRET, + { expiresIn: '30s' }, + ); + const res = await request(app) + .get('/api/auth/check') + .set('Authorization', `Bearer ${staleToken}`); + + expect(res.status).toBe(401); + expect(res.headers['set-cookie']).toBeUndefined(); + }); + + it('refreshes a near-expiry token with a current token_version and carries it forward', async () => { + const user = DatabaseService.getInstance().getUserByUsername(TEST_USERNAME)!; + const currentToken = jwt.sign( + { username: TEST_USERNAME, remember: false, tv: user.token_version }, + TEST_JWT_SECRET, + { expiresIn: '30s' }, + ); + const res = await request(app) + .get('/api/auth/check') + .set('Authorization', `Bearer ${currentToken}`); + + expect(res.status).toBe(200); + const refreshed = extractSessionToken(res.headers['set-cookie']); + expect(refreshed).toBeDefined(); + const decoded = jwt.verify(refreshed!, TEST_JWT_SECRET) as { exp: number; tv?: number }; + expect(decoded.tv).toBe(user.token_version); + expect(decoded.exp * 1000 - Date.now()).toBeGreaterThan(23 * 60 * 60 * 1000); + }); +}); + +describe('"stay signed in" at login', () => { + it('issues a 30-day session when remember is true', async () => { + const res = await request(app) + .post('/api/auth/login') + .send({ username: TEST_USERNAME, password: TEST_PASSWORD, remember: true }); + + expect(res.status).toBe(200); + const cookieEntry = sessionCookieEntries(res.headers['set-cookie'])[0]; + expect(cookieEntry).toBeDefined(); + // The cookie's own Max-Age must match the JWT's exp, or a browser would + // drop the cookie before the token expires, defeating "stay signed in" + // even though the token itself looks correct. + expect(cookieEntry).toMatch(/Max-Age=2592000/); + const token = extractSessionToken(res.headers['set-cookie']); + expect(token).toBeDefined(); + const decoded = jwt.verify(token!, TEST_JWT_SECRET) as { exp: number; iat: number; remember?: boolean }; + expect(decoded.remember).toBe(true); + expect(decoded.exp - decoded.iat).toBeCloseTo(30 * 24 * 60 * 60, -2); + }); + + it('issues the standard 24h session when remember is omitted', async () => { + const res = await request(app) + .post('/api/auth/login') + .send({ username: TEST_USERNAME, password: TEST_PASSWORD }); + + const token = extractSessionToken(res.headers['set-cookie']); + const decoded = jwt.verify(token!, TEST_JWT_SECRET) as { exp: number; iat: number; remember?: boolean }; + expect(decoded.remember).toBe(false); + expect(decoded.exp - decoded.iat).toBeCloseTo(24 * 60 * 60, -2); + }); + + it('carries remember through an MFA challenge to the final session', async () => { + const { secret } = await seedMfaUser('mfa-remember-user', 'mfa-remember-pass'); + + const loginRes = await request(app) + .post('/api/auth/login') + .send({ username: 'mfa-remember-user', password: 'mfa-remember-pass', remember: true }); + expect(loginRes.body.mfaRequired).toBe(true); + const pendingCookieHeader = cookieEntries(loginRes.headers['set-cookie'], 'sencho_mfa_pending')[0]; + expect(pendingCookieHeader).toBeDefined(); + + const code = authenticator.generateSync({ secret, ...TOTP_PARAMS }); + const mfaRes = await request(app) + .post('/api/auth/login/mfa') + .set('Cookie', pendingCookieHeader!) + .send({ code }); + + expect(mfaRes.status).toBe(200); + const finalToken = extractSessionToken(mfaRes.headers['set-cookie']); + expect(finalToken).toBeDefined(); + const decoded = jwt.verify(finalToken!, TEST_JWT_SECRET) as { exp: number; iat: number; remember?: boolean }; + expect(decoded.remember).toBe(true); + expect(decoded.exp - decoded.iat).toBeCloseTo(30 * 24 * 60 * 60, -2); + }); +}); + +describe('reissueSessionAfterTokenBump preserves "stay signed in"', () => { + it('keeps a 30-day session after a password change', async () => { + const bcrypt = (await import('bcrypt')).default; + const passwordHash = await bcrypt.hash('bump-test-pass', 1); + DatabaseService.getInstance().addUser({ username: 'bump-test-user', password_hash: passwordHash, role: 'admin' }); + + const loginRes = await request(app) + .post('/api/auth/login') + .send({ username: 'bump-test-user', password: 'bump-test-pass', remember: true }); + const rememberedCookie = extractSessionToken(loginRes.headers['set-cookie']); + const cookieHeader = sessionCookieEntries(loginRes.headers['set-cookie'])[0]; + + const changeRes = await request(app) + .put('/api/auth/password') + .set('Cookie', cookieHeader!) + .send({ oldPassword: 'bump-test-pass', newPassword: 'bump-test-pass-2' }); + + expect(changeRes.status).toBe(200); + // Exactly one sencho_token Set-Cookie, not one from the sliding refresh + // (pre-bump token_version) followed by a second from the post-bump + // reissue: a second, stale entry would leave any client that reads the + // first Set-Cookie signed out on its very next request. + expect(sessionCookieEntries(changeRes.headers['set-cookie'])).toHaveLength(1); + const reissued = extractSessionToken(changeRes.headers['set-cookie']); + expect(reissued).toBeDefined(); + expect(reissued).not.toBe(rememberedCookie); + const decoded = jwt.verify(reissued!, TEST_JWT_SECRET) as { exp: number; iat: number; remember?: boolean }; + expect(decoded.remember).toBe(true); + expect(decoded.exp - decoded.iat).toBeCloseTo(30 * 24 * 60 * 60, -2); + }); + + it('does not duplicate the session cookie when the password change lands inside the sliding-refresh window', async () => { + const bcrypt = (await import('bcrypt')).default; + const passwordHash = await bcrypt.hash('bump-window-pass', 1); + const db = DatabaseService.getInstance(); + db.addUser({ username: 'bump-window-user', password_hash: passwordHash, role: 'admin' }); + const user = db.getUserByUsername('bump-window-user')!; + + // Hand-sign a near-expiry token so authMiddleware's sliding refresh fires + // on this very request, immediately before the route bumps token_version. + const nearExpiryToken = jwt.sign( + { username: 'bump-window-user', remember: true, tv: user.token_version }, + TEST_JWT_SECRET, + { expiresIn: '30s' }, + ); + + const changeRes = await request(app) + .put('/api/auth/password') + .set('Authorization', `Bearer ${nearExpiryToken}`) + .send({ oldPassword: 'bump-window-pass', newPassword: 'bump-window-pass-2' }); + + expect(changeRes.status).toBe(200); + const entries = sessionCookieEntries(changeRes.headers['set-cookie']); + expect(entries).toHaveLength(1); + const decoded = jwt.verify(extractSessionToken(changeRes.headers['set-cookie'])!, TEST_JWT_SECRET) as { tv?: number }; + expect(decoded.tv).toBe(db.getUserByUsername('bump-window-user')!.token_version); + }); +}); diff --git a/backend/src/__tests__/settings-routes.test.ts b/backend/src/__tests__/settings-routes.test.ts index 83c11dc6..cc3fb9bb 100644 --- a/backend/src/__tests__/settings-routes.test.ts +++ b/backend/src/__tests__/settings-routes.test.ts @@ -241,6 +241,47 @@ describe('prune_on_update (auto-prune after updates)', () => { }); }); +describe('session_sliding_refresh (keep active sessions alive)', () => { + it('defaults to ON in a freshly seeded database', () => { + expect(DatabaseService.getInstance().getGlobalSettings().session_sliding_refresh).toBe('1'); + }); + + it('is exposed through the settings GET projection', async () => { + const res = await request(app).get('/api/settings').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.session_sliding_refresh).toBeDefined(); + }); + + it('accepts a well-formed write and persists it', async () => { + const res = await request(app) + .post('/api/settings') + .set('Cookie', adminCookie) + .send({ key: 'session_sliding_refresh', value: '0' }); + expect(res.status).toBe(200); + expect(DatabaseService.getInstance().getGlobalSettings().session_sliding_refresh).toBe('0'); + // Restore the seeded default so later suites observe the shipped behavior. + DatabaseService.getInstance().updateGlobalSetting('session_sliding_refresh', '1'); + }); + + it('rejects a non-enum value (400) and does not write it', async () => { + const res = await request(app) + .post('/api/settings') + .set('Cookie', adminCookie) + .send({ key: 'session_sliding_refresh', value: 'banana' }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('Validation failed'); + expect(DatabaseService.getInstance().getGlobalSettings().session_sliding_refresh).not.toBe('banana'); + }); + + it('rejects a non-admin write with 403', async () => { + const res = await request(app) + .post('/api/settings') + .set('Cookie', viewerCookie) + .send({ key: 'session_sliding_refresh', value: '0' }); + expect(res.status).toBe(403); + }); +}); + describe('prune_orphaned_scans (purge scans for deleted images/stacks)', () => { it('defaults to ON in a freshly seeded database', () => { expect(DatabaseService.getInstance().getGlobalSettings().prune_orphaned_scans).toBe('1'); @@ -663,19 +704,19 @@ describe('Paid-only setting keys (audit_retention_days)', () => { }); describe('reclaim_hero setting', () => { - it('is allowlisted and seeds to "1" (banner on by default)', async () => { + it('is allowlisted and seeds to "0" (banner off by default)', async () => { const res = await request(app).get('/api/settings').set('Cookie', adminCookie); expect(res.status).toBe(200); - expect(res.body.reclaim_hero).toBe('1'); + expect(res.body.reclaim_hero).toBe('0'); }); it('accepts a well-formed write and rejects a non-enum value', async () => { const ok = await request(app) .patch('/api/settings') .set('Cookie', adminCookie) - .send({ reclaim_hero: '0' }); + .send({ reclaim_hero: '1' }); expect(ok.status).toBe(200); - expect(DatabaseService.getInstance().getGlobalSettings().reclaim_hero).toBe('0'); + expect(DatabaseService.getInstance().getGlobalSettings().reclaim_hero).toBe('1'); const bad = await request(app) .patch('/api/settings') @@ -683,9 +724,9 @@ describe('reclaim_hero setting', () => { .send({ reclaim_hero: 'banana' }); expect(bad.status).toBe(400); expect(bad.body.error).toBe('Validation failed'); - expect(DatabaseService.getInstance().getGlobalSettings().reclaim_hero).toBe('0'); + expect(DatabaseService.getInstance().getGlobalSettings().reclaim_hero).toBe('1'); // Reset for any later reads of the shared test DB. - DatabaseService.getInstance().updateGlobalSetting('reclaim_hero', '1'); + DatabaseService.getInstance().updateGlobalSetting('reclaim_hero', '0'); }); }); diff --git a/backend/src/__tests__/stack-update-post-recheck.test.ts b/backend/src/__tests__/stack-update-post-recheck.test.ts new file mode 100644 index 00000000..81cbd9ef --- /dev/null +++ b/backend/src/__tests__/stack-update-post-recheck.test.ts @@ -0,0 +1,194 @@ +/** + * Manual POST /api/stacks/:name/update must start the health gate before + * registry recheck, never blind-clear update status, and keep Compose success + * as HTTP 200 even when recheck throws. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +const { + mockExecute, + mockRecheckStack, + mockBeginStack, + mockClearStackUpdateStatus, + mockBroadcastEvent, + mockDispatchAlert, +} = vi.hoisted(() => ({ + mockExecute: vi.fn(), + mockRecheckStack: vi.fn(), + mockBeginStack: vi.fn(), + mockClearStackUpdateStatus: vi.fn(), + mockBroadcastEvent: vi.fn(), + mockDispatchAlert: vi.fn(), +})); + +vi.mock('../services/StackUpdateOrchestrator', () => ({ + StackUpdateOrchestrator: { + getInstance: () => ({ execute: mockExecute }), + }, + shortImageId: (id: string) => id.slice(0, 12), +})); + +vi.mock('../services/ImageUpdateService', async () => { + const actual = await vi.importActual( + '../services/ImageUpdateService', + ); + return { + ...actual, + ImageUpdateService: { + isChecksEnabled: () => true, + getInstance: () => ({ recheckStack: mockRecheckStack }), + }, + }; +}); + +vi.mock('../services/HealthGateService', () => ({ + HealthGateService: { + getInstance: () => ({ + beginStack: mockBeginStack, + }), + }, +})); + +vi.mock('../services/FileSystemService', () => ({ + FileSystemService: { + getInstance: () => ({ + getBaseDir: () => '/tmp/compose', + hasComposeFile: vi.fn().mockResolvedValue(true), + }), + }, +})); + +vi.mock('../helpers/policyGate', async () => { + const actual = await vi.importActual( + '../helpers/policyGate', + ); + return { + ...actual, + runPolicyGate: vi.fn().mockResolvedValue(true), + triggerPostDeployScan: vi.fn().mockResolvedValue(undefined), + }; +}); + +let tmpDir: string; +let app: import('express').Express; +let authCookie: string; +let clearSpy: ReturnType | undefined; +let broadcastSpy: ReturnType | undefined; +const callOrder: string[] = []; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + authCookie = await loginAsTestAdmin(app); + + const { DatabaseService } = await import('../services/DatabaseService'); + const { NotificationService } = await import('../services/NotificationService'); + clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus').mockImplementation((...args) => { + callOrder.push('clearStackUpdateStatus'); + return mockClearStackUpdateStatus(...args); + }); + broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation((...args) => { + callOrder.push('broadcastEvent'); + return mockBroadcastEvent(...args); + }); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockImplementation((...args) => { + callOrder.push(`dispatchAlert:${String(args[2] ?? args[0])}`); + return mockDispatchAlert(...args) ?? Promise.resolve({ persisted: true }); + }); +}); + +afterAll(() => { + clearSpy?.mockRestore(); + broadcastSpy?.mockRestore(); + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + callOrder.length = 0; + mockExecute.mockReset(); + mockRecheckStack.mockReset(); + mockBeginStack.mockReset(); + mockClearStackUpdateStatus.mockReset(); + mockBroadcastEvent.mockReset(); + mockDispatchAlert.mockReset().mockResolvedValue({ persisted: true }); + + mockExecute.mockImplementation(async () => { + callOrder.push('execute'); + return { kind: 'stack_compose_done', recoveryId: null }; + }); + mockBeginStack.mockImplementation(() => { + callOrder.push('beginStack'); + return 'gate-1'; + }); + mockRecheckStack.mockImplementation(async () => { + callOrder.push('recheckStack'); + return { outcome: 'cleared', warning: null }; + }); +}); + +describe('POST /api/stacks/:name/update post-compose verification', () => { + it('starts the health gate before recheck, skips clear, and broadcasts after recheck', async () => { + const res = await request(app) + .post('/api/stacks/web/update') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ status: 'Update completed', healthGateId: 'gate-1' }); + expect(res.body.recheckWarning).toBeUndefined(); + expect(callOrder.indexOf('execute')).toBeLessThan(callOrder.indexOf('beginStack')); + expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); + expect(callOrder.indexOf('recheckStack')).toBeLessThan(callOrder.indexOf('broadcastEvent')); + expect(callOrder).not.toContain('clearStackUpdateStatus'); + expect(mockClearStackUpdateStatus).not.toHaveBeenCalled(); + }); + + it('returns recheckWarning when the update condition remains', async () => { + mockRecheckStack.mockImplementation(async () => { + callOrder.push('recheckStack'); + return { + outcome: 'still_present', + warning: 'The update command completed, but Sencho still detects an available image update.', + }; + }); + + const res = await request(app) + .post('/api/stacks/web/update') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(res.status).toBe(200); + expect(res.body.recheckWarning).toBe( + 'The update command completed, but Sencho still detects an available image update.', + ); + }); + + it('keeps HTTP 200 and success notification when recheck throws after Compose', async () => { + mockRecheckStack.mockImplementation(async () => { + callOrder.push('recheckStack'); + throw new Error('registry blew up'); + }); + + const res = await request(app) + .post('/api/stacks/web/update') + .set('Cookie', authCookie) + .send({ skip_scan: true }); + + expect(res.status).toBe(200); + expect(res.body.healthGateId).toBe('gate-1'); + expect(res.body.recheckWarning).toMatch(/could not fully verify/i); + expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); + expect(callOrder).toContain('broadcastEvent'); + // Success path still notifies; failure notification must not fire. + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'info', + 'image_update_applied', + expect.any(String), + expect.objectContaining({ stackName: 'web' }), + ); + expect(mockDispatchAlert.mock.calls.some((c) => c[0] === 'error' && c[1] === 'deploy_failure')).toBe(false); + }); +}); diff --git a/backend/src/__tests__/storage-inventory.test.ts b/backend/src/__tests__/storage-inventory.test.ts index 7b5ea118..ad79bdde 100644 --- a/backend/src/__tests__/storage-inventory.test.ts +++ b/backend/src/__tests__/storage-inventory.test.ts @@ -132,7 +132,7 @@ describe('buildMounts', () => { { type: 'bind', source: '/app/stack/conf', target: '/conf', readOnly: true }, { type: 'named', source: 'shared', target: '/s', readOnly: false }, ], - privileged: false, hasHealthcheck: true, envKeys: [], networks: [], extraHosts: [], labelKeys: [], + privileged: false, hasHealthcheck: true, composeHealthcheck: 'active', envKeys: [], networks: [], extraHosts: [], labelKeys: [], }, ], networks: {}, @@ -161,7 +161,7 @@ describe('assembleStorageInventory', () => { projectName: 'a', services: [{ name: 'app', ports: [], binds: [], namedVolumes: [], storageMounts: [{ type: 'named', source: 'db', target: '/db', readOnly: false }], - privileged: false, hasHealthcheck: true, envKeys: [], networks: [], extraHosts: [], labelKeys: [], + privileged: false, hasHealthcheck: true, composeHealthcheck: 'active', envKeys: [], networks: [], extraHosts: [], labelKeys: [], }], networks: {}, volumes: {}, }; expect(assembleStorageInventory('a', stateful, null, new Map()).stateful).toBe(true); @@ -178,7 +178,7 @@ describe('assembleStorageInventory', () => { projectName: 'a', services: [{ name: 'app', ports: [], binds: [], namedVolumes: [], storageMounts: [{ type: 'bind', source: '/var/run/docker.sock', target: '/var/run/docker.sock', readOnly: false }], - privileged: false, hasHealthcheck: true, envKeys: [], networks: [], extraHosts: [], labelKeys: [], + privileged: false, hasHealthcheck: true, composeHealthcheck: 'active', envKeys: [], networks: [], extraHosts: [], labelKeys: [], }], networks: {}, volumes: {}, }; expect(assembleStorageInventory('a', socketOnly, null, new Map()).stateful).toBe(false); diff --git a/backend/src/__tests__/update-guard-readiness.test.ts b/backend/src/__tests__/update-guard-readiness.test.ts index c92a2b0f..3b15aa7a 100644 --- a/backend/src/__tests__/update-guard-readiness.test.ts +++ b/backend/src/__tests__/update-guard-readiness.test.ts @@ -11,7 +11,7 @@ import { buildServicesSignal, } from '../services/updateGuard/readiness'; import type { ContainerProbe, ReadinessSignal } from '../services/updateGuard/types'; -import type { UpdatePreviewSummary } from '../services/UpdatePreviewService'; +import type { UpdatePreviewImage, UpdatePreviewSummary } from '../services/UpdatePreviewService'; const NOW = 1_750_000_000_000; @@ -37,6 +37,9 @@ const summary = (over: Partial = {}): UpdatePreviewSummary blocked_reason: null, has_build_services: false, rebuild_available: false, + check_status: 'ok', + verification_failed: false, + verification_error: null, ...over, }); @@ -157,6 +160,13 @@ describe('updatePreviewSignal', () => { it('degrades a preview failure to a non-verdict-affecting unknown', () => { expect(updatePreviewSignal('error')).toMatchObject({ status: 'unknown', affectsVerdict: false }); }); + + it('reports detection disabled without treating it as up to date', () => { + const signal = updatePreviewSignal(summary({ detection_disabled: true, has_update: false })); + expect(signal.status).toBe('unknown'); + expect(signal.affectsVerdict).toBe(false); + expect(signal.detail).toMatch(/disabled/i); + }); }); describe('buildServicesSignal', () => { @@ -225,3 +235,77 @@ describe('aggregateVerdict', () => { expect(aggregateVerdict([driftSignal(0), preflightSignal({ activeStatus: 'pass' }), healthchecksSignal([probe()])])).toBe('ready'); }); }); + +describe('updatePreviewSignal verification failure', () => { + it('does not claim no pending update when digest verification failed', () => { + const signal = updatePreviewSignal(summary({ + verification_failed: true, + verification_error: 'Registry unreachable', + })); + expect(signal.status).toBe('unknown'); + expect(signal.detail).toMatch(/Digest verification failed/); + expect(signal.detail).toMatch(/Registry unreachable/); + expect(signal.detail).not.toMatch(/No pending image update detected/); + }); + + it('holds a confirmed update for review, not ok, when another image failed digest verification', () => { + const signal = updatePreviewSignal(summary({ + has_update: true, + semver_bump: 'patch', + update_kind: 'digest', + verification_failed: true, + verification_error: 'Registry unreachable', + })); + expect(signal.status).toBe('attention'); + expect(signal.detail).toMatch(/Registry unreachable/); + }); + + it('holds a pending rebuild for review, not verification-only, when another image failed digest verification', () => { + const signal = updatePreviewSignal(summary({ + rebuild_available: true, + has_build_services: true, + verification_failed: true, + verification_error: 'Registry unreachable', + })); + expect(signal.status).toBe('attention'); + expect(signal.detail).toMatch(/rebuild/); + expect(signal.detail).toMatch(/Registry unreachable/); + }); + + const image = (over: Partial = {}): UpdatePreviewImage => ({ + service: 'web', + image: 'nginx:1', + current_tag: '1', + next_tag: null, + has_update: false, + digest_update: false, + tag_update: false, + semver_bump: 'none', + check_status: 'ok', + check_error: null, + digest_error: null, + ...over, + }); + + it('keeps a single image with its own confirmed update ok even though that same image also failed its own digest check', () => { + // has_update and check_error are independent per image; there is no + // "other image" here, so per-image detail must clear the review hold + // that the aggregate-only fallback would otherwise apply. + const signal = updatePreviewSignal( + summary({ has_update: true, semver_bump: 'patch', update_kind: 'tag', verification_failed: true, verification_error: 'Registry unreachable' }), + [image({ has_update: true, check_error: 'Registry unreachable' })], + ); + expect(signal.status).toBe('ok'); + }); + + it('holds a confirmed update for review when per-image detail proves a genuinely different image failed verification', () => { + const signal = updatePreviewSignal( + summary({ has_update: true, semver_bump: 'patch', update_kind: 'tag', verification_failed: true, verification_error: 'Registry unreachable' }), + [ + image({ service: 'confirmed', has_update: true, check_error: null }), + image({ service: 'failing', has_update: false, check_error: 'Registry unreachable' }), + ], + ); + expect(signal.status).toBe('attention'); + }); +}); diff --git a/backend/src/__tests__/update-guard-service.test.ts b/backend/src/__tests__/update-guard-service.test.ts index df9896b2..2458c00b 100644 --- a/backend/src/__tests__/update-guard-service.test.ts +++ b/backend/src/__tests__/update-guard-service.test.ts @@ -116,6 +116,21 @@ describe('UpdateGuardService.probeContainers', () => { expect(probes[0].name).toBe('app-web-1'); }); + it('treats Test NONE as no effective healthcheck', async () => { + mockListContainers.mockResolvedValue([ + { Id: 'aaa', Names: ['/app-web-1'], State: 'running' }, + ]); + mockGetContainer.mockReturnValue({ + inspect: vi.fn().mockResolvedValue(inspectResult({ + Config: { Healthcheck: { Test: ['NONE'] } }, + })), + }); + + const probes = await UpdateGuardService.getInstance().probeContainers(0, 'app'); + expect(probes).toHaveLength(1); + expect(probes[0].hasHealthcheck).toBe(false); + }); + it('propagates non-404 inspect failures so the whole signal degrades honestly', async () => { mockListContainers.mockResolvedValue([ { Id: 'aaa', Names: ['/app-web-1'], State: 'running' }, @@ -159,7 +174,7 @@ describe('UpdateGuardService.computeUpdateReadiness wiring', () => { summary: { has_update: true, primary_image: 'nginx', current_tag: '1.27.0', next_tag: '1.27.1', semver_bump: 'patch', update_kind: 'tag', blocked: false, blocked_reason: null, - has_build_services: false, rebuild_available: false, + has_build_services: false, rebuild_available: false, check_status: 'ok', verification_failed: false, verification_error: null, }, rollback_target: 'nginx:1.27.0', changelog: null, @@ -190,7 +205,7 @@ describe('UpdateGuardService.computeUpdateReadiness with a serviceName', () => { summary: { has_update: true, primary_image: 'nginx', current_tag: '1.27.0', next_tag: '1.27.1', semver_bump: 'patch', update_kind: 'tag', blocked: false, blocked_reason: null, - has_build_services: false, rebuild_available: false, + has_build_services: false, rebuild_available: false, check_status: 'ok', verification_failed: false, verification_error: null, }, rollback_target: 'nginx:1.27.0', changelog: null, @@ -284,7 +299,7 @@ describe('UpdateGuardService.computeRollbackReadiness moving-tag wiring', () => summary: { has_update: false, primary_image: 'app', current_tag: images[0]?.current_tag ?? null, next_tag: null, semver_bump: 'none', update_kind: 'none', blocked: false, blocked_reason: null, - has_build_services: false, rebuild_available: false, + has_build_services: false, rebuild_available: false, check_status: 'ok', verification_failed: false, verification_error: null, }, rollback_target: 'app:1.2.3', changelog: null, diff --git a/backend/src/__tests__/update-preview-reconcile.test.ts b/backend/src/__tests__/update-preview-reconcile.test.ts new file mode 100644 index 00000000..dcab0d37 --- /dev/null +++ b/backend/src/__tests__/update-preview-reconcile.test.ts @@ -0,0 +1,508 @@ +/** + * Authoritative-negative update-preview reconcile: commitPreviewClear generation + * safety and route side effects (broadcast / fleet cache invalidate). + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +let tmpDir: string; +let app: import('express').Express; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let ImageUpdateService: typeof import('../services/ImageUpdateService').ImageUpdateService; +let UpdatePreviewService: typeof import('../services/UpdatePreviewService').UpdatePreviewService; +let NotificationService: typeof import('../services/NotificationService').NotificationService; +let CacheService: typeof import('../services/CacheService').CacheService; +let adminCookie: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ ImageUpdateService } = await import('../services/ImageUpdateService')); + ({ UpdatePreviewService } = await import('../services/UpdatePreviewService')); + ({ NotificationService } = await import('../services/NotificationService')); + ({ CacheService } = await import('../services/CacheService')); + ({ app } = await import('../index')); + adminCookie = await loginAsTestAdmin(app); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +beforeEach(() => { + vi.restoreAllMocks(); + const raw = (DatabaseService.getInstance() as unknown as { db: { prepare: (s: string) => { run: () => void } } }).db; + raw.prepare('DELETE FROM stack_update_status').run(); +}); + +function negativeOkPreview(stackName = 'web') { + return { + stack_name: stackName, + images: [{ + service: 'web', + image: 'nginx:1.2.3', + current_tag: '1.2.3', + next_tag: null, + has_update: false, + digest_update: false, + tag_update: false, + semver_bump: 'none' as const, + check_status: 'ok' as const, + check_error: null, + digest_error: null, + }], + build_services: [] as string[], + summary: { + has_update: false, + primary_image: 'nginx:1.2.3', + current_tag: '1.2.3', + next_tag: null, + semver_bump: 'none' as const, + update_kind: 'none' as const, + blocked: false, + blocked_reason: null, + has_build_services: false, + rebuild_available: false, + check_status: 'ok' as const, + verification_failed: false, + verification_error: null, + }, + rollback_target: null, + changelog: null, + }; +} + +describe('ImageUpdateService.commitPreviewClear', () => { + it('clears sticky partial rows and returns cleared', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'web', true, 1000, 'partial', 'half', [ + { service: 'web', image: 'web:1', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ]); + const svc = ImageUpdateService.getInstance(); + const observedMem = svc.peekStackWriteGeneration(nodeId, 'web'); + const observedRow = db.getStackUpdateWriteGeneration(nodeId, 'web'); + const result = await svc.commitPreviewClear(nodeId, 'web', observedMem, observedRow); + expect(result).toBe('cleared'); + expect(db.getStackUpdateDetail(nodeId).web).toBeUndefined(); + }); + + it('clears an older confirmed ok+true row', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + const svc = ImageUpdateService.getInstance(); + const writeGen = (svc as unknown as { + reserveStackWriteGeneration: (n: number, s: string) => number; + }).reserveStackWriteGeneration(nodeId, 'web'); + db.upsertStackUpdateStatus(nodeId, 'web', true, 1000, 'ok', null, [ + { service: 'web', image: 'web:1', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ], writeGen); + const observedMem = svc.peekStackWriteGeneration(nodeId, 'web'); + const observedRow = db.getStackUpdateWriteGeneration(nodeId, 'web'); + expect(observedRow).toBe(writeGen); + expect(await svc.commitPreviewClear(nodeId, 'web', observedMem, observedRow)).toBe('cleared'); + expect(db.getStackUpdateDetail(nodeId).web).toBeUndefined(); + }); + + it('clears a persisted ok+true row when in-memory generation was reset (restart)', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + // Simulate a prior process that wrote generation 7, then a restart that + // left only SQLite state (in-memory high-water is 0 for this stack key). + db.upsertStackUpdateStatus(nodeId, 'restart-web', true, 1000, 'ok', null, [ + { service: 'web', image: 'web:1', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ], 7); + const svc = ImageUpdateService.getInstance(); + expect(svc.peekStackWriteGeneration(nodeId, 'restart-web')).toBe(0); + expect(db.getStackUpdateWriteGeneration(nodeId, 'restart-web')).toBe(7); + expect(await svc.commitPreviewClear(nodeId, 'restart-web', 0, 7)).toBe('cleared'); + expect(db.getStackUpdateDetail(nodeId)['restart-web']).toBeUndefined(); + }); + + it('returns absent when no row exists', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + const svc = ImageUpdateService.getInstance(); + expect(await svc.commitPreviewClear(nodeId, 'missing', 0, 0)).toBe('absent'); + }); + + it('retains a row written after the observation watermark', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + const svc = ImageUpdateService.getInstance() as unknown as { + peekStackWriteGeneration: (n: number, s: string) => number; + reserveStackWriteGeneration: (n: number, s: string) => number; + commitPreviewClear: ( + n: number, + s: string, + observedMem: number, + observedRow: number, + ) => Promise<'cleared' | 'stale' | 'absent'>; + withStackWriteLock: ( + n: number, + s: string, + g: number, + write: () => void | Promise, + ) => Promise; + }; + + const observedMem = svc.peekStackWriteGeneration(nodeId, 'race'); + const observedRow = db.getStackUpdateWriteGeneration(nodeId, 'race'); + const scannerGen = svc.reserveStackWriteGeneration(nodeId, 'race'); + expect(scannerGen).toBeGreaterThan(observedMem); + + await svc.withStackWriteLock(nodeId, 'race', scannerGen, () => { + db.upsertStackUpdateStatus(nodeId, 'race', true, Date.now(), 'ok', null, [ + { service: 'web', image: 'web:2', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ], scannerGen); + }); + + expect(await svc.commitPreviewClear(nodeId, 'race', observedMem, observedRow)).toBe('stale'); + expect(db.getStackUpdateDetail(nodeId).race?.hasUpdate).toBe(true); + expect(db.getStackUpdateDetail(nodeId).race?.checkStatus).toBe('ok'); + }); + + it('retains a row whose DB generation advanced after observation', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'adv', true, 1000, 'ok', null, [ + { service: 'web', image: 'web:1', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ], 3); + const svc = ImageUpdateService.getInstance(); + const observedMem = svc.peekStackWriteGeneration(nodeId, 'adv'); + const observedRow = db.getStackUpdateWriteGeneration(nodeId, 'adv'); + expect(observedRow).toBe(3); + + db.upsertStackUpdateStatus(nodeId, 'adv', true, Date.now(), 'ok', null, [ + { service: 'web', image: 'web:2', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ], 4); + + expect(await svc.commitPreviewClear(nodeId, 'adv', observedMem, observedRow)).toBe('absent'); + expect(db.getStackUpdateDetail(nodeId).adv?.hasUpdate).toBe(true); + }); +}); + +describe('GET/POST /api/stacks/:stackName/update-preview reconcile', () => { + it('GET returns detection_disabled preview without calling getPreview when checks are off', async () => { + const db = DatabaseService.getInstance(); + db.updateGlobalSetting('image_update_checks_enabled', '0'); + const getPreview = vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview'); + + try { + const res = await request(app) + .get('/api/stacks/web/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.summary?.detection_disabled).toBe(true); + expect(res.body.summary?.has_update).toBe(false); + expect(res.body.images).toEqual([]); + expect(getPreview).not.toHaveBeenCalled(); + } finally { + db.updateGlobalSetting('image_update_checks_enabled', '1'); + } + }); + + it('POST returns detection_disabled without registry I/O or sticky reconcile when checks are off', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'web', true, 1000, 'partial', 'half'); + db.updateGlobalSetting('image_update_checks_enabled', '0'); + const getPreview = vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview'); + const broadcast = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation(() => undefined); + const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate').mockImplementation(() => undefined); + + try { + const res = await request(app) + .post('/api/stacks/web/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.summary?.detection_disabled).toBe(true); + expect(res.body.summary?.has_update).toBe(false); + expect(res.body.reconciled).toBe(false); + expect(getPreview).not.toHaveBeenCalled(); + expect(db.getStackUpdateDetail(nodeId).web?.hasUpdate).toBe(true); + expect(broadcast).not.toHaveBeenCalled(); + expect(invalidate).not.toHaveBeenCalled(); + } finally { + db.updateGlobalSetting('image_update_checks_enabled', '1'); + } + }); + + it('GET does not mutate sticky state even for authoritative-negative preview', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'web', true, 1000, 'partial', 'half'); + + vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview').mockResolvedValue(negativeOkPreview('web')); + const broadcast = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation(() => undefined); + const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate').mockImplementation(() => undefined); + + const res = await request(app) + .get('/api/stacks/web/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.reconciled).toBeUndefined(); + expect(db.getStackUpdateDetail(nodeId).web?.hasUpdate).toBe(true); + expect(invalidate).not.toHaveBeenCalled(); + expect(broadcast).not.toHaveBeenCalled(); + }); + + it('POST clears sticky state and broadcasts on authoritative-negative preview', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'web', true, 1000, 'partial', 'half'); + + vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview').mockResolvedValue(negativeOkPreview('web')); + const broadcast = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation(() => undefined); + const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate').mockImplementation(() => undefined); + + const res = await request(app) + .post('/api/stacks/web/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.reconciled).toBe(true); + expect(db.getStackUpdateDetail(nodeId).web).toBeUndefined(); + expect(invalidate).toHaveBeenCalledWith('fleet-updates'); + expect(broadcast).toHaveBeenCalledWith(expect.objectContaining({ + type: 'state-invalidate', + scope: 'image-updates', + action: 'update-status-reconciled', + stackName: 'web', + })); + }); + + it('POST clears an older confirmed ok+true row and broadcasts', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + const svc = ImageUpdateService.getInstance() as unknown as { + reserveStackWriteGeneration: (n: number, s: string) => number; + }; + const writeGen = svc.reserveStackWriteGeneration(nodeId, 'web'); + db.upsertStackUpdateStatus(nodeId, 'web', true, 1000, 'ok', null, [ + { service: 'web', image: 'nginx:1', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ], writeGen); + + vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview').mockResolvedValue(negativeOkPreview('web')); + const broadcast = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation(() => undefined); + const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate').mockImplementation(() => undefined); + + const res = await request(app) + .post('/api/stacks/web/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.reconciled).toBe(true); + expect(db.getStackUpdateDetail(nodeId).web).toBeUndefined(); + expect(invalidate).toHaveBeenCalledWith('fleet-updates'); + expect(broadcast).toHaveBeenCalledWith(expect.objectContaining({ + action: 'update-status-reconciled', + stackName: 'web', + })); + }); + + it('POST does not mutate on partial negative preview', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'web', true, 1000, 'partial', 'half'); + + const preview = negativeOkPreview('web'); + vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview').mockResolvedValue({ + ...preview, + images: [{ ...preview.images[0], check_status: 'partial' }], + summary: { ...preview.summary, check_status: 'partial' }, + }); + + const res = await request(app) + .post('/api/stacks/web/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.reconciled).toBe(false); + expect(db.getStackUpdateDetail(nodeId).web?.hasUpdate).toBe(true); + }); + + it('POST does not mutate when an image is not_checkable alongside ok', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'web', true, 1000, 'partial', 'half'); + + const preview = negativeOkPreview('web'); + vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview').mockResolvedValue({ + ...preview, + images: [ + preview.images[0], + { + ...preview.images[0], + service: 'bad', + image: 'not-a-valid-ref', + check_status: 'not_checkable' as const, + }, + ], + summary: { ...preview.summary, check_status: 'partial' as const }, + }); + + const res = await request(app) + .post('/api/stacks/web/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.reconciled).toBe(false); + expect(db.getStackUpdateDetail(nodeId).web?.hasUpdate).toBe(true); + }); + + it('POST does not mutate when check_status is missing from summary rollup fields still fail every-ok', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'web', true, 1000, 'partial', 'half'); + + const preview = negativeOkPreview('web'); + vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview').mockResolvedValue({ + ...preview, + images: [{ ...preview.images[0], check_status: 'partial' as const }], + }); + + const res = await request(app) + .post('/api/stacks/web/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.reconciled).toBe(false); + expect(db.getStackUpdateDetail(nodeId).web?.hasUpdate).toBe(true); + }); + + it('POST does not broadcast when clear finds no row', async () => { + vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview').mockResolvedValue(negativeOkPreview('ghost')); + const broadcast = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation(() => undefined); + const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate').mockImplementation(() => undefined); + + const res = await request(app) + .post('/api/stacks/ghost/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.reconciled).toBe(false); + expect(broadcast).not.toHaveBeenCalled(); + expect(invalidate).not.toHaveBeenCalled(); + }); + + it('retains a confirmed row written after the preview observation', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + const svc = ImageUpdateService.getInstance() as unknown as { + peekStackWriteGeneration: (n: number, s: string) => number; + reserveStackWriteGeneration: (n: number, s: string) => number; + withStackWriteLock: ( + n: number, + s: string, + g: number, + write: () => void | Promise, + ) => Promise; + }; + + const observedBeforePreview = svc.peekStackWriteGeneration(nodeId, 'web'); + let previewCalls = 0; + vi.spyOn(UpdatePreviewService.getInstance(), 'getPreview').mockImplementation(async () => { + previewCalls += 1; + // Simulate a scanner reservation+commit that begins after observation. + const scannerGen = svc.reserveStackWriteGeneration(nodeId, 'web'); + await svc.withStackWriteLock(nodeId, 'web', scannerGen, () => { + db.upsertStackUpdateStatus(nodeId, 'web', true, Date.now(), 'ok', null, [ + { service: 'web', image: 'nginx:9', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ], scannerGen); + }); + expect(scannerGen).toBeGreaterThan(observedBeforePreview); + return negativeOkPreview('web'); + }); + const broadcast = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent').mockImplementation(() => undefined); + const invalidate = vi.spyOn(CacheService.getInstance(), 'invalidate').mockImplementation(() => undefined); + + const res = await request(app) + .post('/api/stacks/web/update-preview') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(res.body.reconciled).toBe(false); + expect(previewCalls).toBe(1); + expect(db.getStackUpdateDetail(nodeId).web?.hasUpdate).toBe(true); + expect(db.getStackUpdateDetail(nodeId).web?.checkStatus).toBe('ok'); + expect(broadcast).not.toHaveBeenCalled(); + expect(invalidate).not.toHaveBeenCalled(); + }); +}); + +describe('generation ordering for preview clear', () => { + it('a newer scanner reservation supersedes an in-flight observation-watermark clear', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'ord', true, 1000, 'partial', 'half'); + const svc = ImageUpdateService.getInstance(); + const observedMem = svc.peekStackWriteGeneration(nodeId, 'ord'); + const observedRow = db.getStackUpdateWriteGeneration(nodeId, 'ord'); + const scannerGen = (svc as unknown as { + reserveStackWriteGeneration: (n: number, s: string) => number; + }).reserveStackWriteGeneration(nodeId, 'ord'); + expect(scannerGen).toBeGreaterThan(observedMem); + + expect(await svc.commitPreviewClear(nodeId, 'ord', observedMem, observedRow)).toBe('stale'); + + const scannerCommitted = await (svc as unknown as { + withStackWriteLock: ( + n: number, + s: string, + g: number, + write: () => void | Promise, + ) => Promise; + }).withStackWriteLock(nodeId, 'ord', scannerGen, () => { + db.upsertStackUpdateStatus(nodeId, 'ord', true, Date.now(), 'ok', null); + }); + expect(scannerCommitted).toBe(true); + expect(db.getStackUpdateDetail(nodeId).ord?.hasUpdate).toBe(true); + expect(db.getStackUpdateDetail(nodeId).ord?.checkStatus).toBe('ok'); + }); + + it('equal-generation scanner reserved before observation cannot rewrite after clear', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id!; + db.upsertStackUpdateStatus(nodeId, 'ord2', true, 1000, 'partial', 'half', [ + { service: 'web', image: 'web:1', hasUpdate: true, checkStatus: 'partial', lastError: 'half' }, + ], 1); + const svc = ImageUpdateService.getInstance() as unknown as { + peekStackWriteGeneration: (n: number, s: string) => number; + reserveStackWriteGeneration: (n: number, s: string) => number; + commitPreviewClear: ( + n: number, + s: string, + observedMem: number, + observedRow: number, + ) => Promise<'cleared' | 'stale' | 'absent'>; + withStackWriteLock: ( + n: number, + s: string, + g: number, + write: () => void | Promise, + ) => Promise; + }; + + // Full scan reserved generation N before its slow registry work. + const scannerGen = svc.reserveStackWriteGeneration(nodeId, 'ord2'); + // Preview observation sees that same watermark. + const observedMem = svc.peekStackWriteGeneration(nodeId, 'ord2'); + const observedRow = db.getStackUpdateWriteGeneration(nodeId, 'ord2'); + expect(observedMem).toBe(scannerGen); + + expect(await svc.commitPreviewClear(nodeId, 'ord2', observedMem, observedRow)).toBe('cleared'); + expect(db.getStackUpdateDetail(nodeId).ord2).toBeUndefined(); + + // Delayed scanner write using the pre-observation reservation must not commit. + const scannerCommitted = await svc.withStackWriteLock(nodeId, 'ord2', scannerGen, () => { + db.upsertStackUpdateStatus(nodeId, 'ord2', true, Date.now(), 'ok', null, [ + { service: 'web', image: 'web:2', hasUpdate: true, checkStatus: 'ok', lastError: null }, + ], scannerGen); + }); + expect(scannerCommitted).toBe(false); + expect(db.getStackUpdateDetail(nodeId).ord2).toBeUndefined(); + }); +}); diff --git a/backend/src/__tests__/update-preview-service.test.ts b/backend/src/__tests__/update-preview-service.test.ts index 1907143e..81b43f05 100644 --- a/backend/src/__tests__/update-preview-service.test.ts +++ b/backend/src/__tests__/update-preview-service.test.ts @@ -6,15 +6,31 @@ import { computeImagePreview, buildSummary, isMovingTag, + listAllRegistryTagsBounded, + isAuthoritativeNegativePreview, + PREVIEW_TAG_LIST_MAX_PAGES, type ComputePreviewDeps, type LocalDigestInfo, } from '../services/UpdatePreviewService'; -import type { DigestComparisonResult } from '../services/registry-api'; +import { selectLocalRepoDigests, type DigestComparisonResult, type ParsedRef, type TagListResult } from '../services/registry-api'; const PLATFORM = { os: 'linux', architecture: 'amd64' }; -function localDigest(digest: string | null): LocalDigestInfo { - return { digest, platform: PLATFORM }; +function localDigest( + digest: string | null | string[], + emptyReason: LocalDigestInfo['emptyReason'] = null, +): LocalDigestInfo { + if (Array.isArray(digest)) return { digests: digest, platform: PLATFORM, emptyReason }; + if (digest) return { digests: [digest], platform: PLATFORM, emptyReason: null }; + return { + digests: [], + platform: PLATFORM, + emptyReason: emptyReason ?? 'not_checkable', + }; +} + +function tagsOk(tags: string[], nextCursor?: string): TagListResult { + return nextCursor ? { ok: true, tags, nextCursor } : { ok: true, tags }; } describe('parseSemverTag', () => { @@ -97,7 +113,7 @@ function makeDeps(overrides: Partial = {}): ComputePreviewDe getCredentials: vi.fn().mockResolvedValue(null), getLocalDigest: vi.fn().mockResolvedValue(localDigest(null)), compareDigest: vi.fn().mockResolvedValue({ kind: 'error', reason: 'not configured' } satisfies DigestComparisonResult), - listRegistryTags: vi.fn().mockResolvedValue([]), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk([])), ...overrides, }; } @@ -107,7 +123,7 @@ describe('computeImagePreview', () => { const deps = makeDeps({ getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), compareDigest: vi.fn().mockResolvedValue({ kind: 'match' }), - listRegistryTags: vi.fn().mockResolvedValue(['1.2.3']), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk(['1.2.3'])), }); const result = await computeImagePreview('web', 'nginx:1.2.3', deps); expect(result.has_update).toBe(false); @@ -119,7 +135,7 @@ describe('computeImagePreview', () => { const deps = makeDeps({ getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), compareDigest: vi.fn().mockResolvedValue({ kind: 'update' }), - listRegistryTags: vi.fn().mockResolvedValue([]), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk([])), }); const result = await computeImagePreview('web', 'nginx:latest', deps); expect(result.has_update).toBe(true); @@ -132,7 +148,7 @@ describe('computeImagePreview', () => { const deps = makeDeps({ getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), compareDigest: vi.fn().mockResolvedValue({ kind: 'match' }), - listRegistryTags: vi.fn().mockResolvedValue(['27.1.4', '27.1.5', '27.2.0']), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk(['27.1.4', '27.1.5', '27.2.0'])), }); const result = await computeImagePreview('engine', 'docker.io/library/docker:27.1.4', deps); expect(result.has_update).toBe(true); @@ -144,7 +160,7 @@ describe('computeImagePreview', () => { const deps = makeDeps({ getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), compareDigest: vi.fn().mockResolvedValue({ kind: 'match' }), - listRegistryTags: vi.fn().mockResolvedValue(['1.2.3', '2.0.0']), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk(['1.2.3', '2.0.0'])), }); const result = await computeImagePreview('db', 'postgres:1.2.3', deps); expect(result.next_tag).toBe('2.0.0'); @@ -155,36 +171,106 @@ describe('computeImagePreview', () => { const deps = makeDeps({ getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), compareDigest: vi.fn().mockResolvedValue({ kind: 'error', reason: 'Registry unreachable' }), - listRegistryTags: vi.fn().mockResolvedValue(['1.2.3', '1.2.4']), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk(['1.2.3', '1.2.4'])), }); const result = await computeImagePreview('web', 'nginx:1.2.3', deps); expect(result.has_update).toBe(true); expect(result.next_tag).toBe('1.2.4'); expect(result.semver_bump).toBe('patch'); + // A confirmed tag-based update overrides the digest hiccup: the overall + // check_status resolves 'ok' and check_error is not surfaced (see the + // "treats digest error + higher tag as a confirmed update" test below). + expect(result.check_error).toBeNull(); }); - it('fails soft to no-update when the comparison resolver errors and no higher tag exists', async () => { + it('surfaces verification failure when the comparison resolver errors and no higher tag exists', async () => { const deps = makeDeps({ getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), compareDigest: vi.fn().mockResolvedValue({ kind: 'error', reason: 'Registry unreachable' }), - listRegistryTags: vi.fn().mockResolvedValue([]), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk([])), }); const result = await computeImagePreview('web', 'nginx:1.2.3', deps); expect(result.has_update).toBe(false); expect(result.next_tag).toBeNull(); expect(result.semver_bump).toBe('none'); + expect(result.check_error).toBe('Registry unreachable'); + expect(result.check_status).toBe('partial'); }); - it('never calls the comparison resolver when no local digest is resolvable', async () => { + it('treats digest error + higher tag as a confirmed update (ok), but keeps digest_error unmasked for the collateral-risk gate', async () => { + const result = await computeImagePreview('web', 'nginx:1.2.3', makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), + compareDigest: vi.fn().mockResolvedValue({ kind: 'error', reason: 'Registry unreachable' }), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk(['1.2.3', '1.2.4'])), + })); + expect(result.has_update).toBe(true); + expect(result.next_tag).toBe('1.2.4'); + expect(result.check_status).toBe('ok'); + // The tag compare independently confirmed the update, so check_status + // masks the digest failure. digest_error must survive that masking: a + // full-stack apply still re-pulls this image's unverified current tag. + expect(result.digest_error).toBe('Registry unreachable'); + }); + + it('treats empty RepoDigests as not_checkable (no verification failure)', async () => { const compareDigest = vi.fn().mockResolvedValue({ kind: 'update' }); const deps = makeDeps({ - getLocalDigest: vi.fn().mockResolvedValue(localDigest(null)), + getLocalDigest: vi.fn().mockResolvedValue(localDigest(null, 'not_checkable')), compareDigest, - listRegistryTags: vi.fn().mockResolvedValue([]), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk([])), }); const result = await computeImagePreview('web', 'nginx:1.2.3', deps); expect(compareDigest).not.toHaveBeenCalled(); expect(result.has_update).toBe(false); + expect(result.check_error).toBeNull(); + }); + + it('surfaces unresolved RepoDigests as verification failure without claiming an update', async () => { + const compareDigest = vi.fn().mockResolvedValue({ kind: 'update' }); + const deps = makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest(null, 'unresolved')), + compareDigest, + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk([])), + }); + const result = await computeImagePreview('web', 'nginx:1.2.3', deps); + expect(compareDigest).not.toHaveBeenCalled(); + expect(result.has_update).toBe(false); + expect(result.check_error).toBe('Could not resolve a local registry digest'); + }); + + it('wires a real unrelated-repository RepoDigest through selectLocalRepoDigests to an unresolved verification failure', async () => { + // getLocalDigest here is not a canned stand-in: it runs the real + // selectLocalRepoDigests against a RepoDigests array whose sole valid + // entry belongs to a different repository, proving the removed + // sole-unmatched-digest fallback stays removed through the actual + // preview-computation path, not just at the registry-api unit level. + const compareDigest = vi.fn().mockResolvedValue({ kind: 'update' }); + const deps = makeDeps({ + getLocalDigest: async (_imageRef: string, parsed: ParsedRef) => { + const digests = selectLocalRepoDigests( + [`ghcr.io/other/image@sha256:${'b'.repeat(64)}`], + parsed, + ); + return { digests, platform: PLATFORM, emptyReason: digests.length === 0 ? 'unresolved' as const : null }; + }, + compareDigest, + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk([])), + }); + const result = await computeImagePreview('web', 'nginx:1.2.3', deps); + expect(compareDigest).not.toHaveBeenCalled(); + expect(result.has_update).toBe(false); + expect(result.check_error).toBe('Could not resolve a local registry digest'); + }); + + it('surfaces inspect failure as verification failure', async () => { + const deps = makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest(null, 'inspect_failed')), + compareDigest: vi.fn(), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk([])), + }); + const result = await computeImagePreview('web', 'nginx:1.2.3', deps); + expect(result.has_update).toBe(false); + expect(result.check_error).toBe('Failed to inspect local image'); }); it('passes the local digest, tag, and platform through to the comparison resolver', async () => { @@ -192,21 +278,51 @@ describe('computeImagePreview', () => { const deps = makeDeps({ getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), compareDigest, - listRegistryTags: vi.fn().mockResolvedValue([]), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk([])), }); await computeImagePreview('web', 'ghcr.io/linuxserver/radarr:latest', deps); - expect(compareDigest).toHaveBeenCalledWith('sha256:aaa', 'ghcr.io', 'linuxserver/radarr', 'latest', PLATFORM, null); + expect(compareDigest).toHaveBeenCalledWith(['sha256:aaa'], 'ghcr.io', 'linuxserver/radarr', 'latest', PLATFORM, null); + }); + + it('forwards every local digest candidate and reports no same-tag rebuild on match', async () => { + const STALE = `sha256:${'f'.repeat(64)}`; + const CURRENT = `sha256:${'e'.repeat(64)}`; + const compareDigest = vi.fn().mockResolvedValue({ kind: 'match' }); + const deps = makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest([STALE, CURRENT])), + compareDigest, + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk([])), + }); + const result = await computeImagePreview('broker', 'redis:8.8.0', deps); + expect(compareDigest).toHaveBeenCalledWith( + [STALE, CURRENT], + 'registry-1.docker.io', + 'library/redis', + '8.8.0', + PLATFORM, + null, + ); + expect(result).toMatchObject({ + has_update: false, + next_tag: null, + semver_bump: 'none', + }); }); }); describe('buildSummary', () => { - const baseImage = (partial: Partial[1][number]>) => ({ + const baseImage = (partial: Partial[1][number]> = {}) => ({ service: 'svc', image: 'nginx:1.0.0', current_tag: '1.0.0', next_tag: null, has_update: false, + digest_update: false, + tag_update: false, semver_bump: 'none' as const, + check_status: 'ok' as const, + check_error: null as string | null, + digest_error: null as string | null, ...partial, }); @@ -219,6 +335,41 @@ describe('buildSummary', () => { expect(preview.summary.blocked).toBe(true); expect(preview.summary.blocked_reason).toMatch(/major/i); expect(preview.summary.semver_bump).toBe('major'); + expect(preview.summary.verification_failed).toBe(false); + expect(preview.summary.verification_error).toBeNull(); + }); + + it('aggregates digest verification failure without claiming a digest rebuild', () => { + const images = [ + baseImage({ + service: 'broker', + image: 'redis:8.8.0', + current_tag: '8.8.0', + check_error: 'Local image platform is unknown; cannot verify multi-arch membership', + }), + ]; + const preview = buildSummary('app', images); + expect(preview.summary.has_update).toBe(false); + expect(preview.summary.update_kind).toBe('none'); + expect(preview.summary.verification_failed).toBe(true); + expect(preview.summary.verification_error).toContain('cannot verify multi-arch membership'); + }); + + it('keeps a tag update when digest verification failed on the same image', () => { + const images = [ + baseImage({ + service: 'web', + has_update: true, + semver_bump: 'patch', + next_tag: '1.0.1', + check_error: 'Registry unreachable', + }), + ]; + const preview = buildSummary('app', images); + expect(preview.summary.has_update).toBe(true); + expect(preview.summary.update_kind).toBe('tag'); + expect(preview.summary.verification_failed).toBe(true); + expect(preview.summary.verification_error).toBe('Registry unreachable'); }); it('picks first updated image as primary', () => { @@ -281,6 +432,15 @@ describe('buildSummary', () => { expect(buildSummary('stacky', images).rollback_target).toBe('postgres:16'); }); + it('computes rollback target for an explicit docker.io/ ref the same as the bare form', () => { + // docker.io is a Docker Hub alias normalized by parseImageRef; the rollback target + // must render the same shortened form as the bare-name case above, not the literal host. + const images = [ + baseImage({ service: 'db', image: 'docker.io/library/postgres:16', has_update: true, semver_bump: 'patch', next_tag: '16', current_tag: '16' }), + ]; + expect(buildSummary('stacky', images).rollback_target).toBe('postgres:16'); + }); + it('computes rollback target for registry with port', () => { const images = [ baseImage({ @@ -334,4 +494,145 @@ describe('buildSummary', () => { const images = [baseImage({ service: 'clean', has_update: false })]; expect(buildSummary('stacky', images).summary.update_kind).toBe('none'); }); + + it('sets check_status=ok for empty and all-ok images', () => { + expect(buildSummary('empty', []).summary.check_status).toBe('ok'); + expect(buildSummary('ok', [baseImage({ check_status: 'ok' })]).summary.check_status).toBe('ok'); + }); + + it('rolls up mixed and failed check_status', () => { + expect(buildSummary('mixed', [ + baseImage({ service: 'a', check_status: 'ok' }), + baseImage({ service: 'b', check_status: 'partial' }), + ]).summary.check_status).toBe('partial'); + expect(buildSummary('fail', [ + baseImage({ service: 'a', check_status: 'failed' }), + baseImage({ service: 'b', check_status: 'failed' }), + ]).summary.check_status).toBe('failed'); + }); +}); + +describe('preview authority', () => { + it('marks digest match + exhausted empty tags as authoritative ok with no update', async () => { + const result = await computeImagePreview('web', 'nginx:1.2.3', makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), + compareDigest: vi.fn().mockResolvedValue({ kind: 'match' }), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk(['1.2.3'])), + })); + expect(result.has_update).toBe(false); + expect(result.check_status).toBe('ok'); + expect(isAuthoritativeNegativePreview(buildSummary('s', [result]))).toBe(true); + }); + + it('marks digest error + successful tag list with no next as partial (not authoritative-negative)', async () => { + const result = await computeImagePreview('web', 'nginx:1.2.3', makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), + compareDigest: vi.fn().mockResolvedValue({ kind: 'error', reason: 'boom' }), + listRegistryTagsResult: vi.fn().mockResolvedValue(tagsOk(['1.2.3'])), + })); + expect(result.has_update).toBe(false); + expect(result.check_status).toBe('partial'); + expect(isAuthoritativeNegativePreview(buildSummary('s', [result]))).toBe(false); + }); + + it('does not treat empty or not_checkable-only previews as authoritative-negative', () => { + expect(isAuthoritativeNegativePreview(buildSummary('empty', []))).toBe(false); + expect(isAuthoritativeNegativePreview(buildSummary('build', [ + { + service: 'app', + image: 'sha256:dead', + current_tag: 'unknown', + next_tag: null, + has_update: false, + digest_update: false, + tag_update: false, + semver_bump: 'none', + check_status: 'not_checkable', + check_error: null, + digest_error: null, + }, + ]))).toBe(false); + }); + + it('marks digest match + tag list failure as partial for semver tags', async () => { + const result = await computeImagePreview('web', 'nginx:1.2.3', makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), + compareDigest: vi.fn().mockResolvedValue({ kind: 'match' }), + listRegistryTagsResult: vi.fn().mockResolvedValue({ + ok: false, + code: 'REGISTRY_UPSTREAM', + message: 'Registry unreachable', + }), + })); + expect(result.has_update).toBe(false); + expect(result.check_status).toBe('partial'); + }); + + it('allows moving/non-semver tags to be authoritative-negative on digest match without tag enum', async () => { + const listFn = vi.fn(); + const result = await computeImagePreview('web', 'nginx:latest', makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), + compareDigest: vi.fn().mockResolvedValue({ kind: 'match' }), + listRegistryTagsResult: listFn, + })); + expect(listFn).not.toHaveBeenCalled(); + expect(result.has_update).toBe(false); + expect(result.check_status).toBe('ok'); + expect(isAuthoritativeNegativePreview(buildSummary('s', [result]))).toBe(true); + }); + + it('detects a newer tag found on a later page', async () => { + const listFn = vi.fn() + .mockResolvedValueOnce(tagsOk(['1.0.0', '1.0.1'], 'cursor-1')) + .mockResolvedValueOnce(tagsOk(['1.1.0'])); + const result = await computeImagePreview('web', 'nginx:1.0.0', makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), + compareDigest: vi.fn().mockResolvedValue({ kind: 'match' }), + listRegistryTagsResult: listFn, + })); + expect(listFn).toHaveBeenCalledTimes(2); + expect(result.has_update).toBe(true); + expect(result.next_tag).toBe('1.1.0'); + expect(result.check_status).toBe('ok'); + }); + + it('treats page-cap with remaining cursor as non-authoritative for semver negatives', async () => { + let page = 0; + const listFn = vi.fn().mockImplementation(async () => { + page += 1; + return tagsOk([`1.0.${page}`], `cursor-${page}`); + }); + const result = await computeImagePreview('web', 'nginx:2.0.0', makeDeps({ + getLocalDigest: vi.fn().mockResolvedValue(localDigest('sha256:aaa')), + compareDigest: vi.fn().mockResolvedValue({ kind: 'match' }), + listRegistryTagsResult: listFn, + })); + expect(listFn).toHaveBeenCalledTimes(PREVIEW_TAG_LIST_MAX_PAGES); + expect(result.has_update).toBe(false); + expect(result.check_status).toBe('partial'); + expect(isAuthoritativeNegativePreview(buildSummary('s', [result]))).toBe(false); + }); + + it('marks invalid refs as not_checkable', async () => { + const result = await computeImagePreview('web', 'sha256:deadbeef', makeDeps()); + expect(result.check_status).toBe('not_checkable'); + expect(result.has_update).toBe(false); + }); +}); + +describe('listAllRegistryTagsBounded', () => { + it('returns incomplete when nextCursor remains after the page cap', async () => { + const listFn = vi.fn().mockResolvedValue(tagsOk(['a'], 'more')); + const outcome = await listAllRegistryTagsBounded(listFn, 'ghcr.io', 'acme/app', null, { maxPages: 2 }); + expect(outcome.kind).toBe('incomplete'); + expect(listFn).toHaveBeenCalledTimes(2); + }); + + it('returns complete when pagination exhausts', async () => { + const listFn = vi.fn() + .mockResolvedValueOnce(tagsOk(['a'], 'c1')) + .mockResolvedValueOnce(tagsOk(['b'])); + const outcome = await listAllRegistryTagsBounded(listFn, 'ghcr.io', 'acme/app', null); + expect(outcome).toEqual({ kind: 'complete', tags: ['a', 'b'] }); + }); }); diff --git a/backend/src/bootstrap/shutdown.ts b/backend/src/bootstrap/shutdown.ts index 10533ffa..fe2ba8af 100644 --- a/backend/src/bootstrap/shutdown.ts +++ b/backend/src/bootstrap/shutdown.ts @@ -6,6 +6,7 @@ import { AutoHealService } from '../services/AutoHealService'; import { HealthGateService } from '../services/HealthGateService'; import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService'; import { FleetSyncRetryService } from '../services/FleetSyncRetryService'; +import { SuppressionRetractionRetryService } from '../services/SuppressionRetractionRetryService'; import { DockerEventManager } from '../services/DockerEventManager'; import { ImageUpdateService } from '../services/ImageUpdateService'; import { SchedulerService } from '../services/SchedulerService'; @@ -37,6 +38,9 @@ export function installShutdownHandlers(server: Server): void { try { HealthGateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] HealthGateService cleanup failed:', (e as Error).message); } try { ServiceUpdateRecoveryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] ServiceUpdateRecoveryService cleanup failed:', (e as Error).message); } try { FleetSyncRetryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] FleetSyncRetryService cleanup failed:', (e as Error).message); } + try { SuppressionRetractionRetryService.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] SuppressionRetractionRetryService cleanup failed:', (e as Error).message); + } try { DockerEventManager.getInstance().stop(); } catch (e) { console.warn('[Shutdown] DockerEventManager cleanup failed:', (e as Error).message); } diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index f3415e6c..5b6bda80 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -15,6 +15,7 @@ import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryS import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService'; import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService'; import { FleetSyncRetryService } from '../services/FleetSyncRetryService'; +import { SuppressionRetractionRetryService } from '../services/SuppressionRetractionRetryService'; import { DockerEventManager } from '../services/DockerEventManager'; import TrivyService, { sweepStaleTrivyTempDirs } from '../services/TrivyService'; import { ImageUpdateService } from '../services/ImageUpdateService'; @@ -154,6 +155,7 @@ export async function startServer(server: Server): Promise { HealthGateService.getInstance().start(); ServiceUpdateRecoveryService.getInstance().start(); FleetSyncRetryService.getInstance().start(); + SuppressionRetractionRetryService.getInstance().start(); ImageUpdateService.getInstance().start(); SchedulerService.getInstance().start(); MfaService.getInstance().start(); @@ -166,7 +168,11 @@ export async function startServer(server: Server): Promise { // Drop the cached /api/meta entry on tunnel reconnect so the next // /api/nodes/:id/meta refetches fresh capabilities and version through // the live loopback bridge instead of waiting for the 3-minute TTL. - PilotTunnelManager.getInstance().on('tunnel-up', invalidateRemoteMetaCache); + // Also flush durable mute-replica retractions that waited for this node. + PilotTunnelManager.getInstance().on('tunnel-up', (nodeId: number) => { + invalidateRemoteMetaCache(nodeId); + void SuppressionRetractionRetryService.getInstance().flushNode(nodeId); + }); // Most async initializers still run in parallel. Docker event monitoring // is sequenced after self identity so it never classifies Sencho's own diff --git a/backend/src/cli/disableSso.ts b/backend/src/cli/disableSso.ts index 63c53d3c..40c799be 100644 --- a/backend/src/cli/disableSso.ts +++ b/backend/src/cli/disableSso.ts @@ -9,42 +9,97 @@ * With no argument it disables every enabled provider. The stored configuration * is preserved (only the enabled flag is cleared) so it can be fixed and * re-enabled from the UI. Written to the audit log with actor `cli`. + * + * When authentication_mode is sso_only and every provider is disabled (no + * argument), this command restores local_and_sso first so the operator is + * never left with SSO-only and zero providers. A named-provider disable that + * would remove the last enabled provider under sso_only is rejected; use the + * no-argument form or enableLocalLogin instead. Disabling one of several + * providers leaves authentication_mode unchanged. */ import { DatabaseService } from '../services/DatabaseService'; +import { + getAuthenticationMode, + setAuthenticationMode, +} from '../helpers/authenticationMode'; import { auditCli, exitWith, type CliResult } from './_shared'; +function restoreLocalLoginIfNeeded(db: DatabaseService): CliResult | null { + const mode = getAuthenticationMode(db); + if (mode !== 'sso_only') return null; + try { + setAuthenticationMode('local_and_sso', db); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { + ok: false, + message: `Failed to re-enable local login before disabling SSO: ${detail}. Providers were left unchanged.`, + }; + } + auditCli(db, '/cli/enable-local-login', 'CLI re-enabled local password authentication before disabling SSO'); + return null; +} + export function disableSso(provider?: string): CliResult { - const db = DatabaseService.getInstance(); + const db = DatabaseService.getInstance(); - if (provider) { - const config = db.getSSOConfig(provider); - if (!config) { - return { ok: false, message: `No SSO config found for provider: ${provider}` }; - } - if (config.enabled !== 1) { - return { ok: true, message: `SSO provider ${provider} is already disabled.` }; - } - db.upsertSSOConfig(provider, false, config.config_json); - auditCli(db, `/cli/disable-sso/${provider}`, `CLI disabled SSO provider ${provider}`); - return { ok: true, message: `Disabled SSO provider ${provider}. Its configuration was preserved.` }; + if (provider) { + const config = db.getSSOConfig(provider); + if (!config) { + return { ok: false, message: `No SSO config found for provider: ${provider}` }; + } + if (config.enabled !== 1) { + return { ok: true, message: `SSO provider ${provider} is already disabled.` }; } - const enabled = db.getEnabledSSOConfigs(); - if (enabled.length === 0) { - return { ok: true, message: 'No SSO providers are currently enabled.' }; + const mode = getAuthenticationMode(db); + if (mode === 'sso_only') { + const enabled = db.getEnabledSSOConfigs(); + if (enabled.length === 1 && enabled[0].provider === provider) { + return { + ok: false, + message: + `Cannot disable the last SSO provider while SSO-only mode is active. ` + + `Run without a provider argument, or run enableLocalLogin first.`, + }; + } } - for (const config of enabled) { - db.upsertSSOConfig(config.provider, false, config.config_json); - } - const names = enabled.map(c => c.provider).join(', '); - auditCli(db, '/cli/disable-sso', `CLI disabled all SSO providers (${enabled.length})`); - return { ok: true, message: `Disabled ${enabled.length} SSO provider(s): ${names}. Configurations were preserved.` }; + + // Named disable leaves authentication_mode unchanged (including sso_only). + db.upsertSSOConfig(provider, false, config.config_json); + auditCli(db, `/cli/disable-sso/${provider}`, `CLI disabled SSO provider ${provider}`); + const modeNote = + mode === 'sso_only' + ? ' Authentication mode remains SSO only; remaining providers stay available.' + : ''; + return { + ok: true, + message: `Disabled SSO provider ${provider}. Its configuration was preserved.${modeNote}`, + }; + } + + const enabled = db.getEnabledSSOConfigs(); + if (enabled.length === 0) { + const modeError = restoreLocalLoginIfNeeded(db); + if (modeError) return modeError; + return { ok: true, message: 'No SSO providers are currently enabled.' }; + } + + const modeError = restoreLocalLoginIfNeeded(db); + if (modeError) return modeError; + + for (const config of enabled) { + db.upsertSSOConfig(config.provider, false, config.config_json); + } + const names = enabled.map(c => c.provider).join(', '); + auditCli(db, '/cli/disable-sso', `CLI disabled all SSO providers (${enabled.length})`); + return { ok: true, message: `Disabled ${enabled.length} SSO provider(s): ${names}. Configurations were preserved.` }; } function main(): void { - exitWith(disableSso(process.argv[2])); + exitWith(disableSso(process.argv[2])); } if (require.main === module) { - main(); + main(); } diff --git a/backend/src/cli/enableLocalLogin.ts b/backend/src/cli/enableLocalLogin.ts new file mode 100644 index 00000000..178ce0da --- /dev/null +++ b/backend/src/cli/enableLocalLogin.ts @@ -0,0 +1,52 @@ +/** + * Emergency CLI: re-enable local password authentication after SSO-only mode + * locks out interactive password login (for example when the identity provider + * is unavailable). + * + * Run via: + * docker compose exec sencho node dist/cli/enableLocalLogin.js + * + * Requires local shell or Docker-host access. Does not contact the identity + * provider. Written to the audit log with actor `cli`. Restart Sencho after + * running so the in-process settings cache picks up the change. + */ +import { DatabaseService } from '../services/DatabaseService'; +import { + getAuthenticationMode, + setAuthenticationMode, +} from '../helpers/authenticationMode'; +import { auditCli, exitWith, type CliResult } from './_shared'; + +export function enableLocalLogin(): CliResult { + const db = DatabaseService.getInstance(); + const current = getAuthenticationMode(db); + if (current === 'local_and_sso') { + return { + ok: true, + message: + 'Local password authentication is already enabled (authentication_mode=local_and_sso).', + }; + } + + try { + setAuthenticationMode('local_and_sso', db); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + return { ok: false, message: `Failed to re-enable local login: ${detail}` }; + } + + auditCli(db, '/cli/enable-local-login', 'CLI re-enabled local password authentication'); + return { + ok: true, + message: + 'Local login re-enabled. Restart Sencho for the change to take effect: docker compose restart sencho', + }; +} + +function main(): void { + exitWith(enableLocalLogin()); +} + +if (require.main === module) { + main(); +} diff --git a/backend/src/helpers/authenticationMode.ts b/backend/src/helpers/authenticationMode.ts new file mode 100644 index 00000000..a355c3da --- /dev/null +++ b/backend/src/helpers/authenticationMode.ts @@ -0,0 +1,37 @@ +import { DatabaseService } from '../services/DatabaseService'; + +export const AUTHENTICATION_MODES = ['local_and_sso', 'sso_only'] as const; +export type AuthenticationMode = (typeof AUTHENTICATION_MODES)[number]; + +export const AUTHENTICATION_MODE_KEY = 'authentication_mode'; +export const DEFAULT_AUTHENTICATION_MODE: AuthenticationMode = 'local_and_sso'; + +/** Read the cached global setting; missing or unknown values default to local_and_sso. */ +export function getAuthenticationMode(db: DatabaseService = DatabaseService.getInstance()): AuthenticationMode { + const raw = db.getGlobalSettings()[AUTHENTICATION_MODE_KEY]; + if (raw === 'sso_only') return 'sso_only'; + return DEFAULT_AUTHENTICATION_MODE; +} + +export function isLocalLoginEnabled(db: DatabaseService = DatabaseService.getInstance()): boolean { + return getAuthenticationMode(db) !== 'sso_only'; +} + +export function setAuthenticationMode( + mode: AuthenticationMode, + db: DatabaseService = DatabaseService.getInstance(), +): void { + db.updateGlobalSetting(AUTHENTICATION_MODE_KEY, mode); +} + +export function isAuthenticationMode(value: unknown): value is AuthenticationMode { + return value === 'local_and_sso' || value === 'sso_only'; +} + +/** True when disabling/deleting this enabled provider would leave zero providers under sso_only. */ +export function wouldRemoveLastProvider(provider: string, currentlyEnabled: boolean): boolean { + if (!currentlyEnabled) return false; + if (isLocalLoginEnabled()) return false; + const enabled = DatabaseService.getInstance().getEnabledSSOConfigs(); + return enabled.length === 1 && enabled[0].provider === provider; +} diff --git a/backend/src/helpers/autoUpdateDigestGate.ts b/backend/src/helpers/autoUpdateDigestGate.ts new file mode 100644 index 00000000..5aa0a2b7 --- /dev/null +++ b/backend/src/helpers/autoUpdateDigestGate.ts @@ -0,0 +1,75 @@ +import type { ImageCheckResult } from '../services/ImageUpdateService'; + +/** Accumulator for auto-update image checks (digest vs tag-only vs error). */ +export interface AutoUpdateDigestGateState { + hasDigestUpdate: boolean; + hasTagOnlyUpdate: boolean; + updatedImages: string[]; + checkErrors: string[]; +} + +export function createAutoUpdateDigestGateState(): AutoUpdateDigestGateState { + return { + hasDigestUpdate: false, + hasTagOnlyUpdate: false, + updatedImages: [], + checkErrors: [], + }; +} + +/** + * Record one image check into the digest gate. Only digest drift is Compose- + * actionable for auto-update; tag bumps stay advisory. + */ +export function recordAutoUpdateImageCheck( + state: AutoUpdateDigestGateState, + imageRef: string, + result: ImageCheckResult, +): void { + if (result.digestUpdate) { + state.hasDigestUpdate = true; + state.updatedImages.push(imageRef); + return; + } + if (result.tagUpdate) { + state.hasTagOnlyUpdate = true; + return; + } + if (result.error || result.checkStatus === 'failed' || result.checkStatus === 'partial') { + state.checkErrors.push(result.error ?? 'Update check incomplete'); + } +} + +/** + * Operator message when a sibling image check failed and a full-stack Compose + * update must not run (it would pull/recreate the unverified image as + * collateral). Null when digest apply may proceed. + */ +export function messageWhenDigestApplyBlockedByCheckErrors( + stackName: string, + state: Pick, +): string | null { + if (!state.hasDigestUpdate || state.checkErrors.length === 0) return null; + return `Stack "${stackName}": WARNING - digest update available but ${state.checkErrors.length} image check(s) failed; skipped auto-update (${state.checkErrors.join('; ')}).`; +} + +/** Operator message when no digest-actionable update was found. */ +export function messageWhenNoDigestUpdate( + stackName: string, + state: Pick, + imageRefCount: number, +): string { + if (state.hasTagOnlyUpdate) { + const errNote = state.checkErrors.length > 0 + ? ` (${state.checkErrors.length} check(s) failed)` + : ''; + return `Stack "${stackName}": newer tag available but Compose pin unchanged; skipped auto-update${errNote}.`; + } + if (state.checkErrors.length > 0 && state.checkErrors.length === imageRefCount) { + return `Stack "${stackName}": WARNING - all image checks failed (${state.checkErrors.join('; ')}). Unable to determine update status.`; + } + if (state.checkErrors.length > 0) { + return `Stack "${stackName}": all reachable images up to date (${state.checkErrors.length} check(s) failed).`; + } + return `Stack "${stackName}": all images up to date.`; +} diff --git a/backend/src/helpers/constants.ts b/backend/src/helpers/constants.ts index 51de5540..5e30597f 100644 --- a/backend/src/helpers/constants.ts +++ b/backend/src/helpers/constants.ts @@ -23,6 +23,11 @@ export const MAX_ASSIGNMENTS = 1000; // Session cookies export const COOKIE_NAME = 'sencho_token'; export const SESSION_COOKIE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours +export const REMEMBER_SESSION_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days, "stay signed in" +// Sliding-refresh window: a user-session token with less than this much life +// left gets silently reissued with a fresh full TTL, so continued activity +// never runs into the hard expiry. See middleware/auth.ts::authMiddleware. +export const SESSION_REFRESH_THRESHOLD_MS = 60 * 60 * 1000; // 1 hour export const MFA_PENDING_COOKIE_NAME = 'sencho_mfa_pending'; export const MFA_PENDING_SCOPE = 'mfa_pending'; export const MFA_PENDING_TTL_MS = 5 * 60 * 1000; // 5 minutes to complete the challenge diff --git a/backend/src/helpers/cookies.ts b/backend/src/helpers/cookies.ts index b040ac5f..066f2689 100644 --- a/backend/src/helpers/cookies.ts +++ b/backend/src/helpers/cookies.ts @@ -1,15 +1,19 @@ import type { Request } from 'express'; -import { SESSION_COOKIE_MAX_AGE_MS } from './constants'; /** True when the request arrived over HTTPS, either directly or via a trusted TLS-terminating proxy. */ export const isSecureRequest = (req: Request): boolean => { return req.secure || req.headers['x-forwarded-proto'] === 'https'; }; -/** Cookie options derived from the current request (secure flag follows the connection). */ +/** + * Cookie options derived from the current request (secure flag follows the + * connection). Lifetime is deliberately not included: each caller sets its own + * `maxAge` (session cookies vary between the default and "stay signed in", the + * MFA-pending cookie is minutes long), so a shared default here would only ever + * be overridden or misread. + */ export const getCookieOptions = (req: Request) => ({ httpOnly: true, secure: isSecureRequest(req), sameSite: 'strict' as const, - maxAge: SESSION_COOKIE_MAX_AGE_MS, }); diff --git a/backend/src/helpers/fleetUpdateCache.ts b/backend/src/helpers/fleetUpdateCache.ts new file mode 100644 index 00000000..b691f967 --- /dev/null +++ b/backend/src/helpers/fleetUpdateCache.ts @@ -0,0 +1,34 @@ +import { CacheService } from '../services/CacheService'; + +/** Hub aggregation cache for GET /api/image-updates/fleet. */ +export const FLEET_UPDATE_CACHE_KEY = 'fleet-updates'; + +/** + * Drop the hub fleet-updates aggregation. Generation-aware via CacheService: + * an in-flight getOrFetch started before this call cannot commit afterward. + */ +export function invalidateFleetUpdateCache(): void { + CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY); +} + +function pathWithoutQuery(pathAfterApi: string): string { + return pathAfterApi.split('?')[0] ?? pathAfterApi; +} + +/** + * True when `pathAfterApi` is a full-stack update route + * (`/stacks/:name/update`), not a service-scoped update/restore. + * `pathAfterApi` is the Express path after the `/api` mount strip + * (for example `/stacks/paperless/update`). + */ +export function isFullStackUpdatePath(pathAfterApi: string): boolean { + return /^\/stacks\/[^/]+\/update\/?$/.test(pathWithoutQuery(pathAfterApi)); +} + +/** + * True when `pathAfterApi` is the stack update-preview route + * (`/stacks/:name/update-preview`). + */ +export function isUpdatePreviewPath(pathAfterApi: string): boolean { + return /^\/stacks\/[^/]+\/update-preview\/?$/.test(pathWithoutQuery(pathAfterApi)); +} diff --git a/backend/src/helpers/healthcheckPresence.ts b/backend/src/helpers/healthcheckPresence.ts new file mode 100644 index 00000000..25355bce --- /dev/null +++ b/backend/src/helpers/healthcheckPresence.ts @@ -0,0 +1,58 @@ +/** + * Structural healthcheck presence classification for Compose YAML objects and + * Docker inspect `Config.Healthcheck.Test` arrays. Returns enums/booleans only; + * never retains or returns Test command text (commands can carry secrets). + */ + +export type ComposeHealthcheckClass = 'active' | 'disabled' | 'absent'; + +/** + * Classify a Compose `healthcheck:` value from the rendered effective model. + * `disable: true` and `test: NONE` / `["NONE"]` are explicit disablement. + * An empty object, `disable: false` alone, or timing-only fields without a + * `test` are absent (fall through to runtime/image evidence), not active. + */ +export function classifyComposeHealthcheck(healthcheck: unknown): ComposeHealthcheckClass { + if (healthcheck == null) return 'absent'; + if (typeof healthcheck !== 'object' || Array.isArray(healthcheck)) return 'absent'; + const hc = healthcheck as Record; + if (hc.disable === true) return 'disabled'; + if (isNoneTest(hc.test)) return 'disabled'; + if (hasActiveTest(hc.test)) return 'active'; + return 'absent'; +} + +/** True when the Compose healthcheck is an active (non-disabled) declaration. */ +export function isComposeHealthcheckActive(healthcheck: unknown): boolean { + return classifyComposeHealthcheck(healthcheck) === 'active'; +} + +/** + * True when Docker's effective healthcheck Test is present and active. + * Empty / missing / `NONE` / `["NONE"]` are inactive. + */ +export function isDockerHealthcheckActive(test: unknown): boolean { + return hasActiveTest(test); +} + +/** True when `test` is a non-empty, non-NONE healthcheck command. */ +function hasActiveTest(test: unknown): boolean { + if (test == null) return false; + if (typeof test === 'string') { + const trimmed = test.trim(); + return trimmed.length > 0 && !isNoneToken(trimmed); + } + if (!Array.isArray(test) || test.length === 0) return false; + if (isNoneTest(test)) return false; + return true; +} + +function isNoneTest(test: unknown): boolean { + if (typeof test === 'string') return isNoneToken(test); + if (!Array.isArray(test) || test.length === 0) return false; + return test.length === 1 && typeof test[0] === 'string' && isNoneToken(test[0]); +} + +function isNoneToken(value: string): boolean { + return value.trim().toUpperCase() === 'NONE'; +} diff --git a/backend/src/helpers/notificationSuppressionSync.ts b/backend/src/helpers/notificationSuppressionSync.ts index a5f641c8..922dc526 100644 --- a/backend/src/helpers/notificationSuppressionSync.ts +++ b/backend/src/helpers/notificationSuppressionSync.ts @@ -1,9 +1,10 @@ -import { DatabaseService, type NotificationSuppressionRule, type Node } from '../services/DatabaseService'; +import { DatabaseService, type NotificationSuppressionRule, type NotificationSuppressionRetraction, type NotificationSuppressionRetractionKind, type Node } from '../services/DatabaseService'; import { NodeRegistry } from '../services/NodeRegistry'; import { LicenseService } from '../services/LicenseService'; import { PROXY_TIER_HEADER } from '../services/license-headers'; import { NOTIFICATION_SUPPRESSION_SCHEDULE_CAPABILITY, + NOTIFICATION_SUPPRESSION_REPLICA_RETRACTION_CAPABILITY, } from '../services/CapabilityRegistry'; import { remoteAdvertisesCapability } from './remoteCapabilities'; import { getErrorMessage } from '../utils/errors'; @@ -20,6 +21,13 @@ function buildRemoteHeaders(apiToken: string): Record { return headers; } +function retractionFor( + kind: NotificationSuppressionRetractionKind, + rule: Pick, +): NotificationSuppressionRetraction { + return { kind, source_updated_at: rule.updated_at }; +} + /** Hub node ids that should receive a replica for this rule (before wire identity normalize). */ export function replicationTargetIds(rule: NotificationSuppressionRule): number[] { return replicationTargets(rule).map((n) => n.id); @@ -35,46 +43,171 @@ export function replicationTargets(rule: NotificationSuppressionRule): Node[] { return remotes; } +/** Every registered remote: permanent deletes must reach historical replicas, not only current scope. */ +export function allRemoteNodes(): Node[] { + return DatabaseService.getInstance().getNodes().filter((n) => n.type === 'remote'); +} + function replicaPayload(rule: NotificationSuppressionRule): NotificationSuppressionRule { return { ...rule, node_id: null }; } -async function pushRuleToNode(node: Node, rule: NotificationSuppressionRule): Promise { - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target?.apiUrl) { - console.warn(`[SuppressionSync] Skipping node "${node.name}": no proxy target`); - return; - } - const baseUrl = target.apiUrl.replace(/\/$/, ''); - const res = await fetch(`${baseUrl}/api/notification-suppression-rules/replica`, { - method: 'POST', - headers: buildRemoteHeaders(target.apiToken), - body: JSON.stringify({ rule: replicaPayload(rule) }), - signal: AbortSignal.timeout(SYNC_TIMEOUT_MS), - }); - if (!res.ok) { - const body = await res.text().catch(() => ''); - throw new Error(`HTTP ${res.status}${body ? `: ${body.slice(0, 200)}` : ''}`); +type RetractionSupport = 'supported' | 'unsupported' | 'unreachable'; + +async function probeRetractionSupport(nodeId: number): Promise { + try { + const meta = await NodeRegistry.getInstance().fetchMetaForNode(nodeId); + // fetchMetaForNode returns OFFLINE_META (online:false, capabilities:[]) on + // transport failure instead of throwing; treat that as unreachable. + if (meta.online === false) return 'unreachable'; + return meta.capabilities.includes(NOTIFICATION_SUPPRESSION_REPLICA_RETRACTION_CAPABILITY) + ? 'supported' + : 'unsupported'; + } catch (err) { + console.warn( + `[SuppressionSync] Retraction capability probe failed for node ${nodeId}; treating as unreachable:`, + getErrorMessage(err, 'unknown'), + ); + return 'unreachable'; } } -async function deleteRuleOnNode(node: Node, ruleId: number): Promise { +function enqueuePending( + node: Node, + ruleId: number, + retraction: NotificationSuppressionRetraction, + lastError: string, +): void { + DatabaseService.getInstance().upsertNotificationSuppressionPendingRetraction({ + rule_id: ruleId, + node_id: node.id, + kind: retraction.kind, + source_updated_at: retraction.source_updated_at, + last_error: lastError, + }); +} + +function clearPending(nodeId: number, ruleId: number): void { + DatabaseService.getInstance().deleteNotificationSuppressionPendingRetraction(ruleId, nodeId); +} + +function resolveRemoteApi(node: Node): { baseUrl: string; apiToken: string } | null { const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target?.apiUrl) { - throw new Error(`no proxy target for node "${node.name}" (id=${node.id})`); + if (!target?.apiUrl) return null; + return { baseUrl: target.apiUrl.replace(/\/$/, ''), apiToken: target.apiToken }; +} + +/** Non-2xx (including opaque 404) is always failure; never treat missing routes as applied. */ +async function throwHttpFailure(res: Response): Promise { + const body = await res.text().catch(() => ''); + throw new Error(`HTTP ${res.status}${body ? `: ${body.slice(0, 200)}` : ''}`); +} + +/** Prefer JSON outcome; fall back to applied for pre-outcome remotes (bare 2xx). */ +async function readOutcome(res: Response): Promise { + try { + const json = (await res.json()) as { outcome?: string }; + return typeof json.outcome === 'string' ? json.outcome : 'applied'; + } catch { + return 'applied'; } - const baseUrl = target.apiUrl.replace(/\/$/, ''); - const res = await fetch(`${baseUrl}/api/notification-suppression-rules/replica/${ruleId}`, { - method: 'DELETE', - headers: buildRemoteHeaders(target.apiToken), +} + +async function pushRuleToNode(node: Node, rule: NotificationSuppressionRule): Promise { + const remote = resolveRemoteApi(node); + if (!remote) { + console.warn(`[SuppressionSync] Skipping node "${node.name}": no proxy target`); + return; + } + const res = await fetch(`${remote.baseUrl}/api/notification-suppression-rules/replica`, { + method: 'POST', + headers: buildRemoteHeaders(remote.apiToken), + body: JSON.stringify({ rule: replicaPayload(rule) }), signal: AbortSignal.timeout(SYNC_TIMEOUT_MS), }); - if (!res.ok && res.status !== 404) { - const body = await res.text().catch(() => ''); - throw new Error(`HTTP ${res.status}${body ? `: ${body.slice(0, 200)}` : ''}`); + if (!res.ok) await throwHttpFailure(res); + const outcome = await readOutcome(res); + if (outcome !== 'applied') { + throw new Error(`replica POST outcome=${outcome}`); } } +/** + * Deliver a replica DELETE. On transport failure, durable-queues the retraction. + * Clears pending only when the remote reports outcome=applied (not ignored_stale). + */ +export async function deleteRuleOnNode( + node: Node, + ruleId: number, + retraction: NotificationSuppressionRetraction, +): Promise<{ outcome: string }> { + const remote = resolveRemoteApi(node); + if (!remote) { + const err = `no proxy target for node "${node.name}" (id=${node.id})`; + enqueuePending(node, ruleId, retraction, err); + throw new Error(err); + } + try { + const res = await fetch(`${remote.baseUrl}/api/notification-suppression-rules/replica/${ruleId}`, { + method: 'DELETE', + headers: buildRemoteHeaders(remote.apiToken), + body: JSON.stringify(retraction), + signal: AbortSignal.timeout(SYNC_TIMEOUT_MS), + }); + if (!res.ok) await throwHttpFailure(res); + const outcome = await readOutcome(res); + if (outcome !== 'applied') { + // ignored_stale and any other non-applied outcome keep the outbox row. + const reason = + outcome === 'ignored_stale' + ? 'remote ignored_stale; rule still present' + : `remote DELETE outcome=${outcome}`; + enqueuePending(node, ruleId, retraction, reason); + return { outcome }; + } + clearPending(node.id, ruleId); + return { outcome }; + } catch (err) { + try { + enqueuePending(node, ruleId, retraction, getErrorMessage(err, String(err))); + } catch (enqueueErr) { + console.error( + `[SuppressionSync] Failed to durable-queue retract rule=${ruleId} node=${node.id}:`, + getErrorMessage(enqueueErr, String(enqueueErr)), + ); + } + throw err; + } +} + +/** + * Recoverable DELETE only when the remote advertises versioned retractions. + * Otherwise enqueue pending and do not bare-delete on incompatible remotes. + * @returns applied when DELETE was accepted; deferred when queued without sending + * or when the remote ignored a stale watermark. + */ +async function deliverRecoverableDelete( + node: Node, + ruleId: number, + retraction: NotificationSuppressionRetraction, + context: string, +): Promise<'applied' | 'deferred'> { + const support = await probeRetractionSupport(node.id); + if (support !== 'supported') { + const reason = + support === 'unsupported' + ? 'remote lacks notification-suppression-replica-retraction; not sending recoverable DELETE' + : 'remote unreachable for retraction capability probe; not sending recoverable DELETE'; + enqueuePending(node, ruleId, retraction, reason); + console.warn( + `[SuppressionSync] ${context} on node "${node.name}" (id=${node.id}): ${reason}; queued pending retraction`, + ); + return 'deferred'; + } + const { outcome } = await deleteRuleOnNode(node, ruleId, retraction); + return outcome === 'applied' ? 'applied' : 'deferred'; +} + async function pushOrCleanupScheduled(node: Node, rule: NotificationSuppressionRule): Promise { const supportsSchedule = await remoteAdvertisesCapability( node.id, @@ -84,36 +217,46 @@ async function pushOrCleanupScheduled(node: Node, rule: NotificationSuppressionR await pushRuleToNode(node, rule); return; } - // Probe false means unsupported OR unreachable. Never POST a scheduled rule - // through the legacy contract (older remotes would mute all day). Attempt DELETE; - // only claim cleanup when DELETE succeeds. + // Never POST a scheduled rule through the legacy contract. Soft-cleanup DELETE + // requires versioned retraction support so incompatible remotes are not bare-deleted. try { - await deleteRuleOnNode(node, rule.id); - console.warn( - `[SuppressionSync] Scheduled rule ${rule.id} not applied on node "${node.name}" (id=${node.id}): ` + - `capability unsupported-or-unreachable; DELETE succeeded and replica was removed`, + const result = await deliverRecoverableDelete( + node, + rule.id, + retractionFor('recoverable', rule), + `Scheduled rule ${rule.id} not applied`, ); + if (result === 'applied') { + console.warn( + `[SuppressionSync] Scheduled rule ${rule.id} not applied on node "${node.name}" (id=${node.id}): ` + + `schedule unsupported-or-unreachable; recoverable DELETE applied`, + ); + } } catch (err) { console.error( `[SuppressionSync] Scheduled rule ${rule.id}: cleanup pending on node "${node.name}" (id=${node.id}); ` + - `capability unsupported-or-unreachable and DELETE failed (${getErrorMessage(err, String(err))}). ` + - `Prior replica may remain until connectivity returns and the rule is re-saved`, + `recoverable DELETE failed (${getErrorMessage(err, String(err))})`, ); } } async function cleanupInvalidScheduleReplica(node: Node, rule: NotificationSuppressionRule): Promise { - // Never POST an invalid schedule (would mute all day on remotes that ignore the field). - // Attempt DELETE so a prior valid/unscheduled replica cannot keep muting. try { - await deleteRuleOnNode(node, rule.id); - console.warn( - `[SuppressionSync] Corrupt schedule on rule ${rule.id}: replica removed on node "${node.name}" (id=${node.id}); not posting`, + const result = await deliverRecoverableDelete( + node, + rule.id, + retractionFor('recoverable', rule), + `Corrupt schedule on rule ${rule.id}`, ); + if (result === 'applied') { + console.warn( + `[SuppressionSync] Corrupt schedule on rule ${rule.id}: recoverable DELETE applied on node "${node.name}" (id=${node.id}); not posting`, + ); + } } catch (err) { console.error( `[SuppressionSync] Corrupt schedule on rule ${rule.id}: cleanup pending on node "${node.name}" (id=${node.id}); ` + - `DELETE failed (${getErrorMessage(err, String(err))}). Prior replica may remain until connectivity returns`, + `DELETE failed (${getErrorMessage(err, String(err))})`, ); } } @@ -160,13 +303,19 @@ export function syncSuppressionRuleUpdateToFleet( const newIds = new Set(replicationTargetIds(updated)); const db = DatabaseService.getInstance(); const staleIds = [...oldIds].filter((id) => !newIds.has(id)); + const staleRetraction = retractionFor('recoverable', updated); void Promise.allSettled([ ...staleIds.map(async (id) => { const node = db.getNode(id); if (!node || node.type !== 'remote') return; try { - await deleteRuleOnNode(node, previous.id); + await deliverRecoverableDelete( + node, + previous.id, + staleRetraction, + `Stale-target retract for rule ${previous.id}`, + ); } catch (err) { console.error( `[SuppressionSync] Failed to delete stale rule ${previous.id} on node "${node.name}":`, @@ -187,14 +336,18 @@ export function syncSuppressionRuleUpdateToFleet( ]); } -/** Best-effort delete of a replicated rule on fleet nodes. */ +/** + * Authoritative delete: fan out permanent retraction to every known remote + * (not only current scope), and durable-queue failures for retry. + */ export function deleteSuppressionRuleFromFleet(rule: NotificationSuppressionRule): void { - const targets = replicationTargets(rule); + const targets = allRemoteNodes(); if (targets.length === 0) return; + const retraction = retractionFor('permanent', rule); void Promise.allSettled( targets.map(async (node) => { try { - await deleteRuleOnNode(node, rule.id); + await deleteRuleOnNode(node, rule.id, retraction); } catch (err) { console.error( `[SuppressionSync] Failed to delete rule ${rule.id} on node "${node.name}":`, @@ -204,3 +357,40 @@ export function deleteSuppressionRuleFromFleet(rule: NotificationSuppressionRule }), ); } + +/** + * Retry durable pending retractions for one node (tunnel-up / reconnect) or all. + * Recoverable rows still require the remote to advertise retraction support. + */ +export async function flushPendingSuppressionRetractions(nodeId?: number): Promise { + const db = DatabaseService.getInstance(); + const pending = db.listNotificationSuppressionPendingRetractions(nodeId); + for (const row of pending) { + const node = db.getNode(row.node_id); + if (!node || node.type !== 'remote') { + db.deleteNotificationSuppressionPendingRetraction(row.rule_id, row.node_id); + continue; + } + const retraction: NotificationSuppressionRetraction = { + kind: row.kind, + source_updated_at: row.source_updated_at, + }; + try { + if (retraction.kind === 'recoverable') { + await deliverRecoverableDelete( + node, + row.rule_id, + retraction, + `Pending recoverable retract for rule ${row.rule_id}`, + ); + } else { + await deleteRuleOnNode(node, row.rule_id, retraction); + } + } catch (err) { + console.error( + `[SuppressionSync] Pending retract retry failed rule=${row.rule_id} node=${node.name}:`, + getErrorMessage(err, String(err)), + ); + } + } +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 6aac5532..55f3b418 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -40,6 +40,7 @@ import { autoHealRouter } from './routes/autoHeal'; import { notificationsRouter, notificationRoutesRouter, notificationSuppressionRouter } from './routes/notifications'; import { consoleRouter } from './routes/console'; import { ssoConfigRouter } from './routes/ssoConfig'; +import { authModeRouter } from './routes/authMode'; import { registriesRouter } from './routes/registries'; import { systemMaintenanceRouter } from './routes/systemMaintenance'; import { volumesRouter } from './routes/volumes'; @@ -139,6 +140,7 @@ app.use('/api/notification-routes', notificationRoutesRouter); app.use('/api/notification-suppression-rules', notificationSuppressionRouter); app.use('/api/system', consoleRouter); app.use('/api/sso/config', ssoConfigRouter); +app.use('/api/sso/auth-mode', authModeRouter); app.use('/api/registries', registriesRouter); app.use('/api/system', systemMaintenanceRouter); app.use('/api/volumes', volumesRouter); diff --git a/backend/src/middleware/auth.ts b/backend/src/middleware/auth.ts index d704d9cc..86434dc2 100644 --- a/backend/src/middleware/auth.ts +++ b/backend/src/middleware/auth.ts @@ -17,6 +17,9 @@ import { MFA_PENDING_COOKIE_NAME, MFA_PENDING_SCOPE, MFA_PENDING_TTL_MS, + SESSION_COOKIE_MAX_AGE_MS, + REMEMBER_SESSION_MAX_AGE_MS, + SESSION_REFRESH_THRESHOLD_MS, } from '../helpers/constants'; import { getCookieOptions } from '../helpers/cookies'; import { looksLikeApiToken } from '../utils/apiTokenFormat'; @@ -79,7 +82,7 @@ export const authMiddleware: RequestHandler = async (req: Request, res: Response const settings = DatabaseService.getInstance().getGlobalSettings(); const jwtSecret = settings.auth_jwt_secret; if (!jwtSecret) throw new Error('No JWT secret'); - const decoded = jwt.verify(token, jwtSecret) as { username?: string; role?: string; scope?: string; tv?: number; user_id?: number; sso?: boolean }; + const decoded = jwt.verify(token, jwtSecret) as { username?: string; role?: string; scope?: string; tv?: number; user_id?: number; sso?: boolean; remember?: boolean; exp?: number }; if (isDebugEnabled()) console.log('[Auth:diag] Token type:', bearerToken ? 'bearer' : 'cookie', 'scope:', decoded.scope || 'user-session'); @@ -161,8 +164,11 @@ export const authMiddleware: RequestHandler = async (req: Request, res: Response } // Token version check: rejects sessions after password change, role change, or admin reset. - // Pre-migration tokens (no tv claim) are accepted for backward compat and expire within 24h. - if (decoded.tv !== undefined && dbUser.token_version !== decoded.tv) { + // A token without a tv claim is a pre-migration legacy token minted when + // token_version was 1. Default to 1 on decode so the token is rejected only + // when a security event (password change, MFA reset, role change, admin + // invalidation) has actually bumped the version since it was issued. + if (dbUser.token_version !== (decoded.tv ?? 1)) { if (isDebugEnabled()) console.log('[Auth:diag] Token version mismatch for:', decoded.username, 'jwt:', decoded.tv, 'db:', dbUser.token_version); console.log('[Auth] Session rejected: token version mismatch for:', decoded.username); res.status(401).json({ error: 'Session invalidated. Please log in again.' }); @@ -173,6 +179,28 @@ export const authMiddleware: RequestHandler = async (req: Request, res: Response // Use the DB role (not the JWT role) so role changes take effect immediately req.user = { username: dbUser.username, role: dbUser.role as UserRole, userId: dbUser.id }; + const remember = decoded.remember === true; + req.sessionRemember = remember; + + // Sliding refresh: a session nearing its expiry gets silently reissued with + // a fresh full TTL (matching whichever TTL, 24h or "stay signed in" 30d, the + // original login chose), so an actively-used tab never runs into the hard + // cutoff. Disabled via the session_sliding_refresh setting (default on) for + // admins who want a strict absolute session ceiling. This is a best-effort + // optimization on an already-authenticated request, so its own try/catch + // keeps a reissue failure from being reported as an invalid token. + if (settings.session_sliding_refresh !== '0' && typeof decoded.exp === 'number') { + const remainingMs = decoded.exp * 1000 - Date.now(); + const shouldRefresh = remainingMs < SESSION_REFRESH_THRESHOLD_MS; + if (isDebugEnabled()) console.log('[Auth:diag] Sliding refresh check:', decoded.username, 'remainingMs:', remainingMs, 'refreshed:', shouldRefresh); + if (shouldRefresh) { + try { + issueSessionCookie(res, req, dbUser, jwtSecret, remember); + } catch (refreshErr) { + console.error('[Auth] Sliding session refresh failed for', dbUser.username, getErrorMessage(refreshErr, 'unknown')); + } + } + } next(); } catch (err) { @@ -182,19 +210,48 @@ export const authMiddleware: RequestHandler = async (req: Request, res: Response } }; -/** Sign a session JWT and set it as an httpOnly cookie. */ +/** + * Drop any already-queued `Set-Cookie` entry for `name` on this response + * before a caller appends a new one. `res.cookie()` appends rather than + * replaces, so a request path that issues the same cookie twice (the + * sliding refresh in `authMiddleware` followed by a token-bump reissue in + * the same response, e.g. a password change inside the refresh window) + * would otherwise send two `Set-Cookie` headers for one name: the first + * carrying an already-superseded `token_version`. Browsers apply the last + * one, but any other client taking the first would treat itself as + * signed out on its very next request. Deduping keeps exactly one, correct + * cookie in the response regardless of call order. + */ +function dropQueuedCookie(res: Response, name: string): void { + const existing = res.getHeader('Set-Cookie'); + if (!existing) return; + const entries = Array.isArray(existing) ? existing : [String(existing)]; + const filtered = entries.filter((entry) => !entry.startsWith(`${name}=`)); + if (filtered.length !== entries.length) res.setHeader('Set-Cookie', filtered); +} + +/** + * Sign a session JWT and set it as an httpOnly cookie. `remember` extends the + * session to `REMEMBER_SESSION_MAX_AGE_MS` (30 days, "stay signed in") instead + * of the default `SESSION_COOKIE_MAX_AGE_MS` (24h); the choice is carried in + * the token's `remember` claim so a later sliding refresh (authMiddleware) or + * post-token-bump reissue (reissueSessionAfterTokenBump) reapplies the same TTL. + */ export function issueSessionCookie( res: Response, req: Request, user: { username: string; role: string; token_version: number }, jwtSecret: string, + remember = false, ): void { + const maxAgeMs = remember ? REMEMBER_SESSION_MAX_AGE_MS : SESSION_COOKIE_MAX_AGE_MS; const token = jwt.sign( - { username: user.username, role: user.role, tv: user.token_version }, + { username: user.username, role: user.role, tv: user.token_version, remember }, jwtSecret, - { expiresIn: '24h' }, + { expiresIn: Math.floor(maxAgeMs / 1000) }, ); - res.cookie(COOKIE_NAME, token, getCookieOptions(req)); + dropQueuedCookie(res, COOKIE_NAME); + res.cookie(COOKIE_NAME, token, { ...getCookieOptions(req), maxAge: maxAgeMs }); } /** @@ -209,10 +266,10 @@ export function issueMfaPendingCookie( req: Request, user: { id: number; username: string }, jwtSecret: string, - opts: { sso?: boolean } = {}, + opts: { sso?: boolean; remember?: boolean } = {}, ): void { const token = jwt.sign( - { scope: MFA_PENDING_SCOPE, user_id: user.id, username: user.username, sso: opts.sso === true }, + { scope: MFA_PENDING_SCOPE, user_id: user.id, username: user.username, sso: opts.sso === true, remember: opts.remember === true }, jwtSecret, { expiresIn: Math.floor(MFA_PENDING_TTL_MS / 1000) }, ); @@ -239,6 +296,6 @@ export function reissueSessionAfterTokenBump(req: Request, res: Response, userId const refreshed = db.getUserById(userId); const settings = db.getGlobalSettings(); if (refreshed && settings.auth_jwt_secret) { - issueSessionCookie(res, req, refreshed, settings.auth_jwt_secret); + issueSessionCookie(res, req, refreshed, settings.auth_jwt_secret, req.sessionRemember === true); } } diff --git a/backend/src/proxy/remoteNodeProxy.ts b/backend/src/proxy/remoteNodeProxy.ts index e3a35185..abef7123 100644 --- a/backend/src/proxy/remoteNodeProxy.ts +++ b/backend/src/proxy/remoteNodeProxy.ts @@ -11,6 +11,7 @@ import { DatabaseService } from '../services/DatabaseService'; import { redactSensitiveText } from '../utils/safeLog'; import { isDebugEnabled } from '../utils/debug'; import { logDebugTiming, templatizeHydrationPath } from '../utils/requestTiming'; +import { invalidateFleetUpdateCache, isFullStackUpdatePath, isUpdatePreviewPath } from '../helpers/fleetUpdateCache'; /** * Per-request hop timing for the critical hydration GETs, kept off the Request @@ -175,6 +176,18 @@ export function createRemoteProxyMiddleware(): RequestHandler { timing.upstreamStatus = proxyRes.statusCode; timing.ttfbMs = Date.now() - timing.startedAt; } + // Hub fleet aggregation is local-only. A successful remote full-stack + // Apply or update-preview reconcile must drop the hub cache so the next + // fleet poll does not revive a verified-cleared card from a stale entry. + const status = proxyRes.statusCode ?? 0; + if ( + req.method === 'POST' + && status >= 200 + && status < 300 + && (isFullStackUpdatePath(req.path) || isUpdatePreviewPath(req.path)) + ) { + invalidateFleetUpdateCache(); + } }, error: (err, req, proxyRes) => { // Finalize the hop timing with an error outcome before the existing diff --git a/backend/src/routes/auth.ts b/backend/src/routes/auth.ts index 817aa2a1..fb58b893 100644 --- a/backend/src/routes/auth.ts +++ b/backend/src/routes/auth.ts @@ -24,6 +24,7 @@ import { import { isSecureRequest } from '../helpers/cookies'; import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; +import { getAuthenticationMode, isLocalLoginEnabled } from '../helpers/authenticationMode'; export const authRouter = Router(); @@ -34,6 +35,8 @@ authRouter.get('/status', async (req: Request, res: Response): Promise => try { const settings = DatabaseService.getInstance().getGlobalSettings(); const needsSetup = !settings.auth_username || !settings.auth_password_hash || !settings.auth_jwt_secret; + const authenticationMode = getAuthenticationMode(); + const localLoginEnabled = authenticationMode !== 'sso_only'; let mfaPending = false; const mfaCookie = req.cookies?.[MFA_PENDING_COOKIE_NAME]; @@ -46,10 +49,10 @@ authRouter.get('/status', async (req: Request, res: Response): Promise => } } - res.json({ needsSetup, mfaPending }); + res.json({ needsSetup, mfaPending, localLoginEnabled, authenticationMode }); } catch (error) { console.error('Error checking setup status:', error); - res.json({ needsSetup: true, mfaPending: false }); + res.json({ needsSetup: true, mfaPending: false, localLoginEnabled: true, authenticationMode: 'local_and_sso' }); } }); @@ -106,6 +109,7 @@ authRouter.post('/setup', authRateLimiter, async (req: Request, res: Response): // Login endpoint authRouter.post('/login', authRateLimiter, async (req: Request, res: Response): Promise => { const { username, password } = req.body; + const remember = req.body.remember === true; if (!username || !password) { res.status(400).json({ error: 'Username and password are required' }); @@ -113,6 +117,13 @@ authRouter.post('/login', authRateLimiter, async (req: Request, res: Response): } try { + if (!isLocalLoginEnabled()) { + res.status(403).json({ + error: 'Local password authentication is disabled. Sign in using SSO.', + }); + return; + } + const db = DatabaseService.getInstance(); const user = db.getUserByUsername(username); @@ -131,13 +142,13 @@ authRouter.post('/login', authRateLimiter, async (req: Request, res: Response): console.log('[MFA:diag] login: path=local user=', user.username, 'mfaEnabled=', !!mfa?.enabled, 'failedAttempts=', mfa?.failed_attempts ?? 0, 'lockedUntil=', mfa?.locked_until ?? null); } if (mfa?.enabled) { - issueMfaPendingCookie(res, req, user, jwtSecret); + issueMfaPendingCookie(res, req, user, jwtSecret, { remember }); console.log('[Auth] Login password OK, MFA challenge pending:', user.username); res.json({ success: true, mfaRequired: true }); return; } - issueSessionCookie(res, req, user, jwtSecret); + issueSessionCookie(res, req, user, jwtSecret, remember); console.log('[Auth] Login successful:', user.username); res.json({ success: true, message: 'Login successful' }); return; diff --git a/backend/src/routes/authMode.ts b/backend/src/routes/authMode.ts new file mode 100644 index 00000000..40dc2e2e --- /dev/null +++ b/backend/src/routes/authMode.ts @@ -0,0 +1,112 @@ +import { Router, type Request, type Response } from 'express'; +import { DatabaseService } from '../services/DatabaseService'; +import { SSOService } from '../services/SSOService'; +import { requireAdmin } from '../middleware/tierGates'; +import { rejectApiTokenScope } from '../middleware/apiTokenScope'; +import { + getAuthenticationMode, + isAuthenticationMode, + setAuthenticationMode, + type AuthenticationMode, +} from '../helpers/authenticationMode'; + +const SCOPE_MESSAGE = 'API tokens cannot change authentication mode.'; + +export const authModeRouter = Router(); + +authModeRouter.get('/', (req: Request, res: Response): void => { + if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + try { + const mode = getAuthenticationMode(); + res.json({ + authenticationMode: mode, + localLoginEnabled: mode !== 'sso_only', + }); + } catch (error) { + console.error('[AuthMode] Failed to read authentication mode:', error); + res.status(500).json({ error: 'Failed to read authentication mode' }); + } +}); + +authModeRouter.put('/', async (req: Request, res: Response): Promise => { + if (rejectApiTokenScope(req, res, SCOPE_MESSAGE)) return; + if (!requireAdmin(req, res)) return; + + const mode = req.body?.mode as unknown; + if (!isAuthenticationMode(mode)) { + res.status(400).json({ error: 'mode must be local_and_sso or sso_only' }); + return; + } + + try { + if (mode === 'local_and_sso') { + setAuthenticationMode('local_and_sso'); + console.log('[AuthMode] Authentication mode set to local_and_sso'); + res.json({ + success: true, + authenticationMode: 'local_and_sso' satisfies AuthenticationMode, + localLoginEnabled: true, + }); + return; + } + + // Entering sso_only: safety gates. + if (req.body?.confirm !== true) { + res.status(400).json({ error: 'confirm must be true to enable SSO-only mode' }); + return; + } + + const db = DatabaseService.getInstance(); + const admin = db.getUser(req.user!.userId); + if (!admin || admin.role !== 'admin') { + res.status(403).json({ error: 'Administrator access required' }); + return; + } + if (admin.auth_provider === 'local') { + res.status(400).json({ + error: 'Sign in with SSO as an administrator before enabling SSO-only mode', + }); + return; + } + + const enabled = db.getEnabledSSOConfigs(); + if (enabled.length === 0) { + res.status(400).json({ error: 'Enable at least one SSO provider before SSO-only mode' }); + return; + } + + const sso = SSOService.getInstance(); + let anyTestPassed = false; + const failures: string[] = []; + for (const config of enabled) { + const result = + config.provider === 'ldap' + ? await sso.testLdapConnection() + : await sso.testOidcDiscovery(config.provider); + if (result.success) { + anyTestPassed = true; + break; + } + failures.push(`${config.provider}: ${result.error ?? 'connection test failed'}`); + } + if (!anyTestPassed) { + res.status(400).json({ + error: 'At least one enabled SSO provider must pass a connection test', + details: failures, + }); + return; + } + + setAuthenticationMode('sso_only'); + console.log('[AuthMode] Authentication mode set to sso_only'); + res.json({ + success: true, + authenticationMode: 'sso_only' satisfies AuthenticationMode, + localLoginEnabled: false, + }); + } catch (error) { + console.error('[AuthMode] Failed to update authentication mode:', error); + res.status(500).json({ error: 'Failed to update authentication mode' }); + } +}); diff --git a/backend/src/routes/dashboard.ts b/backend/src/routes/dashboard.ts index b99797df..a4aedddc 100644 --- a/backend/src/routes/dashboard.ts +++ b/backend/src/routes/dashboard.ts @@ -1,6 +1,7 @@ import { Router, type Request, type Response } from 'express'; import { DatabaseService, type StackRestartSummary } from '../services/DatabaseService'; import { CloudBackupService } from '../services/CloudBackupService'; +import { FileSystemService } from '../services/FileSystemService'; import TrivyService from '../services/TrivyService'; import { effectiveTier } from '../middleware/tierGates'; import { isDebugEnabled } from '../utils/debug'; @@ -49,11 +50,11 @@ export interface ConfigurationStatus { }; } -export function buildLocalConfigurationStatus( +export async function buildLocalConfigurationStatus( nodeId: number, userId: number, tier: LicenseTier, -): ConfigurationStatus { +): Promise { const db = DatabaseService.getInstance(); const agents = db.getAgents(nodeId); @@ -62,7 +63,10 @@ export function buildLocalConfigurationStatus( return { configured: !!a?.url, enabled: a?.enabled ?? false }; }; - const alertRules = db.getStackAlerts().length; + // Scope to stacks that exist on this node. stack_alerts has no node_id; + // intersecting with the node's compose directory is the per-node filter. + const stackNames = new Set(await FileSystemService.getInstance(nodeId).getStacks()); + const alertRules = db.getStackAlerts().filter((a) => stackNames.has(a.stack_name)).length; const notifRoutes = db.getNotificationRoutes(); const healPolicies = db.getAutoHealPolicies(undefined, nodeId); @@ -165,7 +169,7 @@ export function buildLocalConfigurationStatus( } // All routes below are protected by the global authGate mounted at app.use('/api', authGate) -dashboardRouter.get('/configuration', (req: Request, res: Response): void => { +dashboardRouter.get('/configuration', async (req: Request, res: Response): Promise => { try { const debug = isDebugEnabled(); const startedAt = debug ? Date.now() : 0; @@ -173,7 +177,7 @@ dashboardRouter.get('/configuration', (req: Request, res: Response): void => { const userId = req.user?.userId ?? 0; const tier = effectiveTier(req); - const payload = buildLocalConfigurationStatus(nodeId, userId, tier); + const payload = await buildLocalConfigurationStatus(nodeId, userId, tier); if (debug) { console.debug( `[Dashboard:debug] /configuration built in ${Date.now() - startedAt} ms (nodeId=${nodeId})`, diff --git a/backend/src/routes/fleet.ts b/backend/src/routes/fleet.ts index fd81e12c..dc9c6091 100644 --- a/backend/src/routes/fleet.ts +++ b/backend/src/routes/fleet.ts @@ -62,6 +62,8 @@ import { PROXY_TIER_HEADER, deployProvenanceHeaders } from '../services/license- import { LicenseService } from '../services/LicenseService'; const updateTracker = FleetUpdateTrackerService.getInstance(); +/** Sync lock for remote reapply while meta is fetched (before the pollable tracker exists). */ +const remoteReapplyDispatching = new Set(); const EARLY_FAIL_MS = 180 * 1000; // 3 minutes before declaring a probable pull failure // Shown in the Node Updates UI when a node's image is pinned in a way Fleet // cannot repin (digest or an unresolved value). Node-neutral so it reads the @@ -659,7 +661,7 @@ fleetRouter.get('/configuration', authMiddleware, async (req: Request, res: Resp name: node.name, type: 'local', status: 'online', - configuration: buildLocalConfigurationStatus(node.id, userId, localTier), + configuration: await buildLocalConfigurationStatus(node.id, userId, localTier), }; } @@ -1114,6 +1116,7 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp let remoteImagePinKind: ImagePinKind | null = null; let remoteUpdateBlocked = false; let remoteImageChannel: 'community' | 'hardened' | 'unknown' | null = null; + let remoteCapabilities: string[] = []; if (node.type === 'local') { version = gatewayVersion; } else { @@ -1125,8 +1128,18 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp remoteImagePinKind = meta.imagePinKind; remoteUpdateBlocked = meta.updateBlocked; remoteImageChannel = meta.imageChannel; + remoteCapabilities = meta.capabilities ?? []; } + const isReapply = tracker?.operationKind === 'reapply_configuration'; + const earlyFailMsg = isReapply + ? (node.type === 'local' + ? 'Local reapply did not complete. The container may not have restarted; check Docker logs on the host.' + : 'Reapply may have failed. The node is still running and its process start time has not changed.') + : (node.type === 'local' + ? 'Local update did not complete. The container may not have restarted; check Docker logs on the host.' + : 'Update may have failed. The node is still running and its version has not changed.'); + if (tracker?.status === 'updating') { const elapsed = Date.now() - tracker.startedAt; @@ -1136,7 +1149,9 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp if (elapsed > UPDATE_TIMEOUT_MS) { if (debug) console.debug('[Fleet:debug] Node', node.id, 'timed out after', Math.round(elapsed / 1000) + 's'); - resolveTerminal(node, tracker, 'timeout', UPDATE_TIMEOUT_MSG); + resolveTerminal(node, tracker, 'timeout', isReapply + ? 'Node did not come back online within 5 minutes after reapply.' + : UPDATE_TIMEOUT_MSG); } else if (node.type === 'remote') { if (remoteUpdateError) { if (debug) console.debug('[Fleet:debug] Node', node.id, 'reported pull failure:', remoteUpdateError); @@ -1146,10 +1161,9 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp if (debug) console.debug('[Fleet:debug] Node', node.id, 'went offline (restarting)'); updateTracker.set(node.id, { ...tracker, wasOffline: true }); } - } else if (isValidVersion(version) && version !== tracker.previousVersion) { - // Signal 1: a valid, different version. A null/unparseable version - // from a transient /api/meta blip is NOT a version change, so it - // must not complete a still-running, same-process node here. + } else if (!isReapply && isValidVersion(version) && version !== tracker.previousVersion) { + // Signal 1: a valid, different version. Skipped for reapply because + // the authored image/version is not expected to change. if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 1 (version changed):', tracker.previousVersion, '->', version); updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); } else if ( @@ -1173,20 +1187,20 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 3 (offline then online, startedAt unavailable)'); updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); } else if ( + !isReapply && elapsed > 15_000 && isValidVersion(version) && gatewayValid && !semver.lt(version, compareVersion!) ) { // Signal 4: remote is now at or above gateway version (after - // minimum processing time). Catches fast restarts where the 5s - // polling interval misses the offline window and startedAt - // hasn't been observed to change yet. + // minimum processing time). Never used for reapply: an already + // current node would false-complete before the helper runs. if (debug) console.debug('[Fleet:debug] Node', node.id, 'completed via signal 4 (version >= compare target):', version, '>=', compareVersion); updateTracker.set(node.id, updateTracker.resolve(tracker, 'completed')); } else if (elapsed > EARLY_FAIL_MS) { if (debug) console.debug('[Fleet:debug] Node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's - no signals detected'); - resolveTerminal(node, tracker, 'failed', 'Update may have failed. The node is still running and its version has not changed.'); + resolveTerminal(node, tracker, 'failed', earlyFailMsg); } } else if (node.type === 'local') { // Local node has only two failure signals: an explicit pull/spawn @@ -1202,7 +1216,7 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp selfUpdate.clearLastError(); } else if (elapsed > EARLY_FAIL_MS) { if (debug) console.debug('[Fleet:debug] Local node', node.id, 'early fail after', Math.round(elapsed / 1000) + 's'); - resolveTerminal(node, tracker, 'failed', 'Local update did not complete. The container may not have restarted; check Docker logs on the host.'); + resolveTerminal(node, tracker, 'failed', earlyFailMsg); } } } @@ -1286,6 +1300,10 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp updateBlocked, updateBlockedReason, imageChannel, + operationKind: currentTracker?.operationKind ?? null, + canReapplyCompose: node.type === 'local' + ? SelfUpdateService.getInstance().isAvailable() + : remoteOnline && remoteCapabilities.includes('self-update'), }; }), ); @@ -1305,6 +1323,8 @@ fleetRouter.get('/update-status', authMiddleware, async (req: Request, res: Resp skipActive: false, skippedVersion: null, ...EMPTY_PIN_STATUS, + operationKind: null, + canReapplyCompose: false, }; }); @@ -1345,17 +1365,46 @@ fleetRouter.get('/update-status/release-notes', authMiddleware, async (req: Requ // repin a semver-pinned compose to that release. It is omitted otherwise (never // sent as null/invalid), and an older remote that predates this field simply // ignores the extra body key and behaves as before. -function postSystemUpdate(target: { apiUrl: string; apiToken: string }, targetVersion?: string) { +function postSystemEndpoint( + target: { apiUrl: string; apiToken: string }, + endpoint: '/api/system/update' | '/api/system/reapply-compose', + body: Record = {}, +) { const headers: Record = { 'Content-Type': 'application/json' }; if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`; - return fetch(`${target.apiUrl.replace(/\/$/, '')}/api/system/update`, { + return fetch(`${target.apiUrl.replace(/\/$/, '')}${endpoint}`, { method: 'POST', headers, - body: JSON.stringify(targetVersion ? { targetVersion } : {}), + body: JSON.stringify(body), signal: AbortSignal.timeout(10000), }); } +function postSystemUpdate(target: { apiUrl: string; apiToken: string }, targetVersion?: string) { + return postSystemEndpoint(target, '/api/system/update', targetVersion ? { targetVersion } : {}); +} + +function postSystemReapplyCompose(target: { apiUrl: string; apiToken: string }) { + return postSystemEndpoint(target, '/api/system/reapply-compose'); +} + +/** Clear a terminal tracker row, or time out a stale in-flight one. Returns a + * conflict message when another update/reapply is still actively running. */ +function beginTrackerOperation(nodeId: number, conflictError: string): string | null { + const existing = updateTracker.get(nodeId); + if (existing?.status === 'updating') { + if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) { + updateTracker.set(nodeId, updateTracker.resolve(existing, 'timeout', UPDATE_TIMEOUT_MSG)); + } else { + return conflictError; + } + } + if (existing && (existing.status === 'timeout' || existing.status === 'failed' || existing.status === 'completed')) { + updateTracker.delete(nodeId); + } + return null; +} + function parseRemoteUpdateFailure(payload: unknown): { error: string; code?: string } { if (!payload || typeof payload !== 'object') { return { error: 'Remote node rejected update request.' }; @@ -1435,18 +1484,10 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r const requestedTarget = parseRequestedTargetVersion(req, res); if (requestedTarget === null) return; // invalid supplied value; 400 already sent - const existing = updateTracker.get(nodeId); - if (existing?.status === 'updating') { - if (Date.now() - existing.startedAt > UPDATE_TIMEOUT_MS) { - updateTracker.set(nodeId, updateTracker.resolve(existing, 'timeout', UPDATE_TIMEOUT_MSG)); - } else { - res.status(409).json({ error: 'Update already in progress for this node.' }); - return; - } - } - // Clear terminal states to allow retry. - if (existing && (existing.status === 'timeout' || existing.status === 'failed' || existing.status === 'completed')) { - updateTracker.delete(nodeId); + const conflict = beginTrackerOperation(nodeId, 'Update already in progress for this node.'); + if (conflict) { + res.status(409).json({ error: conflict }); + return; } console.log('[Fleet] Update triggered for node', node.name, node.type); @@ -1548,6 +1589,140 @@ fleetRouter.post('/nodes/:nodeId/update', authMiddleware, async (req: Request, r } }); +fleetRouter.post('/nodes/:nodeId/reapply-compose', authMiddleware, async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + try { + const nodeId = parseIntParam(req, res, 'nodeId', 'node ID'); + if (nodeId === null) return; + const db = DatabaseService.getInstance(); + const node = db.getNode(nodeId); + if (!node) { + res.status(404).json({ error: 'Node not found' }); + return; + } + + const conflict = beginTrackerOperation(nodeId, 'An update or reapply is already in progress for this node.'); + if (conflict) { + res.status(409).json({ error: conflict }); + return; + } + + console.log('[Fleet] Compose reapply triggered for node', node.name, node.type); + + if (node.type === 'local') { + const selfUpdate = SelfUpdateService.getInstance(); + if (!selfUpdate.isAvailable()) { + res.status(503).json({ error: 'Compose reapply unavailable on the local node.' }); + return; + } + const claim = await ImageOperationService.getInstance().claimComposeReapply(); + if (!claim.ok) { + res.status(409).json({ error: 'An image operation is already in progress.', code: claim.failureCode }); + return; + } + updateTracker.set( + nodeId, + updateTracker.create('updating', getSenchoVersion(), null, undefined, undefined, 'reapply_configuration'), + ); + res.status(202).json({ message: 'Compose reapply initiated on local node. The server will restart shortly.' }); + setTimeout(() => { + ImageOperationService.getInstance().executeClaimedComposeReapply().catch(error => { + console.error('[ImageOperation] Unexpected compose reapply failure:', error); + }); + }, 500); + return; + } + + // Sync lock before any await so a concurrent reapply gets 409 without + // racing the remote POST. The pollable tracker is created only after meta + // is known (full process identity), immediately before dispatch. + if (remoteReapplyDispatching.has(nodeId)) { + res.status(409).json({ error: 'An update or reapply is already in progress for this node.' }); + return; + } + remoteReapplyDispatching.add(nodeId); + + const failOwnedTracker = ( + error: string, + code?: string, + previousVersion: string | null = null, + previousProcessStart: number | null = null, + ) => { + const current = updateTracker.get(nodeId); + if (current?.status !== 'updating' || current.operationKind !== 'reapply_configuration') return; + updateTracker.set( + nodeId, + updateTracker.create('failed', previousVersion, previousProcessStart, error, code, 'reapply_configuration'), + ); + }; + + try { + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) { + const error = formatNoTargetError(node); + res.status(503).json({ error }); + return; + } + + const meta = await NodeRegistry.getInstance().fetchMetaForNode(node.id); + if (!meta.online) { + const error = 'Remote node is unreachable. Verify the node is running and the API URL is correct.'; + res.status(503).json({ error }); + return; + } + if (!meta.capabilities.includes('self-update')) { + const error = 'Remote node does not support compose reapply. It may need to be updated manually first.'; + res.status(503).json({ error }); + return; + } + // Digest pins and updateBlocked are intentional non-gates: reapply never + // repins the image, so blocked update rows remain eligible. + + // Reserve before the remote POST so a concurrent reapply still sees + // 'updating' after this request leaves the dispatch set in finally. + updateTracker.set( + nodeId, + updateTracker.create( + 'updating', + meta.version, + meta.startedAt, + undefined, + undefined, + 'reapply_configuration', + ), + ); + + const response = await postSystemReapplyCompose(target); + + if (!response.ok) { + const failure = parseRemoteUpdateFailure(await response.json().catch(() => null)); + failOwnedTracker(failure.error, failure.code, meta.version, meta.startedAt); + res.status(502).json(failure); + return; + } + + res.status(202).json({ message: `Compose reapply initiated on ${node.name}.` }); + } finally { + remoteReapplyDispatching.delete(nodeId); + } + } catch (error) { + console.error('[Fleet] Node compose reapply error:', error); + const errorMsg = getErrorMessage(error, 'Failed to trigger compose reapply.'); + const failedNodeId = parseInt(req.params.nodeId as string, 10); + if (!isNaN(failedNodeId)) { + remoteReapplyDispatching.delete(failedNodeId); + const current = updateTracker.get(failedNodeId); + if (current?.status === 'updating' && current.operationKind === 'reapply_configuration') { + updateTracker.set( + failedNodeId, + updateTracker.create('failed', null, null, errorMsg, undefined, 'reapply_configuration'), + ); + } + } + res.status(500).json({ error: 'Failed to trigger compose reapply.' }); + } +}); + fleetRouter.post('/update-all', authMiddleware, async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; try { diff --git a/backend/src/routes/imageUpdates.ts b/backend/src/routes/imageUpdates.ts index 3b4700c7..e3c000ae 100644 --- a/backend/src/routes/imageUpdates.ts +++ b/backend/src/routes/imageUpdates.ts @@ -5,7 +5,13 @@ import DockerController from '../services/DockerController'; import { DatabaseService } from '../services/DatabaseService'; import { NodeRegistry } from '../services/NodeRegistry'; import { CacheService } from '../services/CacheService'; -import { ImageUpdateService } from '../services/ImageUpdateService'; +import { + createAutoUpdateDigestGateState, + messageWhenDigestApplyBlockedByCheckErrors, + messageWhenNoDigestUpdate, + recordAutoUpdateImageCheck, +} from '../helpers/autoUpdateDigestGate'; +import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from '../services/ImageUpdateService'; import { FileSystemService } from '../services/FileSystemService'; import { StackUpdateOrchestrator } from '../services/StackUpdateOrchestrator'; import { StackOpLockService, stackOpSkipMessage } from '../services/StackOpLockService'; @@ -15,6 +21,8 @@ import { HealthGateService } from '../services/HealthGateService'; import { authMiddleware } from '../middleware/auth'; import { requireAdmin } from '../middleware/tierGates'; import { buildPolicyGateOptions } from '../helpers/policyGate'; +import { FLEET_UPDATE_CACHE_KEY, invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache'; +import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; import { summarizeBlockReasons } from '../utils/policy-risk'; import { isValidStackName } from '../utils/validation'; import { sanitizeForLog } from '../utils/safeLog'; @@ -22,7 +30,6 @@ import { logDebugTiming } from '../utils/requestTiming'; import { getErrorMessage } from '../utils/errors'; // Fleet aggregation cache: 2-minute TTL, shared across dashboard tabs. -const FLEET_UPDATE_CACHE_KEY = 'fleet-updates'; const FLEET_CACHE_TTL = 120_000; const REMOTE_NODE_FETCH_TIMEOUT_MS = 5000; @@ -30,7 +37,9 @@ export const imageUpdatesRouter = Router(); imageUpdatesRouter.get('/', authMiddleware, (req: Request, res: Response): void => { try { - const updates = DatabaseService.getInstance().getStackUpdateStatus(req.nodeId); + // Confirmed-only: partial/failed retained has_update rows stay out of the + // boolean map so Fleet and node cards do not treat uncertainty as pending. + const updates = DatabaseService.getInstance().getConfirmedStackUpdateStatus(req.nodeId); res.json(updates); } catch (error) { console.error('Failed to fetch image update status:', error); @@ -68,6 +77,13 @@ imageUpdatesRouter.get('/detail', authMiddleware, (req: Request, res: Response): imageUpdatesRouter.post('/refresh', authMiddleware, (req: Request, res: Response): void => { if (!requireAdmin(req, res)) return; try { + if (!ImageUpdateService.isChecksEnabled()) { + res.status(409).json({ + enabled: false, + error: 'Image update detection is disabled for this node.', + }); + return; + } const triggered = ImageUpdateService.getInstance().triggerManualRefresh(); if (!triggered) { const mins = ImageUpdateService.manualCooldownMinutes; @@ -172,6 +188,26 @@ imageUpdatesRouter.put('/interval', authMiddleware, (req: Request, res: Response } }); +const EnabledPatchSchema = z.object({ + enabled: z.boolean(), +}); + +imageUpdatesRouter.put('/enabled', authMiddleware, (req: Request, res: Response): void => { + if (!requireAdmin(req, res)) return; + const parsed = EnabledPatchSchema.safeParse(req.body); + if (!parsed.success) { + res.status(400).json({ error: 'enabled must be a boolean' }); + return; + } + try { + const status = ImageUpdateService.getInstance().applyChecksEnabled(parsed.data.enabled); + res.json(status); + } catch (error) { + console.error('Failed to update image-update checks enabled:', error); + res.status(500).json({ error: 'Failed to update image update checks setting' }); + } +}); + imageUpdatesRouter.get('/fleet', authMiddleware, async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; try { @@ -184,10 +220,10 @@ imageUpdatesRouter.get('/fleet', authMiddleware, async (req: Request, res: Respo const nr = NodeRegistry.getInstance(); const data: Record> = {}; - // Local nodes: synchronous DB reads. + // Local nodes: synchronous DB reads (confirmed-only projection). for (const node of nodes) { if (node.type === 'local') { - data[node.id] = db.getStackUpdateStatus(node.id); + data[node.id] = db.getConfirmedStackUpdateStatus(node.id); } } @@ -245,6 +281,7 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request, const triggered: number[] = []; const rateLimited: number[] = []; const failed: number[] = []; + const disabled: number[] = []; // ImageUpdateService is a per-instance singleton, so the local node's manual // refresh fires at most once per request regardless of how many local rows @@ -252,7 +289,9 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request, const localNode = nodes.find(n => n.type === 'local'); if (localNode) { try { - if (ImageUpdateService.getInstance().triggerManualRefresh()) { + if (!ImageUpdateService.isChecksEnabled()) { + disabled.push(localNode.id); + } else if (ImageUpdateService.getInstance().triggerManualRefresh()) { triggered.push(localNode.id); } else { rateLimited.push(localNode.id); @@ -297,6 +336,8 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request, const { nodeId, status } = entry.value; if (status >= 200 && status < 300) { triggered.push(nodeId); + } else if (status === 409) { + disabled.push(nodeId); } else if (status === 429) { rateLimited.push(nodeId); } else { @@ -304,8 +345,8 @@ imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request, } } - CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY); - res.json({ triggered, rateLimited, failed }); + invalidateFleetUpdateCache(); + res.json({ triggered, rateLimited, failed, disabled }); }); /** @@ -318,31 +359,58 @@ export const autoUpdateRouter = Router(); autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; try { - const { target } = req.body as { target?: string }; - console.log(`[AutoUpdate] Execute requested: target="${sanitizeForLog(target || '')}"`); - if (!target || typeof target !== 'string') { - res.status(400).json({ error: 'Missing "target" (stack name or "*" for all)' }); + // Honor the node-scoped image-update detection opt-out before any work. + if (!ImageUpdateService.isChecksEnabled()) { + res.json({ result: 'Image update detection is disabled for this node; skipped.' }); return; } + const { target, targets } = req.body as { target?: string; targets?: unknown }; let stackNames: string[]; - if (target === '*') { - stackNames = await FileSystemService.getInstance(req.nodeId).getStacks(); - if (stackNames.length === 0) { - res.json({ result: 'No stacks found on node; skipped.' }); + if (Array.isArray(targets)) { + if (targets.length === 0) { + res.status(400).json({ error: '"targets" must be a non-empty array of stack names' }); return; } + if (targets.length > 500) { + res.status(400).json({ error: '"targets" accepts at most 500 stack names' }); + return; + } + if (!targets.every((t): t is string => typeof t === 'string' && isValidStackName(t))) { + res.status(400).json({ error: 'Invalid stack name in targets' }); + return; + } + // Deduplicate while preserving order. + const seen = new Set(); + stackNames = []; + for (const name of targets) { + if (seen.has(name)) continue; + seen.add(name); + stackNames.push(name); + } + console.log(`[AutoUpdate] Execute requested: targets=${stackNames.length}`); + } else if (typeof target === 'string' && target.length > 0) { + console.log(`[AutoUpdate] Execute requested: target="${sanitizeForLog(target)}"`); + if (target === '*') { + stackNames = await FileSystemService.getInstance(req.nodeId).getStacks(); + if (stackNames.length === 0) { + res.json({ result: 'No stacks found on node; skipped.' }); + return; + } + } else { + if (!isValidStackName(target)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + stackNames = [target]; + } } else { - if (!isValidStackName(target)) { - res.status(400).json({ error: 'Invalid stack name' }); - return; - } - stackNames = [target]; + res.status(400).json({ error: 'Missing "target" (stack name or "*") or "targets" (stack name array)' }); + return; } const docker = DockerController.getInstance(req.nodeId); const imageUpdateService = ImageUpdateService.getInstance(); - const db = DatabaseService.getInstance(); const atomic = true; const results: string[] = []; @@ -365,35 +433,29 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp continue; } - let hasUpdate = false; - const updatedImages: string[] = []; - const checkErrors: string[] = []; + const gate = createAutoUpdateDigestGateState(); for (const imageRef of imageRefs) { try { const result = await imageUpdateService.checkImage(docker, imageRef); - if (result.error) { - checkErrors.push(result.error); - } else if (result.hasUpdate) { - hasUpdate = true; - updatedImages.push(imageRef); - } + recordAutoUpdateImageCheck(gate, imageRef, result); } catch (e) { const errMsg = getErrorMessage(e, String(e)); - checkErrors.push(errMsg); + gate.checkErrors.push(errMsg); console.warn('[AutoUpdate] Failed to check image %s:', sanitizeForLog(imageRef), sanitizeForLog((e as Error)?.message ?? String(e))); } } - if (!hasUpdate) { - if (checkErrors.length > 0 && checkErrors.length === imageRefs.length) { - results.push(`Stack "${stackName}": WARNING - all image checks failed (${checkErrors.join('; ')}). Unable to determine update status.`); - } else if (checkErrors.length > 0) { - results.push(`Stack "${stackName}": all reachable images up to date (${checkErrors.length} check(s) failed).`); - } else { - results.push(`Stack "${stackName}": all images up to date.`); - } + if (!gate.hasDigestUpdate) { + results.push(messageWhenNoDigestUpdate(stackName, gate, imageRefs.length)); continue; } + const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate); + if (checkErrorBlock) { + results.push(checkErrorBlock); + continue; + } + + const { updatedImages } = gate; // Auto-update runs from the scheduler: a policy bypass is never // appropriate. If updated images fail the gate, skip the stack and @@ -425,7 +487,9 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp results.push(stackOpSkipMessage(stackName, lock.existing.action)); continue; } - db.clearStackUpdateStatus(req.nodeId, stackName); + + // Health observation starts immediately after Compose; registry recheck is + // isolated so a verification failure cannot turn Compose success into a failure. const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`); const orchResult = lock.result; const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; @@ -434,6 +498,21 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId); } + // Recheck persists digest-cleared / tag-advisory state. Do not blind-clear. + let recheckWarning: string | undefined; + try { + const recheck = await imageUpdateService.recheckStack(req.nodeId, stackName); + if (recheck.warning) recheckWarning = recheck.warning; + } catch (recheckErr) { + console.warn( + '[AutoUpdate] Post-update recheck failed for %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(recheckErr, 'unknown')), + ); + recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING; + } + + invalidateNodeCaches(req.nodeId); NotificationService.getInstance().broadcastEvent({ type: 'state-invalidate', scope: 'image-updates', @@ -450,15 +529,20 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp { stackName, actor: 'system:image-update' }, ); - results.push(`Stack "${stackName}": updated (${updatedImages.join(', ')}).`); + const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`; + results.push(recheckWarning ? `${base} ${recheckWarning}` : base); } catch (e) { const msg = getErrorMessage(e, String(e)); results.push(`Stack "${stackName}" failed: ${msg}`); - console.error(`[AutoUpdate] Failed for stack "${stackName}":`, e); + console.error( + '[AutoUpdate] Failed for stack %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(msg), + ); } } - CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY); + invalidateFleetUpdateCache(); res.json({ result: results.join('\n') }); } catch (error) { const msg = getErrorMessage(error, 'Auto-update execution failed'); diff --git a/backend/src/routes/license.ts b/backend/src/routes/license.ts index fd3e473b..e640c6c9 100644 --- a/backend/src/routes/license.ts +++ b/backend/src/routes/license.ts @@ -175,3 +175,23 @@ systemUpdateRouter.post('/update', async (req: Request, res: Response): Promise< }); }, 500); }); + +systemUpdateRouter.post('/reapply-compose', async (req: Request, res: Response): Promise => { + if (!requireAdmin(req, res)) return; + const selfUpdate = SelfUpdateService.getInstance(); + if (!selfUpdate.isAvailable()) { + res.status(503).json({ error: 'Compose reapply unavailable. Sencho must be deployed via Docker Compose.' }); + return; + } + const claim = await ImageOperationService.getInstance().claimComposeReapply(); + if (!claim.ok) { + res.status(409).json({ error: 'An image operation is already in progress.', code: claim.failureCode }); + return; + } + res.status(202).json({ message: 'Compose reapply initiated. The server will restart shortly.' }); + setTimeout(() => { + ImageOperationService.getInstance().executeClaimedComposeReapply().catch(error => { + console.error('[ImageOperation] Unexpected compose reapply failure:', error); + }); + }, 500); +}); diff --git a/backend/src/routes/mfa.ts b/backend/src/routes/mfa.ts index 4f8c3050..b933a0d6 100644 --- a/backend/src/routes/mfa.ts +++ b/backend/src/routes/mfa.ts @@ -72,7 +72,7 @@ mfaRouter.post('/login/mfa', authRateLimiter, async (req: Request, res: Response return; } - let decoded: { scope?: string; user_id?: number; username?: string; sso?: boolean }; + let decoded: { scope?: string; user_id?: number; username?: string; sso?: boolean; remember?: boolean }; try { decoded = jwt.verify(pendingCookie, jwtSecret) as typeof decoded; } catch { @@ -173,7 +173,7 @@ mfaRouter.post('/login/mfa', authRateLimiter, async (req: Request, res: Response db.clearMfaFailures(decoded.user_id); clearMfaPendingCookie(res, req); - issueSessionCookie(res, req, user, jwtSecret); + issueSessionCookie(res, req, user, jwtSecret, decoded.remember === true); console.log('[Auth] MFA challenge cleared:', user.username); if (isDebugEnabled()) console.log('[MFA:diag] login/mfa: success user=', user.username, 'durationMs=', Date.now() - startedAt); res.json({ success: true }); diff --git a/backend/src/routes/notifications.ts b/backend/src/routes/notifications.ts index d253dcbc..fa61b0a6 100644 --- a/backend/src/routes/notifications.ts +++ b/backend/src/routes/notifications.ts @@ -1,5 +1,10 @@ import { Router, type Request, type Response } from 'express'; -import { DatabaseService, type NotificationSuppressionAppliesTo, type NotificationSuppressionRule } from '../services/DatabaseService'; +import { + DatabaseService, + type NotificationSuppressionAppliesTo, + type NotificationSuppressionRetraction, + type NotificationSuppressionRule, +} from '../services/DatabaseService'; import { NotificationService, ALL_NOTIFICATION_CATEGORIES, ALL_SUPPRESSIBLE_CATEGORIES } from '../services/NotificationService'; import type { NotificationCategory } from '../services/NotificationService'; import { NodeRegistry } from '../services/NodeRegistry'; @@ -37,6 +42,42 @@ const VALID_SUPPRESSION_CATEGORIES: ReadonlySet = new Set( const VALID_LEVELS = new Set(['info', 'warning', 'error']); const VALID_APPLIES_TO = new Set(['bell', 'external', 'both']); +function isNonNegativeSafeInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +} + +/** + * Omitted / empty body (old hub) -> permanent watermark 0. + * Present body with any keys must be a complete valid retraction or 400. + */ +function parseReplicaRetractionBody( + body: unknown, + res: Response, +): NotificationSuppressionRetraction | false { + if ( + body == null || + (typeof body === 'object' && !Array.isArray(body) && Object.keys(body as object).length === 0) + ) { + return { kind: 'permanent', source_updated_at: 0 }; + } + if (typeof body !== 'object' || Array.isArray(body)) { + res.status(400).json({ error: 'retraction body must be an object' }); + return false; + } + const raw = body as Record; + const kind = raw.kind; + const source = raw.source_updated_at; + if (kind !== 'permanent' && kind !== 'recoverable') { + res.status(400).json({ error: 'kind must be permanent or recoverable' }); + return false; + } + if (!isNonNegativeSafeInteger(source)) { + res.status(400).json({ error: 'source_updated_at must be a non-negative safe integer' }); + return false; + } + return { kind, source_updated_at: source }; +} + function validateNodeId(nodeId: unknown, res: Response): number | null | false { if (nodeId === undefined || nodeId === null) return null; if (typeof nodeId !== 'number' || !Number.isInteger(nodeId)) { @@ -550,8 +591,16 @@ notificationSuppressionRouter.post('/replica', authMiddleware, (req: Request, re isCreate: true, }, res); if (scheduleResult === false) return; + if (!isNonNegativeSafeInteger(rule.created_at)) { + res.status(400).json({ error: 'created_at must be a non-negative safe integer' }); + return; + } + if (!isNonNegativeSafeInteger(rule.updated_at)) { + res.status(400).json({ error: 'updated_at must be a non-negative safe integer' }); + return; + } - DatabaseService.getInstance().upsertNotificationSuppressionRuleReplica({ + const outcome = DatabaseService.getInstance().upsertNotificationSuppressionRuleReplica({ ...rule, stack_patterns: patterns.patterns, label_ids: Array.isArray(rule.label_ids) && rule.label_ids.length > 0 ? rule.label_ids : null, @@ -561,11 +610,13 @@ notificationSuppressionRouter.post('/replica', authMiddleware, (req: Request, re scheduleInvalid: false, enabled: rule.enabled !== false, expires_at: rule.expires_at ?? null, + created_at: rule.created_at, + updated_at: rule.updated_at, // Replicas are always node-agnostic on the receiving node: the hub's node_id // is a hub-local scoping concept and never trustworthy as a foreign key here. node_id: null, }); - res.json({ success: true }); + res.json({ success: true, outcome }); } catch (error) { console.error('Failed to apply suppression rule replica:', error); res.status(500).json({ error: 'Failed to apply suppression rule replica' }); @@ -577,8 +628,10 @@ notificationSuppressionRouter.delete('/replica/:id', authMiddleware, (req: Reque try { const id = parseIntParam(req, res, 'id', 'suppression rule ID'); if (id === null) return; - DatabaseService.getInstance().deleteNotificationSuppressionRule(id); - res.json({ success: true }); + const retraction = parseReplicaRetractionBody(req.body, res); + if (retraction === false) return; + const { outcome } = DatabaseService.getInstance().deleteNotificationSuppressionRule(id, retraction); + res.json({ success: true, outcome }); } catch (error) { console.error('Failed to delete suppression rule replica:', error); res.status(500).json({ error: 'Failed to delete suppression rule replica' }); @@ -727,7 +780,10 @@ notificationSuppressionRouter.delete('/:id', authMiddleware, (req: Request, res: const existing = DatabaseService.getInstance().getNotificationSuppressionRule(id); if (!existing) { res.status(404).json({ error: 'Suppression rule not found' }); return; } - DatabaseService.getInstance().deleteNotificationSuppressionRule(id); + DatabaseService.getInstance().deleteNotificationSuppressionRule(id, { + kind: 'permanent', + source_updated_at: existing.updated_at, + }); deleteSuppressionRuleFromFleet(existing); console.log(`[Suppression] Rule ${id} deleted`); res.json({ success: true }); diff --git a/backend/src/routes/scheduledTasks.ts b/backend/src/routes/scheduledTasks.ts index f621e9a3..ca503995 100644 --- a/backend/src/routes/scheduledTasks.ts +++ b/backend/src/routes/scheduledTasks.ts @@ -35,6 +35,17 @@ function broadcastScheduledTasksChanged(): void { const VALID_PRUNE_TARGETS = ['containers', 'images', 'networks', 'volumes'] as const; const ERR_FLEET_NODE_REQUIRED = 'Fleet update requires node_id.'; +const STACK_LABEL_SELECTOR = 'stack-label'; +const LABEL_NAME_RE = /^[a-zA-Z0-9 -]+$/; + +function isStackLabelSelector(selectorType: unknown): boolean { + return selectorType === STACK_LABEL_SELECTOR; +} + +/** True when this update+fleet task uses a stack-label selector (node_id may be null). */ +function usesStackLabelSelector(action: BackendScheduledAction, targetType: TargetType, selectorType: unknown): boolean { + return action === 'update' && targetType === 'fleet' && isStackLabelSelector(selectorType); +} function parsePositiveNodeId(nodeId: unknown): number | null { if (typeof nodeId !== 'number' && typeof nodeId !== 'string') return null; @@ -98,17 +109,26 @@ function validateContainerTarget(targetType: TargetType, targetId: unknown, node /** * Shared guard for non-stack actions that require a node. Stack actions use - * validateStackTarget because they also require target_id. + * validateStackTarget because they also require target_id. Label-targeted + * fleet updates may omit node_id (entire fleet); pass selectorType so that + * path is allowed. */ -function validateActionNode(action: BackendScheduledAction, targetType: TargetType, nodeId: unknown): string | null { +function validateActionNode( + action: BackendScheduledAction, + targetType: TargetType, + nodeId: unknown, + selectorType?: unknown, +): string | null { if (targetType === 'stack' || targetType === 'container') return null; const def = getScheduledActionDefinition(action); if (!def?.requiresNode) return null; const labelSingular = nodeRequirementLabel(action, targetType); const labelPlural = localNodeRequirementLabel(action); + const labelFleetUpdate = usesStackLabelSelector(action, targetType, selectorType); if (nodeId == null) { + if (labelFleetUpdate) return null; return action === 'update' && targetType === 'fleet' ? ERR_FLEET_NODE_REQUIRED : `${labelSingular} action requires node_id.`; @@ -124,13 +144,15 @@ function validateActionNode(action: BackendScheduledAction, targetType: TargetTy return null; } -/** Shared validation for prune_targets, target_services, prune_label_filter. Returns an error string or null. */ +/** Shared validation for prune_targets, target_services, prune_label_filter, selector_*. Returns an error string or null. */ function validateOptionalFields( action: BackendScheduledAction, targetType: TargetType, prune_targets: unknown, target_services: unknown, prune_label_filter: unknown, + selector_type?: unknown, + selector_value?: unknown, ): string | null { if (prune_targets !== undefined && prune_targets !== null) { if (!Array.isArray(prune_targets) || prune_targets.length === 0 @@ -155,9 +177,39 @@ function validateOptionalFields( return 'prune_label_filter can only be used with prune action'; } } + + const selectorPresent = (selector_type !== undefined && selector_type !== null) + || (selector_value !== undefined && selector_value !== null); + if (selectorPresent) { + if (action !== 'update' || targetType !== 'fleet') { + return 'selector fields can only be used with update action on fleet target'; + } + if (selector_type !== STACK_LABEL_SELECTOR) { + return 'selector_type must be "stack-label"'; + } + if (typeof selector_value !== 'string' || selector_value.trim().length === 0 || selector_value.trim().length > 30) { + return 'selector_value is required and must be 1-30 characters'; + } + if (!LABEL_NAME_RE.test(selector_value.trim())) { + return 'selector_value may only contain letters, numbers, spaces, and hyphens'; + } + } return null; } +function normalizeSelectorFields( + action: BackendScheduledAction, + targetType: TargetType, + selector_type: unknown, + selector_value: unknown, +): { selector_type: string | null; selector_value: string | null } { + if (action === 'update' && targetType === 'fleet' && isStackLabelSelector(selector_type) + && typeof selector_value === 'string' && selector_value.trim()) { + return { selector_type: STACK_LABEL_SELECTOR, selector_value: selector_value.trim() }; + } + return { selector_type: null, selector_value: null }; +} + /** * Validate a cron expression for Scheduled Operations. The scheduler ticks once * per minute, so an expression with a leading seconds field (6 or more fields) @@ -238,7 +290,11 @@ scheduledTasksRouter.get('/', (req: Request, res: Response): void => { scheduledTasksRouter.post('/', (req: Request, res: Response): void => { if (!requireAdmin(req, res)) return; try { - const { name, target_type, target_id, node_id, action, cron_expression, enabled, prune_targets, target_services, prune_label_filter, delete_after_run, run_at } = req.body; + const { + name, target_type, target_id, node_id, action, cron_expression, enabled, + prune_targets, target_services, prune_label_filter, selector_type, selector_value, + delete_after_run, run_at, + } = req.body; if (!name || typeof name !== 'string' || !name.trim()) { res.status(400).json({ error: 'Name is required' }); return; @@ -253,14 +309,16 @@ scheduledTasksRouter.post('/', (req: Request, res: Response): void => { const targetErr = validateActionTarget(action, target_type); if (targetErr) { res.status(400).json({ error: targetErr }); return; } - const nodeErr = validateActionNode(action, target_type, node_id); + const nodeErr = validateActionNode(action, target_type, node_id, selector_type); if (nodeErr) { res.status(400).json({ error: nodeErr }); return; } const stackTargetErr = validateStackTarget(target_type, target_id, node_id); if (stackTargetErr) { res.status(400).json({ error: stackTargetErr }); return; } const containerTargetErr = validateContainerTarget(target_type, target_id, node_id); if (containerTargetErr) { res.status(400).json({ error: containerTargetErr }); return; } - const optionalErr = validateOptionalFields(action, target_type, prune_targets, target_services, prune_label_filter); + const optionalErr = validateOptionalFields( + action, target_type, prune_targets, target_services, prune_label_filter, selector_type, selector_value, + ); if (optionalErr) { res.status(400).json({ error: optionalErr }); return; } const cronErr = validateCronExpression(cron_expression); @@ -282,7 +340,14 @@ scheduledTasksRouter.post('/', (req: Request, res: Response): void => { : (pinnedRunAt ?? scheduler.calculateNextRun(cron_expression)); const normalizedTargetId = target_type === 'stack' || target_type === 'container' ? target_id : null; - const normalizedNodeId = actionRequiresNode(action) ? parsePositiveNodeId(node_id) : null; + const labelSelector = usesStackLabelSelector(action, target_type, selector_type); + const normalizedNodeId = labelSelector + ? (node_id == null || node_id === '' ? null : parsePositiveNodeId(node_id)) + : (actionRequiresNode(action) ? parsePositiveNodeId(node_id) : null); + if (labelSelector && node_id != null && node_id !== '' && normalizedNodeId === null) { + res.status(400).json({ error: 'Fleet update action requires a valid node_id.' }); return; + } + const selectors = normalizeSelectorFields(action, target_type, selector_type, selector_value); const id = DatabaseService.getInstance().createScheduledTask({ name: name.trim(), @@ -302,6 +367,8 @@ scheduledTasksRouter.post('/', (req: Request, res: Response): void => { prune_targets: action === 'prune' && prune_targets ? JSON.stringify(prune_targets) : null, target_services: action === 'restart' && target_type === 'stack' && target_services ? JSON.stringify(target_services) : null, prune_label_filter: action === 'prune' && prune_label_filter ? prune_label_filter.trim() : null, + selector_type: selectors.selector_type, + selector_value: selectors.selector_value, delete_after_run: delete_after_run ? 1 : 0, run_at: pinnedRunAt, }); @@ -340,7 +407,11 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => { const existing = db.getScheduledTask(id); if (!existing) { res.status(404).json({ error: 'Scheduled task not found' }); return; } - const { name, target_type, target_id, node_id, action, cron_expression, enabled, prune_targets, target_services, prune_label_filter, delete_after_run, run_at } = req.body; + const { + name, target_type, target_id, node_id, action, cron_expression, enabled, + prune_targets, target_services, prune_label_filter, selector_type, selector_value, + delete_after_run, run_at, + } = req.body; if (target_type !== undefined && !(VALID_TARGET_TYPES as readonly string[]).includes(target_type)) { res.status(400).json({ error: 'Invalid target_type' }); return; @@ -354,13 +425,18 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => { const finalTargetId = finalTargetType === 'stack' || finalTargetType === 'container' ? (target_id !== undefined ? target_id : existing.target_id) : null; - const finalNodeId = actionRequiresNode(finalAction) + const finalSelectorType = selector_type !== undefined ? selector_type : existing.selector_type; + const finalSelectorValue = selector_value !== undefined ? selector_value : existing.selector_value; + const labelSelector = usesStackLabelSelector(finalAction, finalTargetType, finalSelectorType); + const finalNodeId = labelSelector ? (node_id !== undefined ? node_id : existing.node_id) - : null; + : (actionRequiresNode(finalAction) + ? (node_id !== undefined ? node_id : existing.node_id) + : null); const targetErr = validateActionTarget(finalAction, finalTargetType); if (targetErr) { res.status(400).json({ error: targetErr }); return; } - const nodeErr = validateActionNode(finalAction, finalTargetType, finalNodeId); + const nodeErr = validateActionNode(finalAction, finalTargetType, finalNodeId, finalSelectorType); if (nodeErr) { res.status(400).json({ error: nodeErr }); return; } const stackTargetErr = validateStackTarget(finalTargetType, finalTargetId, finalNodeId); @@ -369,7 +445,11 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => { const containerTargetErr = validateContainerTarget(finalTargetType, finalTargetId, finalNodeId); if (containerTargetErr) { res.status(400).json({ error: containerTargetErr }); return; } - const optionalErr = validateOptionalFields(finalAction, finalTargetType, prune_targets, target_services, prune_label_filter); + const optionalErr = validateOptionalFields( + finalAction, finalTargetType, prune_targets, target_services, prune_label_filter, + selector_type !== undefined ? selector_type : finalSelectorType, + selector_value !== undefined ? selector_value : finalSelectorValue, + ); if (optionalErr) { res.status(400).json({ error: optionalErr }); return; } if (cron_expression !== undefined) { @@ -391,7 +471,17 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => { if (target_id !== undefined || (finalTargetType !== 'stack' && finalTargetType !== 'container')) { updates.target_id = finalTargetId || null; } - if (node_id !== undefined || !actionRequiresNode(finalAction)) { + // Label-targeted fleet updates keep node_id when provided (or existing); + // clear only when the caller explicitly sends null/empty for fleet-wide, or + // when the action no longer requires a node and is not a label selector. + if (labelSelector) { + if (node_id !== undefined) { + updates.node_id = node_id == null || node_id === '' ? null : parsePositiveNodeId(node_id); + if (node_id != null && node_id !== '' && updates.node_id === null) { + res.status(400).json({ error: 'Fleet update action requires a valid node_id.' }); return; + } + } + } else if (node_id !== undefined || !actionRequiresNode(finalAction)) { updates.node_id = finalNodeId != null ? parsePositiveNodeId(finalNodeId) : null; } if (action !== undefined) updates.action = finalAction; @@ -414,6 +504,12 @@ scheduledTasksRouter.put('/:id', (req: Request, res: Response): void => { } else if (finalAction !== 'prune') { updates.prune_label_filter = null; } + if (selector_type !== undefined || selector_value !== undefined + || finalAction !== 'update' || finalTargetType !== 'fleet') { + const selectors = normalizeSelectorFields(finalAction, finalTargetType, finalSelectorType, finalSelectorValue); + updates.selector_type = selectors.selector_type; + updates.selector_value = selectors.selector_value; + } if (delete_after_run !== undefined) updates.delete_after_run = delete_after_run ? 1 : 0; // Persist a re-supplied run_at to its column (a number pins a one-shot; null diff --git a/backend/src/routes/settings.ts b/backend/src/routes/settings.ts index 782b13c2..1370c8e7 100644 --- a/backend/src/routes/settings.ts +++ b/backend/src/routes/settings.ts @@ -36,6 +36,7 @@ const ALLOWED_SETTING_KEYS = new Set([ 'auto_create_missing_external_networks', 'image_update_sidebar_indicators', 'notification_dispatch_retries', + 'session_sliding_refresh', ]); // Keys whose write requires a paid license, not just an admin role. @@ -77,6 +78,7 @@ const SettingsPatchSchema = z.object({ }); } }).transform((v) => String(parseNotificationDispatchRetries(v)!)), + session_sliding_refresh: z.enum(['0', '1']), }).partial(); export const settingsRouter = Router(); diff --git a/backend/src/routes/sso.ts b/backend/src/routes/sso.ts index 1839cf27..274216e9 100644 --- a/backend/src/routes/sso.ts +++ b/backend/src/routes/sso.ts @@ -43,6 +43,7 @@ ssoRouter.get('/providers', (_req: Request, res: Response): void => { ssoRouter.post('/ldap', authRateLimiter, async (req: Request, res: Response): Promise => { try { const { username, password } = req.body; + const remember = req.body.remember === true; if (!username || !password) { res.status(400).json({ error: 'Username and password are required' }); return; @@ -71,13 +72,13 @@ ssoRouter.post('/ldap', authRateLimiter, async (req: Request, res: Response): Pr console.log('[MFA:diag] login: path=ldap user=', user.username, 'mfaEnabled=', !!mfa?.enabled, 'ssoEnforce=', mfa?.sso_enforce_mfa === 1); } if (mfa?.enabled && mfa.sso_enforce_mfa) { - issueMfaPendingCookie(res, req, user, settings.auth_jwt_secret, { sso: true }); + issueMfaPendingCookie(res, req, user, settings.auth_jwt_secret, { sso: true, remember }); console.log(`[SSO] LDAP login password OK, MFA challenge pending: ${user.username}`); res.json({ success: true, mfaRequired: true }); return; } - issueSessionCookie(res, req, user, settings.auth_jwt_secret); + issueSessionCookie(res, req, user, settings.auth_jwt_secret, remember); console.log(`[SSO] LDAP login successful: ${user.username}`); res.json({ success: true, message: 'Login successful' }); } catch (error) { diff --git a/backend/src/routes/ssoConfig.ts b/backend/src/routes/ssoConfig.ts index ec11845d..d0c13f1f 100644 --- a/backend/src/routes/ssoConfig.ts +++ b/backend/src/routes/ssoConfig.ts @@ -4,6 +4,7 @@ import { SSOService, type SSOProviderConfig } from '../services/SSOService'; import { requireAdmin, requireTierForSsoProvider } from '../middleware/tierGates'; import { rejectApiTokenScope } from '../middleware/apiTokenScope'; import { sanitizeForLog } from '../utils/safeLog'; +import { wouldRemoveLastProvider } from '../helpers/authenticationMode'; const VALID_SSO_PROVIDERS = ['ldap', 'oidc_google', 'oidc_github', 'oidc_okta', 'oidc_custom'] as const; const SSO_SCOPE_MESSAGE = 'API tokens cannot access SSO configuration.'; @@ -85,6 +86,15 @@ ssoConfigRouter.put('/:provider', (req: Request, res: Response): void => { } } + const existing = DatabaseService.getInstance().getSSOConfig(provider); + const wasEnabled = existing?.enabled === 1; + if (!config.enabled && wouldRemoveLastProvider(provider, wasEnabled)) { + res.status(400).json({ + error: 'Cannot disable the last SSO provider while SSO-only mode is active. Switch to Local and SSO first, or use the emergency CLI.', + }); + return; + } + SSOService.getInstance().saveProviderConfig(config); console.log(`[SSO] Config updated: ${sanitizeForLog(provider)} ${config.enabled ? 'enabled' : 'disabled'}`); res.json({ success: true, message: 'SSO configuration saved' }); @@ -101,6 +111,15 @@ ssoConfigRouter.delete('/:provider', (req: Request, res: Response): void => { if (rejectInvalidProvider(provider, res)) return; if (!requireTierForSsoProvider(provider, req, res)) return; try { + const existing = DatabaseService.getInstance().getSSOConfig(provider); + const wasEnabled = existing?.enabled === 1; + if (wouldRemoveLastProvider(provider, wasEnabled)) { + res.status(400).json({ + error: 'Cannot delete the last SSO provider while SSO-only mode is active. Switch to Local and SSO first, or use the emergency CLI.', + }); + return; + } + SSOService.getInstance().deleteProviderConfig(provider); console.log(`[SSO] Config deleted: ${sanitizeForLog(provider)}`); res.json({ success: true, message: 'SSO configuration deleted' }); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 6df26f2a..9e311177 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -17,13 +17,17 @@ import { StackUpdateOrchestrator, shortImageId, type OrchestratorResult } from ' import DockerController, { type BulkStackInfo } from '../services/DockerController'; import { DatabaseService, type StackDossierFields } from '../services/DatabaseService'; import { CacheService, type CacheFetchOutcome } from '../services/CacheService'; -import { UpdatePreviewService } from '../services/UpdatePreviewService'; +import { + UpdatePreviewService, + isAuthoritativeNegativePreview, + buildDetectionDisabledPreview, +} from '../services/UpdatePreviewService'; import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService'; import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService'; import { DriftLedgerService, type DriftTemporal } from '../services/DriftLedgerService'; import { ComposeDoctorService } from '../services/ComposeDoctorService'; -import { RULE_IDS } from '../services/preflight/rules'; +import { RULE_IDS, isPreflightNoteFinding } from '../services/preflight/rules'; import { parseServiceImages, isPreflightAckActive } from '../utils/preflight-ack-filter'; import type { PreflightAckExpiryMode } from '../services/DatabaseService'; import { buildStackNetworkFacts } from '../services/network/composeNetworkInspector'; @@ -56,6 +60,11 @@ import { buildPolicyGateOptions, runPolicyGate, triggerPostDeployScan, describeP import { parseComposePreview, type ComposePreview } from '../helpers/composePreview'; import { filterContainersByComposeService } from '../helpers/composeServiceMatch'; import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; +import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache'; +import { + ImageUpdateService, + UPDATE_VERIFICATION_INCOMPLETE_WARNING, +} from '../services/ImageUpdateService'; import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection'; import { resolveStackEnvSources, discoverStackLocalEnvFiles } from '../helpers/envFileResolution'; import { STACK_STATUSES_CACHE_TTL_MS } from '../helpers/constants'; @@ -1408,6 +1417,10 @@ stacksRouter.post('/:stackName/preflight/acknowledgements', async (req: Request, res.status(400).json({ error: 'ruleId must be a known Compose Doctor rule id' }); return; } + if (isPreflightNoteFinding(ruleId)) { + res.status(400).json({ error: 'Informational notes cannot be acknowledged' }); + return; + } const serviceRaw = body.service == null || body.service === '' ? null : String(body.service).trim(); @@ -2243,6 +2256,14 @@ stacksRouter.post('/:stackName/services/:serviceName/restore', async (req: Reque stacksRouter.get('/:stackName/update-preview', async (req: Request, res: Response) => { const stackName = req.params.stackName as string; try { + // Anatomy and other GET consumers must not contact registries while + // node-scoped detection is off. + if (!ImageUpdateService.isChecksEnabled()) { + res.json(buildDetectionDisabledPreview(stackName)); + return; + } + // Read-only: sticky reconciliation lives on POST so UpdateGuard and other + // GET consumers never mutate persisted scanner state. const preview = await UpdatePreviewService.getInstance().getPreview(req.nodeId, stackName); res.json(preview); } catch (error) { @@ -2251,6 +2272,50 @@ stacksRouter.get('/:stackName/update-preview', async (req: Request, res: Respons } }); +stacksRouter.post('/:stackName/update-preview', async (req: Request, res: Response) => { + const stackName = req.params.stackName as string; + try { + if (!ImageUpdateService.isChecksEnabled()) { + // No registry I/O and no sticky reconcile on a synthetic disabled preview. + res.json({ ...buildDetectionDisabledPreview(stackName), reconciled: false }); + return; + } + // Snapshot write-generation watermarks before the read-only preview so a + // later clear can erase older confirmed/sticky rows without racing a + // scanner that reserved or rewrote the row after this observation. + const imageUpdates = ImageUpdateService.getInstance(); + const db = DatabaseService.getInstance(); + const observedMemoryGeneration = imageUpdates.peekStackWriteGeneration(req.nodeId, stackName); + const observedRowGeneration = db.getStackUpdateWriteGeneration(req.nodeId, stackName); + const preview = await UpdatePreviewService.getInstance().getPreview(req.nodeId, stackName); + let reconciled = false; + if (isAuthoritativeNegativePreview(preview)) { + const clearResult = await imageUpdates.commitPreviewClear( + req.nodeId, + stackName, + observedMemoryGeneration, + observedRowGeneration, + ); + if (clearResult === 'cleared') { + reconciled = true; + invalidateFleetUpdateCache(); + NotificationService.getInstance().broadcastEvent({ + type: 'state-invalidate', + scope: 'image-updates', + nodeId: req.nodeId, + stackName, + action: 'update-status-reconciled', + ts: Date.now(), + }); + } + } + res.json({ ...preview, reconciled }); + } catch (error) { + console.error('[Stacks] Update preview reconcile failed: %s', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'unknown'))); + res.status(500).json({ error: 'Failed to compute update preview' }); + } +}); + stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { const stackName = req.params.stackName as string; if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; @@ -2270,7 +2335,27 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { { nodeId: req.nodeId, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: req.user?.username ?? null }, { atomic, terminalWs: getTerminalWs(req.get(DEPLOY_SESSION_HEADER)) }, ); - DatabaseService.getInstance().clearStackUpdateStatus(req.nodeId, stackName); + // Health observation starts immediately after Compose; registry recheck is + // isolated so a verification failure cannot turn Compose success into 500. + ok = true; + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); + const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; + linkStackUpdateRecoveryGate(recoveryId, healthGateId); + + let recheckWarning: string | undefined; + try { + const recheck = await ImageUpdateService.getInstance().recheckStack(req.nodeId, stackName); + if (recheck.warning) recheckWarning = recheck.warning; + } catch (recheckErr) { + console.warn( + '[Stacks] Post-update recheck failed for %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(recheckErr, 'unknown')), + ); + recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING; + } + + invalidateFleetUpdateCache(); invalidateNodeCaches(req.nodeId); NotificationService.getInstance().broadcastEvent({ type: 'state-invalidate', @@ -2282,11 +2367,11 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { }); dlog(`[Stacks] Update completed: ${sanitizeForLog(stackName)}`); if (debug) console.debug(`[Stacks:debug] Update finished in ${Date.now() - t0}ms`); - ok = true; - const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); - const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; - linkStackUpdateRecoveryGate(recoveryId, healthGateId); - res.json({ status: 'Update completed', healthGateId }); + res.json({ + status: 'Update completed', + healthGateId, + ...(recheckWarning ? { recheckWarning } : {}), + }); notifyActionSuccess('image_update_applied', `${stackName} updated`, stackName, req.user?.username ?? 'system'); if (!skipScan) { triggerPostDeployScan(stackName, req.nodeId).catch(err => diff --git a/backend/src/services/CacheService.ts b/backend/src/services/CacheService.ts index c1b5e4f7..f1d3a9f6 100644 --- a/backend/src/services/CacheService.ts +++ b/backend/src/services/CacheService.ts @@ -64,6 +64,9 @@ export class CacheService { private readonly store = new Map>(); private readonly inflight = new Map>(); + /** Per-key write generation. Bumped on invalidate so an older in-flight + * fetcher cannot commit after the key was intentionally cleared. */ + private readonly generations = new Map(); private readonly stats = new Map(); public static getInstance(): CacheService { @@ -121,13 +124,21 @@ export class CacheService { return { value, outcome: 'inflight' }; } + // Capture generation before the fetch so invalidate() during the wait can + // supersede this writer's store commit (and drop the inflight slot so a + // later caller starts a fresh computation). + const generation = this.currentGeneration(key); + // This caller owns the computation; the closure records whether it ended // as a fresh compute or a stale fallback, read after the promise settles. let outcome: CacheFetchOutcome = 'computed'; + const inflightSelf: { promise: Promise | null } = { promise: null }; const promise = (async () => { try { const value = await fetcher(); - this.set(key, value, ttlMs); + if (this.currentGeneration(key) === generation) { + this.set(key, value, ttlMs); + } return value; } catch (err) { if (existing) { @@ -137,9 +148,14 @@ export class CacheService { } throw err; } finally { - this.inflight.delete(key); + // Only clear the inflight slot if we still own it. invalidate() may + // have already deleted this entry and allowed a newer owner. + if (this.inflight.get(key) === inflightSelf.promise) { + this.inflight.delete(key); + } } })(); + inflightSelf.promise = promise; this.inflight.set(key, promise); const value = await promise; @@ -177,17 +193,22 @@ export class CacheService { this.store.set(key, { value, expiresAt: Date.now() + ttlMs }); } - /** Invalidate a single key. */ + /** Invalidate a single key and supersede any in-flight writer for it. */ public invalidate(key: string): void { this.store.delete(key); + this.inflight.delete(key); + this.bumpGeneration(key); } /** Invalidate every key whose namespace matches `namespace`. */ public invalidateNamespace(namespace: string): void { const prefix = `${namespace}:`; - for (const key of this.store.keys()) { + const keys = new Set([...this.store.keys(), ...this.inflight.keys(), ...this.generations.keys()]); + for (const key of keys) { if (key === namespace || key.startsWith(prefix)) { this.store.delete(key); + this.inflight.delete(key); + this.bumpGeneration(key); } } } @@ -196,6 +217,7 @@ export class CacheService { public flush(): void { this.store.clear(); this.inflight.clear(); + this.generations.clear(); this.stats.clear(); } @@ -259,4 +281,14 @@ export class CacheService { if (entry.expiresAt <= now) this.store.delete(key); } } + + private currentGeneration(key: string): number { + return this.generations.get(key) ?? 0; + } + + private bumpGeneration(key: string): number { + const next = this.currentGeneration(key) + 1; + this.generations.set(key, next); + return next; + } } diff --git a/backend/src/services/CapabilityRegistry.ts b/backend/src/services/CapabilityRegistry.ts index ba806c5d..1ef69b2f 100644 --- a/backend/src/services/CapabilityRegistry.ts +++ b/backend/src/services/CapabilityRegistry.ts @@ -37,12 +37,14 @@ export const CAPABILITIES = [ 'notification-routing', 'notification-suppression', 'notification-suppression-schedule', + 'notification-suppression-replica-retraction', 'host-console', 'host-console-community', 'container-exec', 'audit-log', 'scheduled-ops', 'sso', + 'authentication-mode', 'api-tokens', 'users', 'registries', @@ -83,6 +85,14 @@ export const HOST_CONSOLE_COMMUNITY_CAPABILITY = 'host-console-community' as con export const NOTIFICATION_SUPPRESSION_SCHEDULE_CAPABILITY = 'notification-suppression-schedule' as const satisfies Capability; +/** + * Remotes that accept hub-authored `{ kind, source_updated_at }` on replica DELETE + * and persist versioned tombstones. Without this, hubs must not send recoverable + * soft-cleanup DELETEs (pre-tombstone remotes would bare-delete with no guard). + */ +export const NOTIFICATION_SUPPRESSION_REPLICA_RETRACTION_CAPABILITY = + 'notification-suppression-replica-retraction' as const satisfies Capability; + /** Capability for optional `?removeVolumes=true` on POST /stacks/:name/down. */ export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability; diff --git a/backend/src/services/ComposeDoctorService.ts b/backend/src/services/ComposeDoctorService.ts index b662ff80..911d90fb 100644 --- a/backend/src/services/ComposeDoctorService.ts +++ b/backend/src/services/ComposeDoctorService.ts @@ -11,10 +11,12 @@ import { parseComposeDependencies } from '../helpers/composeDependencyParse'; import { parseEffectiveModel, type EffectiveModel } from './preflight/effectiveModel'; import { getExposureContext } from './network/exposureContext'; import type { ExposureIntent } from './network/types'; -import { runRules, SEVERITY_RANK, RULE_IDS, RENDER_FAILED_RULE_ID } from './preflight/rules'; +import { runRules, SEVERITY_RANK, RULE_IDS, RENDER_FAILED_RULE_ID, isPreflightNoteFinding } from './preflight/rules'; import type { BindCheck, NodePortBinding, PreflightContext, PreflightFinding, PreflightReport, PreflightSeverity, PreflightStatus, MissingEnvFile, + ServiceHealthcheckEvidence, } from './preflight/types'; +import { collectServiceHealthcheckEvidence } from './healthcheck/collectServiceHealthcheckEvidence'; import { applyPreflightAcknowledgements, parseServiceImages } from '../utils/preflight-ack-filter'; import { isPathWithinBase } from '../utils/validation'; @@ -38,6 +40,7 @@ function sortFindings(findings: PreflightFinding[]): PreflightFinding[] { function highestOf(findings: PreflightFinding[]): PreflightSeverity | null { let best: PreflightSeverity | null = null; for (const f of findings) { + if (isPreflightNoteFinding(f.ruleId)) continue; if (best === null || SEVERITY_RANK[f.severity] > SEVERITY_RANK[best]) best = f.severity; } return best; @@ -47,8 +50,10 @@ function activeFields( renderable: boolean, findings: PreflightFinding[], ): Pick { - const active = findings.filter(f => !f.acknowledged); - const acknowledgedCount = findings.length - active.length; + // Notes stay in `findings` for display but do not affect All Clear or active severity. + const issueFindings = findings.filter(f => !isPreflightNoteFinding(f.ruleId)); + const active = issueFindings.filter(f => !f.acknowledged); + const acknowledgedCount = issueFindings.length - active.length; const activeHighestSeverity = highestOf(active); const activeStatus: PreflightStatus = !renderable ? 'unrenderable' @@ -265,7 +270,12 @@ export class ComposeDoctorService { } const { nodePorts, existingNetworkNames, existingVolumeNames, existingContainers, nodeStateAvailable } = await this.nodeState(nodeId, fsSvc, stackName); - const bindChecks = model ? await this.resolveBindChecks(model, baseDir) : []; + const [bindChecks, healthchecks] = await Promise.all([ + model ? this.resolveBindChecks(model, baseDir) : Promise.resolve([] as BindCheck[]), + model + ? collectServiceHealthcheckEvidence(nodeId, stackName, model, nodeStateAvailable) + : Promise.resolve({} as Record), + ]); const { stackIntent, serviceIntents, accessUrlPorts, hasAccessUrls, exposureAvailable } = this.exposureState(nodeId, stackName); const selfStack = await isSelfStack(stackName); @@ -292,6 +302,7 @@ export class ComposeDoctorService { hasAccessUrls, exposureAvailable, isSelfStack: selfStack, + healthchecks, }; } diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index ace540a4..5eea339a 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -94,6 +94,20 @@ function parseServicesJson(raw: string | null | undefined): StackServiceStatus[] } } +/** Write generation embedded in services_json; 0 when missing or unreadable. */ +function parseServicesJsonGeneration(raw: string | null | undefined): number { + if (!raw) return 0; + try { + const parsed = JSON.parse(raw) as { version?: unknown; generation?: unknown }; + if (parsed?.version !== SERVICES_JSON_VERSION) return 0; + return typeof parsed.generation === 'number' && Number.isFinite(parsed.generation) + ? parsed.generation + : 0; + } catch { + return 0; + } +} + function stringifyServicesJson(services: StackServiceStatus[], generation: number): string { return JSON.stringify({ version: SERVICES_JSON_VERSION, generation, services }); } @@ -700,6 +714,10 @@ export interface ScheduledTask { prune_targets: string | null; target_services: string | null; prune_label_filter: string | null; + /** Optional dynamic target selector; currently only 'stack-label'. */ + selector_type: string | null; + /** Selector payload (e.g. Stack Label name when selector_type is stack-label). */ + selector_value: string | null; delete_after_run?: number; // Absolute epoch-ms fire time for a one-time ('once') schedule. A 5-field // cron has no year field, so the chosen instant (including year) is persisted @@ -752,6 +770,52 @@ export interface NotificationRoute { export type NotificationSuppressionAppliesTo = 'bell' | 'external' | 'both'; +/** Hub-authored replica retraction: permanent never clears; recoverable is LWW-ordered. */ +export type NotificationSuppressionRetractionKind = 'permanent' | 'recoverable'; + +export interface NotificationSuppressionRetraction { + kind: NotificationSuppressionRetractionKind; + /** Hub rule.updated_at at retraction time; compared only to hub versions, never receiver clocks. */ + source_updated_at: number; +} + +export interface NotificationSuppressionRuleTombstone { + id: number; + deleted_at: number; + kind: NotificationSuppressionRetractionKind; + source_updated_at: number; +} + +export type SuppressionReplicaWriteOutcome = + | 'applied' + | 'ignored_stale' + | 'ignored_permanent_tombstone' + | 'ignored_recoverable_watermark'; + +export type SuppressionReplicaDeleteOutcome = 'applied' | 'ignored_stale'; + +export interface NotificationSuppressionPendingRetraction { + rule_id: number; + node_id: number; + kind: NotificationSuppressionRetractionKind; + source_updated_at: number; + created_at: number; + updated_at: number; + attempts: number; + last_error: string | null; +} + +/** Fail closed: anything other than recoverable is permanent. */ +function normalizeSuppressionRetractionKind(kind: unknown): NotificationSuppressionRetractionKind { + return kind === 'recoverable' ? 'recoverable' : 'permanent'; +} + +/** Coerce tombstone watermarks for merge/read; invalid values become 0. */ +function safeTombstoneSourceUpdatedAt(value: unknown): number { + const n = Number(value); + return Number.isSafeInteger(n) ? n : 0; +} + export interface NotificationSuppressionRule { id: number; name: string; @@ -1876,6 +1940,8 @@ export class DatabaseService { maybeAddCol('scheduled_tasks', 'prune_targets', 'TEXT DEFAULT NULL'); maybeAddCol('scheduled_tasks', 'target_services', 'TEXT DEFAULT NULL'); maybeAddCol('scheduled_tasks', 'prune_label_filter', 'TEXT DEFAULT NULL'); + maybeAddCol('scheduled_tasks', 'selector_type', 'TEXT DEFAULT NULL'); + maybeAddCol('scheduled_tasks', 'selector_value', 'TEXT DEFAULT NULL'); maybeAddCol('scheduled_tasks', 'delete_after_run', 'INTEGER DEFAULT 0'); maybeAddCol('scheduled_tasks', 'run_at', 'INTEGER DEFAULT NULL'); @@ -1952,16 +2018,27 @@ export class DatabaseService { stmt.run('cve_intel_enabled', '1'); stmt.run('mesh_auto_recreate', '0'); stmt.run('prune_on_update', '1'); - stmt.run('reclaim_hero', '1'); + // Managed by /api/sso/auth-mode, not the generic /api/settings route + // (activation needs safety validation). + stmt.run('authentication_mode', 'local_and_sso'); + stmt.run('reclaim_hero', '0'); stmt.run('health_gate_enabled', '1'); stmt.run('health_gate_window_seconds', '90'); stmt.run('image_update_check_interval_minutes', '120'); stmt.run('image_update_check_mode', 'interval'); stmt.run('image_update_check_cron', ''); stmt.run('image_update_sidebar_indicators', '1'); + // Opt-out for background registry polling. Default on so upgrades keep + // current behavior; missing key is also treated as enabled at read time. + stmt.run('image_update_checks_enabled', '1'); stmt.run('notification_dispatch_retries', '0'); stmt.run('env_block_deploy_on_missing_required', '0'); stmt.run('auto_create_missing_external_networks', '0'); + // Silently extend an actively-used session's cookie instead of hard + // expiring it. On by default (matches how most session-based web apps + // behave); admins who want a strict absolute session ceiling can turn + // it off in Settings > Users. + stmt.run('session_sliding_refresh', '1'); // Seed the default local node if none exists const nodeCount = (this.db.prepare('SELECT COUNT(*) as count FROM nodes').get() as any)?.count || 0; @@ -2252,6 +2329,39 @@ export class DatabaseService { ); `); this.tryAddColumn('notification_suppression_rules', 'schedule', 'TEXT NULL'); + // kind + source_updated_at: hub-authored retract ordering. Legacy rows stay + // permanent (fail closed); source_updated_at backfills from deleted_at for + // audit continuity but is never compared to receiver wall clocks on write paths. + this.tryAddColumn( + 'notification_suppression_rule_tombstones', + 'kind', + "TEXT NOT NULL DEFAULT 'permanent'", + ); + this.tryAddColumn( + 'notification_suppression_rule_tombstones', + 'source_updated_at', + 'INTEGER', + ); + this.db.prepare( + `UPDATE notification_suppression_rule_tombstones + SET source_updated_at = deleted_at + WHERE source_updated_at IS NULL`, + ).run(); + this.db.exec(` + CREATE TABLE IF NOT EXISTS notification_suppression_pending_retractions ( + rule_id INTEGER NOT NULL, + node_id INTEGER NOT NULL, + kind TEXT NOT NULL, + source_updated_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + PRIMARY KEY (rule_id, node_id) + ); + CREATE INDEX IF NOT EXISTS idx_supp_pending_retract_node + ON notification_suppression_pending_retractions(node_id); + `); } private migrateNotificationHistoryContext(): void { @@ -2926,7 +3036,32 @@ export class DatabaseService { return this.getNotificationSuppressionRule(result.lastInsertRowid as number)!; } - public upsertNotificationSuppressionRuleReplica(rule: NotificationSuppressionRule): void { + private insertNotificationSuppressionRuleReplicaRow( + rule: NotificationSuppressionRule, + scheduleJson: string | null, + ): void { + this.db.prepare( + `INSERT INTO notification_suppression_rules + (id, name, node_id, stack_patterns, label_ids, categories, levels, applies_to, enabled, expires_at, schedule, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + rule.id, + rule.name, + rule.node_id ?? null, + JSON.stringify(rule.stack_patterns), + rule.label_ids ? JSON.stringify(rule.label_ids) : null, + rule.categories ? JSON.stringify(rule.categories) : null, + rule.levels ? JSON.stringify(rule.levels) : null, + rule.applies_to, + rule.enabled ? 1 : 0, + rule.expires_at ?? null, + scheduleJson, + rule.created_at, + rule.updated_at, + ); + } + + public upsertNotificationSuppressionRuleReplica(rule: NotificationSuppressionRule): SuppressionReplicaWriteOutcome { const scheduleJson = rule.schedule ? JSON.stringify(rule.schedule) : null; const existing = this.getNotificationSuppressionRule(rule.id); if (existing) { @@ -2937,7 +3072,7 @@ export class DatabaseService { `[DatabaseService] Ignoring stale suppression replica write for rule id=${sanitizeForLog(rule.id)} ` + `(incoming updated_at=${sanitizeForLog(rule.updated_at)} <= stored updated_at=${sanitizeForLog(existing.updated_at)})`, ); - return; + return 'ignored_stale'; } this.db.prepare( `UPDATE notification_suppression_rules SET @@ -2958,37 +3093,46 @@ export class DatabaseService { rule.updated_at, rule.id, ); - return; + return 'applied'; } - const tombstone = this.db.prepare( - 'SELECT deleted_at FROM notification_suppression_rule_tombstones WHERE id = ?', - ).get(rule.id) as { deleted_at: number } | undefined; - if (tombstone) { - console.warn( - `[DatabaseService] Ignoring suppression replica write for rule id=${sanitizeForLog(rule.id)}: ` + - `this id was deleted at ${sanitizeForLog(tombstone.deleted_at)} and must not be recreated`, - ); - return; - } - this.db.prepare( - `INSERT INTO notification_suppression_rules - (id, name, node_id, stack_patterns, label_ids, categories, levels, applies_to, enabled, expires_at, schedule, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - ).run( - rule.id, - rule.name, - rule.node_id ?? null, - JSON.stringify(rule.stack_patterns), - rule.label_ids ? JSON.stringify(rule.label_ids) : null, - rule.categories ? JSON.stringify(rule.categories) : null, - rule.levels ? JSON.stringify(rule.levels) : null, - rule.applies_to, - rule.enabled ? 1 : 0, - rule.expires_at ?? null, - scheduleJson, - rule.created_at, - rule.updated_at, - ); + // Re-read tombstone inside the transaction so a concurrent permanent + // retract cannot be cleared after an outer eligibility check. + let outcome: SuppressionReplicaWriteOutcome = 'applied'; + this.transaction(() => { + const tombstone = this.db.prepare( + `SELECT id, deleted_at, kind, source_updated_at + FROM notification_suppression_rule_tombstones WHERE id = ?`, + ).get(rule.id) as NotificationSuppressionRuleTombstone | undefined; + if (tombstone) { + const kind = normalizeSuppressionRetractionKind(tombstone.kind); + // Keep raw Number here: invalid watermarks must fail closed (block recreate), + // not coerce to 0 the way safeTombstoneSourceUpdatedAt does for merges/reads. + const sourceUpdatedAt = Number(tombstone.source_updated_at); + if (kind === 'permanent') { + console.warn( + `[DatabaseService] Ignoring suppression replica write for rule id=${sanitizeForLog(rule.id)}: ` + + `permanent tombstone (source_updated_at=${sanitizeForLog(sourceUpdatedAt)})`, + ); + outcome = 'ignored_permanent_tombstone'; + return; + } + if (!Number.isSafeInteger(sourceUpdatedAt) || rule.updated_at <= sourceUpdatedAt) { + console.warn( + `[DatabaseService] Ignoring suppression replica write for rule id=${sanitizeForLog(rule.id)}: ` + + `recoverable tombstone source_updated_at=${sanitizeForLog(sourceUpdatedAt)}; ` + + `incoming updated_at=${sanitizeForLog(rule.updated_at)} is not newer`, + ); + outcome = 'ignored_recoverable_watermark'; + return; + } + this.db.prepare( + 'DELETE FROM notification_suppression_rule_tombstones WHERE id = ?', + ).run(rule.id); + } + this.insertNotificationSuppressionRuleReplicaRow(rule, scheduleJson); + outcome = 'applied'; + }); + return outcome; } public updateNotificationSuppressionRule( @@ -3018,19 +3162,164 @@ export class DatabaseService { this.db.prepare(`UPDATE notification_suppression_rules SET ${fields.join(', ')} WHERE id = ?`).run(...values); } - public deleteNotificationSuppressionRule(id: number): number { - const changes = this.db.prepare('DELETE FROM notification_suppression_rules WHERE id = ?').run(id).changes; - // Fleet sync has no delivery ordering guarantee: a replica POST for this id can - // still be in flight. Record the delete permanently (ids are AUTOINCREMENT and - // never reused) so upsertNotificationSuppressionRuleReplica refuses to resurrect it. - // Tombstone unconditionally, even when changes is 0: deleteRuleOnNode issues this - // same DELETE for capability/invalid-schedule cleanup against remotes that may never - // have received the rule yet, and a reordered POST behind that DELETE must not create it. + /** + * Delete a suppression rule and record a hub-authored retraction tombstone. + * Optional retraction defaults to permanent/0 (fail closed) for one-arg callers. + * Recoverable DELETEs that are strictly older than a stored row are ignored + * (no durable mutation). Permanent always applies. Row delete + tombstone + * upsert are one transaction. + */ + public deleteNotificationSuppressionRule( + id: number, + retraction: NotificationSuppressionRetraction = { kind: 'permanent', source_updated_at: 0 }, + ): { changes: number; outcome: SuppressionReplicaDeleteOutcome } { + const kind = normalizeSuppressionRetractionKind(retraction.kind); + const sourceUpdatedAt = retraction.source_updated_at; + + return this.transaction(() => { + const existing = this.getNotificationSuppressionRule(id); + if ( + kind === 'recoverable' && + existing && + existing.updated_at > sourceUpdatedAt + ) { + console.warn( + `[DatabaseService] Ignoring stale recoverable suppression DELETE for rule id=${sanitizeForLog(id)}: ` + + `source_updated_at=${sanitizeForLog(sourceUpdatedAt)} < stored updated_at=${sanitizeForLog(existing.updated_at)}`, + ); + return { changes: 0, outcome: 'ignored_stale' }; + } + + const changes = this.db.prepare( + 'DELETE FROM notification_suppression_rules WHERE id = ?', + ).run(id).changes; + + const prior = this.db.prepare( + `SELECT kind, source_updated_at FROM notification_suppression_rule_tombstones WHERE id = ?`, + ).get(id) as { kind: string; source_updated_at: number } | undefined; + + let mergedKind = kind; + let mergedSource = sourceUpdatedAt; + if (prior) { + const priorKind = normalizeSuppressionRetractionKind(prior.kind); + // Permanent wins over recoverable; watermark always takes the max. + mergedKind = + priorKind === 'permanent' || kind === 'permanent' ? 'permanent' : 'recoverable'; + mergedSource = Math.max( + safeTombstoneSourceUpdatedAt(prior.source_updated_at), + sourceUpdatedAt, + ); + } + + this.db.prepare( + `INSERT INTO notification_suppression_rule_tombstones (id, deleted_at, kind, source_updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + deleted_at = excluded.deleted_at, + kind = excluded.kind, + source_updated_at = excluded.source_updated_at`, + ).run(id, Date.now(), mergedKind, mergedSource); + + return { changes, outcome: 'applied' }; + }); + } + + public getNotificationSuppressionRuleTombstone( + id: number, + ): NotificationSuppressionRuleTombstone | undefined { + const row = this.db.prepare( + `SELECT id, deleted_at, kind, source_updated_at + FROM notification_suppression_rule_tombstones WHERE id = ?`, + ).get(id) as NotificationSuppressionRuleTombstone | undefined; + if (!row) return undefined; + return { + id: row.id, + deleted_at: row.deleted_at, + kind: normalizeSuppressionRetractionKind(row.kind), + // Keep || 0 (not safeTombstoneSourceUpdatedAt): public reads historically + // surface any truthy Number() result; merge/write paths coerce separately. + source_updated_at: Number(row.source_updated_at) || 0, + }; + } + + + public upsertNotificationSuppressionPendingRetraction(row: { + rule_id: number; + node_id: number; + kind: NotificationSuppressionRetractionKind; + source_updated_at: number; + last_error?: string; + }): void { + const now = Date.now(); + const kind = normalizeSuppressionRetractionKind(row.kind); + const prior = this.db.prepare( + `SELECT kind, source_updated_at, attempts FROM notification_suppression_pending_retractions + WHERE rule_id = ? AND node_id = ?`, + ).get(row.rule_id, row.node_id) as { kind: string; source_updated_at: number; attempts: number } | undefined; + let mergedKind = kind; + let mergedSource = row.source_updated_at; + let attempts = 1; + if (prior) { + const priorKind = normalizeSuppressionRetractionKind(prior.kind); + mergedKind = + priorKind === 'permanent' || kind === 'permanent' ? 'permanent' : 'recoverable'; + mergedSource = Math.max( + safeTombstoneSourceUpdatedAt(prior.source_updated_at), + row.source_updated_at, + ); + attempts = (prior.attempts || 0) + 1; + } this.db.prepare( - `INSERT INTO notification_suppression_rule_tombstones (id, deleted_at) VALUES (?, ?) - ON CONFLICT(id) DO UPDATE SET deleted_at = excluded.deleted_at`, - ).run(id, Date.now()); - return changes; + `INSERT INTO notification_suppression_pending_retractions + (rule_id, node_id, kind, source_updated_at, created_at, updated_at, attempts, last_error) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(rule_id, node_id) DO UPDATE SET + kind = excluded.kind, + source_updated_at = excluded.source_updated_at, + updated_at = excluded.updated_at, + attempts = excluded.attempts, + last_error = excluded.last_error`, + ).run( + row.rule_id, + row.node_id, + mergedKind, + mergedSource, + now, + now, + attempts, + row.last_error ?? null, + ); + } + + public deleteNotificationSuppressionPendingRetraction(ruleId: number, nodeId: number): void { + this.db.prepare( + 'DELETE FROM notification_suppression_pending_retractions WHERE rule_id = ? AND node_id = ?', + ).run(ruleId, nodeId); + } + + public listNotificationSuppressionPendingRetractions( + nodeId?: number, + ): NotificationSuppressionPendingRetraction[] { + const rows = ( + nodeId == null + ? this.db.prepare( + `SELECT * FROM notification_suppression_pending_retractions ORDER BY updated_at ASC`, + ).all() + : this.db.prepare( + `SELECT * FROM notification_suppression_pending_retractions + WHERE node_id = ? ORDER BY updated_at ASC`, + ).all(nodeId) + ) as Array>; + return rows.map((r) => ({ + rule_id: r.rule_id as number, + node_id: r.node_id as number, + kind: normalizeSuppressionRetractionKind(r.kind), + source_updated_at: Number(r.source_updated_at) || 0, + created_at: r.created_at as number, + updated_at: r.updated_at as number, + attempts: (r.attempts as number) || 0, + last_error: (r.last_error as string | null) ?? null, + })); } // --- Global Settings --- @@ -4436,6 +4725,9 @@ export class DatabaseService { this.deleteRoleAssignmentsByResource('node', String(id)); this.db.prepare('DELETE FROM fleet_sync_status WHERE node_id = ?').run(id); this.db.prepare('DELETE FROM node_update_skips WHERE node_id = ?').run(id); + this.db.prepare( + 'DELETE FROM notification_suppression_pending_retractions WHERE node_id = ?', + ).run(id); this.db.prepare('DELETE FROM nodes WHERE id = ?').run(id); })(); } @@ -4628,6 +4920,13 @@ export class DatabaseService { ).run(nodeId, stackName, lastError, checkedAt, servicesJson); } + /** + * Raw has_update map for scanner retention and notification transitions. + * Ignores check_status: a partial/failed row with has_update=1 stays true + * so ImageUpdateService can preserve sticky state across incomplete runs. + * API/Fleet consumers that need "confirmed update" must use + * getConfirmedStackUpdateStatus instead. + */ public getStackUpdateStatus(nodeId?: number): Record { const rows = nodeId !== undefined ? this.db.prepare('SELECT stack_name, has_update FROM stack_update_status WHERE node_id = ?').all(nodeId) as Array<{ stack_name: string; has_update: number }> @@ -4639,10 +4938,32 @@ export class DatabaseService { return result; } + /** + * Confirmed-update projection for GET /api/image-updates and Fleet local + * aggregation. True only when has_update=1 and the latest check completed + * successfully (check_status='ok'). Partial/failed retained rows are false. + */ + public getConfirmedStackUpdateStatus(nodeId?: number): Record { + const rows = nodeId !== undefined + ? this.db.prepare( + `SELECT stack_name, has_update, check_status FROM stack_update_status WHERE node_id = ?` + ).all(nodeId) as Array<{ stack_name: string; has_update: number; check_status: string | null }> + : this.db.prepare( + `SELECT stack_name, has_update, check_status FROM stack_update_status` + ).all() as Array<{ stack_name: string; has_update: number; check_status: string | null }>; + const result: Record = {}; + for (const row of rows) { + // Match getNodeUpdateSummary / frontend isConfirmedImageUpdate: + // null check_status is treated as ok (legacy rows). + result[row.stack_name] = row.has_update === 1 && (row.check_status ?? 'ok') === 'ok'; + } + return result; + } + /** * Rich per-stack update status (hasUpdate + check outcome + reason) for the - * sidebar/readiness UI. GET /api/image-updates stays the boolean map so the - * cross-version fleet aggregation contract is unaffected. + * sidebar/readiness UI. Confirmed boolean maps use getConfirmedStackUpdateStatus; + * raw prior state for the scanner stays on getStackUpdateStatus. */ public getStackUpdateDetail(nodeId: number): Record { const rows = this.db.prepare( @@ -4675,8 +4996,28 @@ export class DatabaseService { return parseServicesJson(row?.services_json); } - public clearStackUpdateStatus(nodeId: number, stackName: string): void { - this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName); + /** + * Scanner write generation stored with services_json for this stack. + * Used by preview reconcile to skip clearing a row written after the + * preview observation watermark. Returns 0 when missing or unreadable. + */ + public getStackUpdateWriteGeneration(nodeId: number, stackName: string): number { + const row = this.db.prepare( + 'SELECT services_json FROM stack_update_status WHERE node_id = ? AND stack_name = ?' + ).get(nodeId, stackName) as { services_json: string | null } | undefined; + return parseServicesJsonGeneration(row?.services_json); + } + + /** Deletes the full update row (aggregate + services_json). Returns deleted row count. */ + public clearStackUpdateStatus(nodeId: number, stackName: string): number { + const result = this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName); + return result.changes; + } + + /** Deletes every update row for a node. Returns deleted row count. */ + public clearAllStackUpdateStatus(nodeId: number): number { + const result = this.db.prepare('DELETE FROM stack_update_status WHERE node_id = ?').run(nodeId); + return result.changes; } // --- Stack Scan Attempts --- @@ -4718,7 +5059,10 @@ export class DatabaseService { public getNodeUpdateSummary(): Array<{ node_id: number; stacks_with_updates: number }> { return this.db.prepare( - 'SELECT node_id, SUM(has_update) as stacks_with_updates FROM stack_update_status WHERE has_update = 1 GROUP BY node_id' + `SELECT node_id, SUM(has_update) as stacks_with_updates + FROM stack_update_status + WHERE has_update = 1 AND COALESCE(check_status, 'ok') = 'ok' + GROUP BY node_id` ).all() as Array<{ node_id: number; stacks_with_updates: number }>; } @@ -5703,13 +6047,14 @@ export class DatabaseService { public createScheduledTask(task: Omit): number { const result = this.db.prepare( - 'INSERT INTO scheduled_tasks (name, target_type, target_id, node_id, action, cron_expression, enabled, created_by, created_at, updated_at, last_run_at, next_run_at, last_status, last_error, prune_targets, target_services, prune_label_filter, delete_after_run, run_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' + 'INSERT INTO scheduled_tasks (name, target_type, target_id, node_id, action, cron_expression, enabled, created_by, created_at, updated_at, last_run_at, next_run_at, last_status, last_error, prune_targets, target_services, prune_label_filter, selector_type, selector_value, delete_after_run, run_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' ).run( task.name, task.target_type, task.target_id, task.node_id, task.action, task.cron_expression, task.enabled, task.created_by, task.created_at, task.updated_at, task.last_run_at, task.next_run_at, task.last_status, task.last_error, task.prune_targets, task.target_services, - task.prune_label_filter, task.delete_after_run ?? 0, task.run_at ?? null + task.prune_label_filter, task.selector_type ?? null, task.selector_value ?? null, + task.delete_after_run ?? 0, task.run_at ?? null ); return result.lastInsertRowid as number; } @@ -5726,6 +6071,8 @@ export class DatabaseService { last_status: updates.last_status, last_error: updates.last_error, prune_targets: updates.prune_targets, target_services: updates.target_services, prune_label_filter: updates.prune_label_filter, + selector_type: updates.selector_type, + selector_value: updates.selector_value, delete_after_run: updates.delete_after_run, run_at: updates.run_at, }; diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 7c46a91a..00ae367e 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -22,7 +22,7 @@ import { type PruneTarget, } from './prunePlan'; import type { NetworkingNetworkBase } from './network/networkingTypes'; -import { isPathWithinBase } from '../utils/validation'; +import { isPathWithinBase, isValidStackName } from '../utils/validation'; import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; @@ -2230,8 +2230,12 @@ class DockerController { try { const lines = stdout.trim().split('\n').filter(line => line.trim() !== ''); containers = lines.map(line => JSON.parse(line) as ComposePsContainer); - } catch (innerError) { - console.error('Docker Compose JSON Parse Error for %s:', sanitizeForLog(stackName), sanitizeForLog(stderr || (parseError as Error).message)); + } catch { + const detail = stderr || (parseError as Error).message || 'unparseable compose ps output'; + console.error('Docker Compose JSON Parse Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail)); + // Fail closed: garbage stdout is not a successful empty ps. Callers must not + // treat this as "no containers" and run name-matched orphan removal. + throw new Error(`docker compose ps returned unparseable JSON: ${detail}`); } } } @@ -2242,6 +2246,9 @@ class DockerController { * Legacy orphan containers that Compose ps cannot see but would block a deploy * (wrong project labels). When Compose already manages the stack, returns [] so * deploy can rely on selective `compose up` recreation. + * A thrown `compose ps` returns [] (fail closed): name-matched fallback must not + * run, because those IDs may still be healthy Compose runtimes that deploy would + * then destroy before `compose up` (which often fails for the same reason). */ public async getLegacyOrphanContainersByStack(stackName: string): Promise> { const stackDir = path.join(NodeRegistry.getInstance().getComposeDir(this.nodeId), stackName); @@ -2249,26 +2256,36 @@ class DockerController { list.filter((c): c is { Id: string } => typeof c.Id === 'string' && c.Id.length > 0) .map((c) => ({ Id: c.Id })); + let composeContainers: ComposePsContainer[]; try { - const composeContainers = await this.fetchComposePsContainers(stackName, stackDir); - if (composeContainers.length > 0) return []; - return toIds(await this.smartFallback(stackName, stackDir)); + composeContainers = await this.fetchComposePsContainers(stackName, stackDir); } catch (error) { const execError = error as NodeJS.ErrnoException & { stderr?: string }; const mapped = describeSpawnError(execError, { command: 'docker compose ps' }); - const detail = execError.stderr || mapped.message; + const detail = execError.stderr || mapped.message || getErrorMessage(error, 'docker compose ps failed'); console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail)); - try { - return toIds(await this.smartFallback(stackName, stackDir)); - } catch { - return []; - } + // Fail closed: do not run smartFallback (see method JSDoc). + return []; + } + if (composeContainers.length > 0) return []; + // Empty successful ps: only then may smartFallback report leftovers (name + stack-dir evidence). + try { + return toIds(await this.smartFallback(stackName, stackDir)); + } catch (fallbackError) { + console.error( + 'Smart Fallback failed for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(fallbackError, 'smartFallback failed')), + ); + return []; } } /** * Strict Result API for full-stack updates. Never converts Compose-ps + fallback * failure into empty success. Compose-managed containers are never orphan IDs. + * A thrown `compose ps` is classification_failed (fail closed): name-matched + * fallback must not run, because those IDs may still be healthy Compose runtimes. */ public async classifyLegacyOrphansForUpdate( stackName: string, @@ -2302,18 +2319,15 @@ class DockerController { const composeContainers = await this.fetchComposePsContainers(stackName, stackDir); // Compose already manages this stack: no legacy orphan cleanup (same as deploy). if (composeContainers.length > 0) return { status: 'none' }; + // Empty successful ps: only then may smartFallback report leftovers (name + stack-dir evidence). return await fallbackOrphans(); } catch (error) { const execError = error as NodeJS.ErrnoException & { stderr?: string }; const mapped = describeSpawnError(execError, { command: 'docker compose ps' }); const detail = execError.stderr || mapped.message || getErrorMessage(error, 'docker compose ps failed'); console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail)); - // Unlike getLegacyOrphanContainersByStack, never convert dual failure into empty success. - const fallback = await fallbackOrphans(); - if (fallback.status === 'classification_failed') { - return { status: 'classification_failed', error: String(detail) }; - } - return fallback; + // Fail closed: do not run smartFallback (see method JSDoc). + return { status: 'classification_failed', error: String(detail) }; } } @@ -2349,11 +2363,6 @@ class DockerController { }); return await this.enrichContainers(mapped); } - - // SMART FALLBACK: Trigger when docker compose ps returns empty - // This handles legacy containers with incorrect project labels - return await this.enrichContainers(await this.smartFallback(stackName, stackDir)); - } catch (error) { // If command fails (e.g., stack not deployed, invalid YAML, missing env_file, // or host under memory pressure causing posix_spawn to fail with ENOMEM, @@ -2362,9 +2371,23 @@ class DockerController { const mapped = describeSpawnError(execError, { command: 'docker compose ps' }); const detail = execError.stderr || mapped.message; console.error('Docker Compose Error for %s:', sanitizeForLog(stackName), sanitizeForLog(detail)); + } - // Try smart fallback even on error + // Empty successful ps or compose error: soft fallback for UI listing (never break the page). + return this.listViaSoftSmartFallback(stackName, stackDir); + } + + /** UI listing path: smartFallback, or empty list if it throws. */ + private async listViaSoftSmartFallback(stackName: string, stackDir: string) { + try { return await this.enrichContainers(await this.smartFallback(stackName, stackDir)); + } catch (fallbackError) { + console.error( + 'Smart Fallback failed for %s:', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(fallbackError, 'smartFallback failed')), + ); + return this.enrichContainers([]); } } @@ -2388,95 +2411,116 @@ class DockerController { })); } + /** + * True when container labels bind this runtime to `stackDir` (working_dir or + * config_files). Name match alone is not enough for orphan removal. + */ + private static containerHasStackDirEvidence( + labels: Record | undefined, + stackDir: string, + ): boolean { + if (!labels) return false; + const resolvedStackDir = path.resolve(stackDir); + const pathKey = (p: string) => + process.platform === 'win32' ? path.resolve(p).toLowerCase() : path.resolve(p); + const stackKey = pathKey(resolvedStackDir); + + const workingDir = labels['com.docker.compose.project.working_dir']; + if (workingDir && pathKey(workingDir) === stackKey) return true; + + const firstFile = labels['com.docker.compose.project.config_files']?.split(',')[0]?.trim(); + if (!firstFile) return false; + const fileKey = pathKey(firstFile); + return fileKey === stackKey || fileKey.startsWith(stackKey + path.sep); + } + /** * Smart Fallback: Find legacy containers by parsing compose YAML definitions. - * This handles containers that were deployed with incorrect project labels - * that cause `docker compose ps` to ignore them. + * Handles containers with incorrect project labels that `docker compose ps` + * ignores. Name match alone is insufficient: a container must also have + * working_dir or config_files evidence for this stack directory. Errors + * propagate to callers (classify fail-closes; UI listing soft-catches). */ private async smartFallback(stackName: string, stackDir: string): Promise { - try { - // 1. Flexible Compose File Discovery - // Try multiple valid compose file names - const composeFileNames = COMPOSE_FILE_NAMES; - let yamlContent: string | null = null; + if (!isValidStackName(stackName)) { + throw new Error('Invalid stack path'); + } + // Canonical inline js/path-injection barrier at the fs.readFile sink below. + // CodeQL credits neither wrapped isPathWithinBase nor a barrier separated + // from the sink; resolve under the compose root and require containment. + const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(this.nodeId)); + const resolvedStackDir = path.resolve(baseResolved, stackName); + if (!resolvedStackDir.startsWith(baseResolved + path.sep)) { + throw new Error('Invalid stack path'); + } + // Prefer the validated path over the caller-provided stackDir. + stackDir = resolvedStackDir; - for (const fileName of composeFileNames) { - try { - yamlContent = await fs.readFile(path.join(stackDir, fileName), 'utf-8'); - break; // Successfully read a file, stop trying - } catch { - // File doesn't exist, try next + let yamlContent: string | null = null; + for (const fileName of COMPOSE_FILE_NAMES) { + try { + const composePath = path.resolve(stackDir, fileName); + if (!composePath.startsWith(stackDir + path.sep)) { continue; } + yamlContent = await fs.readFile(composePath, 'utf-8'); + break; + } catch { + continue; } - - if (!yamlContent) { - // No compose file found - return []; - } - - const parsedYaml = yaml.parse(yamlContent); - - if (!parsedYaml || !parsedYaml.services) return []; - - // 2. Extract expected container names with legacy prefix support - const expectedNames: string[] = []; - const nameToService = new Map(); - for (const [serviceName, serviceConfig] of Object.entries(parsedYaml.services)) { - const config = serviceConfig as { container_name?: string }; - nameToService.set(serviceName, serviceName); - if (config.container_name) { - expectedNames.push(config.container_name); - nameToService.set(config.container_name, serviceName); - } else { - // Standard v2 naming - expectedNames.push(serviceName); - expectedNames.push(`${stackName}-${serviceName}-1`); - // Legacy project prefix catch - accounts for orphan containers - expectedNames.push(`compose-${serviceName}-1`); - expectedNames.push(`compose_${serviceName}_1`); - } - } - - // 3. Query the raw Docker daemon - const allContainers = await this.docker.listContainers({ all: true }); - - // 4. Match containers by name - const fallbackContainers = allContainers.filter(container => { - // container.Names usually looks like ['/plex'] - return container.Names.some(name => { - const strippedName = name.replace(/^\//, ''); - return expectedNames.includes(strippedName); - }); - }); - - // 5. Map to the frontend interface - return fallbackContainers.map(c => { - const strippedName = c.Names?.[0]?.replace(/^\//, '') ?? ''; - const labelService = c.Labels?.['com.docker.compose.service']; - const service = (typeof labelService === 'string' && labelService.length > 0 - ? labelService - : nameToService.get(strippedName)) ?? ''; - let Ports: { PrivatePort: number, PublicPort: number, Type?: string }[] = []; - if (c.Ports && Array.isArray(c.Ports)) { - Ports = c.Ports - .filter((p: any) => typeof p.PublicPort === 'number' && p.PublicPort > 0) - .map((p: any) => ({ PrivatePort: (p.PrivatePort || 0) as number, PublicPort: p.PublicPort as number, Type: typeof p.Type === 'string' ? p.Type.toLowerCase() : undefined })); - } - return { - Id: c.Id, - Names: c.Names, - Service: service, - State: c.State, - Status: c.Status, - Labels: c.Labels, - Ports - }; - }); - } catch (fallbackError) { - console.error('Smart Fallback failed for %s:', sanitizeForLog(stackName), sanitizeForLog((fallbackError as Error)?.message ?? String(fallbackError))); - return []; } + if (!yamlContent) return []; + + const parsedYaml = yaml.parse(yamlContent); + if (!parsedYaml || !parsedYaml.services) return []; + + const expectedNames: string[] = []; + const nameToService = new Map(); + for (const [serviceName, serviceConfig] of Object.entries(parsedYaml.services)) { + const config = serviceConfig as { container_name?: string }; + nameToService.set(serviceName, serviceName); + if (config.container_name) { + expectedNames.push(config.container_name); + nameToService.set(config.container_name, serviceName); + } else { + expectedNames.push(serviceName); + expectedNames.push(`${stackName}-${serviceName}-1`); + // Legacy project prefixes for orphan containers + expectedNames.push(`compose-${serviceName}-1`); + expectedNames.push(`compose_${serviceName}_1`); + } + } + + const allContainers = await this.docker.listContainers({ all: true }); + const fallbackContainers = allContainers.filter(container => { + const nameMatch = container.Names.some(name => + expectedNames.includes(name.replace(/^\//, '')), + ); + return nameMatch && DockerController.containerHasStackDirEvidence(container.Labels, stackDir); + }); + + return fallbackContainers.map(c => { + const strippedName = c.Names?.[0]?.replace(/^\//, '') ?? ''; + const labelService = c.Labels?.['com.docker.compose.service']; + const service = (typeof labelService === 'string' && labelService.length > 0 + ? labelService + : nameToService.get(strippedName)) ?? ''; + let Ports: { PrivatePort: number, PublicPort: number, Type?: string }[] = []; + if (c.Ports && Array.isArray(c.Ports)) { + Ports = c.Ports + .filter((p: any) => typeof p.PublicPort === 'number' && p.PublicPort > 0) + .map((p: any) => ({ PrivatePort: (p.PrivatePort || 0) as number, PublicPort: p.PublicPort as number, Type: typeof p.Type === 'string' ? p.Type.toLowerCase() : undefined })); + } + return { + Id: c.Id, + Names: c.Names, + Service: service, + State: c.State, + Status: c.Status, + Labels: c.Labels, + Ports + }; + }); } public async streamContainerLogs(containerId: string, req: any, res: any): Promise { diff --git a/backend/src/services/FleetUpdateTrackerService.ts b/backend/src/services/FleetUpdateTrackerService.ts index 8848cb12..3b4bf7e2 100644 --- a/backend/src/services/FleetUpdateTrackerService.ts +++ b/backend/src/services/FleetUpdateTrackerService.ts @@ -1,3 +1,5 @@ +export type FleetOperationKind = 'update' | 'reapply_configuration'; + export interface UpdateTracker { status: 'updating' | 'completed' | 'timeout' | 'failed'; startedAt: number; @@ -11,6 +13,8 @@ export interface UpdateTracker { wasOffline: boolean; /** Timestamp when the tracker transitioned to a terminal state (completed/failed/timeout). */ resolvedAt?: number; + /** Distinguishes version updates from compose reapply so poll heuristics stay correct. */ + operationKind: FleetOperationKind; } export type TerminalStatus = 'completed' | 'failed' | 'timeout'; @@ -61,13 +65,15 @@ export class FleetUpdateTrackerService { return this.trackers.size; } - /** Create a new tracker with `startedAt=now` and resolvedAt set if terminal. */ + /** Create a new tracker with `startedAt=now` and resolvedAt set if terminal. + * `operationKind` defaults to `'update'` so existing call sites stay unchanged. */ public create( status: UpdateTracker['status'], previousVersion: string | null, previousProcessStart: number | null, error?: string, code?: string, + operationKind: FleetOperationKind = 'update', ): UpdateTracker { const now = Date.now(); return { @@ -78,6 +84,7 @@ export class FleetUpdateTrackerService { wasOffline: false, error, code, + operationKind, resolvedAt: status !== 'updating' ? now : undefined, }; } diff --git a/backend/src/services/ImageOperationService.ts b/backend/src/services/ImageOperationService.ts index e0702f76..e2e442c8 100644 --- a/backend/src/services/ImageOperationService.ts +++ b/backend/src/services/ImageOperationService.ts @@ -8,7 +8,7 @@ import type { ImagePinKind } from '../helpers/selfUpdateCompose'; import type { LocalRegistryAccess } from './hardenedEntitlementTypes'; import { getAuthToken, httpRequest } from './registry-api'; -export type ImageOperationKind = 'switch' | 'update' | 'community_update'; +export type ImageOperationKind = 'switch' | 'update' | 'community_update' | 'compose_reapply'; export type ImageOperationState = 'pending_pull' | 'pulling' | 'patching' | 'recreating' | 'succeeded' | 'failed'; type FailureCode = 'self_update_unavailable' | 'entitlement_denied' | 'preflight_mismatch' | 'compose_unavailable' | 'registry_access_unavailable' | 'update_failed' | 'interrupted_by_restart'; @@ -146,32 +146,12 @@ export class ImageOperationService { public async claimCommunityUpdate(options?: { targetVersion?: string }): Promise< { ok: true } | { ok: false; failureCode: 'IMAGE_OPERATION_IN_FLIGHT' } > { - const selfUpdate = SelfUpdateService.getInstance(); - const resolved = await selfUpdate.getResolvedComposeImageForUpdate(); - const operation = this.newOperation( - 'community_update', - resolved?.imageRef ?? null, - options?.targetVersion ?? null, - resolved?.filePath ?? null, - selfUpdate.getComposeServiceName(), - ); - if (!await this.tryClaim(operation)) { - return { ok: false, failureCode: 'IMAGE_OPERATION_IN_FLIGHT' }; - } - // Disk non-terminal state is the concurrency lock; clear the in-memory mutex - // so a later claim can observe the persisted pending operation. - this.claimed = false; - return { ok: true }; + return this.claimComposeOperation('community_update', options?.targetVersion ?? null); } public async executeClaimedCommunityUpdate(options?: { targetVersion?: string }): Promise<{ ok: boolean; failureCode?: string }> { - const operation = await this.getCurrentOperation(); - if (!operation || operation.kind !== 'community_update') { - return { ok: false, failureCode: 'update_failed' }; - } - if (!['pending_pull', 'pulling', 'patching', 'recreating'].includes(operation.state)) { - return { ok: false, failureCode: 'update_failed' }; - } + const operation = await this.getActiveClaimedOperation('community_update'); + if (!operation) return { ok: false, failureCode: 'update_failed' }; const selfUpdate = SelfUpdateService.getInstance(); try { operation.state = 'pulling'; @@ -207,6 +187,37 @@ export class ImageOperationService { return this.executeClaimedCommunityUpdate(options); } + public async claimComposeReapply(): Promise< + { ok: true } | { ok: false; failureCode: 'IMAGE_OPERATION_IN_FLIGHT' } + > { + return this.claimComposeOperation('compose_reapply', null); + } + + public async executeClaimedComposeReapply(): Promise<{ ok: boolean; failureCode?: string }> { + const operation = await this.getActiveClaimedOperation('compose_reapply'); + if (!operation) return { ok: false, failureCode: 'update_failed' }; + const selfUpdate = SelfUpdateService.getInstance(); + try { + // No pull/patch for reapply: jump straight to recreating. + operation.state = 'recreating'; + await this.persist(operation); + this.watchHelperExit(operation); + await selfUpdate.triggerComposeReapply({ + successMarkerFile: this.successMarkerFile(operation), + successMarkerContent: JSON.stringify({ ok: true, operationId: operation.operationId }), + }); + if (selfUpdate.getLastError()) { + await this.fail(operation, 'update_failed'); + return { ok: false, failureCode: 'update_failed' }; + } + return { ok: true }; + } catch (error) { + console.error('[ImageOperation] Compose reapply failed:', error); + await this.fail(operation, 'update_failed'); + return { ok: false, failureCode: 'update_failed' }; + } + } + public async getOperation(operationId: string): Promise { const filePath = this.operationFile(operationId); if (!filePath) return null; @@ -243,8 +254,9 @@ export class ImageOperationService { const markerPath = this.successMarkerFile(operation); for (let elapsed = 0; elapsed < 30_000; elapsed += 1_000) { const markerOk = await this.isSuccessMarkerForOperation(markerPath, operation.operationId); - if (operation.kind === 'community_update') { - // Community success is the marker alone; floating tags may not equal targetImageRef. + if (operation.kind === 'community_update' || operation.kind === 'compose_reapply') { + // Marker-only success: community updates may leave floating tags that do + // not equal targetImageRef, and reapply never sets a target image at all. if (markerOk) { operation.state = 'succeeded'; operation.resolvedAt = new Date().toISOString(); @@ -284,6 +296,35 @@ export class ImageOperationService { }); } + private async claimComposeOperation( + kind: 'community_update' | 'compose_reapply', + targetImageRef: string | null, + ): Promise<{ ok: true } | { ok: false; failureCode: 'IMAGE_OPERATION_IN_FLIGHT' }> { + const selfUpdate = SelfUpdateService.getInstance(); + const resolved = await selfUpdate.getResolvedComposeImageForUpdate(); + const operation = this.newOperation( + kind, + resolved?.imageRef ?? null, + targetImageRef, + resolved?.filePath ?? null, + selfUpdate.getComposeServiceName(), + ); + if (!await this.tryClaim(operation)) { + return { ok: false, failureCode: 'IMAGE_OPERATION_IN_FLIGHT' }; + } + // Disk non-terminal state is the concurrency lock; clear the in-memory mutex + // so a later claim can observe the persisted pending operation. + this.claimed = false; + return { ok: true }; + } + + private async getActiveClaimedOperation(kind: ImageOperationKind): Promise { + const operation = await this.getCurrentOperation(); + if (!operation || operation.kind !== kind) return null; + if (!['pending_pull', 'pulling', 'patching', 'recreating'].includes(operation.state)) return null; + return operation; + } + private newOperation(kind: ImageOperationKind, previousImageRef: string | null, targetImageRef: string | null, composeFilePath: string | null, serviceName: string | null, preflightFingerprint?: string): ImageOperation { return { schemaVersion: 1, diff --git a/backend/src/services/ImageUpdateService.ts b/backend/src/services/ImageUpdateService.ts index 8c958746..25d1f5a0 100644 --- a/backend/src/services/ImageUpdateService.ts +++ b/backend/src/services/ImageUpdateService.ts @@ -8,16 +8,43 @@ import { RegistryService } from './RegistryService'; import { NodeRegistry } from './NodeRegistry'; import { NotificationService } from './NotificationService'; import { sanitizeNotificationMessage } from '../utils/notificationMessage'; -import { parseImageRef, selectLocalRepoDigest, compareLocalToRemoteTag } from './registry-api'; +import { parseImageRef, selectLocalRepoDigests } from './registry-api'; +import { detectImageUpdate, type PreviewImageCheckStatus } from './imageUpdateDetect'; import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; import { buildEffectiveServiceModel } from './effectiveServiceModel'; +import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache'; const BACKFILL_KEY = 'image_update_notifications_backfilled'; +/** Post-update scanner reconciliation outcome for a single stack. */ +export type StackRecheckOutcome = + | 'cleared' + | 'still_present' + | 'verification_incomplete' + | 'verification_failed'; + +export interface StackRecheckResult { + outcome: StackRecheckOutcome; + /** Present when the update condition remains or could not be verified. */ + warning: string | null; +} + +export const UPDATE_STILL_PRESENT_WARNING = + 'The update command completed, but Sencho still detects an available image update.'; + +export const UPDATE_VERIFICATION_INCOMPLETE_WARNING = + 'The update command completed, but Sencho could not fully verify whether an image update remains.'; + export interface ImageCheckResult { hasUpdate: boolean; + /** Same-tag registry digest drift; Compose pull can apply without pin change. */ + digestUpdate?: boolean; + /** Higher semver tag exists; UI may show it but Compose auto-apply cannot pin it. */ + tagUpdate?: boolean; + /** Detector authority; consumed by reduceServiceStatus / writeStackUpdateStatus. */ + checkStatus?: PreviewImageCheckStatus; error?: string; /** * The image is not registry-backed (locally built, or a bare digest ref @@ -28,6 +55,17 @@ export interface ImageCheckResult { notCheckable?: boolean; } +/** + * Normalize check authority for reduction. Test stubs / older callers may omit + * checkStatus: error means failed, else ok. + */ +export function normalizeImageCheckStatus(r: ImageCheckResult): PreviewImageCheckStatus { + if (r.notCheckable) return 'not_checkable'; + if (r.checkStatus) return r.checkStatus; + if (r.error) return 'failed'; + return 'ok'; +} + /** * Snapshot of the scanner returned by GET /api/image-updates/status. * Units differ by field: `intervalMinutes` / `manualCooldownMinutes` are @@ -38,6 +76,8 @@ export interface ImageCheckResult { * `nextCheckAt` is meaningless while `checking` is true. * `mode` is the active scheduling mode; `cronExpression` is the 5-field * expression when mode is 'cron', null otherwise or when unconfigured. + * `enabled` is whether background image-update detection is armed; always + * present on current nodes, optional on the wire for older remotes. */ export interface ImageUpdateStatus { checking: boolean; @@ -49,6 +89,7 @@ export interface ImageUpdateStatus { mode: 'interval' | 'cron'; cronExpression: string | null; sidebarIndicators: boolean; + enabled: boolean; } // ─── Compose file helpers ──────────────────────────────────────────────────── @@ -273,7 +314,8 @@ export function reduceServiceStatus( const checkableResults: ImageCheckResult[] = []; for (const ref of refs) { const result = imageUpdateMap.get(ref); - if (!result || result.notCheckable) continue; + if (!result) continue; + if (normalizeImageCheckStatus(result) === 'not_checkable') continue; checkableResults.push(result); } @@ -284,12 +326,14 @@ export function reduceServiceStatus( }; } - const errored = checkableResults.filter((r) => r.error !== undefined); + const statuses = checkableResults.map(normalizeImageCheckStatus); + const failed = checkableResults.filter((_, i) => statuses[i] === 'failed'); + const partial = checkableResults.filter((_, i) => statuses[i] === 'partial'); const confirmedUpdateThisRun = checkableResults.some( - (r) => r.error === undefined && r.hasUpdate === true, + (r, i) => statuses[i] === 'ok' && r.hasUpdate === true, ); - if (errored.length === checkableResults.length) { + if (failed.length === checkableResults.length) { const priorHasUpdate = prior?.hasUpdate ?? false; return { status: { @@ -298,14 +342,16 @@ export function reduceServiceStatus( ...(dedupedRuntime.length > 0 ? { runtimeImages: dedupedRuntime } : {}), hasUpdate: priorHasUpdate, checkStatus: 'failed', - lastError: errored[0].error ?? 'Update check failed', + lastError: failed[0].error ?? 'Update check failed', }, confirmedUpdateThisRun: false, }; } - const checkStatus: StackServiceStatus['checkStatus'] = errored.length > 0 ? 'partial' : 'ok'; - const lastError = errored.length > 0 ? (errored[0].error ?? null) : null; + const checkStatus: StackServiceStatus['checkStatus'] = + failed.length > 0 || partial.length > 0 ? 'partial' : 'ok'; + const uncertain = [...partial, ...failed]; + const lastError = uncertain.length > 0 ? (uncertain[0].error ?? null) : null; const hasUpdate = checkStatus === 'partial' ? (confirmedUpdateThisRun || (prior?.hasUpdate ?? false)) : confirmedUpdateThisRun; @@ -368,6 +414,7 @@ export class ImageUpdateService { private static readonly INTERVAL_SETTING_KEY = 'image_update_check_interval_minutes'; private static readonly MODE_SETTING_KEY = 'image_update_check_mode'; private static readonly CRON_SETTING_KEY = 'image_update_check_cron'; + private static readonly ENABLED_SETTING_KEY = 'image_update_checks_enabled'; private static readonly JITTER_FRACTION = 0.1; // ±10% so a fleet does not poll in lockstep private static readonly STARTUP_DELAY_MS = 2 * 60 * 1000; // 2 min after boot @@ -410,8 +457,15 @@ export class ImageUpdateService { public start() { if (this.timer) return; - this.polling = true; this.configureFromSettings(); + if (!ImageUpdateService.isChecksEnabled()) { + // Detection opted out: stay stopped across restarts so a boot does + // not re-arm registry polling until the setting is turned back on. + this.polling = false; + this.nextCheckAt = null; + return; + } + this.polling = true; // Interval mode keeps the 2-minute post-boot delay before the first check. // Cron mode honors its schedule: arm at the next cron fire time so a restart // never triggers an out-of-cadence check (e.g. a weekly cron must not run on @@ -434,7 +488,8 @@ export class ImageUpdateService { * cadence without restarting Sencho. Safe to call repeatedly: it always * clears the existing timer first and only arms a new one while polling, so * it never stacks timers and is a no-op (beyond reconfiguring intervalMs) - * when the service is stopped or was never started. + * when the service is stopped or was never started. When checks are + * disabled, clears nextCheckAt and does not arm. */ public restartPolling(): void { this.scheduleGeneration++; @@ -443,13 +498,65 @@ export class ImageUpdateService { this.timer = null; } this.configureFromSettings(); - if (this.polling) { + if (this.polling && ImageUpdateService.isChecksEnabled()) { this.armNext(this.nextDelayMs()); } else { this.nextCheckAt = null; } } + /** + * Whether background image-update detection is enabled. Missing or blank + * keys default to enabled so upgrades and pre-seed races keep polling. + */ + public static isChecksEnabled(): boolean { + try { + const raw = DatabaseService.getInstance().getGlobalSettings()[ImageUpdateService.ENABLED_SETTING_KEY]; + if (raw == null || String(raw).trim() === '') return true; + return raw === '1'; + } catch (e) { + console.warn('[ImageUpdateService] Could not read checks-enabled setting; treating as enabled:', getErrorMessage(e, String(e))); + return true; + } + } + + /** + * Persist the checks-enabled setting and apply the live transition: stop + * + clear findings when turning off; start (or re-arm) when turning on. + * Safe under repeated toggles (scheduleGeneration bump via stop/start). + */ + public applyChecksEnabled(enabled: boolean): ImageUpdateStatus { + const db = DatabaseService.getInstance(); + db.updateGlobalSetting(ImageUpdateService.ENABLED_SETTING_KEY, enabled ? '1' : '0'); + + if (!enabled) { + this.stop(); + // Scanner only writes rows for local nodes. Use the local default + // node ID, not req.nodeId (which may be a remote active node). + const localNodeId = NodeRegistry.getInstance().getDefaultNodeId(); + db.clearAllStackUpdateStatus(localNodeId); + invalidateFleetUpdateCache(); + NotificationService.getInstance().broadcastEvent({ + type: 'state-invalidate', + scope: 'image-updates', + nodeId: localNodeId, + action: 'checks-disabled', + ts: Date.now(), + }); + return this.getStatus(); + } + + // Re-enable: arm a fresh schedule. start() is a no-op if a timer already + // exists; when we were fully stopped, start() arms. When somehow still + // marked polling without a timer, restartPolling re-arms. + if (!this.timer) { + this.start(); + } else { + this.restartPolling(); + } + return this.getStatus(); + } + /** * Reads image_update_check_interval_minutes into intervalMs, clamped to * [15, 1440], falling back to the 2-hour default on a missing, blank, @@ -558,11 +665,16 @@ export class ImageUpdateService { } /** - * Triggers a check immediately, unless one is already running or the - * manual cooldown (MANUAL_COOLDOWN_MS) has not elapsed. - * Returns false if rate-limited, true if a check was started. + * Triggers a check immediately, unless detection is disabled, one is already + * running, or the manual cooldown (MANUAL_COOLDOWN_MS) has not elapsed. + * Returns false if rate-limited or disabled, true if a check was started. + * Callers that need to distinguish disabled from rate-limited must check + * isChecksEnabled() first. */ public triggerManualRefresh(): boolean { + if (!ImageUpdateService.isChecksEnabled()) { + return false; + } const now = Date.now(); if (now - this.lastManualRefreshAt < ImageUpdateService.MANUAL_COOLDOWN_MS) { return false; @@ -582,6 +694,7 @@ export class ImageUpdateService { } public getStatus(): ImageUpdateStatus { + const enabled = ImageUpdateService.isChecksEnabled(); let sidebarIndicators = false; try { const settings = DatabaseService.getInstance().getGlobalSettings(); @@ -590,21 +703,28 @@ export class ImageUpdateService { console.warn('[ImageUpdateService] Failed to read sidebar indicator setting:', e); } return { - checking: this.isRunning, + checking: enabled ? this.isRunning : false, intervalMinutes: Math.round(this.intervalMs / (60 * 1000)), lastCheckedAt: this.lastCheckedAt, - nextCheckAt: this.nextCheckAt, + nextCheckAt: enabled ? this.nextCheckAt : null, manualCooldownMinutes: ImageUpdateService.manualCooldownMinutes, manualCooldownRemainingMs: this.getManualCooldownRemainingMs(), mode: this.mode, cronExpression: this.cronExpression, sidebarIndicators, + enabled, }; } // ─── Core check ────────────────────────────────────────────────────────── private async check() { + if (!ImageUpdateService.isChecksEnabled()) { + if (isDebugEnabled()) { + console.log('[ImageUpdateService:debug] Checks disabled; skipping scan.'); + } + return; + } // The finally block is the sole owner of isRunning, so a scan that // overruns can never have its lock released out from under it. A // previous fixed timer cleared the lock after CHECK_TIMEOUT_MS, which @@ -770,7 +890,11 @@ export class ImageUpdateService { console.error(`[ImageUpdateService] Error checking ${sanitizeForLog(imageRef)}:`, sanitizeForLog((e as Error)?.message ?? String(e))); // getErrorMessage (not raw String(e)) because this value can surface // verbatim in the sidebar tooltip / readiness advisory as lastError. - imageUpdateMap.set(imageRef, { hasUpdate: false, error: getErrorMessage(e, 'Update check failed') }); + imageUpdateMap.set(imageRef, { + hasUpdate: false, + checkStatus: 'failed', + error: getErrorMessage(e, 'Update check failed'), + }); } await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS); } @@ -857,19 +981,28 @@ export class ImageUpdateService { } /** - * Re-check a single stack after a service-scoped update or restore. On a - * render failure the prior row is left untouched and a warning is returned. + * Re-check a single stack after a service-scoped update or restore, or + * after a manual full-stack update. On a render failure the prior row is + * left untouched and a verification_failed result is returned. */ - public async recheckStack(nodeId: number, stackName: string): Promise<{ warning: string | null }> { + public async recheckStack(nodeId: number, stackName: string): Promise { + // While detection is off, skip registry probes and do not write + // stack_update_status (avoids stale findings after re-enable). + if (!ImageUpdateService.isChecksEnabled()) { + return { outcome: 'cleared', warning: null }; + } const generation = this.reserveStackWriteGeneration(nodeId, stackName); const db = DatabaseService.getInstance(); const docker = DockerController.getInstance(nodeId); const model = await buildEffectiveServiceModel(nodeId, stackName); if (!model.renderable) { - return { warning: model.error }; + return { + outcome: 'verification_failed', + warning: model.error || UPDATE_VERIFICATION_INCOMPLETE_WARNING, + }; } - let containers: Array<{ Image?: string; Labels?: Record }> = []; + let containers: Array<{ Image?: string; Labels?: Record }>; try { containers = await withTimeout(docker.getAllContainers(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'getAllContainers'); } catch (e) { @@ -878,6 +1011,12 @@ export class ImageUpdateService { sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(e, 'unknown')), ); + // Do not clear or upsert from declared-image-only checks: runtime + // digests were never observed, so "cleared" would be a false negative. + return { + outcome: 'verification_incomplete', + warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING, + }; } const refs = new Set(); @@ -892,7 +1031,11 @@ export class ImageUpdateService { try { imageUpdateMap.set(imageRef, await this.checkImage(docker, imageRef)); } catch (e) { - imageUpdateMap.set(imageRef, { hasUpdate: false, error: getErrorMessage(e, 'Update check failed') }); + imageUpdateMap.set(imageRef, { + hasUpdate: false, + checkStatus: 'failed', + error: getErrorMessage(e, 'Update check failed'), + }); } await sleep(ImageUpdateService.INTER_IMAGE_DELAY_MS); } @@ -911,15 +1054,34 @@ export class ImageUpdateService { const lastError = stackStatusLastError(services); const now = Date.now(); - await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => { + const committed = await this.withStackWriteLock(nodeId, stackName, generation, async (gen) => { if (checkStatus === 'failed') { db.recordStackCheckFailure(nodeId, stackName, lastError ?? 'Update check failed', now, services, gen); } else { db.upsertStackUpdateStatus(nodeId, stackName, hasUpdate, now, checkStatus, lastError, services, gen); } }); + // A newer scanner reservation dropped this write; do not report cleared. + if (!committed) { + return { + outcome: 'verification_incomplete', + warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING, + }; + } - return { warning: null }; + if (checkStatus === 'partial' || checkStatus === 'failed') { + return { + outcome: 'verification_incomplete', + warning: UPDATE_VERIFICATION_INCOMPLETE_WARNING, + }; + } + if (hasUpdate) { + return { + outcome: 'still_present', + warning: UPDATE_STILL_PRESENT_WARNING, + }; + } + return { outcome: 'cleared', warning: null }; } private stackWriteKey(nodeId: number, stackName: string): string { @@ -963,6 +1125,70 @@ export class ImageUpdateService { return committed; } + /** + * Current per-stack write-generation high-water mark (0 if never reserved). + * Snapshot before a read-only update-preview so commitPreviewClear can + * compare against writes that reserved or committed after observation. + */ + public peekStackWriteGeneration(nodeId: number, stackName: string): number { + return this.stackWriteState.get(this.stackWriteKey(nodeId, stackName))?.generation ?? 0; + } + + /** + * Clear persisted scanner update state after an authoritative-negative + * update preview. `observedMemoryGeneration` and `observedRowGeneration` + * are snapshotted before the preview. + * + * Ordering: + * - If memory generation advanced after observation, abort (stale). + * - If memory generation still equals the observation watermark, advance + * (tombstone) so an equal-generation writer reserved before observation + * cannot commit after the clear (SF-4). + * - If the persisted row generation advanced after observation, keep the row. + * - Otherwise delete partial, failed, and confirmed ok+true rows. + * + * Returns cleared | stale | absent. + */ + public async commitPreviewClear( + nodeId: number, + stackName: string, + observedMemoryGeneration: number, + observedRowGeneration: number, + ): Promise<'cleared' | 'stale' | 'absent'> { + const key = this.stackWriteKey(nodeId, stackName); + let state = this.stackWriteState.get(key); + if (!state) { + state = { chain: Promise.resolve(), generation: observedMemoryGeneration }; + this.stackWriteState.set(key, state); + } + + // A reservation after observation already owns a higher generation. + if (state.generation > observedMemoryGeneration) { + return 'stale'; + } + + // Tombstone the equal watermark so pre-observation writers reserved at + // this generation become stale when they later try to commit. + if (state.generation === observedMemoryGeneration) { + state.generation += 1; + } + const clearGeneration = state.generation; + + let deleted = 0; + const committed = await this.withStackWriteLock(nodeId, stackName, clearGeneration, () => { + const db = DatabaseService.getInstance(); + const detail = db.getStackUpdateDetail(nodeId)[stackName]; + if (!detail) return; + // Compare DB-embedded generations only (same dimension as the + // pre-preview snapshot). Memory peek resets on restart; SQLite does not. + const rowGeneration = db.getStackUpdateWriteGeneration(nodeId, stackName); + if (rowGeneration > observedRowGeneration) return; + deleted = db.clearStackUpdateStatus(nodeId, stackName); + }); + if (!committed) return 'stale'; + return deleted > 0 ? 'cleared' : 'absent'; + } + private runtimeImagesByService( stackName: string, containers: Array<{ Image?: string; Labels?: Record }>, @@ -1028,14 +1254,16 @@ export class ImageUpdateService { const checkable = Array.from(images) .map((img) => imageUpdateMap.get(img)) - .filter((r): r is ImageCheckResult => !!r && !r.notCheckable); - const errored = checkable.filter((r) => r.error !== undefined); - const confirmedHasUpdate = checkable.some((r) => r.error === undefined && r.hasUpdate === true); + .filter((r): r is ImageCheckResult => !!r && normalizeImageCheckStatus(r) !== 'not_checkable'); + const statuses = checkable.map(normalizeImageCheckStatus); + const failed = checkable.filter((_, i) => statuses[i] === 'failed'); + const partial = checkable.filter((_, i) => statuses[i] === 'partial'); + const confirmedHasUpdate = checkable.some((r, i) => statuses[i] === 'ok' && r.hasUpdate === true); - if (checkable.length > 0 && errored.length === checkable.length) { + if (checkable.length > 0 && failed.length === checkable.length) { const committed = await this.withStackWriteLock(nodeId, stackName, generation, async () => { db.recordStackCheckFailure( - nodeId, stackName, errored[0].error ?? 'Update check failed', checkedAt, + nodeId, stackName, failed[0].error ?? 'Update check failed', checkedAt, ); }); return { @@ -1045,8 +1273,10 @@ export class ImageUpdateService { }; } - const checkStatus: StackCheckStatus = errored.length > 0 ? 'partial' : 'ok'; - const lastError = errored.length > 0 ? (errored[0].error ?? null) : null; + const checkStatus: StackCheckStatus = + failed.length > 0 || partial.length > 0 ? 'partial' : 'ok'; + const uncertain = [...partial, ...failed]; + const lastError = uncertain.length > 0 ? (uncertain[0].error ?? null) : null; const hasUpdate = checkStatus === 'partial' ? (confirmedHasUpdate || previousState[stackName] === true) : confirmedHasUpdate; @@ -1065,20 +1295,21 @@ export class ImageUpdateService { public async checkImage(docker: DockerController, imageRef: string): Promise { const parsed = parseImageRef(imageRef); // A bare digest ref (sha256:...) has no tag to track upstream; not applicable. - if (!parsed) return { hasUpdate: false, notCheckable: true }; + if (!parsed) { + return { hasUpdate: false, checkStatus: 'not_checkable', notCheckable: true }; + } if (isDebugEnabled()) { console.log(`[ImageUpdateService] Checking ${imageRef}: registry=${parsed.registry} repo=${parsed.repo} tag=${parsed.tag}`); } - // Look up stored credentials for this registry const credentials = await RegistryService.getInstance().getAuthForRegistry(parsed.registry); if (isDebugEnabled()) { console.log(`[ImageUpdateService] ${imageRef}: credentials ${credentials ? 'found' : 'none'}`); } - // Get local digest and platform from RepoDigests / Os+Architecture - let localDigest: string | null; + // Get local digests and platform from RepoDigests / Os+Architecture + let localDigests: string[]; let platform: { os: string; architecture: string }; try { const inspect = await withTimeout(docker.getDocker().getImage(imageRef).inspect(), ImageUpdateService.SOCKET_TIMEOUT_MS, 'inspect'); @@ -1086,28 +1317,58 @@ export class ImageUpdateService { // No RepoDigests at all: locally built / not registry-backed, so update // status does not apply. - if (repoDigests.length === 0) return { hasUpdate: false, notCheckable: true }; + if (repoDigests.length === 0) { + return { hasUpdate: false, checkStatus: 'not_checkable', notCheckable: true }; + } - localDigest = selectLocalRepoDigest(repoDigests, parsed); + localDigests = selectLocalRepoDigests(repoDigests, parsed); platform = { os: inspect.Os, architecture: inspect.Architecture }; } catch { - return { hasUpdate: false, error: `Failed to inspect local image "${imageRef}"` }; + return { + hasUpdate: false, + checkStatus: 'failed', + error: `Failed to inspect local image "${imageRef}"`, + }; } // RepoDigests were present but none resolved a usable digest: genuinely // ambiguous, so surface it rather than silently call the image up to date. - if (!localDigest) { - return { hasUpdate: false, error: `Could not resolve a local registry digest for "${imageRef}"` }; + if (localDigests.length === 0) { + return { + hasUpdate: false, + checkStatus: 'failed', + error: `Could not resolve a local registry digest for "${imageRef}"`, + }; } - const comparison = await compareLocalToRemoteTag(localDigest, parsed.registry, parsed.repo, parsed.tag, platform, credentials); - if (comparison.kind === 'error') { - return { hasUpdate: false, error: comparison.reason }; + const detection = await detectImageUpdate({ + localDigests, + platform, + registry: parsed.registry, + repo: parsed.repo, + tag: parsed.tag, + credentials, + }); + + const digestLabel = localDigests[0] ? `${localDigests[0].slice(0, 27)}...` : 'none'; + const nextSuffix = detection.nextTag ? ` next=${detection.nextTag}` : ''; + console.log( + `[ImageUpdateService] ${imageRef}: local=${digestLabel} update=${detection.hasUpdate}` + + ` digest=${detection.digestUpdate} tag=${detection.tagUpdate}` + + ` status=${detection.checkStatus}${nextSuffix}`, + ); + + if (detection.checkStatus === 'not_checkable') { + return { hasUpdate: false, checkStatus: 'not_checkable', notCheckable: true }; } - const hasUpdate = comparison.kind === 'update'; - console.log(`[ImageUpdateService] ${imageRef}: local=${localDigest.slice(0, 27)}... update=${hasUpdate}`); - return { hasUpdate }; + return { + hasUpdate: detection.hasUpdate, + digestUpdate: detection.digestUpdate, + tagUpdate: detection.tagUpdate, + checkStatus: detection.checkStatus, + ...(detection.reason ? { error: detection.reason } : {}), + }; } } diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index 98fe7992..18f1ce49 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -10,8 +10,15 @@ import { StackOpLockService, stackOpSkipMessage as skipMessage } from './StackOp import { FileSystemService } from './FileSystemService'; import { HealthGateService } from './HealthGateService'; import { ServiceUpdateRecoveryService } from './ServiceUpdateRecoveryService'; -import { ImageUpdateService } from './ImageUpdateService'; -import type { ImageCheckResult } from './ImageUpdateService'; +import { + createAutoUpdateDigestGateState, + messageWhenDigestApplyBlockedByCheckErrors, + messageWhenNoDigestUpdate, + recordAutoUpdateImageCheck, +} from '../helpers/autoUpdateDigestGate'; +import { ImageUpdateService, UPDATE_VERIFICATION_INCOMPLETE_WARNING } from './ImageUpdateService'; +import { invalidateFleetUpdateCache } from '../helpers/fleetUpdateCache'; +import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; import { isDebugEnabled } from '../utils/debug'; import { getErrorMessage } from '../utils/errors'; import { formatNoTargetError } from '../utils/remoteTarget'; @@ -732,6 +739,10 @@ export class SchedulerService { } private async executeUpdate(task: ScheduledTask): Promise { + if (task.selector_type === 'stack-label') { + return this.executeUpdateByStackLabel(task); + } + if (task.node_id == null) { throw new Error('Auto-update requires node_id'); } @@ -766,14 +777,13 @@ export class SchedulerService { console.log(`[SchedulerService] executeUpdate: ${stackNames.length} stack(s) to check, fleet=${isFleet}, wildcard=${isWildcard}`); } - const db = DatabaseService.getInstance(); const docker = DockerController.getInstance(task.node_id); const imageUpdateService = ImageUpdateService.getInstance(); const results: string[] = []; for (const stackName of stackNames) { try { - const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, db, isFleet || isWildcard); + const output = await this.executeUpdateForStack(stackName, task.node_id, docker, imageUpdateService, isFleet || isWildcard); results.push(output); } catch (e) { const msg = getErrorMessage(e, String(e)); @@ -785,6 +795,140 @@ export class SchedulerService { return results.join('\n'); } + /** + * Resolve live stack-label membership (fleet-wide or one node) and run the + * existing per-stack auto-update path. Remotes receive an explicit stack + * list; they do not evaluate the selector themselves. + */ + private async executeUpdateByStackLabel(task: ScheduledTask): Promise { + const labelName = (task.selector_value ?? '').trim(); + if (!labelName) { + throw new Error('Label-targeted auto-update requires selector_value'); + } + + const { collectFleetLabelSummaries } = await import('../helpers/fleetLabelSummary'); + let summaries = await collectFleetLabelSummaries(); + if (task.node_id != null) { + summaries = summaries.filter(s => s.nodeId === task.node_id); + if (summaries.length === 0) { + throw new Error(`Target node (id=${task.node_id}) no longer exists`); + } + } + + type NodePlan = { + nodeId: number; + nodeName: string; + reachable: boolean; + stacks: string[]; + error?: string; + }; + + const plans: NodePlan[] = []; + const seen = new Set(); + for (const summary of summaries) { + if (!summary.reachable) { + plans.push({ + nodeId: summary.nodeId, + nodeName: summary.nodeName, + reachable: false, + stacks: [], + error: summary.error ?? 'unreachable', + }); + continue; + } + const match = summary.labels.find(l => l.name === labelName); + const stacks: string[] = []; + if (match) { + for (const stackName of match.stackNames) { + const key = `${summary.nodeId}\0${stackName}`; + if (seen.has(key)) continue; + seen.add(key); + stacks.push(stackName); + } + } + plans.push({ + nodeId: summary.nodeId, + nodeName: summary.nodeName, + reachable: true, + stacks, + }); + } + + const lines: string[] = [ + `Selector: stack-label="${labelName}" · scope=${task.node_id == null ? 'entire fleet' : `node ${task.node_id}`}`, + ]; + for (const plan of plans) { + if (!plan.reachable) { + lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): unreachable (${plan.error})`); + } else if (plan.stacks.length === 0) { + lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): no stacks with label "${labelName}"`); + } else { + lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): ${plan.stacks.length} stack(s) → ${plan.stacks.join(', ')}`); + } + } + + const matchedStacks = plans.reduce((n, p) => n + p.stacks.length, 0); + const unreachableCount = plans.filter(p => !p.reachable).length; + if (matchedStacks === 0 && unreachableCount === 0) { + lines.push(`No stacks currently match label "${labelName}"; skipped.`); + return lines.join('\n'); + } + + let materialFailure = unreachableCount > 0; + const work = plans.filter(p => p.reachable && p.stacks.length > 0); + const NODE_CONCURRENCY = 3; + // Label-targeted runs fail closed on material stack failures or + // unreachable scoped nodes (unlike plain node fleet update, which + // historically returns failure lines as a successful run output). + + const looksLikeStackFailure = (text: string): boolean => + /^Stack ".+" failed:/m.test(text); + + const runNode = async (plan: NodePlan): Promise => { + const node = NodeRegistry.getInstance().getNode(plan.nodeId); + try { + if (node?.type === 'remote') { + const remoteOut = await this.executeUpdateRemoteTargets(plan.nodeId, plan.stacks); + lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}) results:\n${remoteOut}`); + if (looksLikeStackFailure(remoteOut)) materialFailure = true; + } else { + const docker = DockerController.getInstance(plan.nodeId); + const imageUpdateService = ImageUpdateService.getInstance(); + const stackLines: string[] = []; + for (const stackName of plan.stacks) { + try { + const out = await this.executeUpdateForStack( + stackName, plan.nodeId, docker, imageUpdateService, true, + ); + stackLines.push(out); + } catch (e) { + materialFailure = true; + const msg = getErrorMessage(e, String(e)); + stackLines.push(`Stack "${stackName}" failed: ${msg}`); + console.error(`[SchedulerService] Label auto-update failed for stack "${stackName}" on node ${plan.nodeId}:`, e); + } + } + lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}) results:\n${stackLines.join('\n')}`); + } + } catch (e) { + materialFailure = true; + const msg = getErrorMessage(e, String(e)); + lines.push(`Node "${plan.nodeName}" (id=${plan.nodeId}): failed (${msg})`); + console.error(`[SchedulerService] Label auto-update failed for node ${plan.nodeId}:`, e); + } + }; + + for (let i = 0; i < work.length; i += NODE_CONCURRENCY) { + const batch = work.slice(i, i + NODE_CONCURRENCY); + await Promise.all(batch.map(runNode)); + } + + if (materialFailure) { + throw new Error(lines.join('\n')); + } + return lines.join('\n'); + } + /** * Proxy auto-update execution to a remote Sencho instance. * The remote node runs the image checks and compose update locally. @@ -823,6 +967,55 @@ export class SchedulerService { } } + /** Proxy auto-update for an explicit stack list on a remote node (label selector). */ + private async executeUpdateRemoteTargets(nodeId: number, targets: string[]): Promise { + const proxyTarget = this.requireRemoteProxyTarget(nodeId); + const baseUrl = proxyTarget.apiUrl.replace(/\/$/, ''); + const proxyHeaders = LicenseService.getInstance().getProxyHeaders(); + if (isDebugEnabled()) { + console.log(`[SchedulerService] executeUpdateRemoteTargets: node=${nodeId} count=${targets.length}`); + } + const startTime = Date.now(); + try { + const response = await fetch(`${baseUrl}/api/auto-update/execute`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${proxyTarget.apiToken}`, + [PROXY_TIER_HEADER]: proxyHeaders.tier, + }, + body: JSON.stringify({ targets }), + signal: AbortSignal.timeout(300_000), + }); + + // Older remotes only accept { target }. Fall back to one call per + // stack so mixed-version fleets still complete the label schedule. + if (response.status === 400) { + const detail = await this.remoteResponseDetail(response); + if (/target/i.test(detail)) { + const parts: string[] = []; + for (const stackName of targets) { + parts.push(await this.executeUpdateRemote(nodeId, stackName)); + } + return parts.join('\n'); + } + throw new Error(this.remoteProxyFailureMessage(nodeId, detail)); + } + + if (!response.ok) { + throw new Error(this.remoteProxyFailureMessage(nodeId, await this.remoteResponseDetail(response))); + } + + const body = await response.json() as { result?: string }; + if (isDebugEnabled()) { + console.log(`[SchedulerService] executeUpdateRemoteTargets: completed in ${Date.now() - startTime}ms`); + } + return body.result || 'Remote auto-update completed (no details returned).'; + } catch (err) { + this.rethrowRemoteProxyError(nodeId, err); + } + } + /** * Proxy a stack lifecycle action to a remote Sencho instance. ComposeService, * DockerController, and FileSystemService are local-only, so for a remote node @@ -1030,9 +1223,12 @@ export class SchedulerService { nodeId: number, docker: DockerController, imageUpdateService: ImageUpdateService, - db: DatabaseService, isWildcard = false ): Promise { + if (!ImageUpdateService.isChecksEnabled()) { + console.log(`[SchedulerService] Stack "${stackName}": image update detection is disabled; skipped.`); + return `Stack "${stackName}": image update detection is disabled; skipped.`; + } const containers = await docker.getContainersByStack(stackName); if (!containers || containers.length === 0) { if (!isWildcard) { @@ -1056,35 +1252,26 @@ export class SchedulerService { console.log(`[SchedulerService] Stack "${stackName}": checking ${imageRefs.length} image(s): ${imageRefs.join(', ')}`); } - let hasUpdate = false; - const updatedImages: string[] = []; - const checkErrors: string[] = []; + const gate = createAutoUpdateDigestGateState(); for (const imageRef of imageRefs) { try { - const result: ImageCheckResult = await imageUpdateService.checkImage(docker, imageRef); - if (result.error) { - checkErrors.push(result.error); - } else if (result.hasUpdate) { - hasUpdate = true; - updatedImages.push(imageRef); - } + const result = await imageUpdateService.checkImage(docker, imageRef); + recordAutoUpdateImageCheck(gate, imageRef, result); } catch (e) { const msg = getErrorMessage(e, String(e)); - checkErrors.push(msg); + gate.checkErrors.push(msg); console.warn(`[SchedulerService] Failed to check image ${sanitizeForLog(imageRef)}:`, sanitizeForLog((e as Error)?.message ?? String(e))); } } - if (!hasUpdate) { - if (checkErrors.length > 0 && checkErrors.length === imageRefs.length) { - return `Stack "${stackName}": WARNING - all image checks failed (${checkErrors.join('; ')}). Unable to determine update status.`; - } - if (checkErrors.length > 0) { - return `Stack "${stackName}": all reachable images up to date (${checkErrors.length} check(s) failed).`; - } - return `Stack "${stackName}": all images up to date.`; + if (!gate.hasDigestUpdate) { + return messageWhenNoDigestUpdate(stackName, gate, imageRefs.length); } + const checkErrorBlock = messageWhenDigestApplyBlockedByCheckErrors(stackName, gate); + if (checkErrorBlock) return checkErrorBlock; + + const { updatedImages } = gate; await this.enforceSchedulerPolicyGate( stackName, @@ -1103,7 +1290,9 @@ export class SchedulerService { ), ); if (!lock.ran) return skipMessage(stackName, lock.existing.action); - db.clearStackUpdateStatus(nodeId, stackName); + + // Health observation starts immediately after Compose; registry recheck is + // isolated so a verification failure cannot turn Compose success into a failure. const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler'); const orchResult = lock.result; const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; @@ -1112,6 +1301,30 @@ export class SchedulerService { StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId); } + // Recheck persists digest-cleared / tag-advisory state. Do not blind-clear. + let recheckWarning: string | undefined; + try { + const recheck = await imageUpdateService.recheckStack(nodeId, stackName); + if (recheck.warning) recheckWarning = recheck.warning; + } catch (recheckErr) { + console.warn( + `[SchedulerService] Post-update recheck failed for ${sanitizeForLog(stackName)}:`, + sanitizeForLog(getErrorMessage(recheckErr, 'unknown')), + ); + recheckWarning = UPDATE_VERIFICATION_INCOMPLETE_WARNING; + } + + invalidateFleetUpdateCache(); + invalidateNodeCaches(nodeId); + NotificationService.getInstance().broadcastEvent({ + type: 'state-invalidate', + scope: 'image-updates', + nodeId, + stackName, + action: 'stack-updated', + ts: Date.now(), + }); + this.safeDispatch( 'info', 'image_update_applied', @@ -1119,7 +1332,8 @@ export class SchedulerService { stackName ); - return `Stack "${stackName}": updated (${updatedImages.join(', ')}).`; + const base = `Stack "${stackName}": updated (${updatedImages.join(', ')}).`; + return recheckWarning ? `${base} ${recheckWarning}` : base; } private async executeScan(task: ScheduledTask): Promise<{ output: string; failed: number }> { diff --git a/backend/src/services/SelfUpdateService.ts b/backend/src/services/SelfUpdateService.ts index b0568277..804f12a7 100644 --- a/backend/src/services/SelfUpdateService.ts +++ b/backend/src/services/SelfUpdateService.ts @@ -205,6 +205,27 @@ export function buildSelfUpdateRunArgs( ]; } +/** + * Build the argv for a throwaway helper that runs `docker compose … config` + * against the host compose project. Reuses the recreate helper's mount layout + * (socket + working dir + host binds) without mounting /app/data, since + * validation is read-only. Pure and exported for unit testing. + */ +export function buildComposeConfigValidateArgs( + ctx: Pick & { configFiles: string }, +): string[] { + const { workingDir, imageName, hostBindMounts, configFiles } = ctx; + const fFlags = configFiles.split(',').flatMap(f => { + const trimmed = f.trim(); + return trimmed ? ['-f', trimmed] : []; + }); + const composeCmd = ['docker compose', ...fFlags.map(shQuote), 'config'].join(' '); + return buildSelfUpdateRunArgs( + { workingDir, imageName, dataDirHost: null, hostBindMounts }, + composeCmd, + ); +} + class SelfUpdateService { private static instance: SelfUpdateService; private canSelfUpdate = false; @@ -527,6 +548,46 @@ class SelfUpdateService { this.spawnHelper(env, composeCopy, options?.successMarkerFile, options?.successMarkerContent); } + /** + * Recreate the Sencho service from the exact current on-disk Compose project + * without pulling or rewriting the image reference. Used by Fleet "Reapply + * configuration". Validates the authored compose via a throwaway helper + * before the last-breath recreate so invalid config fails before shutdown. + */ + async triggerComposeReapply(options?: { + successMarkerFile?: string; + successMarkerContent?: string; + }): Promise { + if (!this.composeContext) return; + const env = this.buildEnv(); + this.lastUpdateError = null; + this.pendingHelperExitError = undefined; + + try { fs.unlinkSync(UPDATE_ERROR_FILE); } catch { /* absent is the steady state */ } + try { fs.unlinkSync(STAGED_PATCH_FILE); } catch { /* absent is the steady state */ } + + const { workingDir, configFiles, imageName, hostBindMounts } = this.composeContext; + console.log('[SelfUpdate] Validating compose configuration before reapply...'); + try { + await execFileAsync( + 'docker', + buildComposeConfigValidateArgs({ workingDir, imageName, hostBindMounts, configFiles }), + { env, timeout: 60_000, maxBuffer: 10 * 1024 * 1024 }, + ); + } catch (error) { + const stderr = (error as { stderr?: Buffer | string })?.stderr?.toString().trim(); + const stdout = (error as { stdout?: Buffer | string })?.stdout?.toString().trim(); + this.lastUpdateError = + stderr || stdout || (error as Error).message || 'Compose configuration validation failed.'; + console.error('[SelfUpdate] Compose reapply validation failed:', this.lastUpdateError); + return; + } + + // No pull and no compose rewrite: the authored image ref is authoritative. + // Skip dangling-image prune (nothing was pulled). + this.spawnHelper(env, undefined, options?.successMarkerFile, options?.successMarkerContent, false); + } + /** * Spawn the "last breath" helper container that recreates Sencho (and, when a * repin is staged, copies the rewritten compose file onto the host first). @@ -538,6 +599,7 @@ class SelfUpdateService { composeCopy?: ComposeCopy, successMarkerFile?: string, successMarkerContent?: string, + pruneOnUpdateOverride?: boolean, ): void { if (!this.composeContext) return; const { workingDir, configFiles, serviceName, imageName, dataDirHost, hostBindMounts } = this.composeContext; @@ -551,8 +613,9 @@ class SelfUpdateService { // Opt-out (default ON): after a clean recreate, prune the dangling image // layers the pull orphaned. Read fresh so this node honors its own setting. const stderrTmp = '/tmp/_sencho_err'; - const pruneOnUpdate = - DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1'; + const pruneOnUpdate = pruneOnUpdateOverride ?? ( + DatabaseService.getInstance().getGlobalSettings()['prune_on_update'] === '1' + ); const composeCmd = buildSelfUpdateComposeCmd( fFlags, serviceName, diff --git a/backend/src/services/SuppressionRetractionRetryService.ts b/backend/src/services/SuppressionRetractionRetryService.ts new file mode 100644 index 00000000..7abdc8ed --- /dev/null +++ b/backend/src/services/SuppressionRetractionRetryService.ts @@ -0,0 +1,75 @@ +import { flushPendingSuppressionRetractions } from '../helpers/notificationSuppressionSync'; +import { isDebugEnabled } from '../utils/debug'; +import { getErrorMessage } from '../utils/errors'; + +const INITIAL_DELAY_MS = 30_000; +const EVAL_INTERVAL_MS = 5 * 60_000; + +/** + * Background retry for durable mute-replica retractions that failed or were + * deferred (offline Pilot/proxy, or remote lacking versioned retraction support). + */ +export class SuppressionRetractionRetryService { + private static instance: SuppressionRetractionRetryService; + private intervalId: NodeJS.Timeout | null = null; + private initialTimer: NodeJS.Timeout | null = null; + private isProcessing = false; + + private constructor() {} + + static getInstance(): SuppressionRetractionRetryService { + if (!SuppressionRetractionRetryService.instance) { + SuppressionRetractionRetryService.instance = new SuppressionRetractionRetryService(); + } + return SuppressionRetractionRetryService.instance; + } + + start(): void { + this.initialTimer = setTimeout(() => { + void this.evaluate(); + this.intervalId = setInterval(() => void this.evaluate(), EVAL_INTERVAL_MS); + }, INITIAL_DELAY_MS); + } + + stop(): void { + if (this.initialTimer) { + clearTimeout(this.initialTimer); + this.initialTimer = null; + } + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + } + } + + /** Full sweep of all pending rows. */ + async evaluate(): Promise { + if (this.isProcessing) return; + this.isProcessing = true; + try { + if (isDebugEnabled()) { + console.debug('[SuppressionRetractionRetry:debug] Evaluating pending retractions'); + } + await flushPendingSuppressionRetractions(); + } catch (err) { + console.error( + '[SuppressionRetractionRetry] evaluate error:', + getErrorMessage(err, String(err)), + ); + } finally { + this.isProcessing = false; + } + } + + /** Targeted flush when a Pilot tunnel or proxy node comes online. */ + async flushNode(nodeId: number): Promise { + try { + await flushPendingSuppressionRetractions(nodeId); + } catch (err) { + console.error( + `[SuppressionRetractionRetry] flushNode ${nodeId} failed:`, + getErrorMessage(err, String(err)), + ); + } + } +} diff --git a/backend/src/services/UpdateGuardService.ts b/backend/src/services/UpdateGuardService.ts index a36c5d12..14d6db53 100644 --- a/backend/src/services/UpdateGuardService.ts +++ b/backend/src/services/UpdateGuardService.ts @@ -3,9 +3,11 @@ import DockerController from './DockerController'; import { DatabaseService } from './DatabaseService'; import { FileSystemService } from './FileSystemService'; import { ComposeDoctorService } from './ComposeDoctorService'; -import { UpdatePreviewService, isMovingTag, filterPreviewForService } from './UpdatePreviewService'; +import { UpdatePreviewService, isMovingTag, filterPreviewForService, buildDetectionDisabledPreview } from './UpdatePreviewService'; +import { ImageUpdateService } from './ImageUpdateService'; import { buildEffectiveServiceModel, type EffectiveServiceModelResult } from './effectiveServiceModel'; import { filterContainersByComposeService } from '../helpers/composeServiceMatch'; +import { isDockerHealthcheckActive } from '../helpers/healthcheckPresence'; import { withTimeout } from '../utils/withTimeout'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; @@ -97,7 +99,7 @@ export class UpdateGuardService { state: inspect.State?.Status ?? info.State ?? 'unknown', health: inspect.State?.Health?.Status ?? null, exitCode: typeof inspect.State?.ExitCode === 'number' ? inspect.State.ExitCode : null, - hasHealthcheck: !!inspect.Config?.Healthcheck?.Test?.length, + hasHealthcheck: isDockerHealthcheckActive(inspect.Config?.Healthcheck?.Test), restartPolicy: inspect.HostConfig?.RestartPolicy?.Name || null, mounts, }; @@ -137,6 +139,12 @@ export class UpdateGuardService { withTimeout(this.probeContainers(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness sibling probe')) : Promise.resolve([]), this.collect('update preview', stackName, async () => { + // Check inside the thunk so the read stays with the getPreview call. + // Stack GET/POST update-preview use the same isChecksEnabled gate. + if (!ImageUpdateService.isChecksEnabled()) { + const disabled = buildDetectionDisabledPreview(stackName); + return serviceName ? filterPreviewForService(disabled, serviceName) : disabled; + } const full = await withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'readiness update preview'); return serviceName ? filterPreviewForService(full, serviceName) : full; }), @@ -152,7 +160,7 @@ export class UpdateGuardService { driftSignal(drift), containersSignal(containers), healthchecksSignal(containers), - updatePreviewSignal(preview === 'error' ? 'error' : preview.summary), + updatePreviewSignal(preview === 'error' ? 'error' : preview.summary, preview === 'error' ? undefined : preview.images), buildServicesSignal(preview === 'error' ? 'error' : preview.build_services), backupSlotSignal(backup, now), diskSignal(typeof disk === 'number' ? { usePercent: disk, limitPercent } : 'error'), @@ -218,8 +226,12 @@ export class UpdateGuardService { this.collect('backup info', stackName, () => fsSvc.getBackupInfo(stackName)), this.collect('backup env summary', stackName, () => fsSvc.getBackupEnvSummary(stackName)), this.collect('stack env presence', stackName, () => fsSvc.envExists(stackName)), - this.collect('update preview', stackName, () => - withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness update preview')), + this.collect('update preview', stackName, async () => { + if (!ImageUpdateService.isChecksEnabled()) { + return buildDetectionDisabledPreview(stackName); + } + return withTimeout(UpdatePreviewService.getInstance().getPreview(nodeId, stackName), INPUT_TIMEOUT_MS, 'rollback readiness update preview'); + }), this.collect('activity history', stackName, async () => { const events = db.getStackActivity(nodeId, stackName, { limit: 50 }); // A successful update is as good a known-good marker as a deploy. diff --git a/backend/src/services/UpdatePreviewService.ts b/backend/src/services/UpdatePreviewService.ts index 42cdc0ae..6fe09339 100644 --- a/backend/src/services/UpdatePreviewService.ts +++ b/backend/src/services/UpdatePreviewService.ts @@ -10,15 +10,43 @@ import { } from './ImageUpdateService'; import { parseImageRef, - selectLocalRepoDigest, + selectLocalRepoDigests, compareLocalToRemoteTag, - listRegistryTags, + listRegistryTagsResult, type ParsedRef, type RegistryCredentials, - type DigestComparisonResult, } from './registry-api'; +import { + detectImageUpdate, + type ImageUpdateDetectResult, + type ListRegistryTagsResultFn, + type PreviewImageCheckStatus, + type SemverBump, + computeSemverBump, + findNextTag, + isMovingTag, + listAllRegistryTagsBounded, + parseSemverTag, + PREVIEW_TAG_LIST_MAX_PAGES, + PREVIEW_TAG_LIST_MAX_TAGS, + PREVIEW_TAG_LIST_PAGE_SIZE, + type TagEnumOutcome, +} from './imageUpdateDetect'; -export type SemverBump = 'none' | 'patch' | 'minor' | 'major' | 'unknown'; +export type { SemverBump, PreviewImageCheckStatus, TagEnumOutcome, ListRegistryTagsResultFn }; +export { + computeSemverBump, + findNextTag, + isMovingTag, + listAllRegistryTagsBounded, + parseSemverTag, + PREVIEW_TAG_LIST_MAX_PAGES, + PREVIEW_TAG_LIST_MAX_TAGS, + PREVIEW_TAG_LIST_PAGE_SIZE, +}; + +/** Stack-level preview authority; absent on older remotes. */ +export type PreviewCheckStatus = 'ok' | 'partial' | 'failed'; export interface UpdatePreviewImage { service: string; @@ -26,7 +54,30 @@ export interface UpdatePreviewImage { current_tag: string; next_tag: string | null; has_update: boolean; + /** Same-tag content drift; Compose pull can apply without pin change. */ + digest_update: boolean; + /** Higher pinned semver exists; advisory until Compose is edited. */ + tag_update: boolean; semver_bump: SemverBump; + /** Authority of this image's checks; not_checkable for invalid refs. */ + check_status: PreviewImageCheckStatus; + /** + * Best operator-facing reason when check_status is not 'ok'/'not_checkable'. + * Never paired with a digest-based has_update claim: callers must treat + * this as verification-failed / unknown, not as "up to date" or rebuild. + */ + check_error: string | null; + /** + * This image's own digest-comparison failure reason, independent of + * check_status. A confirmed tag-based update on the SAME image resolves + * check_status to 'ok' and nulls check_error even when the digest compare + * itself errored (the tag confirmation is authoritative for that image), + * but a full-stack apply still pulls/recreates this image's CURRENT tag + * content, which was never verified. Callers gating collateral risk to a + * full-stack apply must read this field, not check_error, so that + * masking never hides an unverified image from the mixed-state gate. + */ + digest_error: string | null; } export type UpdateKind = 'tag' | 'digest' | 'none'; @@ -50,6 +101,21 @@ export interface UpdatePreviewSummary { has_build_services: boolean; /** True when a manual update can rebuild local build services (always when has_build_services). */ rebuild_available: boolean; + /** + * Whether every checkable image was verified authoritatively. + * Authoritative-negative reconcile requires check_status === 'ok' and !has_update. + * Older remotes omit this field; treat absence as non-authoritative. + */ + check_status: PreviewCheckStatus; + /** True when any image's check_status is not 'ok' (excluding not_checkable). */ + verification_failed: boolean; + /** First image check_error when verification_failed; otherwise null. */ + verification_error: string | null; + /** + * When true, background detection is disabled on this node and the preview + * was not fetched from registries. Optional for older remotes. + */ + detection_disabled?: boolean; } export interface UpdatePreview { @@ -61,74 +127,6 @@ export interface UpdatePreview { changelog: string | null; } -interface SemverParts { - prefix: string; - major: number; - minor: number; - patch: number; - suffix: string; - raw: string; -} - -const SEMVER_RE = /^(v)?(\d+)\.(\d+)\.(\d+)(?:-([A-Za-z][A-Za-z0-9.-]*))?$/; - -export function parseSemverTag(tag: string): SemverParts | null { - const m = tag.match(SEMVER_RE); - if (!m) return null; - return { - prefix: m[1] ?? '', - major: Number(m[2]), - minor: Number(m[3]), - patch: Number(m[4]), - suffix: m[5] ?? '', - raw: tag, - }; -} - -/** - * A tag is "moving" when restoring the compose file would not revert the image - * behind it: `latest`, a branch name, or an unpinned major/minor like `1.25`. - * Only a fully-pinned semver tag (X.Y.Z, optionally `v`-prefixed and/or with a - * `-prerelease` suffix) is treated as immutable, matching how a file rollback - * restores the exact tag. - */ -export function isMovingTag(tag: string): boolean { - return parseSemverTag(tag) === null; -} - -function compareSemver(a: SemverParts, b: SemverParts): number { - if (a.major !== b.major) return a.major - b.major; - if (a.minor !== b.minor) return a.minor - b.minor; - return a.patch - b.patch; -} - -export function findNextTag(currentTag: string, availableTags: string[]): string | null { - const current = parseSemverTag(currentTag); - if (!current) return null; - let best: SemverParts | null = null; - for (const tag of availableTags) { - const parsed = parseSemverTag(tag); - if (!parsed) continue; - if (parsed.prefix !== current.prefix) continue; - if (parsed.suffix !== current.suffix) continue; - if (compareSemver(parsed, current) <= 0) continue; - if (!best || compareSemver(parsed, best) > 0) best = parsed; - } - return best ? best.raw : null; -} - -export function computeSemverBump(currentTag: string, nextTag: string | null): SemverBump { - if (!nextTag) return 'none'; - if (nextTag === currentTag) return 'patch'; - const current = parseSemverTag(currentTag); - const next = parseSemverTag(nextTag); - if (!current || !next) return 'unknown'; - if (next.major > current.major) return 'major'; - if (next.minor > current.minor) return 'minor'; - if (next.patch > current.patch) return 'patch'; - return 'none'; -} - function maxBump(a: SemverBump, b: SemverBump): SemverBump { // Ranking: none < unknown < patch < minor < major. // unknown ranks below real semver bumps so a single unparseable tag never masks @@ -164,18 +162,81 @@ async function loadStackImages( return extractServiceImagesFromCompose(composeContent, merged); } +export type LocalDigestEmptyReason = 'not_checkable' | 'inspect_failed' | 'unresolved'; + export interface LocalDigestInfo { - digest: string | null; + /** All usable RepoDigests for the image ref; compared as a set against the remote tag. */ + digests: string[]; platform: { os: string; architecture: string }; + /** + * Why digests is empty. `not_checkable` mirrors the scanner (no RepoDigests / + * locally built) and must not surface as verification-failed. Inspect failure + * and unresolved selection do. + */ + emptyReason: LocalDigestEmptyReason | null; } export interface ComputePreviewDeps { getLocalDigest: (imageRef: string, parsed: ParsedRef) => Promise; compareDigest: typeof compareLocalToRemoteTag; - listRegistryTags: typeof listRegistryTags; + listRegistryTagsResult: ListRegistryTagsResultFn; getCredentials: (registry: string) => Promise; } +function imageFromDetect( + service: string, + imageRef: string, + currentTag: string, + detected: ImageUpdateDetectResult, +): UpdatePreviewImage { + return { + service, + image: imageRef, + current_tag: currentTag, + next_tag: detected.nextTag, + has_update: detected.hasUpdate, + digest_update: detected.digestUpdate, + tag_update: detected.tagUpdate, + semver_bump: detected.semverBump, + check_status: detected.checkStatus, + check_error: detected.reason, + digest_error: detected.digestError, + }; +} + +function notCheckableImage(service: string, imageRef: string): UpdatePreviewImage { + return { + service, + image: imageRef, + current_tag: 'unknown', + next_tag: null, + has_update: false, + digest_update: false, + tag_update: false, + semver_bump: 'none', + check_status: 'not_checkable', + check_error: null, + digest_error: null, + }; +} + +/** Local digest could not be established at all; never reaches the registry. */ +function failedLocalDigestImage(service: string, imageRef: string, currentTag: string, reason: string): UpdatePreviewImage { + return { + service, + image: imageRef, + current_tag: currentTag, + next_tag: null, + has_update: false, + digest_update: false, + tag_update: false, + semver_bump: 'none', + check_status: 'failed', + check_error: reason, + digest_error: reason, + }; +} + export async function computeImagePreview( service: string, imageRef: string, @@ -183,52 +244,41 @@ export async function computeImagePreview( ): Promise { const parsed = parseImageRef(imageRef); if (!parsed) { - return { - service, - image: imageRef, - current_tag: 'unknown', - next_tag: null, - has_update: false, - semver_bump: 'none', - }; + return notCheckableImage(service, imageRef); } const credentials = await deps.getCredentials(parsed.registry); - - // Digest-based: is a new build of the SAME tag available? A comparison error - // (network failure, malformed manifest) fails soft: it never claims a - // digest-based update, it only skips it. const localInfo = await deps.getLocalDigest(imageRef, parsed); - const [comparison, tags] = await Promise.all([ - localInfo.digest - ? deps.compareDigest(localInfo.digest, parsed.registry, parsed.repo, parsed.tag, localInfo.platform, credentials) - : Promise.resolve({ kind: 'error', reason: 'No local registry digest available' }), - deps.listRegistryTags(parsed.registry, parsed.repo, credentials), - ]); - const digestUpdate = comparison.kind === 'update'; - - // Tag-based: is a higher semver tag available? - const nextTag = findNextTag(parsed.tag, tags); - - const hasUpdate = digestUpdate || nextTag !== null; - let semverBump: SemverBump = 'none'; - let resolvedNext: string | null = null; - if (nextTag) { - resolvedNext = nextTag; - semverBump = computeSemverBump(parsed.tag, nextTag); - } else if (digestUpdate) { - resolvedNext = parsed.tag; - semverBump = 'patch'; + // A locally-built / non-registry-backed image (no RepoDigests at all) is + // not_checkable, matching ImageUpdateService.checkImage: it must never be + // funneled into detectImageUpdate's "no local digest" error path, which + // would misreport it as a verification failure instead of not applicable. + if (localInfo.emptyReason === 'not_checkable') { + return notCheckableImage(service, imageRef); } - - return { - service, - image: imageRef, - current_tag: parsed.tag, - next_tag: resolvedNext, - has_update: hasUpdate, - semver_bump: semverBump, - }; + // Inspect failure and unresolved RepoDigests never reach the registry, and + // detectImageUpdate's generic "no local digest" reason would blur these two + // distinct causes together; keep the specific reason, matching what + // ImageUpdateService.checkImage reports for the same two cases. + if (localInfo.emptyReason === 'inspect_failed') { + return failedLocalDigestImage(service, imageRef, parsed.tag, 'Failed to inspect local image'); + } + if (localInfo.emptyReason === 'unresolved') { + return failedLocalDigestImage(service, imageRef, parsed.tag, 'Could not resolve a local registry digest'); + } + const detected = await detectImageUpdate({ + localDigests: localInfo.digests, + platform: localInfo.platform, + registry: parsed.registry, + repo: parsed.repo, + tag: parsed.tag, + credentials, + deps: { + compareDigest: deps.compareDigest, + listRegistryTagsResult: deps.listRegistryTagsResult, + }, + }); + return imageFromDetect(service, imageRef, parsed.tag, detected); } function buildRollbackTarget(image: string, currentTag: string): string | null { @@ -244,6 +294,16 @@ function buildRollbackTarget(image: string, currentTag: string): string | null { return `${base}:${currentTag}`; } +export function rollupPreviewCheckStatus(images: UpdatePreviewImage[]): PreviewCheckStatus { + const checkable = images.filter((i) => i.check_status !== 'not_checkable'); + if (checkable.length === 0) return 'ok'; + const allFailed = checkable.every((i) => i.check_status === 'failed'); + if (allFailed) return 'failed'; + const allOk = checkable.every((i) => i.check_status === 'ok'); + if (allOk) return 'ok'; + return 'partial'; +} + export function buildSummary( stackName: string, images: UpdatePreviewImage[], @@ -266,6 +326,7 @@ export function buildSummary( : updated.some(i => i.next_tag !== null && i.next_tag !== i.current_tag) ? 'tag' : 'digest'; + const verificationError = images.find(i => i.check_error)?.check_error ?? null; return { stack_name: stackName, images, @@ -281,12 +342,24 @@ export function buildSummary( blocked_reason: blocked ? 'Major version jumps require human review before applying.' : null, has_build_services: hasBuildServices, rebuild_available: hasBuildServices, + check_status: rollupPreviewCheckStatus(images), + verification_failed: verificationError !== null, + verification_error: verificationError, }, rollback_target: primary ? buildRollbackTarget(primary.image, primary.current_tag) : null, changelog: null, }; } +/** True when a preview is safe to clear sticky scanner state. */ +export function isAuthoritativeNegativePreview(preview: UpdatePreview): boolean { + // Every declared image must be explicitly ok. Mixed ok + not_checkable must + // not clear sticky rows that may still track unresolved services. + return preview.images.length > 0 + && preview.images.every((i) => i.check_status === 'ok') + && preview.summary.has_update === false; +} + /** Filter a full-stack preview down to one service's images and recompute the summary from that subset. */ export function filterPreviewForService(preview: UpdatePreview, serviceName: string): UpdatePreview { const images = preview.images.filter(i => i.service === serviceName); @@ -294,6 +367,33 @@ export function filterPreviewForService(preview: UpdatePreview, serviceName: str return buildSummary(preview.stack_name, images, buildServices); } +/** Minimal preview when node-scoped image update detection is disabled. */ +export function buildDetectionDisabledPreview(stackName: string): UpdatePreview { + return { + stack_name: stackName, + images: [], + build_services: [], + summary: { + has_update: false, + primary_image: null, + current_tag: null, + next_tag: null, + semver_bump: 'none', + update_kind: 'none', + blocked: false, + blocked_reason: null, + has_build_services: false, + rebuild_available: false, + check_status: 'ok', + verification_failed: false, + verification_error: null, + detection_disabled: true, + }, + rollback_target: null, + changelog: null, + }; +} + export class UpdatePreviewService { private static instance: UpdatePreviewService; @@ -317,21 +417,44 @@ export class UpdatePreviewService { const deps: ComputePreviewDeps = { getCredentials: (registry) => RegistryService.getInstance().getAuthForRegistry(registry), compareDigest: compareLocalToRemoteTag, - listRegistryTags, + listRegistryTagsResult, getLocalDigest: async (imageRef: string, parsed: ParsedRef): Promise => { try { const inspect = await docker.getDocker().getImage(imageRef).inspect(); const repoDigests: string[] = inspect.RepoDigests ?? []; - const digest = selectLocalRepoDigest(repoDigests, parsed); - return { digest, platform: { os: inspect.Os, architecture: inspect.Architecture } }; - } catch { - return { digest: null, platform: { os: '', architecture: '' } }; + if (repoDigests.length === 0) { + return { + digests: [], + platform: { os: inspect.Os, architecture: inspect.Architecture }, + emptyReason: 'not_checkable', + }; + } + const digests = selectLocalRepoDigests(repoDigests, parsed); + return { + digests, + platform: { os: inspect.Os, architecture: inspect.Architecture }, + emptyReason: digests.length === 0 ? 'unresolved' : null, + }; + } catch (err) { + console.error('[UpdatePreview] local image inspect failed for %s', imageRef, err); + return { digests: [], platform: { os: '', architecture: '' }, emptyReason: 'inspect_failed' }; } }, }; + // Memoize service-independent detection by image ref so shared images + // hit the registry once, then attach each service name separately. + const detectByRef = new Map>(); const results = await Promise.all( - stackImages.map(({ service, image }) => computeImagePreview(service, image, deps)), + stackImages.map(async ({ service, image }) => { + let shared = detectByRef.get(image); + if (!shared) { + shared = computeImagePreview('_shared_', image, deps); + detectByRef.set(image, shared); + } + const base = await shared; + return { ...base, service }; + }), ); return buildSummary(stackName, results, buildServices); } diff --git a/backend/src/services/effectiveServiceModel.ts b/backend/src/services/effectiveServiceModel.ts index ed74330f..4dec1b5a 100644 --- a/backend/src/services/effectiveServiceModel.ts +++ b/backend/src/services/effectiveServiceModel.ts @@ -18,6 +18,7 @@ */ import { ComposeService } from './ComposeService'; import { parseMissingRequiredVars } from '../helpers/envVarParse'; +import { isComposeHealthcheckActive } from '../helpers/healthcheckPresence'; import { getErrorMessage } from '../utils/errors'; import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; @@ -75,9 +76,7 @@ function parseServiceSpec(name: string, raw: unknown): EffectiveServiceSpec { const svc = (raw ?? {}) as Record; const deploy = (svc.deploy && typeof svc.deploy === 'object') ? svc.deploy as Record : undefined; const healthcheck = svc.healthcheck; - const hasHealthcheck = !!healthcheck - && typeof healthcheck === 'object' - && (healthcheck as Record).disable !== true; + const hasHealthcheck = isComposeHealthcheckActive(healthcheck); return { name, declaredImage: asString(svc.image) ?? null, diff --git a/backend/src/services/healthcheck/collectServiceHealthcheckEvidence.ts b/backend/src/services/healthcheck/collectServiceHealthcheckEvidence.ts new file mode 100644 index 00000000..c63c74a7 --- /dev/null +++ b/backend/src/services/healthcheck/collectServiceHealthcheckEvidence.ts @@ -0,0 +1,210 @@ +/** + * Collect per-service effective healthcheck evidence for Compose Doctor. + * Structural facts only: never returns or logs Healthcheck.Test command text. + */ +import DockerController from '../DockerController'; +import { filterContainersByComposeService } from '../../helpers/composeServiceMatch'; +import { isDockerHealthcheckActive } from '../../helpers/healthcheckPresence'; +import type { EffectiveModel } from '../preflight/effectiveModel'; +import type { ServiceHealthcheckEvidence } from '../preflight/types'; +import { mapWithConcurrency } from '../../utils/mapWithConcurrency'; +import { getErrorMessage } from '../../utils/errors'; +import { sanitizeForLog } from '../../utils/safeLog'; + +const INSPECT_CONCURRENCY = 8; + +type ListedContainer = { + Id: string; + Names?: string[]; + Labels?: Record; + Image?: string; +}; + +type ReplicaProbe = { + hasHealthcheck: boolean; + imageMatches: boolean; + inspectFailed: boolean; +}; + +type ImageEvidence = 'inherited' | 'absent' | 'unverifiable'; + +function evidence( + state: ServiceHealthcheckEvidence['state'], + origin: ServiceHealthcheckEvidence['origin'], + consistentReplicas: boolean | null, +): ServiceHealthcheckEvidence { + return { state, origin, consistentReplicas }; +} + +/** + * Resolve effective healthcheck coverage for each service in the model. + * When `nodeStateAvailable` is false, services that still need Docker evidence + * become unverifiable without listing or inspecting containers/images. + */ +export async function collectServiceHealthcheckEvidence( + nodeId: number, + stackName: string, + model: EffectiveModel, + nodeStateAvailable: boolean, +): Promise> { + const out: Record = {}; + + const needsDocker = model.services.some(s => + s.composeHealthcheck !== 'active' && s.composeHealthcheck !== 'disabled'); + + let listed: ListedContainer[] = []; + let listFailed = false; + // Compose's top-level `name:` becomes com.docker.compose.project; that often + // differs from the Sencho stack directory name used as stackName. + const projectLabel = model.projectName || stackName; + if (nodeStateAvailable && needsDocker) { + try { + const docker = DockerController.getInstance(nodeId).getDocker(); + listed = await docker.listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${projectLabel}`] }, + }) as ListedContainer[]; + } catch (err) { + listFailed = true; + console.warn( + '[ComposeDoctor] Healthcheck container list failed for %s:', + sanitizeForLog(projectLabel), + sanitizeForLog(getErrorMessage(err, 'unknown')), + ); + } + } + + for (const svc of model.services) { + if (svc.composeHealthcheck === 'active') { + out[svc.name] = evidence('compose-declared', 'compose', null); + continue; + } + if (svc.composeHealthcheck === 'disabled') { + out[svc.name] = evidence('explicitly-disabled', 'compose', null); + continue; + } + + if (!nodeStateAvailable) { + out[svc.name] = evidence('unverifiable', 'none', null); + continue; + } + + if (listFailed) { + // Container list failed, but a local image inspect may still succeed. + out[svc.name] = await evidenceFromLocalImage(nodeId, svc.image, null); + continue; + } + + const scoped = filterContainersByComposeService(listed, svc.name); + if (scoped.length > 0) { + const replicas = await mapWithConcurrency(scoped, INSPECT_CONCURRENCY, (c) => + probeReplica(nodeId, c, svc.image)); + const fromRuntime = await resolveRuntimeEvidence(replicas); + if (fromRuntime) { + out[svc.name] = fromRuntime; + continue; + } + } + + // No suitable runtime evidence: local image or unverifiable. + out[svc.name] = await evidenceFromLocalImage(nodeId, svc.image, null); + } + + return out; +} + +/** + * Decide from inspected, image-matched replicas. + * Returns null when the caller should fall through to a generic local-image lookup + * (all inspects failed, or every replica is a stale/mismatched image). + */ +function resolveRuntimeEvidence( + replicas: ReplicaProbe[], +): ServiceHealthcheckEvidence | null { + const inspected = replicas.filter(r => !r.inspectFailed); + if (inspected.length === 0) return null; + + const usable = inspected.filter(r => r.imageMatches); + if (usable.length === 0) return null; + + const withHc = usable.filter(r => r.hasHealthcheck).length; + const withoutHc = usable.length - withHc; + const partial = inspected.length < replicas.length; + + if (withHc > 0 && withoutHc > 0) { + return evidence('inconsistent-replicas', 'runtime', false); + } + + if (withHc === usable.length) { + // Incomplete inspection: do not claim full coverage. + if (partial) return evidence('unverifiable', 'runtime', null); + return evidence('runtime-inherited', 'runtime', true); + } + + // All usable replicas lack an effective healthcheck. Do not upgrade a verified + // runtime gap to local-image-inherited; live replicas are authoritative. + if (partial) return evidence('unverifiable', 'runtime', null); + return evidence('absent', 'runtime', true); +} + +async function evidenceFromLocalImage( + nodeId: number, + image: string | undefined, + consistentReplicas: boolean | null, +): Promise { + if (!image) return evidence('unverifiable', 'none', consistentReplicas); + + const imageKind = await inspectLocalImage(nodeId, image); + if (imageKind === 'inherited') { + return evidence('local-image-inherited', 'local-image', consistentReplicas); + } + if (imageKind === 'absent') { + return evidence('absent', 'local-image', consistentReplicas); + } + return evidence('unverifiable', 'none', consistentReplicas); +} + +async function probeReplica( + nodeId: number, + listed: ListedContainer, + declaredImage: string | undefined, +): Promise { + try { + const docker = DockerController.getInstance(nodeId).getDocker(); + const inspect = await docker.getContainer(listed.Id).inspect(); + const test = inspect.Config?.Healthcheck?.Test; + const hasHealthcheck = isDockerHealthcheckActive(test); + const runtimeImage = typeof inspect.Config?.Image === 'string' ? inspect.Config.Image : listed.Image; + const imageMatches = !declaredImage + || !runtimeImage + || runtimeImage === declaredImage; + return { hasHealthcheck, imageMatches, inspectFailed: false }; + } catch (err) { + if ((err as { statusCode?: number })?.statusCode === 404) { + return { hasHealthcheck: false, imageMatches: false, inspectFailed: true }; + } + console.warn( + '[ComposeDoctor] Healthcheck container inspect failed:', + sanitizeForLog(getErrorMessage(err, 'unknown')), + ); + return { hasHealthcheck: false, imageMatches: false, inspectFailed: true }; + } +} + +async function inspectLocalImage( + nodeId: number, + imageRef: string, +): Promise { + try { + const { inspect } = await DockerController.getInstance(nodeId).inspectImage(imageRef); + const test = (inspect as { Config?: { Healthcheck?: { Test?: unknown } } })?.Config?.Healthcheck?.Test; + return isDockerHealthcheckActive(test) ? 'inherited' : 'absent'; + } catch (err) { + console.warn( + '[ComposeDoctor] Healthcheck image inspect failed for %s:', + sanitizeForLog(imageRef), + sanitizeForLog(getErrorMessage(err, 'unknown')), + ); + return 'unverifiable'; + } +} diff --git a/backend/src/services/imageUpdateDetect.ts b/backend/src/services/imageUpdateDetect.ts new file mode 100644 index 00000000..111e7002 --- /dev/null +++ b/backend/src/services/imageUpdateDetect.ts @@ -0,0 +1,293 @@ +/** + * Shared image-update detection used by both persisted sidebar status + * (ImageUpdateService.checkImage) and Fleet/Anatomy preview + * (UpdatePreviewService.computeImagePreview). + * + * An update is available when either: + * 1. the local digest no longer matches the registry manifest for the + * currently declared tag (digestUpdate; Compose-actionable), or + * 2. a higher pinned semver tag exists in a complete bounded tag list + * (tagUpdate; advisory only until Compose is edited). + */ +import { + compareLocalToRemoteTag, + listRegistryTagsResult, + type DigestComparisonResult, + type RegistryCredentials, + type TagListResult, +} from './registry-api'; + +export type SemverBump = 'none' | 'patch' | 'minor' | 'major' | 'unknown'; + +/** Per-image check confidence for preview authority and scanner persistence. */ +export type PreviewImageCheckStatus = 'ok' | 'partial' | 'failed' | 'not_checkable'; + +interface SemverParts { + prefix: string; + major: number; + minor: number; + patch: number; + suffix: string; + raw: string; +} + +const SEMVER_RE = /^(v)?(\d+)\.(\d+)\.(\d+)(?:-([A-Za-z][A-Za-z0-9.-]*))?$/; + +/** Max pages when enumerating tags for a pinned-semver authoritative-negative. */ +export const PREVIEW_TAG_LIST_MAX_PAGES = 20; +/** Max tags accumulated across pages for the same purpose. */ +export const PREVIEW_TAG_LIST_MAX_TAGS = 2000; +/** Per-page size passed to listRegistryTagsResult. */ +export const PREVIEW_TAG_LIST_PAGE_SIZE = 100; + +export function parseSemverTag(tag: string): SemverParts | null { + const m = tag.match(SEMVER_RE); + if (!m) return null; + return { + prefix: m[1] ?? '', + major: Number(m[2]), + minor: Number(m[3]), + patch: Number(m[4]), + suffix: m[5] ?? '', + raw: tag, + }; +} + +/** + * A tag is "moving" when restoring the compose file would not revert the image + * behind it: `latest`, a branch name, or an unpinned major/minor like `1.25`. + * Only a fully-pinned semver tag (X.Y.Z, optionally `v`-prefixed and/or with a + * `-prerelease` suffix) is treated as immutable, matching how a file rollback + * restores the exact tag. + */ +export function isMovingTag(tag: string): boolean { + return parseSemverTag(tag) === null; +} + +function compareSemver(a: SemverParts, b: SemverParts): number { + if (a.major !== b.major) return a.major - b.major; + if (a.minor !== b.minor) return a.minor - b.minor; + return a.patch - b.patch; +} + +export function findNextTag(currentTag: string, availableTags: string[]): string | null { + const current = parseSemverTag(currentTag); + if (!current) return null; + let best: SemverParts | null = null; + for (const tag of availableTags) { + const parsed = parseSemverTag(tag); + if (!parsed) continue; + if (parsed.prefix !== current.prefix) continue; + if (parsed.suffix !== current.suffix) continue; + if (compareSemver(parsed, current) <= 0) continue; + if (!best || compareSemver(parsed, best) > 0) best = parsed; + } + return best ? best.raw : null; +} + +export function computeSemverBump(currentTag: string, nextTag: string | null): SemverBump { + if (!nextTag) return 'none'; + if (nextTag === currentTag) return 'patch'; + const current = parseSemverTag(currentTag); + const next = parseSemverTag(nextTag); + if (!current || !next) return 'unknown'; + if (next.major > current.major) return 'major'; + if (next.minor > current.minor) return 'minor'; + if (next.patch > current.patch) return 'patch'; + return 'none'; +} + +export type ListRegistryTagsResultFn = ( + registry: string, + repo: string, + credentials?: RegistryCredentials | null, + opts?: { limit?: number; cursor?: string }, +) => Promise; + +export type TagEnumOutcome = + | { kind: 'complete'; tags: string[] } + | { kind: 'incomplete'; tags: string[]; reason: string } + | { kind: 'error'; reason: string } + | { kind: 'skipped' }; + +/** + * Enumerate tags with bounded pagination. Hitting the page/tag cap while a + * nextCursor remains is incomplete (non-authoritative), not a successful empty + * or "no newer tag" result. + */ +export async function listAllRegistryTagsBounded( + listFn: ListRegistryTagsResultFn, + registry: string, + repo: string, + credentials: RegistryCredentials | null, + opts: { maxPages?: number; maxTags?: number; pageSize?: number } = {}, +): Promise { + const maxPages = opts.maxPages ?? PREVIEW_TAG_LIST_MAX_PAGES; + const maxTags = opts.maxTags ?? PREVIEW_TAG_LIST_MAX_TAGS; + const pageSize = opts.pageSize ?? PREVIEW_TAG_LIST_PAGE_SIZE; + const tags: string[] = []; + let cursor: string | undefined; + for (let page = 0; page < maxPages; page++) { + const result = await listFn(registry, repo, credentials, { limit: pageSize, cursor }); + if (!result.ok) { + return { kind: 'error', reason: result.message }; + } + tags.push(...result.tags); + if (tags.length > maxTags) { + return { + kind: 'incomplete', + tags: tags.slice(0, maxTags), + reason: `Tag list exceeded ${maxTags} tags before pagination completed`, + }; + } + if (!result.nextCursor) { + return { kind: 'complete', tags }; + } + cursor = result.nextCursor; + } + return { + kind: 'incomplete', + tags, + reason: `Tag list exceeded ${maxPages} pages before pagination completed`, + }; +} + +export interface ImageUpdateDetectInput { + localDigests: readonly string[]; + platform: { os: string; architecture: string }; + registry: string; + repo: string; + tag: string; + credentials: RegistryCredentials | null; +} + +export interface ImageUpdateDetectResult { + hasUpdate: boolean; + digestUpdate: boolean; + tagUpdate: boolean; + nextTag: string | null; + digestError: string | null; + tagEnumKind: 'complete' | 'incomplete' | 'error' | 'skipped'; + tagEnumReason: string | null; + checkStatus: PreviewImageCheckStatus; + /** Best operator-facing uncertainty reason for lastError / tooltips. */ + reason: string | null; + semverBump: SemverBump; +} + +export interface DetectImageUpdateDeps { + compareDigest?: typeof compareLocalToRemoteTag; + listRegistryTagsResult?: ListRegistryTagsResultFn; +} + +export function resolveImageCheckStatus(args: { + digest: DigestComparisonResult['kind']; + tagApplicable: boolean; + tagOutcome: TagEnumOutcome; + hasUpdate: boolean; +}): PreviewImageCheckStatus { + const { digest, tagApplicable, tagOutcome, hasUpdate } = args; + + if (digest === 'update') return 'ok'; + + if (!tagApplicable) { + return digest === 'match' ? 'ok' : 'failed'; + } + + if (tagOutcome.kind === 'complete') { + if (hasUpdate) return 'ok'; + if (digest === 'match') return 'ok'; + return 'partial'; + } + + if (tagOutcome.kind === 'incomplete') { + if (hasUpdate) return 'ok'; + return 'partial'; + } + + if (hasUpdate) return 'partial'; + if (digest === 'match') return 'partial'; + return 'failed'; +} + +function uncertaintyReason( + checkStatus: PreviewImageCheckStatus, + digestError: string | null, + tagEnumReason: string | null, +): string | null { + if (checkStatus === 'ok' || checkStatus === 'not_checkable') return null; + return tagEnumReason ?? digestError; +} + +/** + * Core availability check shared by preview and persistence. + */ +export async function detectImageUpdate(args: ImageUpdateDetectInput & { + deps?: DetectImageUpdateDeps; +}): Promise { + const compareDigest = args.deps?.compareDigest ?? compareLocalToRemoteTag; + const listFn = args.deps?.listRegistryTagsResult ?? listRegistryTagsResult; + const tagApplicable = !isMovingTag(args.tag); + + const comparisonPromise: Promise = args.localDigests.length > 0 + ? compareDigest( + args.localDigests, + args.registry, + args.repo, + args.tag, + args.platform, + args.credentials, + ) + : Promise.resolve({ kind: 'error', reason: 'No local registry digest available' }); + + const tagPromise: Promise = tagApplicable + ? listAllRegistryTagsBounded(listFn, args.registry, args.repo, args.credentials) + : Promise.resolve({ kind: 'skipped' }); + + const [comparison, tagOutcome] = await Promise.all([comparisonPromise, tagPromise]); + + let nextTag: string | null = null; + if (tagOutcome.kind === 'complete' || tagOutcome.kind === 'incomplete') { + nextTag = findNextTag(args.tag, tagOutcome.tags); + } + + const digestUpdate = comparison.kind === 'update'; + const tagUpdate = nextTag !== null; + const hasUpdate = digestUpdate || tagUpdate; + + let resolvedNext: string | null = null; + let semverBump: SemverBump = 'none'; + if (nextTag) { + resolvedNext = nextTag; + semverBump = computeSemverBump(args.tag, nextTag); + } else if (digestUpdate) { + resolvedNext = args.tag; + semverBump = 'patch'; + } + + const digestError = comparison.kind === 'error' ? comparison.reason : null; + const tagEnumKind = tagOutcome.kind; + const tagEnumReason = tagOutcome.kind === 'incomplete' || tagOutcome.kind === 'error' + ? tagOutcome.reason + : null; + + const checkStatus = resolveImageCheckStatus({ + digest: comparison.kind, + tagApplicable, + tagOutcome, + hasUpdate, + }); + + return { + hasUpdate, + digestUpdate, + tagUpdate, + nextTag: resolvedNext, + digestError, + tagEnumKind, + tagEnumReason, + checkStatus, + reason: uncertaintyReason(checkStatus, digestError, tagEnumReason), + semverBump, + }; +} diff --git a/backend/src/services/preflight/effectiveModel.ts b/backend/src/services/preflight/effectiveModel.ts index e5bf0efd..c66eab55 100644 --- a/backend/src/services/preflight/effectiveModel.ts +++ b/backend/src/services/preflight/effectiveModel.ts @@ -6,6 +6,8 @@ * are handled by the caller, not here. */ +import { classifyComposeHealthcheck } from '../../helpers/healthcheckPresence'; + /** A host-published port range declared by a service (start==end for one port). */ export interface EffPortSpec { startPort: number; @@ -55,7 +57,13 @@ export interface EffService { privileged: boolean; networkMode?: string; restart?: string; + /** + * True when the rendered Compose model declares an active healthcheck. + * False for absent, `disable: true`, and `test: NONE` / `["NONE"]`. + */ hasHealthcheck: boolean; + /** Compose-layer classification used by healthcheck evidence collection. */ + composeHealthcheck: 'active' | 'disabled' | 'absent'; /** Raw deploy block (preflight uses key presence; Drift also reads restart_policy.condition). Undefined = none. */ deploy?: Record; containerName?: string; @@ -393,10 +401,7 @@ export function parseEffectiveModel(parsed: unknown, fallbackProjectName: string : []; const { binds, named } = parseVolumes(svc.volumes); const storageMounts = parseStorageMounts(svc.volumes, svc.tmpfs); - const healthcheck = svc.healthcheck; - const hasHealthcheck = !!healthcheck - && typeof healthcheck === 'object' - && (healthcheck as Record).disable !== true; + const composeHealthcheck = classifyComposeHealthcheck(svc.healthcheck); services.push({ name, image: str(svc.image), @@ -407,7 +412,8 @@ export function parseEffectiveModel(parsed: unknown, fallbackProjectName: string privileged: svc.privileged === true, networkMode: str(svc.network_mode), restart: str(svc.restart), - hasHealthcheck, + hasHealthcheck: composeHealthcheck === 'active', + composeHealthcheck, deploy: (svc.deploy && typeof svc.deploy === 'object') ? svc.deploy as Record : undefined, containerName: str(svc.container_name), user: str(svc.user), diff --git a/backend/src/services/preflight/rules.ts b/backend/src/services/preflight/rules.ts index 2e39aef1..074644a1 100644 --- a/backend/src/services/preflight/rules.ts +++ b/backend/src/services/preflight/rules.ts @@ -7,6 +7,18 @@ import { classifyMissingExternalNetworks } from '../network/missingExternalNetwo /** Higher number = more severe. Used to derive a run's overall status. */ export const SEVERITY_RANK: Record = { info: 0, warning: 1, high: 2, blocker: 3 }; +/** + * Rule IDs that are informational notes, not issue findings. They appear in the + * report for context but do not affect All Clear, active severity, or Update Guard. + */ +export const PREFLIGHT_NOTE_RULE_IDS: ReadonlySet = new Set([ + 'healthcheck-inherited', +]); + +export function isPreflightNoteFinding(ruleId: string): boolean { + return PREFLIGHT_NOTE_RULE_IDS.has(ruleId); +} + /** The one rule whose message doubles as the report's render error. Shared so the * service that reconstructs renderError from it cannot drift from the rule id. */ export const RENDER_FAILED_RULE_ID = 'render-failed'; @@ -373,15 +385,102 @@ const noHealthcheck: PreflightRule = { run(ctx) { if (!ctx.model) return []; return ctx.model.services - .filter(s => !s.hasHealthcheck) + .filter(s => ctx.healthchecks[s.name]?.state === 'absent') + .map(s => { + const origin = ctx.healthchecks[s.name]?.origin; + let from = 'available evidence'; + if (origin === 'runtime') from = 'currently running containers'; + else if (origin === 'local-image') from = 'the locally available image'; + return { + ruleId: 'no-healthcheck', + severity: 'warning' as const, + title: 'No effective healthcheck detected', + message: `Service "${s.name}" has no effective healthcheck in ${from}, so Docker and Sencho cannot tell when it is actually ready.`, + sourcePath: s.name, + service: s.name, + remediation: 'Add a healthcheck to the Compose service, or use an image that defines one.', + }; + }); + }, +}; + +const healthcheckDisabled: PreflightRule = { + id: 'healthcheck-disabled', + run(ctx) { + if (!ctx.model) return []; + return ctx.model.services + .filter(s => ctx.healthchecks[s.name]?.state === 'explicitly-disabled') .map(s => ({ - ruleId: 'no-healthcheck', + ruleId: 'healthcheck-disabled', severity: 'warning' as const, - title: 'No healthcheck', - message: `Service "${s.name}" declares no healthcheck, so Docker and Sencho cannot tell when it is actually ready (the image may still define one).`, + title: 'Healthcheck explicitly disabled', + message: `Service "${s.name}" disables its healthcheck in the Compose model (disable: true or test: NONE), so Docker will not report readiness for this service.`, sourcePath: s.name, service: s.name, - remediation: 'Add a healthcheck, or confirm the image provides one.', + remediation: 'Remove the disablement, or replace it with an active healthcheck if the service should report readiness.', + })); + }, +}; + +const healthcheckInherited: PreflightRule = { + id: 'healthcheck-inherited', + run(ctx) { + if (!ctx.model) return []; + return ctx.model.services + .filter(s => { + const state = ctx.healthchecks[s.name]?.state; + return state === 'runtime-inherited' || state === 'local-image-inherited'; + }) + .map(s => { + const origin = ctx.healthchecks[s.name]?.origin; + const from = origin === 'runtime' + ? 'Docker is using the healthcheck from the currently running container image' + : 'Docker is using the healthcheck defined by the locally available container image'; + return { + ruleId: 'healthcheck-inherited', + severity: 'info' as const, + title: 'Healthcheck inherited from image', + message: `Service "${s.name}" does not declare a healthcheck in Compose. ${from}.`, + sourcePath: s.name, + service: s.name, + remediation: 'Optionally declare the healthcheck in Compose so its configuration remains explicit and independently controlled.', + }; + }); + }, +}; + +const healthcheckUnverifiable: PreflightRule = { + id: 'healthcheck-unverifiable', + run(ctx) { + if (!ctx.model) return []; + return ctx.model.services + .filter(s => ctx.healthchecks[s.name]?.state === 'unverifiable') + .map(s => ({ + ruleId: 'healthcheck-unverifiable', + severity: 'info' as const, + title: 'Healthcheck inheritance could not be verified', + message: `Service "${s.name}" has no Compose healthcheck, and Sencho could not verify whether the running container or a local image provides one (Docker unreachable, image missing locally, or incomplete inspection).`, + sourcePath: s.name, + service: s.name, + remediation: 'Ensure the declared image is present locally, or add an explicit Compose healthcheck. Sencho does not pull images during Doctor runs.', + })); + }, +}; + +const healthcheckInconsistent: PreflightRule = { + id: 'healthcheck-inconsistent', + run(ctx) { + if (!ctx.model) return []; + return ctx.model.services + .filter(s => ctx.healthchecks[s.name]?.state === 'inconsistent-replicas') + .map(s => ({ + ruleId: 'healthcheck-inconsistent', + severity: 'warning' as const, + title: 'Replica healthcheck coverage is inconsistent', + message: `Service "${s.name}" has running replicas with mixed effective healthcheck coverage, so readiness is not uniform across replicas.`, + sourcePath: s.name, + service: s.name, + remediation: 'Recreate the service so every replica uses the same image and healthcheck configuration.', })); }, }; @@ -803,6 +902,10 @@ export const PREFLIGHT_RULES: PreflightRule[] = [ imageLatest, noRestartPolicy, noHealthcheck, + healthcheckDisabled, + healthcheckInherited, + healthcheckUnverifiable, + healthcheckInconsistent, deploySwarmOnly, nodeStateUnavailable, externalNetworkMissing, diff --git a/backend/src/services/preflight/types.ts b/backend/src/services/preflight/types.ts index 2f2a2fc3..36ccfa42 100644 --- a/backend/src/services/preflight/types.ts +++ b/backend/src/services/preflight/types.ts @@ -84,6 +84,34 @@ export interface BindCheck { ownerUid: number | null; } +/** Effective healthcheck coverage state for one Compose service. */ +export type HealthcheckEvidenceState = + | 'compose-declared' + | 'explicitly-disabled' + | 'runtime-inherited' + | 'local-image-inherited' + | 'absent' + | 'unverifiable' + | 'inconsistent-replicas'; + +/** Which layer produced the decisive healthcheck evidence. */ +export type HealthcheckEvidenceOrigin = + | 'compose' + | 'runtime' + | 'local-image' + | 'none'; + +/** + * Structural healthcheck evidence for one service. Never carries Test command + * text (commands can include credentials or interpolated secrets). + */ +export interface ServiceHealthcheckEvidence { + state: HealthcheckEvidenceState; + origin: HealthcheckEvidenceOrigin; + /** null when replica consistency does not apply (no runtime replicas inspected). */ + consistentReplicas: boolean | null; +} + /** * Everything the pure rule functions need, computed once by the service so the * rules stay synchronous and individually testable. No field ever holds an @@ -131,4 +159,9 @@ export interface PreflightContext { exposureAvailable: boolean; /** True when this stack is the running Sencho instance on the node. */ isSelfStack: boolean; + /** + * Per-service effective healthcheck evidence (Compose, runtime, local image). + * Empty when the model is null. Structural facts only; never Test command text. + */ + healthchecks: Record; } diff --git a/backend/src/services/registry-api.ts b/backend/src/services/registry-api.ts index 2f93b04d..13125668 100644 --- a/backend/src/services/registry-api.ts +++ b/backend/src/services/registry-api.ts @@ -35,7 +35,12 @@ export function parseImageRef(imageRef: string): ParsedRef | null { if (slashIdx !== -1) { const firstPart = imageRef.slice(0, slashIdx); if (firstPart.includes('.') || firstPart.includes(':') || firstPart === 'localhost') { - registry = firstPart; + // docker.io / index.docker.io are Docker Hub aliases; normalize them to the + // actual registry API host so requests never hit the marketing domain (which + // redirects instead of serving /v2/) and the library/ auto-prefix below still applies. + registry = (firstPart === 'docker.io' || firstPart === 'index.docker.io') + ? 'registry-1.docker.io' + : firstPart; rest = imageRef.slice(slashIdx + 1); } } @@ -214,7 +219,12 @@ const MANIFEST_ACCEPT = [ 'application/vnd.oci.image.manifest.v1+json', ].join(', '); -/** docker.io has three hostnames that all address the same registry. */ +/** + * docker.io has three hostnames that all address the same registry. Folds two + * already-parsed ParsedRef.registry values down to a shared form for equality/cache-key + * comparisons; not the canonical form parseImageRef assigns (which normalizes to + * 'registry-1.docker.io'). + */ function canonicalRegistry(host: string): string { if (host === 'docker.io' || host === 'index.docker.io' || host === 'registry-1.docker.io') { return 'docker.io'; @@ -242,14 +252,16 @@ export function repoDigestMatchesRef(repoDigest: string, parsed: ParsedRef): boo const SHA256_DIGEST_RE = /^sha256:[0-9a-f]{64}$/i; /** - * Deterministic local RepoDigest selection shared by the scanner and the - * update preview: the first entry whose repository matches the parsed - * image ref, else the sole remaining valid entry, else null. A truncated - * or malformed digest (not a complete "sha256:" + 64 hex chars) is never - * selected, so a corrupted RepoDigests entry surfaces as "could not - * resolve" rather than a false match or update. + * All usable local RepoDigests for a parsed image ref, shared by the scanner + * and update preview. Returns every valid digest whose repository matches the + * ref (deduped, first-seen order), else []. A truncated or malformed digest is + * never selected, and a valid digest belonging to an unrelated repository + * (e.g. left over from a retag) is never selected either, so either surfaces + * as "could not resolve" rather than a guessed match or update. Callers must + * compare every candidate: Docker can list a stale index digest ahead of the + * current one on the same image. */ -export function selectLocalRepoDigest(repoDigests: string[], parsed: ParsedRef): string | null { +export function selectLocalRepoDigests(repoDigests: readonly string[], parsed: ParsedRef): string[] { const valid = repoDigests .map((entry) => { const at = entry.indexOf('@'); @@ -257,9 +269,28 @@ export function selectLocalRepoDigest(repoDigests: string[], parsed: ParsedRef): }) .filter((e): e is { entry: string; digest: string } => e !== null && SHA256_DIGEST_RE.test(e.digest)); - const matched = valid.find((e) => repoDigestMatchesRef(e.entry, parsed)); - if (matched) return matched.digest; - return valid.length === 1 ? valid[0].digest : null; + const matched: string[] = []; + const seen = new Set(); + for (const e of valid) { + if (!repoDigestMatchesRef(e.entry, parsed)) continue; + const key = e.digest.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + matched.push(e.digest); + } + // No digest matches the configured repository: comparing an unrelated + // repository's digest (e.g. a retag) risks a false update against a + // registry state that has nothing to do with the image actually + // declared. Report unresolved instead of guessing. + return matched; +} + +/** + * Scalar view of {@link selectLocalRepoDigests}: the first candidate, or null. + * Prefer the plural form when comparing against a remote tag. + */ +export function selectLocalRepoDigest(repoDigests: readonly string[], parsed: ParsedRef): string | null { + return selectLocalRepoDigests(repoDigests, parsed)[0] ?? null; } /** Outcome of a remote-digest lookup: the digest, or a human-readable reason it failed. */ @@ -754,24 +785,41 @@ async function classifyManifest( } /** - * Compare a local image digest to the registry's current manifest for a tag. - * `platform` is the local image's Os/Architecture (from `docker image inspect`), - * required to safely match against an index's platform descriptors; without it, - * an index mismatch is an error rather than a speculative match. Never retries - * against the mutable tag once a primary digest is established: classification - * always targets that digest. + * Compare local image digests to the registry's current manifest for a tag. + * Any candidate that equals the remote primary or is a member of that primary's + * index counts as current (Docker often lists a stale index digest ahead of the + * current one). `platform` is the local image's Os/Architecture (from + * `docker image inspect`), required to safely match against an index's platform + * descriptors; without it, an index mismatch is an error rather than a + * speculative match. Never retries against the mutable tag once a primary + * digest is established: classification always targets that digest. + * + * `update` is returned only after a successful, complete remote classification + * with no candidate matching the primary, an exact member, or a same-platform + * runnable descriptor that the index actually offers. When the index has no + * descriptor labeled for the local platform, a platform-less leaf (which OCI + * permits) is trusted as this platform's content only when it is the ONLY + * kind of descriptor present (nothing else in the index claims a different + * platform); an index that mixes platform-less leaves with descriptors + * labeled for other platforms cannot attribute the unlabeled ones and errors + * instead. Empty/all-malformed candidates, unknown platform when platform + * matching is required, an index with no runnable content for the local + * platform at all (including an empty or fully-filtered index), and + * classification failures also return `error`. */ export async function compareLocalToRemoteTag( - localDigest: string, + localDigests: readonly string[], registry: string, repo: string, tag: string, platform: { os: string; architecture: string }, credentials?: RegistryCredentials | null, ): Promise { - if (!SHA256_DIGEST_RE.test(localDigest)) { + const candidates = localDigests.filter((d) => SHA256_DIGEST_RE.test(d)); + if (candidates.length === 0) { return { kind: 'error', reason: 'Local digest is malformed or truncated' }; } + const candidateSet = new Set(candidates.map((d) => d.toLowerCase())); const ref = `${registry}/${repo}:${tag}`; const probe = await probeManifestForRef(registry, repo, tag, credentials, ref); @@ -781,7 +829,7 @@ export async function compareLocalToRemoteTag( if (!SHA256_DIGEST_RE.test(primaryDigest)) { return { kind: 'error', reason: `Registry returned a malformed digest for ${ref}` }; } - if (localDigest === primaryDigest) return { kind: 'match' }; + if (candidateSet.has(primaryDigest.toLowerCase())) return { kind: 'match' }; let classification: ManifestClassification; try { @@ -792,15 +840,35 @@ export async function compareLocalToRemoteTag( if (classification.kind === 'single') return { kind: 'update' }; - if (classification.exactDigests.includes(localDigest)) return { kind: 'match' }; + if (classification.exactDigests.some((d) => candidateSet.has(d.toLowerCase()))) return { kind: 'match' }; if (!platform.os || !platform.architecture) { return { kind: 'error', reason: `Local image platform is unknown; cannot verify multi-arch membership for ${ref}` }; } - const isMember = classification.descriptors.some( - (d) => d.os === platform.os && d.architecture === platform.architecture && d.digest === localDigest, + const platformDescriptors = classification.descriptors.filter( + (d) => d.os === platform.os && d.architecture === platform.architecture, ); + if (platformDescriptors.length === 0) { + // No descriptor is labeled for this platform. exactDigests leaves omit + // platform entirely (OCI allows this), so they cannot be attributed to + // any specific platform; their mere presence does not prove content + // for THIS one, especially when other descriptors in the same index + // are explicitly labeled for a different platform. Only when every + // descriptor in the index is unlabeled (classification.descriptors is + // empty) does a leaf's presence plausibly represent this platform's + // content, since nothing else claims a different one; that is the one + // case where reporting `update` instead of failing closed is safe. + if (classification.exactDigests.length === 0) { + return { kind: 'error', reason: `Remote image index has no ${platform.os}/${platform.architecture} variant for ${ref}` }; + } + if (classification.descriptors.length > 0) { + return { kind: 'error', reason: `Remote image index has no confirmed ${platform.os}/${platform.architecture} variant for ${ref}` }; + } + return { kind: 'update' }; + } + + const isMember = platformDescriptors.some((d) => candidateSet.has(d.digest.toLowerCase())); return isMember ? { kind: 'match' } : { kind: 'update' }; } @@ -852,20 +920,30 @@ function parseNextCursor(linkHeader: string | string[] | undefined): string | un } /** - * Typed tag list for the Resources registry browser. Never collapses auth - * failures into an empty array (that would hide credential problems). + * Typed tag list for the Resources registry browser and update-preview authority. + * Never collapses auth failures into an empty array (that would hide credential + * problems and falsely look like a successful empty listing). `credentials` is + * optional: `getAuthToken` already resolves an anonymous pull token for public + * repositories on registries whose `WWW-Authenticate` challenge grants one + * without credentials (Docker Hub unconditionally; others via the standard + * token-service challenge), so a public repository still returns a real tag + * list with none configured. */ export async function listRegistryTagsResult( registry: string, repo: string, - credentials: RegistryCredentials, + credentials?: RegistryCredentials | null, opts: { limit?: number; cursor?: string } = {}, ): Promise { const limit = Math.min(Math.max(opts.limit ?? 50, 1), 100); try { const token = await getAuthToken(registry, repo, credentials); if (!token) { - return { ok: false, code: 'REGISTRY_UNAUTHORIZED', message: 'Registry rejected credentials' }; + return { + ok: false, + code: 'REGISTRY_UNAUTHORIZED', + message: credentials ? 'Registry rejected credentials' : 'Registry did not issue an anonymous token for this repository', + }; } const headers: Record = { Accept: 'application/json', Authorization: `Bearer ${token}` }; const params = new URLSearchParams({ n: String(limit) }); @@ -896,13 +974,12 @@ export async function listRegistryTagsResult( } } -/** Compatibility wrapper for update-preview: empty list on any failure. */ +/** Compatibility wrapper for callers that only need tags: empty list on any failure. */ export async function listRegistryTags( registry: string, repo: string, credentials?: RegistryCredentials | null, ): Promise { - if (!credentials) return []; const result = await listRegistryTagsResult(registry, repo, credentials); return result.ok ? result.tags : []; } diff --git a/backend/src/services/updateGuard/readiness.ts b/backend/src/services/updateGuard/readiness.ts index a6d824a5..2a04d1d7 100644 --- a/backend/src/services/updateGuard/readiness.ts +++ b/backend/src/services/updateGuard/readiness.ts @@ -1,5 +1,5 @@ import type { PreflightStatus } from '../preflight/types'; -import type { UpdatePreviewSummary } from '../UpdatePreviewService'; +import type { UpdatePreviewImage, UpdatePreviewSummary } from '../UpdatePreviewService'; import type { ContainerProbe, ReadinessSignal, @@ -109,11 +109,39 @@ export function healthchecksSignal(input: ContainerProbe[] | Errored): Readiness return { ...base, detail: parts.join(' ') }; } -export function updatePreviewSignal(input: UpdatePreviewSummary | Errored): ReadinessSignal { +/** + * True when a full-stack apply would pull/recreate an image whose digest + * verification failed as collateral of applying a DIFFERENT image's confirmed + * update or a local rebuild. `has_update` and `check_error` are independent + * per image (a tag-based update can be confirmed via the registry's tag list + * even when that same image's own digest comparison errored), so this + * excludes the case where the only verification failure belongs to the very + * image whose update is confirmed: that image's own stale-digest check does + * not block moving it to a newer tag. Falls back to the aggregate summary + * flags when per-image detail is unavailable. + */ +function hasUnverifiedOtherImage(summary: UpdatePreviewSummary, images: UpdatePreviewImage[] | undefined): boolean { + if (!images || images.length === 0) { + return summary.verification_failed && (summary.has_update || summary.rebuild_available); + } + const hasPureFailureImage = images.some((i) => Boolean(i.check_error) && !i.has_update); + if (!hasPureFailureImage) return false; + return images.some((i) => i.has_update) || summary.rebuild_available; +} + +export function updatePreviewSignal(input: UpdatePreviewSummary | Errored, images?: UpdatePreviewImage[]): ReadinessSignal { const base = { id: 'update_preview' as const, title: 'Pending update' }; if (input === 'error') { return { ...base, status: 'unknown', affectsVerdict: false, detail: 'The update preview is unavailable.' }; } + if (input.detection_disabled) { + return { + ...base, + status: 'unknown', + affectsVerdict: false, + detail: 'Image update detection is disabled for this node.', + }; + } if (input.blocked) { return { ...base, @@ -129,20 +157,50 @@ export function updatePreviewSignal(input: UpdatePreviewSummary | Errored): Read if (input.has_update && input.semver_bump === 'unknown') { return { ...base, status: 'warning', affectsVerdict: true, detail: 'An image update is pending but the version change could not be classified.' }; } + // A DIFFERENT image in the stack failing digest verification holds any + // pending action (a tag/digest update or a local-build rebuild) for review, + // since a full-stack apply would pull/recreate the unverified image as + // collateral. has_update and rebuild_available are handled symmetrically + // here to match isReviewRequiredUpdatePreview on the frontend. + const reviewRequired = hasUnverifiedOtherImage(input, images); if (input.has_update) { const kind = input.update_kind === 'digest' ? 'a same-tag image refresh' : `a ${input.semver_bump} update`; const buildNote = input.has_build_services ? ' Local build services will also be rebuilt from source.' : ''; + if (reviewRequired) { + const verifyNote = input.verification_error ? `: ${input.verification_error}` : '.'; + return { + ...base, + status: 'attention', + affectsVerdict: true, + detail: `Pending: ${kind}.${buildNote} Another image failed digest verification${verifyNote} Review before a full-stack update.`, + }; + } return { ...base, status: 'ok', affectsVerdict: true, detail: `Pending: ${kind}.${buildNote}` }; } if (input.rebuild_available) { const n = input.has_build_services ? 'Local build service(s)' : 'Build'; + const rebuildNote = `${n} require a rebuild from source; the update rebuilds images and recreates containers.`; + if (reviewRequired) { + const verifyNote = input.verification_error ? `: ${input.verification_error}` : '.'; + return { + ...base, + status: 'attention', + affectsVerdict: true, + detail: `${rebuildNote} Another image failed digest verification${verifyNote} Review before a full-stack update.`, + }; + } + return { ...base, status: 'warning', affectsVerdict: true, detail: rebuildNote }; + } + if (input.verification_failed) { return { ...base, - status: 'warning', + status: 'unknown', affectsVerdict: true, - detail: `${n} require a rebuild from source; the update rebuilds images and recreates containers.`, + detail: input.verification_error + ? `Digest verification failed: ${input.verification_error}` + : 'Digest verification failed; Sencho is not claiming a rebuild.', }; } return { ...base, status: 'ok', affectsVerdict: true, detail: 'No pending image update detected; the update re-pulls and recreates with current tags.' }; diff --git a/backend/src/types/express.ts b/backend/src/types/express.ts index abd63dbc..6007ef5d 100644 --- a/backend/src/types/express.ts +++ b/backend/src/types/express.ts @@ -19,6 +19,8 @@ declare global { mfaPendingUserId?: number; /** True when the pending MFA session originated from an SSO login (LDAP or OIDC) rather than a password login. */ mfaPendingSso?: boolean; + /** True when the caller's current user-session cookie was issued with "stay signed in". Read by reissueSessionAfterTokenBump so a password/MFA change doesn't silently shorten a remembered session. */ + sessionRemember?: boolean; /** Cached remote-proxy target resolved by `remoteNodeProxy`'s outer gate so the http-proxy router/proxyReq callbacks do not re-resolve. */ proxyTarget?: { apiUrl: string; apiToken: string }; /** Trusted deploy provenance from machine auth or gateway overwrite. */ diff --git a/backend/src/utils/audit-summaries.ts b/backend/src/utils/audit-summaries.ts index 8d2b854c..4ecdb21b 100644 --- a/backend/src/utils/audit-summaries.ts +++ b/backend/src/utils/audit-summaries.ts @@ -42,6 +42,7 @@ export const AUDIT_ROUTE_SUMMARIES: Record = { 'POST /system/networks/delete': 'Deleted networks', 'POST /system/networks': 'Created network', 'POST /system/console-token': 'Generated console token', + 'POST /system/reapply-compose': 'Triggered compose reapply', // Node management 'POST /nodes': 'Added node', @@ -91,6 +92,7 @@ export const AUDIT_ROUTE_SUMMARIES: Record = { 'DELETE /fleet/snapshots': 'Deleted fleet backup', 'POST /fleet/snapshots/*/restore': 'Restored fleet backup', 'POST /fleet/nodes/*/update': 'Triggered fleet node update', + 'POST /fleet/nodes/*/reapply-compose': 'Triggered fleet node compose reapply', 'POST /fleet/update-all': 'Triggered fleet-wide update', 'POST /fleet/role/reanchor': 'Re-anchored fleet replica', 'POST /fleet/role/demote': 'Demoted fleet replica to control', @@ -106,6 +108,7 @@ export const AUDIT_ROUTE_SUMMARIES: Record = { 'PUT /sso/config': 'Updated SSO configuration', 'DELETE /sso/config': 'Deleted SSO configuration', 'POST /sso/config/*/test': 'Tested SSO configuration', + 'PUT /sso/auth-mode': 'Updated authentication mode', // API tokens 'POST /api-tokens': 'Created API token', diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index 97043bb4..4cd08e0e 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -7,14 +7,14 @@ description: Threshold and event alerts for your fleet, dispatched to Discord, S Sencho watches each node it manages for container crashes, host pressure, scheduled-task results, and update availability, then surfaces every signal in two places: the in-app notification bell at the top of the shell and an external channel you configure. This page covers everything from configuring channels to writing stack and service threshold rules, routing alerts to dedicated channels with routing rules, and tuning retention. - Settings · Notifications · Channels panel with NODE Local in the header, a Delivery retries row showing Extra attempts 0 and a Save retries button, Discord Slack Webhook and Apprise tabs with Apprise selected, Enabled off, an empty Apprise endpoint placeholder, and Test beside Save. + Settings · Notifications · Channels panel with NODE Local in the header, Discord Slack Webhook and Apprise tabs with Apprise selected, Enabled off, an empty Apprise endpoint placeholder, Test beside Save, and a Delivery retries section below showing Extra attempts 0 and a Save retries button. ## Notification channels Open **Settings · Notifications · Channels** to configure Discord, Slack, custom webhook, and Apprise channels. Each channel is per-node, so switching the active node via the node picker reloads the panel against that node's stored settings. The masthead carries a `CHANNELS` stat showing how many of the four slots are enabled. -Above the channel tabs, **Delivery retries** sets how many extra in-process attempts (0 to 3) Sencho makes after a transient delivery failure on that node. The default is `0` (single-shot). Extra attempts wait a fixed one second between tries. Admin role is required to change the value. +Below the channel tabs, **Delivery retries** sets how many extra in-process attempts (0 to 3) Sencho makes after a transient delivery failure on that node. The default is `0` (single-shot). Extra attempts wait a fixed one second between tries. Admin role is required to change the value. Each Discord, Slack, and Webhook tab carries an **Enabled** toggle, a **Webhook URL** input (HTTPS only), and **Test** / **Save**. The Apprise tab uses an **Apprise endpoint** instead: keyed `/notify/{key}` shows optional **Tags**; stateless `/notify` shows **Destination URLs**. The kicker on each tab toggles between `enabled` and `off` so you can see at a glance which slots are wired up. diff --git a/docs/features/appearance.mdx b/docs/features/appearance.mdx index f66047e6..31fc1ab0 100644 --- a/docs/features/appearance.mdx +++ b/docs/features/appearance.mdx @@ -97,10 +97,10 @@ The **Navigation** group chooses how the desktop top bar presents page destinati - **Navigation style** - **Smart bar** (recommended default): keeps a short set of primary destinations visible and moves the rest into a grouped **More** menu. - - **Classic bar**: shows the full horizontal destination strip. - - **Compact launcher**: puts destinations in a left-side launcher menu and optionally pins up to five **quick links** on the bar. + - **Classic bar**: shows the full horizontal destination strip. Choosing Classic shows a callout that Classic bar will be removed soon; the preference is kept until then. + - **Compact launcher**: puts destinations in a left-side launcher menu and optionally pins up to seven **quick links** on the bar. - **Top navigation labels** (Classic and Smart): shows text beside top navigation icons. Turn it off for an icon-only bar; destinations stay reachable by tooltip, accessible name, and the command palette. Phone layout always keeps labels. With labels off, **Top navigation alignment** places the icon-only bar left or centered. -- **Quick links** (Compact launcher): labeled pins after the launcher, with a trailing **+** that opens reachable unpinned destinations. Right-click a pin and choose Remove, or manage the full list under Appearance. Up to five pins; recommended defaults start you with four. +- **Quick links** (Compact launcher): labeled pins after the launcher, with a trailing **+** that opens reachable unpinned destinations. Right-click a pin and choose Remove, or manage the full list under Appearance. Up to seven pins; recommended defaults start you with four. Deploy-progress behavior and the diff-preview-before-save step are stack workflow preferences, so they live in **Settings → Infrastructure → Stacks**, not here. diff --git a/docs/features/auto-update-policies.mdx b/docs/features/auto-update-policies.mdx index c830a2fc..2bac30d4 100644 --- a/docs/features/auto-update-policies.mdx +++ b/docs/features/auto-update-policies.mdx @@ -34,13 +34,13 @@ When nothing is pending, the board renders a single Shield-icon panel with the h ## Detection cadence -Sencho polls your registries on a configurable schedule to detect available image updates and raise notifications. This detection cadence is configurable under **Settings > Automation > Image update checks**: choose a fixed interval (every 15 minutes to once a day) or set a cron expression for precise scheduling (e.g. "every Monday at 03:00"). The default is every 2 hours on an interval schedule, and changing it takes effect immediately, with no restart. +Sencho polls your registries on a configurable schedule to detect available image updates and raise notifications. This detection cadence is configurable under **Settings > Automation > Image update checks**: choose a fixed interval (every 15 minutes to once a day) or set a cron expression for precise scheduling (e.g. "every Monday at 03:00"). The default is every 2 hours on an interval schedule, and changing it takes effect immediately, with no restart. You can also turn image update checks off for a node when another tool is the update authority; explicit stack Update and redeploy actions stay available. Detection is separate from applying updates: - **Registry detection** is this interval. It only looks for newer images and raises an "update available" notification the first time a stack goes from up to date to having an update. -- **Scheduled auto-update tasks** apply updates on their own cron schedule, independent of the detection interval. See [Scheduling auto-updates](#scheduling-auto-updates). -- **Apply now** updates a single stack on demand, regardless of either schedule. +- **Scheduled auto-update tasks** pull and recreate on their own cron schedule when the Compose-pinned tag has same-tag digest drift. A higher semver tag that is not yet written into Compose stays advisory on the readiness board and sidebar. See [Scheduling auto-updates](#scheduling-auto-updates). +- **Apply now** pulls and recreates the tags currently written in Compose (same-tag digest rebuilds). It does not rewrite a higher semver pin. The interval is node-scoped: each node runs its own scanner on its own cadence, and across a fleet Sencho staggers the runs slightly so the nodes do not all poll at the same instant. Setting the interval requires an admin account. @@ -68,36 +68,40 @@ Once the cause is resolved, the next check (on the interval, or via **Recheck**) ## Workflow 1. Open **Update** from the top nav strip. -2. Skim the card grid. The badge tells you the risk at a glance: `Safe · patch` is green, `Review · minor` is amber, `Blocked · major` is red, and a digest-only rebuild on a non-semver tag shows the gray `Digest rebuild` badge. -3. For a safe update, click **Apply now** on the card to pull and recreate the stack immediately. -4. For a major bump, review the changelog preview and the upstream release notes. **Apply now** is disabled on the readiness board for blocked cards; to apply a major bump after review, use the stack's lifecycle **Update** action (right-click the stack in the sidebar, or open the kebab menu and choose **Update**, or click **Deploy** in the stack editor). +2. Skim the card grid. The badge tells you the risk at a glance: `Safe · patch` is green, `Review · minor` is amber, `Blocked · major` is red, a digest-only rebuild on a non-semver tag shows the gray `Digest rebuild` badge, `Check uncertain` appears when the last preview check was incomplete or failed, and `Newer tag · edit Compose` marks a higher tag that requires editing the Compose pin. +3. For a **Digest rebuild** (same Compose tag, new registry content), click **Apply now** on the card to pull and recreate the stack immediately. **Apply now** stays disabled for tag-only advisories and for uncertain checks. +4. For a higher pinned semver tag (a version diff such as `1.2.3` → `1.2.4`, including major bumps), edit the Compose `image:` reference to the next tag, then deploy the stack from the editor. **Apply now** and the lifecycle **Update** action pull the currently pinned tag only; they do not rewrite the pin. +5. For a major bump, review the changelog preview and upstream release notes before changing the pin and deploying. **Apply now** is disabled on blocked cards. -On multi-service stacks, the Updates view can also apply a single service when that service has a confirmed image update. Scheduled auto-update, webhook pull, and bulk update always refresh the full stack. -5. Use **Recheck** in the hero to force an immediate registry poll across every reachable node. A 2-minute per-node cooldown applies, and the toast tells you how many nodes were triggered, rate-limited, or failed. +On multi-service stacks, the Updates view can also apply a single service when that service has a confirmed same-tag digest update. Scheduled auto-update, webhook pull, and bulk update always refresh the full stack. +6. Use **Recheck** in the hero to force an immediate registry poll across every reachable node. A 2-minute per-node cooldown applies, and the toast tells you how many nodes were triggered, rate-limited, or failed. -**Apply now** runs the same update path as every other update trigger on a stack: Sencho takes an atomic backup of the compose and env files first, pulls the updated registry images (or rebuilds from source for services that declare `build:`, then pulls the rest), and recreates the containers. See [Controlling a running stack](/features/stack-management#controlling-a-running-stack) for exactly how a build-aware update differs from a plain pull. Once the containers are up, the [post-update health gate](/features/health-gated-updates#the-post-update-health-gate) observes them for a configurable window and records a passed, failed, or unknown verdict on the stack's timeline, the same as a manual update from the editor toolbar or sidebar. +**Apply now** runs the same update path as other Compose-pull triggers on a stack: Sencho takes an atomic backup of the compose and env files first, pulls the updated registry images for the tags already written in Compose (or rebuilds from source for services that declare `build:`, then pulls the rest), and recreates the containers. See [Controlling a running stack](/features/stack-management#controlling-a-running-stack) for exactly how a build-aware update differs from a plain pull. Once the containers are up, the [post-update health gate](/features/health-gated-updates#the-post-update-health-gate) observes them for a configurable window and records a passed, failed, or unknown verdict on the stack's timeline, the same as a manual update from the editor toolbar or sidebar. After Apply finishes, Sencho rechecks the stack's update status and keeps the readiness card when an image update is still detected or verification cannot confirm clearance. ## Risk badges | Badge | Color | When it appears | |-------|-------|-----------------| -| `Safe · patch` | Green (Shield icon) | Patch-level semver bump (e.g. `1.2.3` to `1.2.4`) | -| `Review · minor` | Amber (AlertTriangle icon) | Minor semver bump (e.g. `1.2.3` to `1.3.0`) | -| `Blocked · major` | Red (ShieldAlert icon) | Major semver bump (e.g. `1.2.3` to `2.0.0`). **Apply now** is disabled; the card surfaces the reason "Major version jumps require human review before applying." | -| `Digest rebuild` | Gray | Non-semver tag (e.g. `main`, `stable`) with an updated digest (tag name unchanged; content changed; see [Tags vs digests](/features/vulnerability-scanning#tags-vs-digests)) | +| `Safe · patch` | Green (Shield icon) | Patch-level semver bump (e.g. `1.2.3` to `1.2.4`). Resolve by editing the Compose pin, then deploying. | +| `Review · minor` | Amber (AlertTriangle icon) | Minor semver bump (e.g. `1.2.3` to `1.3.0`). Resolve by editing the Compose pin, then deploying. | +| `Blocked · major` | Red (ShieldAlert icon) | Major semver bump (e.g. `1.2.3` to `2.0.0`). **Apply now** is disabled; the card surfaces the reason "Major version jumps require human review before applying." After review, edit the Compose pin, then deploy. | +| `Digest rebuild` | Gray | Non-semver tag (e.g. `main`, `stable`) with an updated digest (tag name unchanged; content changed; see [Tags vs digests](/features/vulnerability-scanning#tags-vs-digests)). **Apply now** pulls this rebuild. | +| `Check uncertain` | Amber | Preview check incomplete or failed; **Apply now** stays disabled until a full successful check. | +| `Newer tag · edit Compose` | Amber | A higher tag exists than the Compose pin; edit Compose (do not use **Apply now**). | + +Blocked major bumps still surface in scheduled check runs so you stay informed, but **Apply now** stays disabled until you review them, change the Compose pin, and deploy. A separate inline `Rebuild available` label replaces the version diff when only the digest changed (same tag, new image). The risk badge on those cards still reflects the underlying semver classification reported by the registry. -Blocked updates still surface in scheduled check runs so you stay informed, but the apply button is disabled until you review them manually. - ## Per-stack control Auto-update is opt-in per stack. A stack participates in unattended updates only when an enabled scheduled task covers it. To leave a stack out (databases, self-built images, anything pinned to a fixed tag), simply do not create a schedule for it. - **Per-stack schedule.** Create a **Auto-update Stack** task targeting that stack alone. Only this stack is updated when the cron fires. -- **Fleet-wide schedule.** Create a **Auto-update All Stacks on Node** task targeting a node. Every stack on that node is checked and updated when new images are available. If you do not want every stack covered, create per-stack schedules instead. +- **Fleet-wide schedule.** Create a **Auto-update All Stacks on Node** task targeting a node. Every stack on that node is checked when the cron fires, and stacks with same-tag digest drift are pulled and recreated. If you do not want every stack covered, create per-stack schedules instead. +- **Label schedule.** Create an **Auto-update stacks by label** task, pick a Stack Label, and choose Entire fleet or one node. At each run Sencho resolves the stacks that currently carry that label and updates those with newer images. Assigning or removing the label changes the next run without editing the schedule. - **Stack list dot.** Image-update *detection* runs on the configured interval (every 2 hours by default) regardless of whether any schedule is configured. The sidebar dot and the readiness board still show available updates so you can decide what to do with them. -- **Manual updates are always available.** The lifecycle **Update** action on a stack applies an update on demand, independent of any scheduled task. +- **Manual updates are always available.** The lifecycle **Update** action pulls and recreates the tags currently written in Compose, independent of any scheduled task. It does not rewrite a higher-tag advisory into the Compose file. ## Cleaning up after updates @@ -119,7 +123,9 @@ Auto-update is a first-class action in the Schedules view. To create a recurring The task lives alongside restart, prune, snapshot, and scan tasks in the same timeline and table. Run history, notifications, and the Run Now button behave the same as for every other scheduled action. See [Scheduled Operations](/features/scheduled-operations) for details. -A scheduled run applies an update the same way **Apply now** does: an atomic backup first, then the pull or build-aware rebuild, then the [post-update health gate](/features/health-gated-updates) observing the result. A schedule never bypasses a [deploy enforcement](/features/deploy-enforcement) policy: a run that a policy would block fails with the policy violation recorded in the run history, rather than applying partway. +A scheduled run (including **Run Now**) only auto-applies when Compose already pins the tag that has new registry content: same-tag digest drift, the gray **Digest rebuild** case. Sencho then follows the same update path as **Apply now**: an atomic backup first, then the pull or build-aware rebuild, then the [post-update health gate](/features/health-gated-updates) observing the result. + +When detection finds a higher semver tag while Compose still pins the older one (for example Compose says `nginx:1.2.3` and the registry also publishes `1.2.4`), the readiness board and sidebar keep showing the advisory update. The schedule does not rewrite the Compose image reference, so it skips Compose pull and recreate until you change the pin in Compose and deploy. A schedule never bypasses a [deploy enforcement](/features/deploy-enforcement) policy: a run that a policy would block fails with the policy violation recorded in the run history, rather than applying partway. ## Multi-node support @@ -133,7 +139,7 @@ For each stack with a pending image update, Sencho computes a preview by: 1. Parsing the compose file to enumerate every service that pulls a registry image (a service declaring only `build:`, with no `image:` key, has no registry reference to check and is excluded from this preview). 2. Calling the registry with your configured credentials to fetch the current tag list and remote digest. -3. Picking the highest semver tag greater than the current tag (keeping the same prefix and suffix). If the highest available tag matches the current one but the remote digest has changed, the card surfaces as a `Rebuild available` update. Sencho checks both a newer semver tag and a digest change behind the same tag; see [Tags vs digests](/features/vulnerability-scanning#tags-vs-digests). +3. Picking the highest semver tag greater than the current tag (keeping the same prefix and suffix). If a higher tag exists, the card surfaces as a tag-only advisory (`Newer tag · edit Compose`); scheduled auto-update and **Apply now** do not rewrite Compose pins. If the highest available tag matches the current one but the remote digest has changed, the card surfaces as a Compose-actionable `Rebuild available` / digest update. Sencho checks both a newer semver tag and a digest change behind the same tag; see [Tags vs digests](/features/vulnerability-scanning#tags-vs-digests). 4. Scoring the overall stack by the most severe image bump. Any major bump marks the stack as blocked. 5. Normalizing Docker Hub library paths so credentials and changelog lookups resolve correctly. @@ -148,7 +154,7 @@ A stack that mixes registry images and `build:` services still gets a card, scor Sencho reads changelog metadata from the registry's manifest and OCI annotations. Registries that do not publish this metadata (most private registries and many self-hosted ones) render the card without a changelog. The risk badge is still accurate because it is computed from the tag itself. - The stack has a major version bump and is blocked on the readiness board by policy: major updates never auto-apply without human review. To apply after reviewing the upstream release notes, use the stack's lifecycle **Update** action from the sidebar kebab or right-click menu, or open the stack editor and click **Deploy**. + The stack has a major version bump and is blocked on the readiness board by policy: major updates never auto-apply without human review. After reviewing the upstream release notes, edit the Compose `image:` pin to the reviewed next tag, then deploy the stack from the editor. **Apply now** and lifecycle **Update** pull the currently pinned tag only; they do not bump the pin. The registry call is either still pending or it failed. Click **Recheck** in the hero to retry. If the stack uses private-registry credentials, confirm they are still valid in **Settings > Registries**. @@ -157,7 +163,15 @@ A stack that mixes registry images and `build:` services still gets a card, scor Image update detection runs on the configured interval (every 2 hours by default) on each node, and the readiness board uses the same cached status. Trigger **Recheck** to force a fresh check across every reachable node, or shorten the interval under **Settings > Automation > Image update checks**. - No schedule covers that stack. Open **Schedules** in the top nav, create a new **Auto-update Stack** task targeting the stack (or an **Auto-update All Stacks on Node** task on its node), and pick a cron. The next firing will include the stack, or you can trigger an immediate run from the row. + First confirm a schedule covers that stack: open **Schedules**, create or enable an **Auto-update Stack** task (or an **Auto-update All Stacks on Node** task on its node), and pick a cron. The next firing includes the stack, or you can trigger an immediate run from the row. + + If a schedule already covers the stack and the run history says newer tags are available but the Compose pin is unchanged, detection found a higher tag than the image reference in Compose. Scheduled auto-update only pulls same-tag digest rebuilds; it does not bump `image:1.2.3` to `image:1.2.4` for you. Edit the Compose image pin to the next tag, then deploy. + + + The registry publishes a higher pinned semver tag than the one written in Compose. The readiness board and sidebar keep that advisory visible. Scheduled auto-update leaves Compose as the source of truth, so it skips pull and recreate until the pin is updated. Edit the Compose `image:` reference to the next tag, then deploy the stack. **Apply now** and lifecycle **Update** cannot rewrite the pin. + + + Compose still pins the older tag. Apply pulls and recreates that pinned tag only. To move to the next tag shown on the card, edit the Compose `image:` reference, then deploy. One or more nodes that are marked online in your fleet did not respond within the request timeout. Pending updates from those nodes are not shown until they come back. Check the node's status from the Fleet view and the network path between this Sencho instance and the unreachable node. diff --git a/docs/features/compose-doctor.mdx b/docs/features/compose-doctor.mdx index 4fdf56ac..66ddffc3 100644 --- a/docs/features/compose-doctor.mdx +++ b/docs/features/compose-doctor.mdx @@ -5,7 +5,7 @@ description: Run a preflight check on a stack before you deploy. Compose Doctor The **Doctor** tab in the right-hand **Anatomy** panel answers one question before you apply a change: *what will Docker actually run, and is it safe on this node?* Compose Doctor renders the effective Compose model (the fully resolved result after interpolation, includes, profiles, `.env`, and `env_file` are applied) and then runs a set of deterministic checks against it and the live Docker state on the node it would deploy to. -The check is advisory: on its own it never blocks a deploy or changes a stack. (One rule is the exception: see [Self-management](#self-management) below for the one case where Sencho actively blocks an action, independent of this report.) It runs on demand: press **run preflight** and Sencho renders the model, runs all 32 checks, and stores the result so the tab still shows it the next time you open the stack. +The check is advisory: on its own it never blocks a deploy or changes a stack. (One rule is the exception: see [Self-management](#self-management) below for the one case where Sencho actively blocks an action, independent of this report.) It runs on demand: press **run preflight** and Sencho renders the model, runs all 36 checks, and stores the result so the tab still shows it the next time you open the stack. ## Where to find it @@ -24,7 +24,7 @@ Every preflight run follows three steps: 1. **Render** the effective model. Sencho calls `docker compose config` on the stack, which resolves all variable interpolation, `include` directives, profile overrides, and `env_file` references into a single, normalized model. 2. **Snapshot** live Docker state. Sencho reads which host ports are in use, which containers are running, and which named networks and volumes exist on the target node. -3. **Run 32 deterministic rules** against the combination. Each rule is pure and produces zero or more findings with a severity, a message, and a suggested fix. +3. **Run 36 deterministic rules** against the combination. Each rule is pure and produces zero or more findings with a severity, a message, and a suggested fix. Sencho stores exactly one run per stack per node, so a new run immediately overwrites the previous one; there is no history. @@ -58,9 +58,9 @@ The summary card at the top of the Doctor tab reflects the overall outcome of th | **High risk** | Highest active finding is high risk | Amber card with a count of active findings by severity | | **Warning** | Highest active finding is a warning | Blue card with a count of active findings by severity | | **Info** | All active findings are informational | Muted card with a count of active findings by severity | -| **Acknowledged** | Every finding from the last run has been acknowledged | Muted card noting all findings are acknowledged | +| **All clear · findings acknowledged** | Model rendered; every finding from the last run has been acknowledged | Green card with "No active findings remain. One or more detected issues were reviewed and acknowledged by an authorized operator." | -The summary card also shows when the run happened and who triggered it: "ran 5 minutes ago by admin". If any findings are acknowledged, the line adds a count: "3 active (1 blocker · 2 warning) · 2 acknowledged". +The summary card also shows when the run happened and who triggered it: "ran 5 minutes ago by admin". When active findings remain alongside acknowledgements, the line adds a count: "3 active (1 blocker · 2 warning) · 2 acknowledged". A small colored dot appears on the **Doctor** tab label when the last run's active findings include a blocker (red) or a high-risk finding (amber). Click the **X** on the summary card to dismiss it; this also clears the tab dot. The dismissal is remembered per stack and node, and clears itself automatically the moment the finding set changes (a rule clearing, a new rule firing, or a severity change), so a stale dismissal can never hide a genuinely new problem. It never clears just by opening the tab or re-running preflight to the same result, and it does not sync across browsers or teammates: it is stored in your browser only. @@ -73,7 +73,7 @@ A small colored dot appears on the **Doctor** tab label when the last run's acti ## What it checks -All 32 rules are listed below, organized by topic. +All 36 rules are listed below, organized by topic. ### Model rendering @@ -119,7 +119,11 @@ All 32 rules are listed below, organized by topic. |------|----------|----------------| | Image uses a moving tag | Warning | A service uses `:latest` or a tag-less image reference, making deploys non-reproducible and subject to unexpected changes. | | No restart policy | Warning | A service has no restart policy and will not come back after a crash or host reboot. | -| No healthcheck | Warning | A service declares no healthcheck in the Compose model. The image itself may define one, but Sencho cannot see it from the model alone. | +| No effective healthcheck detected | Warning | Sencho verified that the service has no effective healthcheck in the Compose model, running containers, or a locally available image. | +| Healthcheck explicitly disabled | Warning | The Compose model disables the healthcheck (`disable: true` or `test: NONE`). | +| Healthcheck inherited from image | Note | The Compose model does not declare a healthcheck, but the running container or a local image provides one. Coverage is treated as satisfied (All Clear); the note explains the origin. | +| Healthcheck inheritance could not be verified | Info | No Compose healthcheck is declared, and Sencho could not verify inheritance (Docker unreachable, image not present locally, or incomplete inspection). Doctor never pulls images. | +| Replica healthcheck coverage is inconsistent | Warning | Running replicas for the service disagree on effective healthcheck coverage. | ### Compose semantics @@ -163,7 +167,7 @@ These rules activate when the stack publishes at least one host port. They use t ## Exposure intent checks -Five of the 32 rules cross-reference the stack's exposure intent and the access URLs documented in the Stack Dossier. These rules only fire when the stack publishes at least one host port. +Five of the 36 rules cross-reference the stack's exposure intent and the access URLs documented in the Stack Dossier. These rules only fire when the stack publishes at least one host port. To resolve exposure-related findings: @@ -186,7 +190,7 @@ The networking-relevant rules on this page (host mode, exposure intent, port con ## Node-state checks and graceful degradation -Six of the 32 rules require live Docker state to run: five are in the Node state category (external networks and volumes, new-resource notices, and container_name collision) and one is "Host port is already in use" in Port conflicts. All six are skipped when the Docker daemon is unreachable. +Six of the 36 rules require live Docker state to run: five are in the Node state category (external networks and volumes, new-resource notices, and container_name collision) and one is "Host port is already in use" in Port conflicts. All six are skipped when the Docker daemon is unreachable. Healthcheck inheritance checks also read Docker when Compose does not declare a healthcheck; those degrade to an Info finding when the daemon or image is unavailable. When the daemon is unreachable: @@ -260,7 +264,7 @@ There is no tier gate: Compose Doctor is available on all plans. - **Advisory only, with one exception.** Compose Doctor's report never blocks a deploy or changes any stack configuration; act on findings or ignore them. The one exception is the self-management guard: Sencho refuses generic deploy, update, stop, down, and delete actions on its own compose project regardless of whether that finding is acknowledged. See [Self-management](#self-management). - **Dismissing the summary is local to your browser.** The **X** on the summary card and the Doctor tab dot share a per-stack, per-node dismissal stored in your browser's local storage. It is not synced across devices or between teammates, so a dismissal you make is invisible to anyone else looking at the same stack. - **Bind-mount checks are scoped.** Only paths that resolve inside the node's Compose base directory can be checked for existence and ownership. Absolute host paths outside that directory (such as `/mnt/media`) are not reported as missing. -- **Healthcheck rule cannot see image-level healthchecks.** The no-healthcheck rule fires when the Compose model does not declare a healthcheck. Many images define one internally that Sencho cannot see from the rendered model; treat the finding as a prompt to confirm the image provides one. +- **Healthcheck coverage uses Compose, runtime, and local image evidence.** Doctor checks the rendered Compose model first, then running containers for the service, then the locally available declared image. It never pulls an image. When evidence is incomplete, it reports that inheritance could not be verified instead of claiming a definitive absence. - **One run stored per node.** There is no history. Each new run overwrites the previous one for that stack on that node. - **Node-state rules require a reachable Docker daemon.** See [Node-state checks and graceful degradation](#node-state-checks-and-graceful-degradation). - **Port conflict check excludes the checked stack.** Host ports already held by the stack being checked are ignored, so redeploying a running stack does not generate a false conflict with itself. @@ -276,7 +280,7 @@ There is no tier gate: Compose Doctor is available on all plans. Compose Doctor can only check paths that resolve inside the node's Compose base directory, such as a relative `./data` mount. An absolute host path like `/mnt/media` is outside what Sencho can see from inside its container and is never reported as missing. A missing relative path is a real finding: Docker would create it as a root-owned directory on deploy. - The no-healthcheck rule fires when the Compose model does not declare a healthcheck. Many images define their own healthcheck internally, which Sencho cannot see from the model alone. Treat these findings as a prompt to confirm the image provides a healthcheck rather than a hard problem. + Doctor reports missing healthcheck coverage only after checking the Compose model, running containers for the service, and any locally available declared image. An Info finding means inheritance could not be verified (for example the image is not present locally). Doctor does not pull images during a run. Add an explicit Compose healthcheck, or make sure the image is available on the node and run preflight again. Preflight ignores ports already held by the stack being checked, so redeploying a running stack does not flag its own bindings. A conflict finding means a different stack or an unmanaged container holds that host port on this node. diff --git a/docs/features/deep-links.mdx b/docs/features/deep-links.mdx index 5e1ec9e1..7b6bd2d5 100644 --- a/docs/features/deep-links.mdx +++ b/docs/features/deep-links.mdx @@ -95,7 +95,7 @@ On a phone, `/nodes/local/stacks//files` opens the compose editor instead Some in-app state is intentionally not encoded: -- **Stack Anatomy sub-tabs** (Anatomy, Activity, Dossier, Drift, Environment, and the rest) stay in memory only. Refresh returns you to the default Anatomy tab for that stack. A [Networking](/features/networking) finding's action can open a stack directly on its Doctor, Dossier, or Drift tab; this is in-app navigation, not a separate URL, so the same refresh behavior applies. +- **Stack Anatomy sub-tabs** (Anatomy, Activity, Doctor, Drift, Dossier, Environment, and the rest) stay in memory only. Refresh returns you to the default Anatomy tab for that stack. A [Networking](/features/networking) finding's action can open a stack directly on its Doctor, Dossier, or Drift tab; this is in-app navigation, not a separate URL, so the same refresh behavior applies. ## Tips diff --git a/docs/features/editor.mdx b/docs/features/editor.mdx index 28c7bdf4..a0a11be4 100644 --- a/docs/features/editor.mdx +++ b/docs/features/editor.mdx @@ -28,7 +28,7 @@ The top card on the left holds the stack's identity and primary controls. The action bar runs every state transition for the whole stack. The primary buttons (**Start**, **Restart**, **Stop**, **Take down** when running, **Update**) require the `stack:deploy` permission; the **Delete** entry in the kebab dropdown requires the `stack:delete` permission. The bar still appears when only **Delete** is authorised so the operator has a way to remove the stack. - If a stack is the Sencho instance you are currently signed into, its deploy/delete actions are protected: clicking any of them opens a **Sencho instance protected** dialog instead of running the action. Update Sencho from **Fleet → Node Update**, or move Sencho's own compose project outside `COMPOSE_DIR` to manage it as a normal stack. + If a stack is the Sencho instance you are currently signed into, stop, take down, delete, update, and rollback stay protected: clicking them opens a **Sencho instance protected** dialog. Eligible admins can apply on-disk Compose changes with **Save & Reapply** in this editor (same recreate procedure as Fleet Node Updates), or open **Fleet → Node Updates**. To manage the stack as a normal stack, move Sencho's compose project outside `COMPOSE_DIR`. | Button | Behavior | @@ -58,7 +58,7 @@ Below the action bar, the **CONTAINERS** section lists every container the stack | **Status badge** | `✓` (green, healthy or running), `✗` (red, exited or unhealthy), `…` (yellow, starting). | | **Name** | The Docker container name; falls back to the first 12 characters of the container ID. | | **Uptime / state** | `up 2h 15m` for running containers, the raw state for everything else. | -| **Healthcheck label** | `healthcheck passing`, `healthcheck failing`, or `healthcheck starting`, only when a healthcheck is defined. | +| **Healthcheck label** | `healthy`, `unhealthy`, or `starting`, only when a healthcheck is defined. | | **Port mapping** | The first detected web-UI port, formatted `host → container/proto`. The mapping itself is a link that opens the service in a new tab, with a **Copy URL** button beside it. The address uses the active node's host and switches to `https` for port 443. | | **Action buttons** | **Image source links**, **View logs**, **Open bash shell** (admin only), **Service actions**. Per-container image references and registry links live here, not in the header. | | **Live stats** | CPU, memory, and net I/O with rolling sparklines. Only rendered while the container is running. Stats refresh on the same 1500 ms cadence as the dashboard. | @@ -79,21 +79,21 @@ Stacks with more than one container gain a summary strip above the container lis The right column shows the **Anatomy panel** by default: a read-only summary of the compose file alongside a scrollable tab row for other stack views. - Anatomy panel header strip showing the Anatomy, Activity, Dossier, Drift, Environment, Networking, Doctor, and Storage tab row with Files and Edit compose shortcuts on the right + Anatomy panel header strip showing the Anatomy, Activity, Doctor, Drift, Dossier, Environment, Networking, and Storage tab row with Files and Edit compose shortcuts on the right -The tab row always shows four tabs: **Anatomy**, **Activity**, **Dossier**, and **Drift**. Four more tabs appear when the active node advertises the matching capability. +The tab row always shows four tabs: **Anatomy**, **Activity**, **Drift**, and **Dossier**. When Compose Doctor is available on the node, **Doctor** sits between Activity and Drift. Four more capability-gated tabs appear when the active node advertises them. | Tab | Always present? | What it shows | |-----|----------------|--------------| | **Anatomy** | Yes | Read-only compose file summary. | | **Activity** | Yes | Operational event timeline for this stack. See [Stack Activity](/features/stack-activity). | -| **Dossier** | Yes | Exportable Markdown of the anatomy combined with operator notes. See [Stack Dossier](/features/stack-dossier). | +| **Doctor** | When `compose-doctor` capability is present | Preflight check results grouped by severity. The tab gains a red dot for blocker findings and an amber dot for high-risk findings. See [Compose Doctor](/features/compose-doctor). | | **Drift** | Yes | Live comparison of the declared compose against the running containers. See [Stack Drift](/features/stack-drift). | +| **Dossier** | Yes | Exportable Markdown of the anatomy combined with operator notes. See [Stack Dossier](/features/stack-dossier). | | **Environment** | When `env-inventory` capability is present | Variable inventory across all env files, with status for each variable. See [Environment Guardrails](/features/environment-guardrails). | | **Compose Labels** | When `container-label-inventory` capability is present | Declared Compose labels vs runtime container labels per service. See [Docker Label Audit](/features/docker-label-audit). | | **Networking** | When `compose-networking` capability is present | Port exposure summary per service with intent classification. See [Compose Networking](/features/compose-networking). | -| **Doctor** | When `compose-doctor` capability is present | Preflight check results grouped by severity. The tab gains a red dot for blocker findings and an amber dot for high-risk findings. See [Compose Doctor](/features/compose-doctor). | | **Storage** | When `compose-storage` capability is present | Mount inventory with portability assessment and snapshot coverage. See [Compose Storage](/features/compose-storage). | The **Files** shortcut and the **Edit compose** button sit at the right end of the strip and stay available regardless of which tab is active. @@ -144,19 +144,20 @@ The `.env` editor renders a teal banner above the textarea reminding you that va ### Save options -As soon as the compose editor opens, the toolbar shows a split button. The primary action is **Save & Deploy**; the dropdown chevron reveals two more. +As soon as the compose editor opens, the toolbar shows a split button. The primary action is **Save & Deploy** for ordinary stacks. On Sencho's own Compose-managed stack, eligible admins see **Save & Reapply** instead. The dropdown chevron reveals two more actions. | Action | Effect | |--------|--------| | **Save & Deploy** | Writes the file to disk, then runs `docker compose up -d` to apply changes. | -| **Save Only** | Writes the file to disk without restarting any containers. Changes take effect on the next deploy. | +| **Save & Reapply** | Shown only for admins on the Compose-managed Sencho self-stack when reapply is available. Writes the file, then confirms and recreates Sencho from the current on-disk Compose project without selecting a newer image. Same procedure as Fleet **Reapply configuration**. | +| **Save Only** | Writes the file to disk without restarting any containers. Changes take effect on the next deploy or reapply. | | **Discard Changes** | Reverts the active file (compose or env) to the last saved version. Unsaved edits in that file are lost. | The same controls apply to the `compose.yaml` and `.env` editors. ## Diff preview before save -When **Diff preview before save** is enabled in **Settings → Infrastructure → Stacks**, clicking **Save & Deploy** or **Save Only** opens a side-by-side diff modal before anything is written to disk. The left pane is the on-disk content; the right pane is your unsaved edits with additions highlighted in green. The footer reads `ON DISK → UNSAVED` so the panes are unambiguous. +When **Diff preview before save** is enabled in **Settings → Infrastructure → Stacks**, clicking **Save & Deploy**, **Save & Reapply**, or **Save Only** opens a side-by-side diff modal before anything is written to disk. The left pane is the on-disk content; the right pane is your unsaved edits with additions highlighted in green. The footer reads `ON DISK → UNSAVED` so the panes are unambiguous. Diff preview modal showing side-by-side YAML diff with the unsaved version on the right and an ON DISK to UNSAVED legend in the footer @@ -164,7 +165,7 @@ When **Diff preview before save** is enabled in **Settings → Infrastructure Review the diff, then: -- Click the primary button (**Save & deploy** when triggered from **Save & Deploy**, or **Save** when triggered from **Save Only**) to confirm and write the changes. +- Click the primary button (**Save & deploy** or **Save & reapply** when triggered from that primary action, or **Save** when triggered from **Save Only**) to confirm and write the changes. - Click **Cancel** to return to the editor without saving. If there are no unsaved changes the modal is skipped and the save proceeds directly. The toggle is off by default and stored per browser, so each device remembers its own setting. @@ -177,7 +178,7 @@ On a narrow screen the stack opens as a full-screen detail with **Health**, **Lo Mobile compose editor with a Cancel button, the compose.yaml label, a monospace text field showing the compose file, a small-edits note, and Save and Save and Deploy buttons -The mobile editor is a lightweight monospace text field rather than Monaco. Tap **compose** or **.env** at the top to choose the file. The **.env** toggle appears only when the stack has an env file, and the file picker is locked while you have unsaved edits so switching files cannot drop them. The footer carries the same **Save** and **Save & Deploy** actions, and every protection is shared with desktop: the diff preview, save-conflict handling, and the unsaved-changes prompt all behave the same way. **Cancel** leaves the editor and asks before discarding unsaved edits. +The mobile editor is a lightweight monospace text field rather than Monaco. Tap **compose** or **.env** at the top to choose the file. The **.env** toggle appears only when the stack has an env file, and the file picker is locked while you have unsaved edits so switching files cannot drop them. The footer carries the same **Save** and **Save & Deploy** (or **Save & Reapply** when eligible) actions, and every protection is shared with desktop: the diff preview, save-conflict handling, and the unsaved-changes prompt all behave the same way. **Cancel** leaves the editor and asks before discarding unsaved edits. A note at the bottom of the editor is a reminder that mobile editing is meant for small corrections such as bumping an image tag or fixing a value. For large compose rewrites, open the stack on a desktop. Editing requires the `stack:edit` permission. @@ -252,7 +253,7 @@ Sencho tries `/bin/bash` first and transparently falls back to `/bin/sh` if bash The editor blocks a silent loss of in-progress edits. Click **Cancel** to return to the original node with your edits intact; click **Discard** to abandon them and proceed to the other node. - That stack is running the Sencho instance you are signed into, and its deploy/delete actions are blocked to prevent locking yourself out. Update Sencho from **Fleet → Node Update** instead. To manage the stack normally, move its compose project to a directory outside `COMPOSE_DIR`. + That stack is running the Sencho instance you are signed into. Stop, take down, delete, update, and rollback stay blocked. Eligible admins can apply Compose edits with **Save & Reapply** in the editor, or use **Fleet → Node Updates**. To manage the stack normally, move its compose project to a directory outside `COMPOSE_DIR`. The stats WebSocket failed to open, or closed unexpectedly. This usually means the Docker daemon on the node is unreachable. Container status, action buttons, and logs continue to work; only live CPU / memory / network rates pause. diff --git a/docs/features/fleet-view.mdx b/docs/features/fleet-view.mdx index d93dabb3..61856b69 100644 --- a/docs/features/fleet-view.mdx +++ b/docs/features/fleet-view.mdx @@ -280,9 +280,9 @@ The table lists every registered node, filtered by the search box at the top. Co | **Type** | `local` or `remote` outline pill | | **Current** | The node's reported Sencho version, in mono. Reads `unknown` if the node has not reported (offline, unreachable, or never connected). | | **Latest** | The newest published Sencho release. Highlighted when newer than Current. | -| **Status** | Either an `Up to date` success badge, an `Update` button (per-row), an in-progress / failed badge with retry and dismiss controls, or a `Skipped` badge when the version has been deferred. | +| **Status** | Either an `Up to date` success badge, an `Update` button when a newer release is available, an icon-only **Reapply configuration** control (tooltip) for Compose-managed nodes (including up-to-date rows), an in-progress / failed badge with retry and dismiss controls, or a `Skipped` badge when the version has been deferred. | -The latest-version label is resolved from the GitHub Releases API (with a Docker Hub fallback) and cached for 30 minutes. **Recheck** flushes the cache and re-resolves immediately. +The latest-version label is resolved from the GitHub Releases API (with a Docker Hub fallback) and cached for 30 minutes. **Recheck** flushes the cache and re-resolves immediately. See [Remote Updates · Reapply configuration](/features/remote-updates#reapply-configuration) for what reapply does and when to use it. ### Skipping a version diff --git a/docs/features/overview.mdx b/docs/features/overview.mdx index 681c3f68..bb42b92d 100644 --- a/docs/features/overview.mdx +++ b/docs/features/overview.mdx @@ -13,7 +13,7 @@ Sencho is a self-hosted cockpit for Docker Compose. The catalog below groups Sen ## Core workflow - A stack open in the editor: the action toolbar, the container card with CPU and memory sparklines, the live log stream, and the right-hand panel on the Networking tab showing exposure intent, network memberships, published ports, and a runtime drift check. The full tab strip (Anatomy, Activity, Dossier, Drift, Environment, Networking, Doctor, Storage) is visible across the top of the panel. + A stack open in the editor: the action toolbar, the container card with CPU and memory sparklines, the live log stream, and the right-hand panel on the Networking tab showing exposure intent, network memberships, published ports, and a runtime drift check. The full tab strip (Anatomy, Activity, Doctor, Drift, Dossier, Environment, Networking, Storage) is visible across the top of the panel. ### Stack management diff --git a/docs/features/rbac.mdx b/docs/features/rbac.mdx index 6e93aa86..ae047376 100644 --- a/docs/features/rbac.mdx +++ b/docs/features/rbac.mdx @@ -184,6 +184,12 @@ Sencho enforces user changes immediately by versioning JWT tokens at the user re Cookies and Bearer tokens go through the same auth middleware, so the same rules apply to API-token-based sessions where a token is bound to a user. +### Session lifetime + +A signed-in session lasts 24 hours by default, or 30 days if **Stay signed in** was checked at sign-in. Either way, an actively-used session renews itself: any request made within an hour of expiry silently reissues the cookie for a fresh full lifetime, so you are never signed out mid-session just for staying active. Only real inactivity, or one of the events in the table above, ends a session. + +Admins can turn this renewal off from **Settings > Users > Session policy** (**Keep active sessions alive**) to enforce a strict, fixed ceiling on every session regardless of activity. It is on by default. + ## SSO auto-provisioning With SSO configured on Admiral, users authenticate through an identity provider (LDAP, Custom OIDC, Google, GitHub, Okta). On their first successful sign-in, Sencho auto-creates a user record. SSO accounts appear in the Users list alongside local accounts and can be edited the same way; only the password and (optionally) the role differ. @@ -227,6 +233,9 @@ Entries include the acting user, IP address, HTTP method and path, response stat Token-version bumps invalidate sessions. Two events do this: an admin changed the user's password, or an admin reset their 2FA. Both rotate the user's token version, so every JWT issued before the rotation is rejected on the next request. The user can sign in again with their (possibly new) password. Role changes do **not** sign the user out; they take effect on the next request without rotating the token version. + + Check whether **Session policy > Keep active sessions alive** was turned off in **Settings > Users**. With it off, every session hits a strict, fixed 24-hour (or 30-day, with **Stay signed in**) ceiling regardless of activity. Turn it back on so an active session renews itself instead of hard-expiring, or have the user check **Stay signed in** at their next sign-in for a longer session between visits. + Two causes. **One,** the assignment was created on Admiral but the license has since dropped to Community. The permission resolver only consults scoped assignments when the effective tier is Admiral; on Community the scope is ignored and the user falls back to their global role. **Two,** the resource type or name on the assignment does not match the request's resource. Re-open the user in the edit form and check the existing-scope row matches the stack name (case-sensitive) exactly. diff --git a/docs/features/remote-updates.mdx b/docs/features/remote-updates.mdx index c1de64d4..221b9d0a 100644 --- a/docs/features/remote-updates.mdx +++ b/docs/features/remote-updates.mdx @@ -52,6 +52,21 @@ When an update is available, this same card gains a warning **Update available** The gateway switches to a fast 5-second polling loop while any node is in the `Updating` state, so the badge advances in near real time without waiting for the next 30-second fleet refresh. +## Reapply configuration + +When a node is already on the current Sencho release, **Update** is hidden, but Compose-managed nodes still expose a **Reapply configuration** icon control (tooltip on hover) in the Node updates sheet. Use this after you change the node's on-disk Compose project (environment variables, mounts, ports, labels, limits, healthcheck, networks, or socket-proxy settings) and need Sencho to recreate itself from that project without selecting a newer release. + +Reapply: + +- Uses the same Compose project context as a version update (multiple Compose files, `env_file`, configs, secrets, and bind mounts). +- Preserves the image reference declared in Compose. Sencho does not resolve a newer release or rewrite the pin. +- Works for digest-pinned installs, because no repin step runs. +- Requires the admin role. + +Confirming a local reapply opens a dialog that states the node will recreate from its current Compose configuration, the dashboard may briefly disconnect, no newer version is selected, and the configured image reference is not rewritten. Confirming a remote reapply uses the same required acknowledgement (recreate from current Compose, no newer version, no image rewrite) before the request is sent; Fleet then shows a **Reapplying** badge until the node restarts or the operation fails. + +Eligible admins can also run the same procedure from the Compose editor: on Sencho's own stack the primary save action becomes **Save & Reapply**, which saves the file first, then opens the same confirmation and fleet reapply path. + ## Updating the local (gateway) node Updating the gateway is special because the dashboard is hosted by the very container that is about to restart. Clicking **Update** on the local row, or **Update to vX.Y.Z** on the Local card, opens a confirmation dialog (kicker **LOCAL · UPDATE**, title **Update local node**, with **Cancel** and **Update & restart** buttons) before anything happens on disk. The body text depends on how the compose file pins the image: for a semver pin it names the exact rewrite (for example, "This install pins `saelix/sencho:0.94.1`. Updating rewrites it to `saelix/sencho:0.95.0`..."); for a floating tag it reads more generally ("Pulls Sencho v0.95.0 and restarts the server..."). Both variants end with the same note that the dashboard briefly disconnects and reconnects automatically. diff --git a/docs/features/resources.mdx b/docs/features/resources.mdx index 8596496e..599455df 100644 --- a/docs/features/resources.mdx +++ b/docs/features/resources.mdx @@ -11,13 +11,13 @@ The **Resources** tab shows everything Docker is storing on your host, broken do ## Reclaim hero -When there is reclaimable disk space (unused images, stopped containers, or dangling volumes), an amber banner leads the view with the total amount you can free and a `·`-separated breakdown of what contributes to it (for example, `2 unused images · 10 dangling volumes`). Click **Review & prune** to open a confirmation dialog that lists the exact items that will be removed (capped in the preview with an "and N more" note when the list is long). Confirm only after the plan is ready; Sencho rechecks the list at execute time and skips anything that is no longer eligible. +Once enabled for a node, an amber banner leads the view whenever there is reclaimable disk space (unused images, stopped containers, or dangling volumes), showing the total amount you can free and a `·`-separated breakdown of what contributes to it (for example, `2 unused images · 10 dangling volumes`). Click **Review & prune** to open a confirmation dialog that lists the exact items that will be removed (capped in the preview with an "and N more" note when the list is long). Confirm only after the plan is ready; Sencho rechecks the list at execute time and skips anything that is no longer eligible. The hero stays hidden when there is nothing to reclaim, keeping the view focused on the rest of your inventory. To set the banner aside without pruning, use the **×** in its top-right corner. It stays hidden on that browser until the reclaimable total grows past the amount it held when you dismissed it, so a small, stubborn remainder will not keep reappearing while a genuine new build-up still surfaces. -To keep the banner off for a node entirely, open **Settings → Monitoring → Docker & Storage** and switch off **Show reclaimable-space banner**. It is on by default and applies per node. +The banner is off by default and applies per node. To turn it on, open **Settings → Monitoring → Docker & Storage** and switch on **Show reclaimable-space banner**. The banner and the **Review & prune** action are admin-only. Read-only roles still see the rest of the page but cannot trigger destructive operations. diff --git a/docs/features/scheduled-operations.mdx b/docs/features/scheduled-operations.mdx index 79100e88..6f1fab49 100644 --- a/docs/features/scheduled-operations.mdx +++ b/docs/features/scheduled-operations.mdx @@ -26,7 +26,7 @@ Open the **Schedules** tab from the top navigation bar. The page opens on the Ti The Timeline plots every firing of every enabled task across a rolling 24-hour window starting from the current minute. - **Masthead.** A `NEXT 24 HOURS` kicker, an italic display heading, the window's start and end timestamps in a monospace range, and a right-anchored **Next** pill that reads out the time and task name of the next firing and a relative countdown. -- **Five lanes.** Lifecycle (label blue), Updates (success green), Security (label purple), Upkeep (warning amber), and Backups (brand cyan). The Lifecycle lane holds stack lifecycle actions (Backup Stack Compose Files, Start / Bring Up Stack, Restart Stack, Stop Stack, Take Stack Down) and standalone container actions (Restart Container, Stop Container, Start Container); Updates holds per-node and fleet image updates; Security holds vulnerability scans; Upkeep holds node resource prunes; Backups holds fleet snapshots. +- **Five lanes.** Lifecycle (label blue), Updates (success green), Security (label purple), Upkeep (warning amber), and Backups (brand cyan). The Lifecycle lane holds stack lifecycle actions (Backup Stack Compose Files, Start / Bring Up Stack, Restart Stack, Stop Stack, Take Stack Down) and standalone container actions (Restart Container, Stop Container, Start Container); Updates holds per-stack, per-node, and label-targeted image updates; Security holds vulnerability scans; Upkeep holds node resource prunes; Backups holds fleet snapshots. - **Pills.** One pill per firing within the window, positioned proportionally to the firing's time. Each pill shows the firing time and a target: the stack for stack actions, the container name for container actions, the selected node for prune and scan, and "Entire fleet" for a fleet snapshot. Hover a pill for the full detail (action, task name, and node). Pills are color-matched to their lane. Click a pill to open the run history sheet for that task. - **Now rail.** A glowing vertical rail at the current minute, anchored to the left of the track at page open and drifting right as time passes (the page recomputes positions periodically). - **Axis.** Six monospace time ticks run along the bottom, evenly spaced through the window. @@ -61,6 +61,7 @@ The All tasks toggle swaps the lane track for a sortable table. | **Restart Stack** | A specific stack (optionally specific services) on a specific node | Restarts all or selected containers in the stack. | | **Auto-update Stack** | A specific stack on a specific node | Checks each image in the stack for a newer tag and recreates the stack if any image has an update. See [Auto-Update Policies](/features/auto-update-policies) for the companion review board. | | **Auto-update All Stacks on Node** | A specific node | Runs the auto-update check across every stack on the node. Pair with **Auto-update Stack** rows when you want different cadences for specific stacks. | +| **Auto-update stacks by label** | A Stack Label across the entire fleet, or on one selected node | At each run, resolves stacks that currently carry the chosen Stack Label and updates those with newer images. Membership is dynamic: assigning or removing the label changes the next run without editing the schedule. | | **Create Fleet Snapshot** | The whole fleet | Creates a versioned, fleet-wide snapshot of every node's compose files and `.env` files. See [Fleet Backups](/features/fleet-backups). | | **Prune Node Resources** | A local node | Prunes containers, images, networks, and volumes (any subset), optionally filtered by a Docker label. Runs on local nodes only. | | **Scan Node Images** | A local node | Runs Trivy against every image on the node and persists the results. Requires Trivy to be installed on the target node ([Installing Trivy](/operations/trivy-setup)). Runs on local nodes only. | @@ -93,6 +94,7 @@ Conditional fields per action: - **Stack actions** (Backup Stack Compose Files, Start / Bring Up Stack, Restart Stack, Auto-update Stack, Stop Stack, Take Stack Down) add a **Node** combobox and a **Stack** combobox. When the selected stack's compose file defines more than one service, Restart Stack additionally renders a **Services** checkbox grid so you can scope the restart to a subset instead of the entire stack; single-service stacks skip the grid since there is nothing to scope. - **Container actions** (Restart Container, Stop Container, Start Container) add a **Node** combobox and a **Container** combobox listing every container on that node (running and stopped). The picker shows each container's name, state, and image. When the container is not part of a Sencho stack, helper text explains that the schedule targets the container by node and name. - **Auto-update All Stacks on Node** adds a **Node** combobox. The helper text "Checks every stack on the selected node and updates stacks with newer images" appears above, next to the Runtime change badge. +- **Auto-update stacks by label** adds a Stack Label name field, a scope control (Entire fleet or Selected node), an optional Node combobox when scoped to one node, and a read-only current-match preview. The preview is informational; membership is resolved again at each run. Saving with zero current matches is allowed. - **Scan Node Images** adds a **Node** combobox listing local nodes only. The helper text "Runs Trivy against images on the selected local node and records the findings" and Read-only badge appear above. - **Prune Node Resources** adds a **Node** combobox listing local nodes only, then a **Prune Targets** group (Containers, Images, Networks, Volumes; all selected by default) and a **Label Filter** input for scoping the prune to resources matching a Docker label. - **Create Fleet Snapshot** shows a read-only **Scope: Entire fleet** summary instead of a Node or Stack picker, because it captures every node. diff --git a/docs/features/sso.mdx b/docs/features/sso.mdx index b15a0523..cac68d3b 100644 --- a/docs/features/sso.mdx +++ b/docs/features/sso.mdx @@ -4,9 +4,9 @@ sidebarTitle: SSO and LDAP description: Authenticate with your existing identity provider, including LDAP, Google, GitHub, Okta, and any spec-compliant OIDC provider. --- -Sencho lets your team sign in with the identity provider you already use instead of maintaining a second set of credentials. SSO works **alongside** password authentication; it does not replace it. +Sencho lets your team sign in with the identity provider you already use instead of maintaining a second set of credentials. By default, SSO works alongside local password authentication. You can switch to **SSO only** so interactive local password login is disabled and only configured LDAP or OIDC providers are accepted. -SSO is available on every Sencho tier. Custom OIDC and the preset providers for Google, GitHub, and Okta work on every tier; LDAP and Active Directory require Sencho Admiral. +SSO provider configuration and authentication mode are available on every Sencho tier. Custom OIDC and the preset providers for Google, GitHub, and Okta work on every tier; LDAP and Active Directory require Sencho Admiral. ## Supported providers @@ -78,7 +78,23 @@ SSO can be configured two ways: ### Via Settings UI -Admins manage SSO providers in **Settings → Access → SSO** (admin only; hidden on remote nodes). The masthead shows the SCOPE (global), the number of configured **PROVIDERS**, and how many are **ENABLED**. The page lists every provider as a collapsible card with a label, an **enable / disable** toggle pill on the right, and an **Active** badge on the header when the provider is on. +Admins manage SSO providers in **Settings → Access → SSO** (admin only). The masthead shows the SCOPE (global), the number of configured **PROVIDERS**, and how many are **ENABLED**. The page lists every provider as a collapsible card with a label, an **enable / disable** toggle pill on the right, and an **Active** badge on the header when the provider is on. + +At the top of the page, **Authentication mode** chooses how interactive login works: + +- **Local and SSO** (default): local username/password login remains available alongside configured providers. +- **SSO only**: local password login is disabled. The login page shows only LDAP and/or OIDC providers. Direct calls to the password login endpoint are rejected. When exactly one OIDC provider is enabled and LDAP is not, the login page redirects straight to that provider's authorization endpoint. Multiple OIDC providers still show chooser buttons. A failed SSO attempt that returns to the login page with an error stays on the page so the message is visible. + +SSO only cannot be enabled until at least one provider is enabled, a connection test succeeds, and the signed-in administrator authenticated through SSO with the Admin role. Confirm the outage-risk warning before saving. Existing sessions stay valid until they expire or are revoked. + +If the identity provider is unavailable after SSO only is enabled, restore local password login from the host: + +```bash +docker compose exec sencho node dist/cli/enableLocalLogin.js +docker compose restart sencho +``` + +Restart is required so the running process reloads the setting. The same command is listed under **Settings → System → Recovery**. SSO settings panel listing the five identity providers as collapsible cards with enable / disable toggles @@ -310,7 +326,17 @@ If not set, Sencho auto-detects the URL from the request's `Host` header and pro - Verify the provider is **enabled** (toggle on, showing the **Active** badge) in **Settings → Access → SSO** and that the configuration saved successfully. The login page fetches the list of enabled providers when it loads; hard-refresh the tab if changes were just made. + Confirm the provider is enabled (toggle on, showing the Active badge) under **Settings → Access → SSO** and that the configuration saved successfully. The login page fetches the list of enabled providers when it loads; hard-refresh the tab if changes were just made. + + + From the host that runs Sencho, re-enable local password login and restart so the setting takes effect: + + ```bash + docker compose exec sencho node dist/cli/enableLocalLogin.js + docker compose restart sencho + ``` + + Then sign in with a local administrator account and repair the identity provider configuration before enabling SSO only again. diff --git a/docs/features/stack-drift.mdx b/docs/features/stack-drift.mdx index 2297f106..81cf7404 100644 --- a/docs/features/stack-drift.mdx +++ b/docs/features/stack-drift.mdx @@ -121,11 +121,11 @@ Drift events also appear in the stack's **Activity** tab alongside deploys, rest The tab bar in the stack detail view showing Anatomy, Activity, Dossier, Drift, Environment, and Compose Labels tabs with Drift selected, and separate Files and Edit compose buttons to the right 1. Click any stack in the sidebar to open it. -2. In the right-hand panel, switch to the **Drift** tab. If the tab row doesn't fit the panel width, scroll it to reveal Drift alongside the stack's other tabs (Anatomy, Activity, Dossier, Environment, Networking, and so on). +2. In the right-hand panel, switch to the **Drift** tab. If the tab row doesn't fit the panel width, scroll it to reveal Drift alongside the stack's other tabs (Anatomy, Activity, Doctor, Dossier, Environment, Networking, and so on). 3. Read the status badge and any findings. 4. Click **re-check** after editing the Compose file or after a manual Docker operation to refresh the ledger. diff --git a/docs/features/stack-labels.mdx b/docs/features/stack-labels.mdx index bf8944b5..040cbb8c 100644 --- a/docs/features/stack-labels.mdx +++ b/docs/features/stack-labels.mdx @@ -202,6 +202,9 @@ When the chosen label name exists with different colors on different nodes, the The full reference for every cross-node action, including Stop by label and Bulk label assign. + + Schedule auto-updates that target stacks by Stack Label across the fleet or on one node. + The complete Mute Rules field reference and how suppression interacts with routing. diff --git a/docs/features/stack-management.mdx b/docs/features/stack-management.mdx index 08cdf81d..31ed3e2c 100644 --- a/docs/features/stack-management.mdx +++ b/docs/features/stack-management.mdx @@ -144,8 +144,9 @@ Each row leads with a two-character status indicator that summarizes the stack's Additional indicators appear to the right of the stack name: -- A pulsing fuchsia dot flags that an image update is available. -- A muted alert icon replaces the dot when the update check ran but failed to reach the registry or errored. +- A pulsing fuchsia dot flags that an image update is available and the latest check completed successfully. That signal covers Compose-actionable digest drift (same tag, new registry content) as well as a newer tag that requires editing the Compose pin. +- A muted warning icon appears when the last check was incomplete. When an update was also detected or previously recorded, the tooltip notes that the full stack could not be verified. When no update is recorded, the tooltip notes that update status could not be fully verified. Incomplete results are not counted under **Updates**. +- A muted alert icon appears when the update check failed (and, when an earlier update was recorded, notes that the previous status was retained). - A branch icon signals that the Git source's upstream branch has moved ahead of the working copy. ### Search and filter chips @@ -159,7 +160,7 @@ The search box above the list filters stacks by name. The chip row below it filt - **All**: every stack discovered in `COMPOSE_DIR`. - **Up**: stacks where every container is running. - **Down**: stacks with at least one stopped or exited container. -- **Updates**: stacks with a pending image update. +- **Updates**: stacks with a confirmed pending image update (latest check completed successfully). Incomplete or failed checks are excluded from this chip. Each chip carries a live count. Click **Hide filters** in the top-right of the row to collapse the chips when you want a denser list. Search and chips combine: filtering by **Updates** and typing a few characters returns the intersection. @@ -185,8 +186,8 @@ The header answers two questions at a glance: - **What is this?** A breadcrumb (`LOCAL · STACKS · NAME`) and the stack name as the title. - **Is it healthy?** A state pill to the right of the title reports the live state: - - `running · healthy` (green) when at least one container reports a passing healthcheck. - - `running · unhealthy` (red) when any container reports a failing healthcheck. + - `running · healthy` (green) when at least one container reports Docker health status `healthy`. + - `running · unhealthy` (red) when any container reports Docker health status `unhealthy`. - `running · starting` (amber) during the Docker healthcheck start period. - `running` (green) when no healthcheck is defined. - `exited` (red) when no containers are up. @@ -209,15 +210,17 @@ The same links button appears on update cards in [Auto-Update Policies](/feature Below the header, each container in the stack gets a single row that answers "is this piece working, and how do I reach it?" When the stack has multiple containers, a summary strip appears above the list showing total, running, paused, and unhealthy counts, along with a **Compact / Detailed** toggle. Compact mode shows status, name, uptime, port, and action buttons; detailed mode (the default) adds CPU, memory, and network I/O sparklines. +While the stack detail stays open, Sencho keeps those rows aligned with Docker: health transitions and container lifecycle changes update the cards without reloading the compose file, environment files, or logs. If automatic refresh cannot reach the node for a stretch, a small stale notice appears with a Retry action so you can pull state manually. + Container card for the plex service showing health badge, uptime, port mapping, open link, action icons, and CPU, memory, and network stat tiles Each row includes: -- **Health badge.** A colored glyph reports the Docker healthcheck state: `✓` green for passing, `✗` red for failing, `…` amber while the healthcheck start period is in flight. Containers without a `healthcheck:` block show a neutral `✓`. +- **Health badge.** A colored glyph reports the Docker healthcheck state: `✓` green for `healthy`, `✗` red for `unhealthy`, `…` amber while the healthcheck start period is in flight. Containers without a `healthcheck:` block show a neutral `✓`. - **Container name** in mono. -- **Meta line.** Uptime (`up 12 hours`) and the primary port mapping (`8989 → 8989/tcp`). +- **Meta line.** Uptime (`up 12 hours`), the Docker health status when defined (`healthy`, `unhealthy`, or `starting`), and the primary port mapping (`8989 → 8989/tcp`). - **Open link.** When the container publishes a port, the mapping itself is a link (`8989 → 8989/tcp ↗`) that opens the service in a new tab, with a **Copy URL** button beside it. The address uses the active node's host and switches to `https` for port 443. Recognised multi-port apps open their web path automatically (for example, Plex opens `/web`). - **Live stat tiles.** Three tiles show CPU, memory, and network I/O with a rolling sparkline. The sparkline uses the cyan data color and refreshes roughly every 1.5 seconds. - **Action icons.** The image source links button (see above), plus shortcuts to **View logs**, open a bash shell, and (on single-service stacks) the per-container Start / Stop / Restart kebab. @@ -256,7 +259,7 @@ The structured viewer holds up to 10,000 lines; older entries are dropped from t ## Anatomy panel -The right column of the stack view is a tabbed panel: Anatomy, Activity, Dossier, Drift, Environment, Compose Labels, Networking, Doctor, and Storage. Tabs appear only when the data they need is available, and the row scrolls horizontally (with a chevron button) when there are more tabs than fit. +The right column of the stack view is a tabbed panel: Anatomy, Activity, Doctor, Drift, Dossier, Environment, Compose Labels, Networking, and Storage. Tabs appear only when the data they need is available, and the row scrolls horizontally (with a chevron button) when there are more tabs than fit. Anatomy panel with the Anatomy tab active, showing services, ports, volumes, restart, env_file, network, and source rows plus the tab row @@ -281,7 +284,13 @@ Each row maps one compose concept to the value it resolves to right now: A footer card under the rows surfaces the first published port as a clickable **EXPOSED** link, so you can jump straight to the running app. -If an image update is available, or the stack declares one or more services with a local `build:` section, an inline banner appears below the rows. Registry updates name each image with a pending update and show its version transition (for example `27.1.4 -> 27.1.5`); the detail line below reads `patch · safe to apply` (green), `minor · review recommended` (amber), or `major · breaking changes possible` (rose), followed by an **apply** button. Build-only stacks show **Rebuild available** instead of a version bump, with a **Rebuild & Update** button. A stack that mixes registry images and local builds still shows a single banner, with the rebuild note folded into the same detail line. Major bumps use the rose styling and are worth reviewing before applying. +If an image update is available, or the stack declares one or more services with a local `build:` section, an inline banner appears below the rows. + +- **Digest rebuild / rebuild**: same-tag registry content changed, or a local `build:` service needs rebuilding. The banner shows the version context and an **apply** (or **Rebuild & Update**) button when the check completed successfully. +- **Newer tag**: a higher semver tag exists than the pin in Compose. The detail line reads `newer tag · edit Compose pin`. There is no **apply** button; Compose pull does not rewrite image pins. Edit the Compose file (or use your usual pin-change workflow), then deploy. +- **Incomplete or failed check** with no confirmed update: a separate amber banner explains that status is uncertain until a full check succeeds. + +For actionable registry updates, the detail line also reads `patch · safe to apply` (green), `minor · review recommended` (amber), or `major · breaking changes possible` (rose). Build-only stacks show **Rebuild available** instead of a version bump, with a **Rebuild & Update** button. A stack that mixes registry images and local builds still shows a single banner, with the rebuild note folded into the same detail line. Major bumps use the rose styling and are worth reviewing before applying. Rebuilds can take longer than a registry pull and depend on the local Dockerfile context, network access, and base-image availability. Atomic rollback restores compose and env files only; previously built image layers are not rolled back automatically. diff --git a/docs/getting-started/introduction.mdx b/docs/getting-started/introduction.mdx index cd1a11d8..5793fc79 100644 --- a/docs/getting-started/introduction.mdx +++ b/docs/getting-started/introduction.mdx @@ -52,7 +52,7 @@ Opening a stack gives you the day-to-day workspace: - Running stacks expose **Restart**, **Stop**, **Take down**, and **Update**; stopped stacks expose **Start** and **Update**. The overflow menu holds less frequent actions such as rollback, config scan, and delete. - Container rows show health, uptime, published ports, live CPU and memory, network activity, logs, and service actions. - The logs panel can run in **Structured** mode or **Raw terminal** mode. -- The right panel provides tabs for **Anatomy**, **Activity**, **Dossier**, **Drift**, **Environment**, **Compose Labels**, **Networking**, **Doctor**, and **Storage**, with **Files** and **Edit** controls for browsing stack files and editing compose or env content. +- The right panel provides tabs for **Anatomy**, **Activity**, **Doctor**, **Drift**, **Dossier**, **Environment**, **Compose Labels**, **Networking**, and **Storage**, with **Files** and **Edit** controls for browsing stack files and editing compose or env content. ## Fleet operations diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 5ad6963a..35ded748 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -149,6 +149,69 @@ components: type: boolean example: true + UpdatePreviewImage: + type: object + required: + [service, image, current_tag, next_tag, has_update, digest_update, tag_update, semver_bump, check_status] + properties: + service: { type: string } + image: { type: string } + current_tag: { type: string } + next_tag: { type: ["string", "null"] } + has_update: { type: boolean } + digest_update: + type: boolean + description: Same-tag registry content drift; Compose pull can apply without changing the pin. + tag_update: + type: boolean + description: A higher pinned semver exists; advisory until Compose is edited. + semver_bump: + type: string + enum: [none, unknown, patch, minor, major] + check_status: + type: string + enum: [ok, partial, failed, not_checkable] + + UpdatePreviewSummary: + type: object + required: + [has_update, primary_image, current_tag, next_tag, semver_bump, update_kind, blocked, blocked_reason, has_build_services, rebuild_available, check_status] + properties: + has_update: { type: boolean } + primary_image: { type: ["string", "null"] } + current_tag: { type: ["string", "null"] } + next_tag: { type: ["string", "null"] } + semver_bump: + type: string + enum: [none, unknown, patch, minor, major] + update_kind: + type: string + enum: [tag, digest, none] + blocked: { type: boolean } + blocked_reason: { type: ["string", "null"] } + has_build_services: { type: boolean } + rebuild_available: { type: boolean } + check_status: + type: string + enum: [ok, partial, failed] + + UpdatePreview: + type: object + required: [stack_name, images, build_services, summary, rollback_target, changelog] + properties: + stack_name: { type: string } + images: + type: array + items: + $ref: "#/components/schemas/UpdatePreviewImage" + build_services: + type: array + items: { type: string } + summary: + $ref: "#/components/schemas/UpdatePreviewSummary" + rollback_target: { type: ["string", "null"] } + changelog: { type: ["string", "null"] } + LabelSource: type: string description: Provenance of a label. `unknown` when a container or image could not be inspected. @@ -1712,6 +1775,12 @@ paths: Id of the post-update health gate observation started for this update, for use with the health-gate endpoint. Null when the health gate is disabled on the node. + recheckWarning: + type: string + description: | + Present when Compose succeeded but post-update image + verification still reports an available update or could + not fully verify clearance. Optional for older nodes. "403": $ref: "#/components/responses/Forbidden" "500": @@ -1788,6 +1857,68 @@ paths: "500": $ref: "#/components/responses/InternalError" + /api/stacks/{stackName}/update-preview: + get: + operationId: getStackUpdatePreview + tags: [Stacks] + summary: Compute image update preview (read-only) + description: | + Computes the current registry update preview for the stack without + mutating persisted scanner state. Use POST when the client should + reconcile sticky update indicators after an authoritative-negative + result. Requires `stack:read` permission. + parameters: + - $ref: "#/components/parameters/stackName" + - $ref: "#/components/parameters/nodeId" + responses: + "200": + description: Update preview. + content: + application/json: + schema: + $ref: "#/components/schemas/UpdatePreview" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "500": + $ref: "#/components/responses/InternalError" + post: + operationId: reconcileStackUpdatePreview + tags: [Stacks] + summary: Compute update preview and reconcile sticky state + description: | + Computes the same preview as GET. When every image reports + `check_status: ok` and `has_update` is false (mixed `ok` + + `not_checkable` does not clear), clears sticky confirmed update rows + for the stack and sets `reconciled: true`. Requires `stack:read` + permission. + parameters: + - $ref: "#/components/parameters/stackName" + - $ref: "#/components/parameters/nodeId" + responses: + "200": + description: Update preview, with reconcile outcome. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/UpdatePreview" + - type: object + required: [reconciled] + properties: + reconciled: + type: boolean + description: | + True when sticky update rows were cleared after an + authoritative-negative preview. + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "500": + $ref: "#/components/responses/InternalError" + /api/stacks/{stackName}/update-readiness: get: operationId: getStackUpdateReadiness diff --git a/docs/reference/contact.mdx b/docs/reference/contact.mdx index da79ba9b..bc080cc1 100644 --- a/docs/reference/contact.mdx +++ b/docs/reference/contact.mdx @@ -12,7 +12,7 @@ Sencho provides dedicated email channels for different types of inquiries. Use t Priority email support, reserved for Admiral license holders: Monday to Friday, 09:00-17:00 America/New_York, with a one-business-day first-response target. This is not a contractual SLA or 24/7 service. -Every user, Community and Admiral alike, can reach [Documentation](https://docs.sencho.io) and [GitHub Issues](https://github.com/studio-saelix/sencho/issues) from **Settings → Help → Support** in the app. Admiral adds the priority email channel above on the same page. +Every user, Community and Admiral alike, can reach [Documentation](https://docs.sencho.io), [GitHub Issues](https://github.com/studio-saelix/sencho/issues), and [Discord](https://discord.gg/rvXAszRGSc) from **Settings → Help → Support** in the app. Admiral adds the priority email channel above on the same page. ## General inquiries diff --git a/docs/reference/settings.mdx b/docs/reference/settings.mdx index 42ce8d58..1a41b03d 100644 --- a/docs/reference/settings.mdx +++ b/docs/reference/settings.mdx @@ -145,7 +145,7 @@ A live preview card shows a sample fleet-status tile so you can see a color choi | Control | What it does | |---------|--------------| -| **Navigation style** | **Smart bar** (recommended default): primary destinations stay visible in the top bar and the rest live under **More**. **Classic bar** keeps the full horizontal strip of destinations. **Compact launcher** puts every destination in a menu, with optional quick links. | +| **Navigation style** | **Smart bar** (recommended default): primary destinations stay visible in the top bar and the rest live under **More**. **Classic bar** keeps the full horizontal strip of destinations (retiring soon; a callout appears while it is selected). **Compact launcher** puts every destination in a menu, with optional quick links. | | **Top navigation labels** | On by default. Shows text labels beside the top navigation icons; turn off for a more compact bar with icons only. | Deploy-progress behavior and the diff-preview-before-save step are stack workflow preferences and live in their own [Stacks](#stacks) section under Infrastructure. @@ -293,7 +293,7 @@ Configure the reclaimable-space alert, the reclaimable-space banner, and automat | Setting | Default | Description | |---------|---------|-------------| | **Reclaimable Docker data threshold** | 5 GiB | Alert when reclaimable Docker data (images, volumes, build cache that `docker prune` could free) exceeds this size. Set to `0` to disable the alert. | -| **Show reclaimable-space banner** | On | Show the reclaimable-space banner at the top of the Resource Hub when this node has unused images, stopped containers, or dangling volumes to clear. | +| **Show reclaimable-space banner** | Off | Show the reclaimable-space banner at the top of the Resource Hub when this node has unused images, stopped containers, or dangling volumes to clear. | ### Image cleanup @@ -403,7 +403,7 @@ Quick reference: Configure external destinations for alert notifications. Four agent types are available on separate tabs: **Discord**, **Slack**, **Webhook**, and **Apprise**. The masthead publishes a **CHANNELS** pill showing how many agents are enabled (for example, `2/4`). -**Delivery retries** (admin-only) sets how many extra in-process attempts (0 to 3, default 0) this node makes after a transient channel failure, with a fixed one-second delay between attempts. There is no durable queue; ambiguous network failures can produce duplicate notifications. +Below the channel tabs, **Delivery retries** (admin-only) sets how many extra in-process attempts (0 to 3, default 0) this node makes after a transient channel failure, with a fixed one-second delay between attempts. There is no durable queue; ambiguous network failures can produce duplicate notifications. For each agent: @@ -489,11 +489,16 @@ Configure how often this node polls container registries to detect available ima | Setting | Default | Description | |---------|---------|-------------| -| **Scheduling mode** | Interval | **Interval**: check every fixed period. **Cron**: check on a precise cron schedule (runs in the node's local timezone). | +| **Enable image update checks** | On | When on, this node polls registries on schedule, raises update notifications, and feeds Home, sidebar, Anatomy, and Fleet Readiness. Turn off when another tool is the update authority for this node. Explicit stack Update, pull, and redeploy remain available. | +| **Scheduling mode** | Interval | **Interval**: check every fixed period. **Cron**: check on a precise cron schedule (runs in the node's local timezone). Greyed out while image update checks are off. | | **Check interval** | 2 hours | How often to poll registries when in Interval mode. Presets: 15 min, 30 min, 1 h, 2 h, 6 h, 12 h, 24 h. Selecting a new preset saves immediately. | | **Cron expression** | - | A standard five-field cron expression (for example, `0 3 * * 1` for every Monday at 03:00). A human-readable description appears below the field as you type. Click **Save schedule** to apply. | -The section footer shows the last-checked timestamp and when the next check is scheduled. +The section footer shows the last-checked timestamp and when the next check is scheduled. When checks are off, the footer reads **Next check: disabled** and retains the last-checked time from before detection was turned off. + + + Nodes running older versions of Sencho do not expose the enable toggle. Upgrade the node to turn detection off. + ### Sidebar @@ -721,6 +726,7 @@ Links to help resources, with an additional channel for Admiral operators. |----------|-------------| | **Documentation** | Opens docs.sencho.io. | | **GitHub Issues** | Report bugs and request features on GitHub. | +| **Discord** | Chat with the community and the team. | ### Admiral support diff --git a/e2e/desktop-navigation.spec.ts b/e2e/desktop-navigation.spec.ts index eaf6bedf..58195ca9 100644 --- a/e2e/desktop-navigation.spec.ts +++ b/e2e/desktop-navigation.spec.ts @@ -43,7 +43,8 @@ test.describe('Desktop navigation styles', () => { await setTopNavMode(page, 'smart'); await expect(page.locator('[data-sn-chrome="topbar"]')).toHaveAttribute('data-sn-nav-mode', 'smart'); await page.getByRole('button', { name: 'More navigation' }).click(); - await expect(page.locator('.font-heading').filter({ hasText: 'More' })).toBeVisible(); + await expect(page.getByRole('menuitem', { name: /Logs/i })).toBeVisible(); + await expect(page.locator('.font-heading').filter({ hasText: 'More' })).toHaveCount(0); await page.getByRole('menuitem', { name: /Logs/i }).click(); await expect(page.locator('body')).toContainText(/Logs|Central|Observability/i); }); diff --git a/e2e/sidebar-stack-truncate.spec.ts b/e2e/sidebar-stack-truncate.spec.ts index 55a824f7..943457ba 100644 --- a/e2e/sidebar-stack-truncate.spec.ts +++ b/e2e/sidebar-stack-truncate.spec.ts @@ -209,7 +209,7 @@ test.describe('Sidebar stack name truncation', () => { }); }); - test('update dot wins over check-failed on a long stack name', async ({ page }) => { + test('failed icon wins over retained hasUpdate on a long stack name', async ({ page }) => { await page.route('**/api/image-updates/detail', async (route) => { await route.fulfill({ status: 200, @@ -230,7 +230,7 @@ test.describe('Sidebar stack name truncation', () => { await assertRowLayout(page, UPDATE_STACK, { expectTruncated: true, - trailingKind: 'update', + trailingKind: 'failed', }); }); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 624f98e4..86c1ddcc 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -4004,16 +4004,16 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", + "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/brace-expansion/node_modules/balanced-match": { diff --git a/frontend/package.json b/frontend/package.json index 1c79e79e..c41c755d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -65,7 +65,7 @@ "overrides": { "dompurify": "^3.4.11", "@babel/core": "^7.29.6", - "brace-expansion": "^5.0.7" + "brace-expansion": "^5.0.8" }, "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/frontend/src/components/AutoUpdateReadinessView.tsx b/frontend/src/components/AutoUpdateReadinessView.tsx index 8b2cc687..e02a50fd 100644 --- a/frontend/src/components/AutoUpdateReadinessView.tsx +++ b/frontend/src/components/AutoUpdateReadinessView.tsx @@ -7,6 +7,18 @@ import { toast } from '@/components/ui/toast-store'; import { apiFetch, fetchForNode } from '@/lib/api'; import { formatTimeAgo } from '@/lib/relativeTime'; import type { ImageUpdateStatus, StackUpdateInfo } from '@/types/imageUpdates'; +import { isAuthoritativeNegativePreview } from '@/types/imageUpdates'; +import { fetchUpdatePreview } from '@/lib/fetchUpdatePreview'; +import { + isActionableUpdatePreview, + isClearedUpdatePreview, + isLegacyPreview, + isPreviewUncertain, + isReviewRequiredUpdatePreview, + isServiceApplyActionable, + isTagOnlyAdvisory, + isVerificationOnlyPreview, +} from '@/lib/updatePreviewActionability'; import { useNodes } from '@/context/NodeContext'; import { useIsMobile } from '@/hooks/use-is-mobile'; import { Masthead, Kicker } from '@/components/mobile/mobile-ui'; @@ -24,7 +36,14 @@ interface UpdatePreviewImage { current_tag: string; next_tag: string | null; has_update: boolean; + digest_update?: boolean; + tag_update?: boolean; semver_bump: SemverBump; + /** Absent on older remotes; backend uses !== 'not_checkable' for checkability. */ + check_status?: 'ok' | 'partial' | 'failed' | 'not_checkable'; + check_error?: string | null; + /** This image's own digest-comparison failure; not masked by a confirmed tag update. */ + digest_error?: string | null; } type UpdateKind = 'tag' | 'digest' | 'none'; @@ -43,6 +62,10 @@ interface UpdatePreview { blocked_reason: string | null; has_build_services?: boolean; rebuild_available?: boolean; + /** Absent on older remotes; treat missing as non-authoritative. */ + check_status?: 'ok' | 'partial' | 'failed'; + verification_failed?: boolean; + verification_error?: string | null; }; build_services?: string[]; rollback_target: string | null; @@ -57,6 +80,11 @@ function declaredServiceCount(preview: UpdatePreview | null | undefined): number return names.size; } +/** Append `: reason` when present, otherwise end the lead-in with a period. */ +function withErrorDetail(lead: string, error: string | null | undefined): string { + return error ? `${lead}: ${error}` : `${lead}.`; +} + export interface StackCard { stack: string; nodeId: number; @@ -72,6 +100,9 @@ export interface StackCard { // Name of the service currently applying a per-service update on this card, // or null when none is in flight. Distinct from `applying` (full-stack). applyingService: string | null; + // Post-Apply verification note when Compose succeeded but clearance could + // not be confirmed (distinct from a failed preview fetch). + verificationNote: string | null; } interface NodeGroup { @@ -104,15 +135,20 @@ export function CadenceStrip({ cadence, className }: { cadence: ImageUpdateStatu if (!cadence) return null; + const checksOff = cadence.enabled === false; const lastChecked = cadence.lastCheckedAt != null ? formatTimeAgo(cadence.lastCheckedAt) : 'never'; - const nextCheck = cadence.checking - ? 'checking now' - : cadence.nextCheckAt != null - ? formatRelative(cadence.nextCheckAt) - : 'not scheduled'; - const cooldown = cooling - ? `Recheck available in ${Math.ceil(remainingMs / 1000)}s` - : 'Recheck ready'; + const nextCheck = checksOff + ? 'disabled' + : cadence.checking + ? 'checking now' + : cadence.nextCheckAt != null + ? formatRelative(cadence.nextCheckAt) + : 'not scheduled'; + const cooldown = checksOff + ? 'Detection off' + : cooling + ? `Recheck available in ${Math.ceil(remainingMs / 1000)}s` + : 'Recheck ready'; return (
@@ -148,7 +184,19 @@ function formatClock(ts: number | null): string { }); } -function RiskBadge({ bump, blocked }: { bump: SemverBump; blocked: boolean }) { +function RiskBadge({ + bump, + blocked, + reviewRequired, + uncertain, + tagOnly, +}: { + bump: SemverBump; + blocked: boolean; + reviewRequired?: boolean; + uncertain?: boolean; + tagOnly?: boolean; +}) { if (blocked || bump === 'major') { return ( @@ -157,6 +205,29 @@ function RiskBadge({ bump, blocked }: { bump: SemverBump; blocked: boolean }) { ); } + if (reviewRequired) { + return ( + + + Review · unverified + + ); + } + if (uncertain) { + return ( + + + Check uncertain + + ); + } + if (tagOnly) { + return ( + + Newer tag · edit Compose + + ); + } if (bump === 'minor') { return ( @@ -212,9 +283,10 @@ function StackReadinessCard({ onApply: (stack: string, nodeId: number) => void; onApplyService?: (stack: string, nodeId: number, serviceName: string) => void; }) { - const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled } = card; + const { stack, nodeId, preview, previewLoaded, scheduledTask, applying, applyingService, autoUpdateEnabled, verificationNote } = card; const loading = !previewLoaded; - const failed = previewLoaded && preview === null; + const uncertain = previewLoaded && !!verificationNote; + const failed = previewLoaded && preview === null && !verificationNote; const blocked = preview?.summary.blocked ?? false; const bump = preview?.summary.semver_bump ?? 'none'; const updatingImages = preview?.images.filter(i => i.has_update) ?? []; @@ -223,6 +295,15 @@ function StackReadinessCard({ // build-only), not preview.images.length (shared tags collapse that list). const showServiceApply = canServiceUpdate && declaredServiceCount(preview) > 1 && updatingImageCount > 0; const nextRun = scheduledTask?.next_run_at ?? null; + const verificationOnly = isVerificationOnlyPreview(preview); + const reviewRequired = isReviewRequiredUpdatePreview(preview); + // Full-stack apply is held for review when another image in the same stack + // failed digest verification: applying would pull/recreate that image as + // collateral. Per-service apply targets only images with their own confirmed + // update, so it is not gated by a different image's verification failure. + const applyDisabled = !isActionableUpdatePreview(preview) + || applying + || applyingService !== null; return ( @@ -242,12 +323,24 @@ function StackReadinessCard({ Auto: Off )} - {previewLoaded && preview && } + {previewLoaded && preview && ( + + )}
{loading ? (
Checking registry...
+ ) : uncertain ? ( +
+ {verificationNote} +
) : failed ? (
Preview failed. Registry may be unreachable. @@ -256,20 +349,45 @@ function StackReadinessCard({ (() => { const p = preview!; const blockedReason = p.summary.blocked_reason; + const verificationFailed = Boolean(p.summary.verification_failed); + const verificationError = p.summary.verification_error; + let applyTitle: string | undefined; + if (blocked) applyTitle = blockedReason ?? undefined; + else if (verificationOnly) applyTitle = 'Digest verification failed'; + else if (reviewRequired) applyTitle = 'Another image in this stack failed digest verification; apply the confirmed service individually or resolve verification first.'; + + let headline: ReactNode; + if (verificationFailed && !p.summary.has_update) { + headline = ( +
+ {withErrorDetail('Digest verification failed', verificationError)} +
+ ); + } else if (p.summary.update_kind === 'digest') { + headline = ( +
+ {p.summary.current_tag} + + Rebuild available + +
+ ); + } else { + headline = ( + + ); + } + return ( <> - {p.summary.update_kind === 'digest' ? ( -
- {p.summary.current_tag} - - Rebuild available - + {headline} + {verificationFailed && p.summary.has_update && ( +
+ {withErrorDetail('Digest check could not be verified', verificationError)}
- ) : ( - )}
@@ -302,7 +420,7 @@ function StackReadinessCard({ variant="outline" className="h-6 gap-1 rounded-md px-2 text-[11px]" onClick={() => onApplyService?.(stack, nodeId, img.service)} - disabled={blocked || applying || applyingService !== null} + disabled={blocked || applying || applyingService !== null || !isServiceApplyActionable(preview, img.service)} > {applyingService === img.service ? 'Applying...' : 'Apply'} @@ -329,8 +447,8 @@ function StackReadinessCard({
{!previewLoaded ? (
Checking registry...
+ ) : uncertain ? ( +
{verificationNote}
) : failed ? (
Preview failed. Registry may be unreachable.
- ) : ( - <> - {preview!.summary.update_kind === 'digest' ? ( + ) : (() => { + const p = preview!; + const verificationFailed = Boolean(p.summary.verification_failed); + const verificationError = p.summary.verification_error; + + let headline: ReactNode; + if (verificationFailed && !p.summary.has_update) { + headline = ( +
+ {withErrorDetail('Digest verification failed', verificationError)} +
+ ); + } else if (p.summary.update_kind === 'digest') { + headline = (
- {preview!.summary.current_tag} + {p.summary.current_tag} Rebuild available
- ) : ( - - )} -
{preview!.summary.primary_image ?? '-'}
-
- {lead && {lead}}{rest} -
- {showServiceApply && ( -
- {updatingImages.map(img => ( -
- {img.service} - -
- ))} + ); + } else { + headline = ; + } + + return ( + <> + {headline} + {verificationFailed && p.summary.has_update && ( +
+ {withErrorDetail('Digest check could not be verified', verificationError)} +
+ )} +
{p.summary.primary_image ?? '-'}
+
+ {lead && {lead}}{rest}
- )} -
- - {nextRun ? <>{formatClock(nextRun)} · {formatRelative(nextRun)} : (blocked ? 'Held for review' : 'No schedule')} - - -
- - )} + {showServiceApply && ( +
+ {updatingImages.map(img => ( +
+ {img.service} + +
+ ))} +
+ )} +
+ + {nextRun ? <>{formatClock(nextRun)} · {formatRelative(nextRun)} : (blocked || reviewRequired ? 'Held for review' : 'No schedule')} + + +
+ + ); + })()}
); } @@ -679,14 +842,17 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) // Local-node check failures: surfaced separately because the fleet map is // boolean and the card grid only lists stacks with a confirmed update. + // Sticky has_update during a failed check is not a verified rebuild, so those + // stacks stay in the advisory only. + let failedLocalStacks = new Set(); if (detailRes.ok) { const detail = await detailRes.json() as Record; - setCheckFailures( - Object.entries(detail) - .filter(([, info]) => info.checkStatus === 'failed') - .map(([stack, info]) => ({ stack, reason: info.lastError })) - .sort((a, b) => a.stack.localeCompare(b.stack)), - ); + const failures = Object.entries(detail) + .filter(([, info]) => info.checkStatus === 'failed') + .map(([stack, info]) => ({ stack, reason: info.lastError })) + .sort((a, b) => a.stack.localeCompare(b.stack)); + failedLocalStacks = new Set(failures.map(f => f.stack)); + setCheckFailures(failures); } else { // Clear stale failures rather than persist them across a load, but log: // an empty advisory must not silently stand in for "detail unavailable". @@ -730,7 +896,9 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) const node = currentNodes.find(n => n.id === nodeId); if (!node) continue; const stacks = Object.entries(stackMap) - .filter(([, hasUpdate]) => hasUpdate) + .filter(([stack, hasUpdate]) => + hasUpdate && !(node.type === 'local' && failedLocalStacks.has(stack)), + ) .map(([stack]) => stack) .sort(); if (stacks.length === 0) continue; @@ -753,6 +921,7 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) applying: false, applyingService: null, autoUpdateEnabled: scheduledTask !== null, + verificationNote: null, }; }); initialGroups.push({ @@ -779,9 +948,11 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) const previews = await Promise.all( flatPairs.map(async ({ nodeId, stack }) => { try { - const res = await fetchForNode(`/stacks/${encodeURIComponent(stack)}/update-preview`, nodeId); - if (!res.ok) return null; - return await res.json() as UpdatePreview; + const result = await fetchUpdatePreview(stack, { + fetchImpl: (path, init) => fetchForNode(path, nodeId, init), + }); + if (!result.ok || !result.preview) return null; + return result.preview as UpdatePreview; } catch { return null; } @@ -794,14 +965,52 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) previewByKey.set(`${pair.nodeId}::${pair.stack}`, previews[idx]); }); - setGroups(initialGroups.map(g => ({ - ...g, - cards: g.cards.map(c => ({ - ...c, - preview: previewByKey.get(`${c.nodeId}::${c.stack}`) ?? null, - previewLoaded: true, - })), - }))); + const previewAdvisory: { stack: string; reason: string | null }[] = []; + const groupsWithPreview = initialGroups + .map(g => { + const cards: StackCard[] = []; + for (const c of g.cards) { + const preview = previewByKey.get(`${c.nodeId}::${c.stack}`) ?? null; + if (isVerificationOnlyPreview(preview)) { + previewAdvisory.push({ + stack: g.nodeType === 'remote' ? `${c.stack} (${g.nodeName})` : c.stack, + reason: preview?.summary.verification_error ?? 'Digest verification failed', + }); + continue; + } + // Sticky fleet booleans can outlive a successful no-update preview. + // Drop those cards without treating them as check failures. + if (isClearedUpdatePreview(preview)) continue; + // A legacy preview (missing verification_failed entirely) is kept + // rather than cleared, but that alone renders as a pending card + // with nothing to explain it; flag why in the advisory too. Skip + // this when the preview is already actionable on its own terms + // (the remote's own has_update/rebuild_available): the card + // already speaks for itself, and pairing it with "could not be + // checked" would contradict the Apply affordance right next to it. + if (isLegacyPreview(preview) && !isActionableUpdatePreview(preview)) { + previewAdvisory.push({ + stack: g.nodeType === 'remote' ? `${c.stack} (${g.nodeName})` : c.stack, + reason: 'This node\'s preview predates digest verification reporting', + }); + } + cards.push({ ...c, preview, previewLoaded: true }); + } + return { ...g, cards }; + }) + .filter(g => g.cards.length > 0); + + if (previewAdvisory.length > 0) { + setCheckFailures(prev => { + const byStack = new Map(prev.map(f => [f.stack, f])); + for (const entry of previewAdvisory) { + if (!byStack.has(entry.stack)) byStack.set(entry.stack, entry); + } + return [...byStack.values()].sort((a, b) => a.stack.localeCompare(b.stack)); + }); + } + + setGroups(groupsWithPreview); } catch (err) { if (token !== loadTokenRef.current) return; toast.error((err as Error)?.message || 'Failed to load readiness'); @@ -920,13 +1129,25 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) } // Reload authoritative preview so summary / Apply affordances stay accurate. try { - const res = await fetchForNode(`/stacks/${encodeURIComponent(stack)}/update-preview`, nodeId); - if (res.ok) { - const next = await res.json() as UpdatePreview; - setCardField(c => c.stack === stack && c.nodeId === nodeId, { preview: next, previewLoaded: true }); + const result = await fetchUpdatePreview(stack, { + fetchImpl: (path, init) => fetchForNode(path, nodeId, init), + }); + if (result.ok && result.preview) { + const next = result.preview as UpdatePreview; + if (isAuthoritativeNegativePreview(next)) { + setGroups(prev => prev + .map(g => g.nodeId === nodeId + ? { ...g, cards: g.cards.filter(c => c.stack !== stack) } + : g) + .filter(g => g.cards.length > 0)); + } else { + setCardField(c => c.stack === stack && c.nodeId === nodeId, { preview: next, previewLoaded: true }); + } + } else { + console.error(`[AutoUpdateReadinessView] post-apply update-preview failed (${result.status})`); } - } catch { - // Preview refresh is best-effort; the update itself already succeeded. + } catch (err) { + console.error('[AutoUpdateReadinessView] post-apply update-preview refresh failed', err); } return { ok: true as const, @@ -948,8 +1169,26 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) ...g, cards: g.cards.map(c => predicate(c) ? { ...c, ...patch } : c), }))); + const matchCard = (c: StackCard) => c.stack === stack && c.nodeId === nodeId; + const removeCard = () => setGroups(prev => prev + .map(g => g.nodeId === nodeId + ? { ...g, cards: g.cards.filter(c => c.stack !== stack) } + : g) + .filter(g => g.cards.length > 0)); + const retainPreviewFailed = () => setCardField(matchCard, { + applying: false, + preview: null, + previewLoaded: true, + verificationNote: null, + }); + const retainUncertain = (note: string) => setCardField(matchCard, { + applying: false, + preview: null, + previewLoaded: true, + verificationNote: note, + }); - setCardField(c => c.stack === stack && c.nodeId === nodeId, { applying: true }); + setCardField(matchCard, { applying: true, verificationNote: null }); const loadingId = toast.loading(`Applying update to ${stack}...`); try { const res = await fetchForNode( @@ -961,15 +1200,57 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) const data = await res.json().catch(() => ({ error: 'Update failed' })); throw new Error(data.error ?? 'Update failed'); } - toast.success(`${stack} updated successfully`); - setGroups(prev => prev - .map(g => g.nodeId === nodeId - ? { ...g, cards: g.cards.filter(c => c.stack !== stack) } - : g) - .filter(g => g.cards.length > 0)); + const body = await res.json().catch(() => ({})) as { recheckWarning?: unknown }; + const recheckWarning = typeof body.recheckWarning === 'string' ? body.recheckWarning : undefined; + if (recheckWarning) toast.info(recheckWarning); + else toast.success(`${stack} updated successfully`); + + // Authoritative live preview decides card removal. When it disagrees with + // a backend recheckWarning (preview cleared, persisted check uncertain), + // keep an uncertain card so Fleet does not diverge from the sidebar. + try { + const previewRes = await fetchForNode( + `/stacks/${encodeURIComponent(stack)}/update-preview`, + nodeId, + ); + if (!previewRes.ok) { + retainPreviewFailed(); + return; + } + const next = await previewRes.json() as UpdatePreview; + if (typeof next?.summary?.has_update !== 'boolean') { + retainPreviewFailed(); + return; + } + // Drop only when the live preview proves nothing remains (tag-only + // advisories stay pending via isClearedUpdatePreview). + const cleared = isAuthoritativeNegativePreview(next) || isClearedUpdatePreview(next); + if (!cleared) { + if (next.summary.has_update && !recheckWarning) { + toast.info( + 'The update command completed, but Sencho still detects an available image update.', + ); + } + setCardField(matchCard, { + applying: false, + preview: next, + previewLoaded: true, + verificationNote: recheckWarning ?? null, + }); + return; + } + if (recheckWarning) { + retainUncertain(recheckWarning); + return; + } + removeCard(); + } catch (previewErr) { + console.error('[AutoUpdate] post-Apply preview reconciliation failed', previewErr); + retainPreviewFailed(); + } } catch (err) { toast.error((err as Error)?.message || 'Update failed'); - setCardField(c => c.stack === stack && c.nodeId === nodeId, { applying: false }); + setCardField(matchCard, { applying: false }); } finally { toast.dismiss(loadingId); } @@ -978,14 +1259,12 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) const flatCards = useMemo(() => groups.flatMap(g => g.cards), [groups]); const { total, ready } = useMemo(() => { const t = flatCards.length; - // "Ready" means a schedule covers the stack, the preview loaded without - // error, and no major-bump blocked it. Without a covering schedule the - // stack cannot apply automatically regardless of preview state. + // Schedule-covered and actionable (confirmed update/rebuild, not blocked). const r = flatCards.filter(c => c.autoUpdateEnabled && c.previewLoaded && c.preview !== null - && !c.preview.summary.blocked, + && isActionableUpdatePreview(c.preview), ).length; return { total: t, ready: r }; }, [flatCards]); @@ -999,15 +1278,25 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
0} - meta={total > 0 ? `${ready} ready · ${total - ready} in review` : 'all stacks current'} + state={cadence?.enabled === false + ? 'Disabled' + : total === 0 + ? (checkFailures.length > 0 ? 'No verified updates' : 'Up to date') + : `${total} pending`} + stateTone={cadence?.enabled === false + ? 'brand' + : total === 0 && checkFailures.length === 0 ? 'success' : 'warning'} + live={total > 0 && cadence?.enabled !== false} + meta={cadence?.enabled === false + ? 'image update detection off' + : total > 0 + ? `${ready} ready · ${total - ready} in review` + : (checkFailures.length > 0 ? 'some checks unresolved' : 'all stacks current')} right={headerActions} />
- @@ -1024,9 +1313,19 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
Loading readiness...
) : groups.length === 0 ? (
-
) : ( groups.map(group => ( @@ -1052,6 +1351,8 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps) nodeCount={groups.length} refreshing={refreshing} onRefresh={handleRefresh} + unresolvedChecks={checkFailures.length > 0} + detectionDisabled={cadence?.enabled === false} /> @@ -1070,10 +1371,18 @@ function AutoUpdateReadinessContent({ headerActions }: AutoUpdateReadinessProps)
) : groups.length === 0 ? (
-
) : ( diff --git a/frontend/src/components/ComposeDiffPreviewDialog.tsx b/frontend/src/components/ComposeDiffPreviewDialog.tsx index b1cefbc5..8f0f03a2 100644 --- a/frontend/src/components/ComposeDiffPreviewDialog.tsx +++ b/frontend/src/components/ComposeDiffPreviewDialog.tsx @@ -3,6 +3,7 @@ import { DiffEditor } from '@/lib/monacoLoader'; import { Loader2 } from 'lucide-react'; import { Modal, ModalHeader, ModalFooter } from '@/components/ui/modal'; import { Button } from '@/components/ui/button'; +import type { ComposeDiffActionLabel } from '@/components/resolveComposeDiffActionLabel'; export interface ComposeDiffPreviewDialogProps { open: boolean; @@ -12,7 +13,7 @@ export interface ComposeDiffPreviewDialogProps { language: 'yaml' | 'ini'; original: string; modified: string; - actionLabel: 'Save' | 'Save & deploy'; + actionLabel: ComposeDiffActionLabel; confirming: boolean; isDarkMode: boolean; onConfirm: () => void | Promise; diff --git a/frontend/src/components/EditorLayout.tsx b/frontend/src/components/EditorLayout.tsx index be3611eb..53a1a1d1 100644 --- a/frontend/src/components/EditorLayout.tsx +++ b/frontend/src/components/EditorLayout.tsx @@ -17,11 +17,15 @@ import { useUrlSync } from './EditorLayout/hooks/useUrlSync'; import { shouldClearPendingDetailStack } from './EditorLayout/mobile-pending-detail'; import { useOverlayState } from './EditorLayout/hooks/useOverlayState'; import { useStackActions, NODE_SWITCH_PENDING_TOKEN } from './EditorLayout/hooks/useStackActions'; +import { useSelectedStackLiveRefresh } from './EditorLayout/hooks/useSelectedStackLiveRefresh'; import { useTheme } from '@/hooks/use-theme'; import { ThemeQuickSwitch } from './theme/ThemeQuickSwitch'; import { useNotifications } from './EditorLayout/hooks/useNotifications'; import { useContainerStats } from './EditorLayout/hooks/useContainerStats'; import { useSidebarContextMenu } from './EditorLayout/hooks/useSidebarContextMenu'; +import { useActiveNodeReapplyEligibility } from './EditorLayout/hooks/useActiveNodeReapplyEligibility'; +import { resolveCanSaveAndReapply } from './EditorLayout/resolveCanSaveAndReapply'; +import { useComposeReapplyAction } from './FleetView/hooks/useComposeReapplyAction'; import { NodeSwitcher } from './NodeSwitcher'; import { GlobalCommandPalette, @@ -185,6 +189,12 @@ export default function EditorLayout() { createDialogOpen, setCreateDialogOpen, } = overlayState; + const { canReapply: canReapplyCompose } = useActiveNodeReapplyEligibility(activeNode?.id); + const composeReapply = useComposeReapplyAction(); + const isSelfStackSelected = selectedFile ? stackSelfFlags[selectedFile] === true : false; + // Ordinary stacks keep Save & Deploy even when the node supports compose reapply. + const canSaveAndReapply = resolveCanSaveAndReapply(isAdmin, canReapplyCompose, isSelfStackSelected); + // Which mode the create dialog opens on (always empty after import tab removal). const [createDialogInitialMode, setCreateDialogInitialMode] = useState('empty'); const [adoptDialogOpen, setAdoptDialogOpen] = useState(false); @@ -291,11 +301,23 @@ export default function EditorLayout() { canOfferVolumeRemoval, onDeletedOpenStack: () => onDeletedOpenStackRef.current(), removeNotificationsForStack, + isAdmin, + canReapplyCompose, }); // Wire the ref now that stackActions is available resetEditorStateRef.current = stackActions.resetEditorState; + const { syncStale: containersSyncStale, retrySync: retryContainersSync } = useSelectedStackLiveRefresh({ + selectedFile, + activeNodeId: activeNode?.id, + isDetailVisible: activeView === 'editor', + containers, + composeContent: content, + containersLoadStatus, + refreshSelectedContainers: stackActions.refreshSelectedContainers, + }); + // A failed health gate routes into the existing recovery affordance: record // a failure for the stack so RecoveryChip/RecoveryPanel offer the same // explicit, user-confirmed rollback as any failed operation. Keyed by gate @@ -623,6 +645,8 @@ export default function EditorLayout() { containersLoadStatus={containersLoadStatus} containersLoadError={containersLoadError} onRetryContainersLoad={() => { void stackActions.retryContainersLoad(); }} + containersSyncStale={containersSyncStale} + onRetrySync={retryContainersSync} containerStats={containerStats} containerStatsError={containerStatsError} content={content} @@ -679,15 +703,16 @@ export default function EditorLayout() { requestDeleteStack={stackActions.requestDeleteStack} requestTakeDownStack={stackActions.requestTakeDownStack} showTakeDown={selectedFile ? stackActions.getStackMenuVisibility(selectedFile).showTakeDown : false} - isSelfStack={selectedFile ? stackSelfFlags[selectedFile] === true : false} + isSelfStack={isSelfStackSelected} + canSaveAndReapply={canSaveAndReapply} recoveryResult={selectedFile ? lastActionResult[selectedFile] : undefined} onRefreshState={async () => { if (!selectedFile) return; const name = selectedFile.replace(/\.(yml|yaml)$/, ''); - const ok = await stackActions.refreshSelectedContainers(name, selectedFile); + const outcome = await stackActions.refreshSelectedContainers(name, selectedFile); await refreshStacks(true); - if (ok) toast.success('Refreshed container state.'); - else toast.error('Could not refresh container state.'); + if (outcome === 'ok') toast.success('Refreshed container state.'); + else if (outcome === 'failed') toast.error('Could not refresh container state.'); }} onDismissRecovery={() => { if (selectedFile) dismissActionResult(selectedFile); }} panelStartedAt={panelStartedAt} @@ -1037,6 +1062,8 @@ export default function EditorLayout() { gitSourceOpen={gitSourceOpen} setGitSourceOpen={setGitSourceOpen} canSelfUpdate={hasCapability('self-update')} + composeReapply={composeReapply} + canSaveAndReapply={canSaveAndReapply} canOfferVolumeRemoval={canOfferVolumeRemoval} onOpenFleetNodeUpdates={() => { if (isMobile) { diff --git a/frontend/src/components/EditorLayout/CreateStackDialog.tsx b/frontend/src/components/EditorLayout/CreateStackDialog.tsx index 66944243..f49c4c72 100644 --- a/frontend/src/components/EditorLayout/CreateStackDialog.tsx +++ b/frontend/src/components/EditorLayout/CreateStackDialog.tsx @@ -487,59 +487,57 @@ export function CreateStackDialog({ open, onOpenChange, onStackCreated, onStacks {createMode === 'docker-run' && (
- - -
- - setNewStackName(e.target.value)} - disabled={creatingFromDockerRun} - /> + +
+ + setNewStackName(e.target.value)} + disabled={creatingFromDockerRun} + /> +
+
+ +