From a0170d325205f5bc7fc52cdb6fedaa04176da309 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 10 Jul 2026 01:46:08 +0100 Subject: [PATCH] Block silent downgrades on apply and give the updater a sanctioned rollback path The in-app updater validated download URL and channel but never compared the target against the running version, so any valid older release asset URL installed silently while the UI presented it as an update. ApplyUpdate now rejects targets at or below the running version on both the community and Pro broker paths before any history entry or download, with an explicit allowDowngrade opt-in on POST /api/updates/apply for sanctioned cases. The rollback half already existed but nothing reached it: createBackup retains three backups, restoreBackup works, and history records BackupPath, yet no endpoint or UI called restoreBackup. RollbackToBackup restores the retained backup recorded on a history entry after re-validating the path against the managed backup roots, shares the update-in-flight slot with ApplyUpdate, records an Action rollback history entry linked to the source update, marks that update rolled_back, streams a restoring stage through the existing status/SSE machinery, and restarts via the exit-for-systemd path. POST /api/updates/rollback carries it with the same RequireAdmin plus settings:write gating as apply. Rollback is purely local, so the Pro edition gate never applies to it. Settings now has the update history surface that was missing entirely: nothing called /api/updates/history before. The Updates panel lists recent updates with a Roll back action on successful entries whose backup is still retained, behind a confirmation dialog naming the restore version, and the rollback rides updateStore's shared pending-apply marker for the post-restart toast. restoreBackup also honors PULSE_INSTALL_DIR now instead of hardcoding /opt/pulse, matching createBackup. Contract deltas ride along: api-contracts picks up the rollback transport and downgrade-conflict semantics, agent-lifecycle and storage-recovery pin rollback as server self-update plumbing, ai-runtime and cloud-paid pin the update watcher stage vocabulary as non-assistant non-paid shell chrome, and frontend-primitives adds UpdateHistorySection as the history/rollback presentation owner with matching architecture proofs. --- .../v6/internal/subsystems/agent-lifecycle.md | 6 +- .../v6/internal/subsystems/ai-runtime.md | 5 + .../v6/internal/subsystems/api-contracts.md | 15 +- .../v6/internal/subsystems/cloud-paid.md | 6 + .../subsystems/deployment-installability.md | 34 ++ .../subsystems/frontend-primitives.md | 8 +- .../internal/subsystems/storage-recovery.md | 7 +- frontend-modern/src/App.tsx | 12 +- .../src/__tests__/App.architecture.test.ts | 17 + .../src/api/__tests__/updates.test.ts | 22 ++ frontend-modern/src/api/updates.ts | 39 ++ .../Settings/UpdateHistorySection.tsx | 232 ++++++++++++ .../Settings/UpdatesSettingsPanel.tsx | 5 + .../__tests__/UpdateHistorySection.test.tsx | 117 ++++++ .../__tests__/settingsArchitecture.test.ts | 21 ++ .../src/stores/__tests__/updates.test.ts | 70 ++++ frontend-modern/src/stores/updates.ts | 49 ++- internal/api/route_inventory_test.go | 1 + internal/api/router_routes_registration.go | 1 + internal/api/updates.go | 99 ++++- internal/api/updates_test.go | 148 ++++++++ internal/updates/manager.go | 196 +++++++++- internal/updates/manager_rollback_test.go | 341 ++++++++++++++++++ 23 files changed, 1434 insertions(+), 17 deletions(-) create mode 100644 frontend-modern/src/components/Settings/UpdateHistorySection.tsx create mode 100644 frontend-modern/src/components/Settings/__tests__/UpdateHistorySection.test.tsx create mode 100644 internal/updates/manager_rollback_test.go diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 6dd6cc66e..220b01b76 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -160,7 +160,11 @@ off it. The same boundary covers the server updater's deployment adapters behind `GET /api/updates/plan` in `internal/api/updates.go`: they are plan providers only, with the real apply in the `internal/updates/manager.go` pipeline, and lifecycle surfaces must not read them as an agent-side apply, -update, or rollback transport. Workflow starter counts on that endpoint, +update, or rollback transport. The server updater's own downgrade guard and +`POST /api/updates/rollback` backup-restore endpoint are equally server +self-update plumbing: they roll the Pulse server binary and its local +backups, never agent binaries, and agent lifecycle surfaces must not key +enrollment, update liveness, or fleet-control semantics off them. Workflow starter counts on that endpoint, contextual Assistant/external-agent collaboration counts inside the Assistant step, the content-free Patrol control starter split, and Patrol control completed-loop, resolved-loop, or `patrolControlValueState` proof mirrored to diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 3abb1421f..749a67ba5 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -4319,6 +4319,11 @@ The retired monitored-system capacity banner follows the same shell rule: volume warnings just because settings or support surfaces still expose monitored-system grouping data. Assistant state and shell notices stay independent from retired infrastructure-volume commerce. +The global update progress watcher in `frontend-modern/src/App.tsx` is +likewise server-updater shell chrome, not assistant surface: its in-progress +stage vocabulary mirrors the backend updater pipeline (including the +`restoring` stage emitted by update rollback), and assistant state, drawer +ownership, and AI runtime surfaces must not key off those update stages. That same shared shell boundary must respect blocking modal ownership. `frontend-modern/src/App.tsx` and `frontend-modern/src/AppLayout.tsx` may use the shared dialog runtime to hide the closed assistant launcher and close the diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 16625d55a..9f2af0092 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -1589,8 +1589,19 @@ payload shape change when the portal presents compact client rows. The updater registry behind `GET /api/updates/plan` is a plan-provider seam only (`SupportsApply`, `PrepareUpdate`, `GetDeploymentType`); apply and rollback semantics ride the manager pipeline behind - `POST /api/updates/apply`, and the updates API exposes no per-deployment - adapter execute or rollback transport. + `POST /api/updates/apply` and `POST /api/updates/rollback`, and the + updates API exposes no per-deployment adapter execute or rollback + transport. + `POST /api/updates/apply` rejects target versions at or below the running + version with a conflict response unless the request body sets the + explicit `allowDowngrade` flag, so a stale-but-valid release asset URL is + a refused downgrade rather than a silent install. + `POST /api/updates/rollback` takes the `eventId` of an update history + entry, is gated admin plus `settings:write` exactly like apply, restores + the retained backup recorded on that entry through the manager pipeline, + and answers with the same started-acknowledgement shape as apply; + conflict responses cover pruned or missing backups and Docker + deployments, and not-found covers unknown history entries. 85. `pkg/aicontracts/fix_execution.go` shared with `ai-runtime`: the public approved-fix execution contract is both an AI runtime approved-action boundary and a canonical API dependency contract for Patrol and enterprise auto-fix binders. 86. `pkg/aicontracts/investigation.go` shared with `ai-runtime`: the public Patrol investigation record and finding contract is both an AI runtime handoff boundary and a canonical API payload contract for Patrol, Assistant, unified findings, persistence, and audit surfaces. 87. `pkg/aicontracts/orchestrator_deps.go` shared with `ai-runtime`: the public investigation orchestrator dependency contract is both an AI runtime handoff boundary and a canonical API payload contract for Assistant and Patrol tool-call history. diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index 921f49743..b41503f9e 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -2081,6 +2081,12 @@ theme synchronization, and authenticated runtime startup, and as org switching and kiosk-safe navigation. Future hosted browser bootstrap work must extend that split rather than pulling org bootstrap and app chrome back into one monolithic route component. +The global update progress watcher inside that entry shell is self-hosted +server-update chrome: its stage vocabulary tracks the backend updater +pipeline (downloading through restarting, including the `restoring` stage +emitted by update rollback) and it carries no hosted entitlement, org, or +paid-gating semantics; hosted and paid surfaces must not key tenant or +billing behavior off update progress stages. That same authenticated shell split must also respect shared blocking dialogs: hosted chrome may not leave the Pulse Assistant launcher or an already-open assistant drawer interactive behind a modal that currently owns the viewport. diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index c5458d4b6..72574849b 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -2278,3 +2278,37 @@ for that volume before launching the wrapper, recover the same state during uninstall, and keep the persisted boot copy aligned with updater-owned runtime binary replacements instead of assuming `/usr/local/bin` survives reboot on QTS/QuTS hero. + +The in-app updater's apply pipeline now owns a downgrade guard on the normal +apply path. A syntactically valid release asset URL can name a release older +than the running binary, so `internal/updates/manager.go` `ApplyUpdate` must +reject any resolved target version at or below the running version, on both +the community release-asset path and the Pro broker path, before any history +entry is written or byte is downloaded. Sanctioned downgrades are an explicit +opt-in through the `AllowDowngrade` request flag carried by +`POST /api/updates/apply`, never a silent side effect of a stale download +URL. The guard fails open only for versions that do not parse as semver +(development builds), which stay covered by the existing URL and channel +validation. `internal/updates/manager_rollback_test.go` and the apply handler +tests in `internal/api/updates_test.go` are the direct proof surface for this +rule. + +That same updater boundary now also owns the sanctioned rollback path from +retained update backups. `RollbackToBackup` in `internal/updates/manager.go` +restores the backup directory recorded on an update history entry after +re-validating that the path still names a managed update backup on disk, +shares the single update-in-flight slot with `ApplyUpdate`, records the +rollback as its own history entry with `Action` `rollback` and a +`RelatedEventID` back to the rolled-back update, marks that source entry +`rolled_back`, streams progress through the existing update status and SSE +machinery as the `restoring` stage, and restarts through the same +exit-for-systemd path as a normal update. The transport surface is +`POST /api/updates/rollback`, admin plus `settings:write` gated exactly like +apply, and the Settings update history table in +`frontend-modern/src/components/Settings/UpdateHistorySection.tsx` is the +user-facing rollback surface. Rollback is a purely local restore: it must not +touch the Pro download broker or any edition gate, so it behaves identically +on community and Pro binaries. The rollback tests in +`internal/updates/manager_rollback_test.go`, the rollback handler tests in +`internal/api/updates_test.go`, and the route inventory pin for +`/api/updates/rollback` are the proof surface for this path. diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index c313ef40b..e22fef67e 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -3873,7 +3873,13 @@ top-level settings shell, while `frontend-modern/src/components/Settings/updatesSettingsModel.ts` plus `frontend-modern/src/utils/updatesPresentation.ts` own the deployment-specific install guide, copy-command block, and update-channel/install -model data plus customer-facing update status/action copy. The panel shell must +model data plus customer-facing update status/action copy. +`frontend-modern/src/components/Settings/UpdateHistorySection.tsx` joins that +split as the presentation owner for the update history table and the +rollback confirmation dialog: the panel shell mounts it as a section and must +not inline history rows, rollback gating, or rollback confirmation copy +itself, and the section starts rollbacks through the shared +`updateStore.rollbackUpdate` action rather than its own POST path. The panel shell must not rebuild copy-to-clipboard command cards, deployment instruction trees, or update-surface wording inline. `CopyCommandBlock` must use the shared `copyToClipboard` helper so install/update/agent snippets keep the same diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index bbb477230..064c99826 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -548,7 +548,12 @@ recovery scope, or a storage/recovery-owned secret source. so recovery semantics never fork by edition. That machinery lives solely in the `internal/updates/manager.go` apply pipeline; the deployment adapters behind the update-plan endpoint are plan providers only and own - no download, restore, or rollback path. + no download, restore, or rollback path. The sanctioned rollback behind + `POST /api/updates/rollback` restores the updater's own retained + pre-update backup (binary, data, config, env, VERSION) through that same + manager pipeline; it is server self-update recovery, purely local with no + broker or edition fork, and storage/recovery surfaces must not read it as + guest backup, datastore restore, or recovery-point coverage. Proxmox-side LXC Docker inventory wiring may also pass through `internal/api/router.go` and Proxmox agent install-command generation, but storage and recovery may consume the resulting app-container/resource diff --git a/frontend-modern/src/App.tsx b/frontend-modern/src/App.tsx index 7c3ccf22f..7780c316d 100644 --- a/frontend-modern/src/App.tsx +++ b/frontend-modern/src/App.tsx @@ -140,9 +140,19 @@ async function preloadAppShellRoutes() { } // Helper to detect if an update is actively in progress (not just checking for updates) +// Mirrors the stage names emitted by internal/updates/manager.go updateStatus: +// the apply pipeline plus 'restoring' from the rollback path. function isUpdateInProgress(status: string | undefined): boolean { if (!status) return false; - const inProgressStates = ['downloading', 'verifying', 'extracting', 'installing', 'restarting']; + const inProgressStates = [ + 'downloading', + 'verifying', + 'extracting', + 'backing-up', + 'applying', + 'restoring', + 'restarting', + ]; return inProgressStates.includes(status); } diff --git a/frontend-modern/src/__tests__/App.architecture.test.ts b/frontend-modern/src/__tests__/App.architecture.test.ts index 6430193fd..5636b1fa5 100644 --- a/frontend-modern/src/__tests__/App.architecture.test.ts +++ b/frontend-modern/src/__tests__/App.architecture.test.ts @@ -212,6 +212,23 @@ describe('App architecture', () => { expect(appSource).not.toContain('monitoredSystemLimitWarningBanner'); }); + it('keeps the update progress watcher aligned with the backend updater stages', () => { + // The in-progress stage list must mirror internal/updates/manager.go + // updateStatus emissions, including the rollback path's restoring stage, + // so the progress modal auto-opens for every real update or rollback. + expect(appSource).toContain("'downloading',"); + expect(appSource).toContain("'verifying',"); + expect(appSource).toContain("'extracting',"); + expect(appSource).toContain("'backing-up',"); + expect(appSource).toContain("'applying',"); + expect(appSource).toContain("'restoring',"); + expect(appSource).toContain("'restarting',"); + // 'checking' is a probe, not an apply: it must never pop the modal, and + // the never-emitted legacy 'installing' stage must not return. + expect(appSource).not.toContain("'checking',"); + expect(appSource).not.toContain("'installing'"); + }); + it('keeps integration browser proofs off the retired AI route', () => { const routesSource = readFileSync(join(integrationTestsDir, 'routes.ts'), 'utf8'); const retiredRouteNavigations = readIntegrationTestSources(integrationTestsDir).flatMap( diff --git a/frontend-modern/src/api/__tests__/updates.test.ts b/frontend-modern/src/api/__tests__/updates.test.ts index 2f9a7135f..c377590f7 100644 --- a/frontend-modern/src/api/__tests__/updates.test.ts +++ b/frontend-modern/src/api/__tests__/updates.test.ts @@ -61,6 +61,28 @@ describe('UpdatesAPI', () => { }); }); + it('rejects empty rollback event ID before making a request', async () => { + await expect(UpdatesAPI.rollbackUpdate(' ')).rejects.toThrow('Event ID is required'); + expect(apiFetchJSONMock).not.toHaveBeenCalled(); + }); + + it('trims event ID before rollback request', async () => { + apiFetchJSONMock.mockResolvedValueOnce({ status: 'started', message: 'ok' } as any); + await UpdatesAPI.rollbackUpdate(' 01JZEXAMPLE '); + + expect(apiFetchJSONMock).toHaveBeenCalledWith('/api/updates/rollback', { + method: 'POST', + body: JSON.stringify({ eventId: '01JZEXAMPLE' }), + }); + }); + + it('encodes the update-history limit', async () => { + apiFetchJSONMock.mockResolvedValueOnce([] as any); + await UpdatesAPI.listUpdateHistory(5); + + expect(apiFetchJSONMock).toHaveBeenCalledWith('/api/updates/history?limit=5'); + }); + it('encodes update-plan version and channel safely', async () => { apiFetchJSONMock.mockResolvedValueOnce({ canAutoUpdate: true } as any); await UpdatesAPI.getUpdatePlan('v1.2.3-rc.1+build', 'rc'); diff --git a/frontend-modern/src/api/updates.ts b/frontend-modern/src/api/updates.ts index a1094c1dd..a4503dfaa 100644 --- a/frontend-modern/src/api/updates.ts +++ b/frontend-modern/src/api/updates.ts @@ -61,6 +61,32 @@ export interface UpdatePlan { readiness?: UpdateReadiness; } +export interface UpdateHistoryEntryError { + message: string; + code?: string; + details?: string; +} + +export interface UpdateHistoryEntry { + event_id: string; + timestamp: string; + action: string; + channel: string; + version_from: string; + version_to: string; + deployment_type: string; + initiated_by: string; + initiated_via: string; + status: string; + duration_ms: number; + backup_path?: string; + log_path?: string; + error?: UpdateHistoryEntryError; + download_bytes?: number; + related_event_id?: string; + notes?: string; +} + const requireNonEmpty = (value: string, fieldName: string): string => { const trimmed = value.trim(); if (!trimmed) { @@ -89,6 +115,19 @@ export class UpdatesAPI { }); } + static async rollbackUpdate(eventId: string): Promise<{ status: string; message: string }> { + const normalizedEventId = requireNonEmpty(eventId, 'Event ID'); + return apiFetchJSON('/api/updates/rollback', { + method: 'POST', + body: JSON.stringify({ eventId: normalizedEventId }), + }); + } + + static async listUpdateHistory(limit = 20): Promise { + const search = new URLSearchParams({ limit: String(limit) }); + return apiFetchJSON(`/api/updates/history?${search.toString()}`); + } + static async getUpdateStatus(): Promise { return apiFetchJSON('/api/updates/status'); } diff --git a/frontend-modern/src/components/Settings/UpdateHistorySection.tsx b/frontend-modern/src/components/Settings/UpdateHistorySection.tsx new file mode 100644 index 000000000..d17fa825b --- /dev/null +++ b/frontend-modern/src/components/Settings/UpdateHistorySection.tsx @@ -0,0 +1,232 @@ +import { Component, For, Show, createResource, createSignal } from 'solid-js'; +import HistoryIcon from 'lucide-solid/icons/history'; +import { UpdatesAPI, type UpdateHistoryEntry } from '@/api/updates'; +import { updateStore } from '@/stores/updates'; +import { Button } from '@/components/shared/Button'; +import { Dialog } from '@/components/shared/Dialog'; +import { LoadingSpinner } from '@/components/shared/LoadingSpinner'; + +const HISTORY_LIMIT = 20; + +const formatVersionLabel = (version: string): string => { + const trimmed = version.trim(); + if (!trimmed) return 'unknown'; + return trimmed.startsWith('v') ? trimmed : `v${trimmed}`; +}; + +const formatTimestamp = (timestamp: string): string => { + const date = new Date(timestamp); + return Number.isNaN(date.getTime()) ? timestamp : date.toLocaleString(); +}; + +const actionLabel = (entry: UpdateHistoryEntry): string => { + switch (entry.action) { + case 'rollback': + return 'Rollback'; + default: + return entry.initiated_by === 'auto' ? 'Automatic update' : 'Update'; + } +}; + +const statusPresentation = (status: string): { label: string; className: string } => { + switch (status) { + case 'success': + return { label: 'Succeeded', className: 'text-emerald-600 dark:text-emerald-400' }; + case 'failed': + return { label: 'Failed', className: 'text-red-600 dark:text-red-400' }; + case 'in_progress': + return { label: 'In progress', className: 'text-blue-600 dark:text-blue-400' }; + case 'rolled_back': + return { label: 'Rolled back', className: 'text-amber-600 dark:text-amber-400' }; + case 'cancelled': + return { label: 'Cancelled', className: 'text-muted' }; + default: + return { label: status, className: 'text-muted' }; + } +}; + +// A rollback restores the backup taken before this update was applied, so it +// is only offered where that backup is still retained on disk (the backend +// clears backup_path when retention prunes it) and the update actually landed. +const canRollBack = (entry: UpdateHistoryEntry): boolean => + entry.action === 'update' && entry.status === 'success' && Boolean(entry.backup_path); + +export const UpdateHistorySection: Component = () => { + const [history, { refetch }] = createResource(() => UpdatesAPI.listUpdateHistory(HISTORY_LIMIT)); + const [confirmEntry, setConfirmEntry] = createSignal(null); + const [isStartingRollback, setIsStartingRollback] = createSignal(false); + + const startRollback = async () => { + const entry = confirmEntry(); + if (!entry || isStartingRollback()) return; + + setIsStartingRollback(true); + try { + const accepted = await updateStore.rollbackUpdate({ + eventId: entry.event_id, + fromVersion: updateStore.versionInfo()?.version || entry.version_to, + toVersion: entry.version_from, + }); + if (accepted) { + // The global update progress watcher picks the rollback up from the + // status stream and owns the restart/reload flow from here. + setConfirmEntry(null); + } else { + // The store already surfaced the error toast; refresh in case the + // entry state changed underneath us (e.g. backup pruned). + void refetch(); + } + } finally { + setIsStartingRollback(false); + } + }; + + return ( +
+
+

+

+

+ Updates applied through Pulse, with rollback to the backup taken before each one. +

+
+ + + + Loading update history... +
+ } + > + Update history is unavailable right now.

} + > + 0} + fallback={ +

No updates have been applied through Pulse yet.

+ } + > +
+ + + + + + + + + + + + + + + + + + + + {(entry) => ( + + + + + + + + )} + + +
WhenActionVersionResult
+ {formatTimestamp(entry.timestamp)} + {actionLabel(entry)} + {formatVersionLabel(entry.version_from)} →{' '} + {formatVersionLabel(entry.version_to)} + + {statusPresentation(entry.status).label} + + + + +
+
+ + + + + { + if (!isStartingRollback()) setConfirmEntry(null); + }} + panelClass="max-w-lg" + closeOnBackdrop={!isStartingRollback()} + ariaLabel="Confirm rollback" + > + + {(entry) => ( +
+
+

+ Roll back to Pulse {formatVersionLabel(entry().version_from)}? +

+
+
+

+ This restores the binary, configuration, and data from the backup taken before + the update to {formatVersionLabel(entry().version_to)} on{' '} + {formatTimestamp(entry().timestamp)}. +

+

+ Settings and alert changes made since that backup will be reverted, and Pulse + will restart to complete the rollback. +

+
+
+ + +
+
+ )} + + + + ); +}; diff --git a/frontend-modern/src/components/Settings/UpdatesSettingsPanel.tsx b/frontend-modern/src/components/Settings/UpdatesSettingsPanel.tsx index b87821937..a26b8165d 100644 --- a/frontend-modern/src/components/Settings/UpdatesSettingsPanel.tsx +++ b/frontend-modern/src/components/Settings/UpdatesSettingsPanel.tsx @@ -14,6 +14,7 @@ import Package from 'lucide-solid/icons/package'; import type { UpdateInfo, VersionInfo, UpdatePlan } from '@/api/updates'; import { buildDockerImageTag, buildLinuxAmd64DownloadCommand } from '@/components/updateVersion'; import { UpdateInstallGuide } from '@/components/Settings/UpdateInstallGuide'; +import { UpdateHistorySection } from '@/components/Settings/UpdateHistorySection'; import { getUpdateChannelCardOptions, type UpdateChannelOptionValue, @@ -365,6 +366,10 @@ export const UpdatesSettingsPanel: Component = (props + +
+ +
); }; diff --git a/frontend-modern/src/components/Settings/__tests__/UpdateHistorySection.test.tsx b/frontend-modern/src/components/Settings/__tests__/UpdateHistorySection.test.tsx new file mode 100644 index 000000000..f56bc6b56 --- /dev/null +++ b/frontend-modern/src/components/Settings/__tests__/UpdateHistorySection.test.tsx @@ -0,0 +1,117 @@ +import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { UpdateHistoryEntry } from '@/api/updates'; + +const mockListUpdateHistory = vi.fn(); +const mockStoreRollbackUpdate = vi.fn(); + +vi.mock('@/api/updates', () => ({ + UpdatesAPI: { + listUpdateHistory: (...args: unknown[]) => mockListUpdateHistory(...args), + }, +})); + +vi.mock('@/stores/updates', () => ({ + updateStore: { + versionInfo: () => ({ version: '6.0.5' }), + rollbackUpdate: (...args: unknown[]) => mockStoreRollbackUpdate(...args), + }, +})); + +import { UpdateHistorySection } from '../UpdateHistorySection'; + +const baseEntry: UpdateHistoryEntry = { + event_id: '01JZSUCCESS', + timestamp: '2026-07-09T14:39:00Z', + action: 'update', + channel: 'stable', + version_from: '6.0.4', + version_to: '6.0.5', + deployment_type: 'systemd', + initiated_by: 'user', + initiated_via: 'ui', + status: 'success', + duration_ms: 30000, + backup_path: '/var/lib/pulse/backup-20260709-143900', +}; + +describe('UpdateHistorySection', () => { + beforeEach(() => { + mockListUpdateHistory.mockReset(); + mockStoreRollbackUpdate.mockReset(); + }); + + afterEach(() => { + cleanup(); + }); + + it('shows the empty state when no updates were applied', async () => { + mockListUpdateHistory.mockResolvedValue([]); + + render(() => ); + + expect( + await screen.findByText('No updates have been applied through Pulse yet.'), + ).toBeInTheDocument(); + }); + + it('offers rollback only for successful updates with a retained backup', async () => { + const entries: UpdateHistoryEntry[] = [ + baseEntry, + // Backup pruned by retention: backend cleared backup_path. + { + ...baseEntry, + event_id: '01JZPRUNED', + version_from: '6.0.3', + version_to: '6.0.4', + backup_path: undefined, + }, + // Failed update: nothing to return to. + { + ...baseEntry, + event_id: '01JZFAILED', + status: 'failed', + error: { message: 'checksum verification failed' }, + }, + // A recorded rollback never offers another rollback. + { + ...baseEntry, + event_id: '01JZROLLBACK', + action: 'rollback', + version_from: '6.0.5', + version_to: '6.0.4', + }, + ]; + mockListUpdateHistory.mockResolvedValue(entries); + + render(() => ); + + await screen.findByText('Failed'); + expect(screen.getAllByRole('button', { name: 'Roll back' })).toHaveLength(1); + expect(screen.getByText('Rollback')).toBeInTheDocument(); + }); + + it('confirms which version a rollback restores before starting it', async () => { + mockListUpdateHistory.mockResolvedValue([baseEntry]); + mockStoreRollbackUpdate.mockResolvedValue(true); + + render(() => ); + + fireEvent.click(await screen.findByRole('button', { name: 'Roll back' })); + + expect(await screen.findByText('Roll back to Pulse v6.0.4?')).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Roll back to v6.0.4' })); + + await waitFor(() => + expect(mockStoreRollbackUpdate).toHaveBeenCalledWith({ + eventId: '01JZSUCCESS', + fromVersion: '6.0.5', + toVersion: '6.0.4', + }), + ); + await waitFor(() => + expect(screen.queryByText('Roll back to Pulse v6.0.4?')).not.toBeInTheDocument(), + ); + }); +}); diff --git a/frontend-modern/src/components/Settings/__tests__/settingsArchitecture.test.ts b/frontend-modern/src/components/Settings/__tests__/settingsArchitecture.test.ts index ac90aa638..1a76b47fe 100644 --- a/frontend-modern/src/components/Settings/__tests__/settingsArchitecture.test.ts +++ b/frontend-modern/src/components/Settings/__tests__/settingsArchitecture.test.ts @@ -37,6 +37,7 @@ import securityAuthPanelSource from '../SecurityAuthPanel.tsx?raw'; import securityOverviewPanelSource from '../SecurityOverviewPanel.tsx?raw'; import systemLogsPanelSource from '../SystemLogsPanel.tsx?raw'; import updatesSettingsPanelSource from '../UpdatesSettingsPanel.tsx?raw'; +import updateHistorySectionSource from '../UpdateHistorySection.tsx?raw'; import updateInstallGuideSource from '../UpdateInstallGuide.tsx?raw'; import agentProfilesPanelSource from '../AgentProfilesPanel.tsx?raw'; import infrastructureWorkspaceSource from '../InfrastructureWorkspace.tsx?raw'; @@ -1083,6 +1084,26 @@ describe('settings architecture guardrails', () => { ); }); + it('keeps update history and rollback on the dedicated section boundary', () => { + // The panel shell mounts the history section; it must not inline history + // rows or rollback confirmation itself. + expect(updatesSettingsPanelSource).toContain( + "import { UpdateHistorySection } from '@/components/Settings/UpdateHistorySection';", + ); + expect(updatesSettingsPanelSource).toContain(''); + expect(updatesSettingsPanelSource).not.toContain('listUpdateHistory'); + expect(updatesSettingsPanelSource).not.toContain('rollbackUpdate'); + // The section starts rollbacks through the shared store action, never its + // own POST, and always behind an explicit confirmation dialog gated to + // successful updates whose backup is still retained. + expect(updateHistorySectionSource).toContain('updateStore.rollbackUpdate'); + expect(updateHistorySectionSource).not.toContain('apiFetchJSON'); + expect(updateHistorySectionSource).toContain('ariaLabel="Confirm rollback"'); + expect(updateHistorySectionSource).toContain( + "entry.action === 'update' && entry.status === 'success' && Boolean(entry.backup_path)", + ); + }); + it('keeps infrastructure on a source-manager landing with route-backed dialogs', () => { expect(infrastructureWorkspaceSource).toContain( "import { ConnectionEditor } from './ConnectionEditor/ConnectionEditor';", diff --git a/frontend-modern/src/stores/__tests__/updates.test.ts b/frontend-modern/src/stores/__tests__/updates.test.ts index c6e82e191..18ed575bc 100644 --- a/frontend-modern/src/stores/__tests__/updates.test.ts +++ b/frontend-modern/src/stores/__tests__/updates.test.ts @@ -4,6 +4,7 @@ import { STORAGE_KEYS } from '@/utils/localStorage'; const mockGetVersion = vi.fn(); const mockCheckForUpdates = vi.fn(); const mockApplyUpdate = vi.fn(); +const mockRollbackUpdate = vi.fn(); const mockNotifySuccess = vi.fn(); const mockNotifyError = vi.fn(); @@ -12,6 +13,7 @@ vi.mock('@/api/updates', () => ({ getVersion: (...args: unknown[]) => mockGetVersion(...args), checkForUpdates: (...args: unknown[]) => mockCheckForUpdates(...args), applyUpdate: (...args: unknown[]) => mockApplyUpdate(...args), + rollbackUpdate: (...args: unknown[]) => mockRollbackUpdate(...args), }, })); @@ -55,6 +57,7 @@ describe('updateStore', () => { mockGetVersion.mockReset(); mockCheckForUpdates.mockReset(); mockApplyUpdate.mockReset(); + mockRollbackUpdate.mockReset(); mockNotifySuccess.mockReset(); mockNotifyError.mockReset(); }); @@ -170,6 +173,47 @@ describe('updateStore', () => { }); }); + describe('rollbackUpdate', () => { + const rollbackParams = { + eventId: '01JZEXAMPLE', + fromVersion: 'v1.1.0', + toVersion: 'v1.0.0', + }; + + it('posts the rollback and records a rollback pending marker before it', async () => { + mockRollbackUpdate.mockResolvedValue({ status: 'started', message: '' }); + + const updateStore = await loadUpdateStore(); + const started = await updateStore.rollbackUpdate(rollbackParams); + + expect(started).toBe(true); + expect(mockRollbackUpdate).toHaveBeenCalledWith('01JZEXAMPLE'); + const persisted = JSON.parse(localStorage.getItem(STORAGE_KEYS.UPDATES) ?? '{}'); + expect(persisted.pendingApply).toMatchObject({ + fromVersion: 'v1.1.0', + toVersion: 'v1.0.0', + action: 'rollback', + }); + expect(mockNotifyError).not.toHaveBeenCalled(); + }); + + it('clears the marker and toasts the backend error when the rollback is rejected', async () => { + mockRollbackUpdate.mockRejectedValue( + new Error('no retained backup for this update; it may have been pruned by backup retention'), + ); + + const updateStore = await loadUpdateStore(); + const started = await updateStore.rollbackUpdate(rollbackParams); + + expect(started).toBe(false); + expect(mockNotifyError).toHaveBeenCalledWith( + 'no retained backup for this update; it may have been pruned by backup retention', + ); + const persisted = JSON.parse(localStorage.getItem(STORAGE_KEYS.UPDATES) ?? '{}'); + expect(persisted.pendingApply).toBeUndefined(); + }); + }); + describe('post-update confirmation', () => { const seedPendingApply = (fromVersion: string, toVersion: string) => { localStorage.setItem( @@ -218,6 +262,32 @@ describe('updateStore', () => { expect(mockNotifySuccess).toHaveBeenCalledWith('Updated to v1.1.0'); }); + it('uses rollback wording when a rollback marker confirms', async () => { + localStorage.setItem( + STORAGE_KEYS.UPDATES, + JSON.stringify({ + lastCheck: 0, + pendingApply: { + fromVersion: 'v1.1.0', + toVersion: 'v1.0.0', + startedAt: Date.now(), + action: 'rollback', + }, + }), + ); + mockGetVersion.mockResolvedValue({ ...baseVersionInfo, version: 'v1.0.0' }); + mockCheckForUpdates.mockResolvedValue({ + ...baseUpdateInfo, + available: false, + currentVersion: 'v1.0.0', + }); + + const updateStore = await loadUpdateStore(); + await updateStore.checkForUpdates(true); + + expect(mockNotifySuccess).toHaveBeenCalledWith('Rolled back to v1.0.0'); + }); + it('clears the marker silently when the version did not change', async () => { seedPendingApply('v1.0.0', 'v1.1.0'); mockGetVersion.mockResolvedValue({ ...baseVersionInfo, version: 'v1.0.0' }); diff --git a/frontend-modern/src/stores/updates.ts b/frontend-modern/src/stores/updates.ts index 23a00ed50..07fbadc73 100644 --- a/frontend-modern/src/stores/updates.ts +++ b/frontend-modern/src/stores/updates.ts @@ -17,6 +17,9 @@ interface PendingApply { fromVersion: string; toVersion: string; startedAt: number; + // Distinguishes the post-restart toast wording; absent on markers written + // before rollbacks existed, which read as updates. + action?: 'update' | 'rollback'; } interface UpdateState { @@ -79,7 +82,7 @@ const normalizeUpdateInfo = (value: unknown): UpdateInfo | undefined => { const normalizePendingApply = (value: unknown): PendingApply | undefined => { if (!isRecord(value)) return undefined; - const { fromVersion, toVersion, startedAt } = value; + const { fromVersion, toVersion, startedAt, action } = value; if ( typeof fromVersion !== 'string' || typeof toVersion !== 'string' || @@ -88,7 +91,12 @@ const normalizePendingApply = (value: unknown): PendingApply | undefined => { return undefined; } - return { fromVersion, toVersion, startedAt }; + return { + fromVersion, + toVersion, + startedAt, + ...(action === 'update' || action === 'rollback' ? { action } : {}), + }; }; const normalizeUpdateState = (value: unknown): UpdateState => { @@ -195,9 +203,13 @@ export const withTransientRetry = async ( } }; -const markApplyStarted = (fromVersion: string, toVersion: string) => { +const markApplyStarted = ( + fromVersion: string, + toVersion: string, + action: 'update' | 'rollback' = 'update', +) => { const state = loadState(); - saveState({ ...state, pendingApply: { fromVersion, toVersion, startedAt: Date.now() } }); + saveState({ ...state, pendingApply: { fromVersion, toVersion, startedAt: Date.now(), action } }); }; const clearPendingApply = () => { @@ -224,7 +236,8 @@ const confirmPendingApply = (currentVersion: string, state: UpdateState) => { saveState(state); if (currentVersion && currentVersion !== pending.fromVersion) { - notificationStore.success(`Updated to ${formatVersionLabel(currentVersion)}`); + const verb = pending.action === 'rollback' ? 'Rolled back to' : 'Updated to'; + notificationStore.success(`${verb} ${formatVersionLabel(currentVersion)}`); } }; @@ -248,6 +261,31 @@ const applyUpdate = async (): Promise => { } }; +// Sanctioned rollback of a recorded update: restores the retained backup on +// the selected history entry. Reuses the pending-apply marker so the boot +// after the post-rollback restart confirms the version moved, with rollback +// wording on the toast. Returns true when the backend accepted the request. +const rollbackUpdate = async (params: { + eventId: string; + fromVersion: string; + toVersion: string; +}): Promise => { + markApplyStarted(params.fromVersion, params.toVersion, 'rollback'); + try { + await UpdatesAPI.rollbackUpdate(params.eventId); + return true; + } catch (error) { + clearPendingApply(); + logger.error('Failed to start rollback', error); + notificationStore.error( + error instanceof Error && error.message + ? error.message + : 'Unable to start the rollback. Please try again.', + ); + return false; + } +}; + // Check for updates const checkForUpdates = async (force = false): Promise => { // Don't check if already checking @@ -406,6 +444,7 @@ export const updateStore = { // Actions checkForUpdates, applyUpdate, + rollbackUpdate, dismissUpdate, clearDismissed, diff --git a/internal/api/route_inventory_test.go b/internal/api/route_inventory_test.go index 5bd343a3e..a9d376757 100644 --- a/internal/api/route_inventory_test.go +++ b/internal/api/route_inventory_test.go @@ -447,6 +447,7 @@ var allRouteAllowlist = []string{ "GET /api/onboarding/deep-link", "/api/updates/check", "/api/updates/apply", + "/api/updates/rollback", "/api/updates/status", "/api/updates/stream", "/api/updates/plan", diff --git a/internal/api/router_routes_registration.go b/internal/api/router_routes_registration.go index 2e01eaf14..8bb34f403 100644 --- a/internal/api/router_routes_registration.go +++ b/internal/api/router_routes_registration.go @@ -125,6 +125,7 @@ func (r *Router) registerConfigSystemRoutes(updateHandlers *UpdateHandlers) { // Update routes r.mux.HandleFunc("/api/updates/check", RequireAdmin(r.config, RequireScope(config.ScopeSettingsRead, updateHandlers.HandleCheckUpdates))) r.mux.HandleFunc("/api/updates/apply", RequireAdmin(r.config, RequireScope(config.ScopeSettingsWrite, updateHandlers.HandleApplyUpdate))) + r.mux.HandleFunc("/api/updates/rollback", RequireAdmin(r.config, RequireScope(config.ScopeSettingsWrite, updateHandlers.HandleRollbackUpdate))) r.mux.HandleFunc("/api/updates/status", RequireAdmin(r.config, RequireScope(config.ScopeSettingsRead, updateHandlers.HandleUpdateStatus))) r.mux.HandleFunc("/api/updates/stream", RequireAdmin(r.config, RequireScope(config.ScopeSettingsRead, updateHandlers.HandleUpdateStream))) r.mux.HandleFunc("/api/updates/plan", RequireAdmin(r.config, RequireScope(config.ScopeSettingsRead, updateHandlers.HandleGetUpdatePlan))) diff --git a/internal/api/updates.go b/internal/api/updates.go index da6774994..08c641c38 100644 --- a/internal/api/updates.go +++ b/internal/api/updates.go @@ -36,6 +36,7 @@ type UpdateHandlers struct { type UpdateManager interface { CheckForUpdatesWithChannel(ctx context.Context, channel string) (*updates.UpdateInfo, error) ApplyUpdate(ctx context.Context, req updates.ApplyUpdateRequest) error + RollbackToBackup(ctx context.Context, req updates.RollbackRequest) error GetStatus() updates.UpdateStatus GetSSECachedStatus() (updates.UpdateStatus, time.Time) AddSSEClient(w http.ResponseWriter, clientID string) *updates.SSEClient @@ -134,6 +135,9 @@ func (h *UpdateHandlers) HandleApplyUpdate(w http.ResponseWriter, r *http.Reques var req struct { DownloadURL string `json:"downloadUrl"` + // AllowDowngrade opts in to installing a target at or below the + // running version; without it the manager rejects downgrades. + AllowDowngrade bool `json:"allowDowngrade"` } if err := decodeStrictJSONBody(r.Body, &req); err != nil { @@ -184,10 +188,11 @@ func (h *UpdateHandlers) HandleApplyUpdate(w http.ResponseWriter, r *http.Reques } applyReq := updates.ApplyUpdateRequest{ - DownloadURL: req.DownloadURL, - Channel: channel, - InitiatedBy: updates.InitiatedByUser, - InitiatedVia: updates.InitiatedViaUI, + DownloadURL: req.DownloadURL, + Channel: channel, + InitiatedBy: updates.InitiatedByUser, + InitiatedVia: updates.InitiatedViaUI, + AllowDowngrade: req.AllowDowngrade, } result := make(chan error, 1) @@ -255,7 +260,8 @@ func classifyApplyUpdateStartError(err error) (int, string) { return http.StatusConflict, "Update already in progress" case strings.Contains(errMsg, "download url is required"), strings.Contains(errMsg, "invalid download url"): return http.StatusBadRequest, "Invalid download URL" - case strings.Contains(errMsg, "stable channel cannot install prerelease builds"): + case strings.Contains(errMsg, "stable channel cannot install prerelease builds"), + strings.Contains(errMsg, "not newer than the running version"): return http.StatusConflict, err.Error() case strings.Contains(errMsg, "cannot be applied in docker environment"), strings.Contains(errMsg, "manual migration required"), @@ -266,6 +272,89 @@ func classifyApplyUpdateStartError(err error) (int, string) { } } +// HandleRollbackUpdate handles rollback requests: it restores the retained +// backup recorded on the selected update history entry. +func (h *UpdateHandlers) HandleRollbackUpdate(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Limit request body to 8KB to prevent memory exhaustion + r.Body = http.MaxBytesReader(w, r.Body, 8*1024) + defer r.Body.Close() + + var req struct { + EventID string `json:"eventId"` + } + + if err := decodeStrictJSONBody(r.Body, &req); err != nil { + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + req.EventID = strings.TrimSpace(req.EventID) + + if req.EventID == "" { + http.Error(w, "Event ID is required", http.StatusBadRequest) + return + } + + rollbackReq := updates.RollbackRequest{ + EventID: req.EventID, + InitiatedBy: updates.InitiatedByUser, + InitiatedVia: updates.InitiatedViaUI, + } + result := make(chan error, 1) + + // Start rollback in background with a new context (not request context which gets canceled) + go func() { + result <- h.manager.RollbackToBackup(context.Background(), rollbackReq) + }() + + select { + case err := <-result: + if err != nil { + statusCode, msg := classifyRollbackStartError(err) + if statusCode >= http.StatusInternalServerError { + log.Error().Err(err).Str("event_id", req.EventID).Msg("Failed to start rollback") + } else { + log.Warn().Err(err).Str("event_id", req.EventID).Msg("Rollback request rejected") + } + http.Error(w, msg, statusCode) + return + } + case <-time.After(applyUpdateStartAckTimeout): + } + + // Return success immediately + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]string{ + "status": "started", + "message": "Rollback process started", + }); err != nil { + log.Error().Err(err).Msg("Failed to encode rollback start response") + } +} + +func classifyRollbackStartError(err error) (int, string) { + errMsg := strings.ToLower(strings.TrimSpace(err.Error())) + switch { + case strings.Contains(errMsg, "already in progress"): + return http.StatusConflict, "Update already in progress" + case strings.Contains(errMsg, "event id is required"): + return http.StatusBadRequest, "Event ID is required" + case strings.Contains(errMsg, "history entry not found"): + return http.StatusNotFound, "Update history entry not found" + case strings.Contains(errMsg, "no retained backup"), + strings.Contains(errMsg, "backup no longer exists"), + strings.Contains(errMsg, "not a managed update backup"), + strings.Contains(errMsg, "cannot be applied in docker environment"): + return http.StatusConflict, err.Error() + default: + return http.StatusInternalServerError, "Failed to start rollback" + } +} + // HandleUpdateStatus handles update status requests with rate limiting func (h *UpdateHandlers) HandleUpdateStatus(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { diff --git a/internal/api/updates_test.go b/internal/api/updates_test.go index e653197cd..280d461b5 100644 --- a/internal/api/updates_test.go +++ b/internal/api/updates_test.go @@ -19,6 +19,7 @@ import ( type MockUpdateManager struct { CheckForUpdatesFunc func(ctx context.Context, channel string) (*updates.UpdateInfo, error) ApplyUpdateFunc func(ctx context.Context, req updates.ApplyUpdateRequest) error + RollbackToBackupFunc func(ctx context.Context, req updates.RollbackRequest) error GetStatusFunc func() updates.UpdateStatus GetSSECachedStatusFunc func() (updates.UpdateStatus, time.Time) AddSSEClientFunc func(w http.ResponseWriter, clientID string) *updates.SSEClient @@ -39,6 +40,13 @@ func (m *MockUpdateManager) ApplyUpdate(ctx context.Context, req updates.ApplyUp return nil } +func (m *MockUpdateManager) RollbackToBackup(ctx context.Context, req updates.RollbackRequest) error { + if m.RollbackToBackupFunc != nil { + return m.RollbackToBackupFunc(ctx, req) + } + return nil +} + func (m *MockUpdateManager) GetStatus() updates.UpdateStatus { if m.GetStatusFunc != nil { return m.GetStatusFunc() @@ -155,6 +163,146 @@ func TestHandleApplyUpdate_Success(t *testing.T) { // Note: ApplyUpdate runs in background, so we just check it was accepted } +func TestHandleApplyUpdate_PassesAllowDowngrade(t *testing.T) { + received := make(chan updates.ApplyUpdateRequest, 1) + mockManager := &MockUpdateManager{ + ApplyUpdateFunc: func(ctx context.Context, req updates.ApplyUpdateRequest) error { + received <- req + return nil + }, + } + + h := NewUpdateHandlers(mockManager, nil) + w := httptest.NewRecorder() + body := `{"downloadUrl": "https://example.com/update.tar.gz", "allowDowngrade": true}` + r := httptest.NewRequest(http.MethodPost, "/updates/apply", strings.NewReader(body)) + + h.HandleApplyUpdate(w, r) + + if w.Code != http.StatusOK { + t.Fatalf("Expected status 200, got %d: %s", w.Code, w.Body.String()) + } + select { + case req := <-received: + if !req.AllowDowngrade { + t.Fatal("expected AllowDowngrade to be passed through to the manager request") + } + case <-time.After(time.Second): + t.Fatal("ApplyUpdate was not invoked") + } +} + +func TestHandleRollbackUpdate_Success(t *testing.T) { + received := make(chan updates.RollbackRequest, 1) + mockManager := &MockUpdateManager{ + RollbackToBackupFunc: func(ctx context.Context, req updates.RollbackRequest) error { + received <- req + return nil + }, + } + + h := NewUpdateHandlers(mockManager, nil) + w := httptest.NewRecorder() + body := `{"eventId": "01JZEXAMPLE"}` + r := httptest.NewRequest(http.MethodPost, "/updates/rollback", strings.NewReader(body)) + + h.HandleRollbackUpdate(w, r) + + if w.Code != http.StatusOK { + t.Fatalf("Expected status 200, got %d: %s", w.Code, w.Body.String()) + } + var resp map[string]string + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp["status"] != "started" { + t.Fatalf("expected started status, got %q", resp["status"]) + } + select { + case req := <-received: + if req.EventID != "01JZEXAMPLE" { + t.Fatalf("expected event ID to be passed through, got %q", req.EventID) + } + if req.InitiatedBy != updates.InitiatedByUser || req.InitiatedVia != updates.InitiatedViaUI { + t.Fatalf("expected user/ui initiation, got %q/%q", req.InitiatedBy, req.InitiatedVia) + } + case <-time.After(time.Second): + t.Fatal("RollbackToBackup was not invoked") + } +} + +func TestHandleRollbackUpdate_InvalidRequests(t *testing.T) { + mockManager := &MockUpdateManager{ + RollbackToBackupFunc: func(ctx context.Context, req updates.RollbackRequest) error { + t.Fatal("RollbackToBackup should not be called for invalid requests") + return nil + }, + } + h := NewUpdateHandlers(mockManager, nil) + + t.Run("rejects non-POST", func(t *testing.T) { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/updates/rollback", nil) + h.HandleRollbackUpdate(w, r) + if w.Code != http.StatusMethodNotAllowed { + t.Fatalf("Expected status %d, got %d", http.StatusMethodNotAllowed, w.Code) + } + }) + + t.Run("rejects unknown fields", func(t *testing.T) { + w := httptest.NewRecorder() + body := `{"eventId":"01JZEXAMPLE","unexpected":true}` + r := httptest.NewRequest(http.MethodPost, "/updates/rollback", strings.NewReader(body)) + h.HandleRollbackUpdate(w, r) + if w.Code != http.StatusBadRequest { + t.Fatalf("Expected status %d, got %d", http.StatusBadRequest, w.Code) + } + }) + + t.Run("rejects missing event ID", func(t *testing.T) { + w := httptest.NewRecorder() + body := `{"eventId":" "}` + r := httptest.NewRequest(http.MethodPost, "/updates/rollback", strings.NewReader(body)) + h.HandleRollbackUpdate(w, r) + if w.Code != http.StatusBadRequest { + t.Fatalf("Expected status %d, got %d", http.StatusBadRequest, w.Code) + } + }) +} + +func TestHandleRollbackUpdate_ErrorMapping(t *testing.T) { + cases := []struct { + name string + managerErr error + wantCode int + }{ + {"entry not found", errors.New("update history entry not found: x"), http.StatusNotFound}, + {"backup pruned", errors.New("no retained backup for this update; it may have been pruned by backup retention"), http.StatusConflict}, + {"backup missing on disk", errors.New("backup no longer exists on disk: /var/lib/pulse/backup-1"), http.StatusConflict}, + {"unmanaged path", errors.New("backup path is not a managed update backup: /etc/passwd"), http.StatusConflict}, + {"docker", errors.New("rollback cannot be applied in Docker environment"), http.StatusConflict}, + {"in progress", errors.New("update already in progress"), http.StatusConflict}, + {"unexpected", errors.New("disk exploded"), http.StatusInternalServerError}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mockManager := &MockUpdateManager{ + RollbackToBackupFunc: func(ctx context.Context, req updates.RollbackRequest) error { + return tc.managerErr + }, + } + h := NewUpdateHandlers(mockManager, nil) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/updates/rollback", strings.NewReader(`{"eventId":"01JZEXAMPLE"}`)) + h.HandleRollbackUpdate(w, r) + if w.Code != tc.wantCode { + t.Fatalf("expected status %d, got %d: %s", tc.wantCode, w.Code, w.Body.String()) + } + }) + } +} + func TestHandleApplyUpdate_AlreadyInProgress(t *testing.T) { mockManager := &MockUpdateManager{ ApplyUpdateFunc: func(ctx context.Context, req updates.ApplyUpdateRequest) error { diff --git a/internal/updates/manager.go b/internal/updates/manager.go index 4a17aa8a5..34d5facc1 100644 --- a/internal/updates/manager.go +++ b/internal/updates/manager.go @@ -201,6 +201,19 @@ type ApplyUpdateRequest struct { InitiatedBy InitiatedBy InitiatedVia InitiatedVia Notes string + // AllowDowngrade permits installing a target at or below the running + // version. The normal apply path rejects those so a valid-but-older + // release asset URL cannot silently downgrade; sanctioned rollbacks go + // through RollbackToBackup instead. + AllowDowngrade bool +} + +// RollbackRequest describes a rollback of a recorded update, restoring the +// retained backup captured before that update was applied. +type RollbackRequest struct { + EventID string + InitiatedBy InitiatedBy + InitiatedVia InitiatedVia } // NewManager creates a new update manager @@ -604,10 +617,23 @@ func (m *Manager) ApplyUpdate(ctx context.Context, req ApplyUpdateRequest) error } else { targetVersion, validationErr := ValidateApplyTargetVersion(channel, req.DownloadURL) if validationErr != nil { + m.updateStatus("error", 10, "Update rejected", validationErr) return validationErr } artifact = resolvedUpdateArtifact{downloadURL: req.DownloadURL, version: targetVersion} } + + // A valid release asset URL can still point at an older release than the + // running binary; without this guard the "update" would silently install a + // downgrade. Sanctioned downgrades either restore a retained backup via + // RollbackToBackup or set AllowDowngrade explicitly on the request. + if !req.AllowDowngrade { + if err := ensureApplyTargetIsNewer(currentInfo.Version, artifact.version); err != nil { + m.updateStatus("error", 10, "Update rejected", err) + return err + } + } + initiatedBy := req.InitiatedBy if initiatedBy == "" { initiatedBy = InitiatedByUser @@ -1663,9 +1689,177 @@ func (m *Manager) createBackup(ctx context.Context) (string, error) { return backupDir, nil } +// ensureApplyTargetIsNewer rejects update targets at or below the running +// version so a valid-but-older release asset URL cannot silently downgrade. +// Versions that do not parse as semver (development builds) are left to the +// existing URL and channel validation. +func ensureApplyTargetIsNewer(currentVersion, targetVersion string) error { + current, err := ParseVersion(currentVersion) + if err != nil { + return nil + } + target, err := ParseVersion(targetVersion) + if err != nil { + return nil + } + if target.Compare(current) <= 0 { + return fmt.Errorf("target version %s is not newer than the running version %s; use the update history rollback to return to an earlier version", target.String(), current.String()) + } + return nil +} + +// validateRetainedBackupDir confirms a history-recorded backup path still +// points at an existing managed update backup directory before anything is +// restored from it. The history file is server-owned state, but the path is +// re-checked against the managed backup roots so a corrupted or hand-edited +// history entry cannot direct a restore from an arbitrary filesystem location. +func validateRetainedBackupDir(raw string) (string, error) { + cleaned := filepath.Clean(strings.TrimSpace(raw)) + managed := false + for _, root := range managedUpdateBackupRoots() { + if filepath.Dir(cleaned) != root { + continue + } + if strings.HasPrefix(filepath.Base(cleaned), managedUpdateBackupPrefix(root)) { + managed = true + break + } + } + if !managed { + return "", fmt.Errorf("backup path is not a managed update backup: %s", cleaned) + } + + info, err := os.Stat(cleaned) + if err != nil { + if os.IsNotExist(err) { + return "", fmt.Errorf("backup no longer exists on disk: %s", cleaned) + } + return "", fmt.Errorf("stat backup directory %q: %w", cleaned, err) + } + if !info.IsDir() { + return "", fmt.Errorf("backup path is not a directory: %s", cleaned) + } + return cleaned, nil +} + +// RollbackToBackup restores the retained backup recorded on an update history +// entry, records the rollback as its own history entry, and restarts through +// the same exit-for-systemd path as ApplyUpdate. It is a purely local +// restore: no release download, broker call, or edition gate is involved, so +// it behaves identically on community and Pro binaries. +func (m *Manager) RollbackToBackup(ctx context.Context, req RollbackRequest) error { + if m.history == nil { + return fmt.Errorf("update history is not available") + } + eventID := strings.TrimSpace(req.EventID) + if eventID == "" { + return fmt.Errorf("event ID is required") + } + + source, err := m.history.GetEntry(eventID) + if err != nil { + return fmt.Errorf("update history entry not found: %s", eventID) + } + if strings.TrimSpace(source.BackupPath) == "" { + return fmt.Errorf("no retained backup for this update; it may have been pruned by backup retention") + } + backupDir, err := validateRetainedBackupDir(source.BackupPath) + if err != nil { + return err + } + + currentInfo, _ := GetCurrentVersion() + if currentInfo.IsDocker { + return fmt.Errorf("rollback cannot be applied in Docker environment") + } + + // Rollback and update share the single in-flight slot: restoring a backup + // while an update is replacing the same files would corrupt both. + m.updateMu.Lock() + if m.updateInFlight { + m.updateMu.Unlock() + return fmt.Errorf("update already in progress") + } + m.updateInFlight = true + m.updateMu.Unlock() + defer func() { + m.updateMu.Lock() + m.updateInFlight = false + m.updateMu.Unlock() + }() + + initiatedBy := req.InitiatedBy + if initiatedBy == "" { + initiatedBy = InitiatedByUser + } + initiatedVia := req.InitiatedVia + if initiatedVia == "" { + initiatedVia = InitiatedViaAPI + } + + m.updateStatus("restoring", 20, fmt.Sprintf("Restoring Pulse %s from backup...", source.VersionFrom)) + + start := time.Now() + rollbackEventID := m.createHistoryEntry(ctx, UpdateHistoryEntry{ + Action: "rollback", + Channel: source.Channel, + VersionFrom: currentInfo.Version, + VersionTo: source.VersionFrom, + DeploymentType: currentInfo.DeploymentType, + InitiatedBy: initiatedBy, + InitiatedVia: initiatedVia, + Status: StatusInProgress, + BackupPath: backupDir, + RelatedEventID: source.EventID, + }) + + var runErr error + defer func() { + if rollbackEventID == "" { + return + } + status := StatusSuccess + if runErr != nil { + status = StatusFailed + } + m.completeHistoryEntry(ctx, rollbackEventID, status, start, runErr) + }() + + if err := m.restoreBackup(backupDir); err != nil { + restoreErr := fmt.Errorf("failed to restore backup: %w", err) + m.updateStatus("error", 40, "Failed to restore backup", restoreErr) + runErr = restoreErr + return restoreErr + } + + // The update this backup predates is no longer the running install. + m.updateHistoryEntry(ctx, eventID, func(entry *UpdateHistoryEntry) { + entry.Status = StatusRolledBack + }) + + m.updateStatus("restarting", 95, "Restarting service...") + + // Schedule a clean exit after a short delay - systemd will restart us + if !dockerUpdatesAllowed() { + go func() { + time.Sleep(2 * time.Second) + log.Info().Msg("Exiting for restart after rollback") + os.Exit(0) + }() + } else { + log.Info().Msg("Skipping process exit after rollback (mock/CI mode)") + } + + m.updateStatus("completed", 100, "Rollback completed, restarting...") + return nil +} + // restoreBackup restores from a backup func (m *Manager) restoreBackup(backupDir string) error { - pulseDir := "/opt/pulse" + pulseDir := os.Getenv("PULSE_INSTALL_DIR") + if pulseDir == "" { + pulseDir = "/opt/pulse" + } // Restore directories dirsToRestore := []string{"data", "config"} diff --git a/internal/updates/manager_rollback_test.go b/internal/updates/manager_rollback_test.go new file mode 100644 index 000000000..d5612f2ef --- /dev/null +++ b/internal/updates/manager_rollback_test.go @@ -0,0 +1,341 @@ +package updates + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestEnsureApplyTargetIsNewer(t *testing.T) { + cases := []struct { + name string + current string + target string + wantErr bool + }{ + {"newer patch allowed", "6.0.4", "6.0.5", false}, + {"older patch blocked", "6.0.5", "6.0.4", true}, + {"same version blocked", "6.0.5", "6.0.5", true}, + {"rc to stable of same base allowed", "6.0.5-rc.4", "6.0.5", false}, + {"stable to rc of same base blocked", "6.0.5", "6.0.5-rc.4", true}, + {"older rc blocked", "6.0.5-rc.4", "6.0.5-rc.3", true}, + {"newer rc allowed", "6.0.5-rc.4", "6.0.5-rc.5", false}, + {"v prefix handled", "v6.0.5", "v6.0.6", false}, + {"unparseable current allowed", "not-a-version", "6.0.4", false}, + {"unparseable target allowed", "6.0.5", "not-a-version", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ensureApplyTargetIsNewer(tc.current, tc.target) + if tc.wantErr && err == nil { + t.Fatalf("expected downgrade error for current=%s target=%s", tc.current, tc.target) + } + if !tc.wantErr && err != nil { + t.Fatalf("unexpected error for current=%s target=%s: %v", tc.current, tc.target, err) + } + if err != nil && !strings.Contains(err.Error(), "not newer than the running version") { + t.Fatalf("expected canonical downgrade message, got %q", err.Error()) + } + }) + } +} + +func TestApplyUpdateRejectsDowngradeBeforeDownload(t *testing.T) { + t.Setenv("PULSE_ALLOW_DOCKER_UPDATES", "true") + t.Setenv("PULSE_DATA_DIR", t.TempDir()) + t.Setenv("PULSE_INSTALL_DIR", t.TempDir()) + + oldBuildVersion := BuildVersion + BuildVersion = "6.0.5" + t.Cleanup(func() { BuildVersion = oldBuildVersion }) + + manager := &Manager{} + + err := manager.ApplyUpdate(context.Background(), ApplyUpdateRequest{ + DownloadURL: "https://github.com/rcourtman/Pulse/releases/download/v6.0.4/pulse-v6.0.4-linux-amd64.tar.gz", + }) + if err == nil { + t.Fatal("expected downgrade to be rejected") + } + if !strings.Contains(err.Error(), "not newer than the running version") { + t.Fatalf("expected downgrade rejection, got %v", err) + } + + status := manager.GetStatus() + if status.Status != "error" { + t.Fatalf("expected error status after rejected downgrade, got %q", status.Status) + } +} + +func TestApplyUpdateAllowDowngradeSkipsGuard(t *testing.T) { + t.Setenv("PULSE_ALLOW_DOCKER_UPDATES", "true") + t.Setenv("PULSE_DATA_DIR", t.TempDir()) + t.Setenv("PULSE_INSTALL_DIR", t.TempDir()) + + // Serve 404s locally so the apply fails fast at the download stage + // without touching the real release host. + server := httptest.NewServer(http.NotFoundHandler()) + t.Cleanup(server.Close) + t.Setenv("PULSE_UPDATE_SERVER", server.URL) + + oldBuildVersion := BuildVersion + BuildVersion = "6.0.5" + t.Cleanup(func() { BuildVersion = oldBuildVersion }) + + manager := &Manager{} + + // With the opt-in set the request must get past the downgrade guard; it + // then fails later at the download step instead. + err := manager.ApplyUpdate(context.Background(), ApplyUpdateRequest{ + DownloadURL: server.URL + "/releases/download/v6.0.4/pulse-v6.0.4-linux-amd64.tar.gz", + AllowDowngrade: true, + }) + if err == nil { + t.Fatal("expected apply to fail at a later stage in this environment") + } + if strings.Contains(err.Error(), "not newer than the running version") { + t.Fatalf("expected the downgrade guard to be skipped, got %v", err) + } +} + +func TestValidateRetainedBackupDir(t *testing.T) { + dataDir := t.TempDir() + t.Setenv("PULSE_DATA_DIR", dataDir) + + valid := filepath.Join(dataDir, "backup-20260710-010101") + if err := os.MkdirAll(valid, 0755); err != nil { + t.Fatalf("mkdir valid backup: %v", err) + } + + t.Run("accepts managed backup", func(t *testing.T) { + got, err := validateRetainedBackupDir(valid) + if err != nil { + t.Fatalf("expected managed backup to validate: %v", err) + } + if got != filepath.Clean(valid) { + t.Fatalf("expected %q, got %q", valid, got) + } + }) + + t.Run("rejects path outside managed roots", func(t *testing.T) { + outside := filepath.Join(t.TempDir(), "backup-20260710-010101") + if err := os.MkdirAll(outside, 0755); err != nil { + t.Fatalf("mkdir outside backup: %v", err) + } + if _, err := validateRetainedBackupDir(outside); err == nil { + t.Fatal("expected unmanaged path to be rejected") + } + }) + + t.Run("rejects wrong prefix inside managed root", func(t *testing.T) { + wrongPrefix := filepath.Join(dataDir, "snapshots-20260710-010101") + if err := os.MkdirAll(wrongPrefix, 0755); err != nil { + t.Fatalf("mkdir wrong prefix: %v", err) + } + if _, err := validateRetainedBackupDir(wrongPrefix); err == nil { + t.Fatal("expected wrong-prefix path to be rejected") + } + }) + + t.Run("rejects missing directory", func(t *testing.T) { + missing := filepath.Join(dataDir, "backup-19990101-000000") + _, err := validateRetainedBackupDir(missing) + if err == nil || !strings.Contains(err.Error(), "no longer exists") { + t.Fatalf("expected missing-backup error, got %v", err) + } + }) +} + +func TestRollbackToBackupRestoresFilesAndRecordsHistory(t *testing.T) { + t.Setenv("PULSE_ALLOW_DOCKER_UPDATES", "true") + dataDir := t.TempDir() + installDir := t.TempDir() + t.Setenv("PULSE_DATA_DIR", dataDir) + t.Setenv("PULSE_INSTALL_DIR", installDir) + + // Backup contents captured before the recorded update. The backup holds + // no "pulse" binary on purpose: restoring one would overwrite the running + // test executable via os.Executable. + backupDir := filepath.Join(dataDir, "backup-20260710-020202") + if err := os.MkdirAll(filepath.Join(backupDir, "config"), 0755); err != nil { + t.Fatalf("mkdir backup config: %v", err) + } + if err := os.WriteFile(filepath.Join(backupDir, "config", "system.json"), []byte(`{"restored":true}`), 0600); err != nil { + t.Fatalf("write backup config: %v", err) + } + if err := os.WriteFile(filepath.Join(backupDir, ".env"), []byte("RESTORED=1\n"), 0600); err != nil { + t.Fatalf("write backup env: %v", err) + } + if err := os.WriteFile(filepath.Join(backupDir, "VERSION"), []byte("6.0.4\n"), 0644); err != nil { + t.Fatalf("write backup VERSION: %v", err) + } + + // Current install state that the rollback must replace. + if err := os.MkdirAll(filepath.Join(installDir, "config"), 0755); err != nil { + t.Fatalf("mkdir install config: %v", err) + } + if err := os.WriteFile(filepath.Join(installDir, "config", "system.json"), []byte(`{"restored":false}`), 0600); err != nil { + t.Fatalf("write install config: %v", err) + } + + history, err := NewUpdateHistory(t.TempDir()) + if err != nil { + t.Fatalf("NewUpdateHistory: %v", err) + } + sourceEventID, err := history.CreateEntry(context.Background(), UpdateHistoryEntry{ + Action: "update", + Status: StatusSuccess, + VersionFrom: "6.0.4", + VersionTo: "6.0.5", + BackupPath: backupDir, + }) + if err != nil { + t.Fatalf("CreateEntry: %v", err) + } + + manager := &Manager{history: history} + + if err := manager.RollbackToBackup(context.Background(), RollbackRequest{EventID: sourceEventID}); err != nil { + t.Fatalf("RollbackToBackup: %v", err) + } + + restoredConfig, err := os.ReadFile(filepath.Join(installDir, "config", "system.json")) + if err != nil { + t.Fatalf("read restored config: %v", err) + } + if !strings.Contains(string(restoredConfig), `"restored":true`) { + t.Fatalf("expected config to be restored from backup, got %s", restoredConfig) + } + restoredEnv, err := os.ReadFile(filepath.Join(installDir, ".env")) + if err != nil { + t.Fatalf("read restored .env: %v", err) + } + if !strings.Contains(string(restoredEnv), "RESTORED=1") { + t.Fatalf("expected .env to be restored, got %s", restoredEnv) + } + + source, err := history.GetEntry(sourceEventID) + if err != nil { + t.Fatalf("GetEntry source: %v", err) + } + if source.Status != StatusRolledBack { + t.Fatalf("expected source entry to be marked rolled_back, got %q", source.Status) + } + + entries := history.ListEntries(HistoryFilter{Action: "rollback"}) + if len(entries) != 1 { + t.Fatalf("expected one rollback history entry, got %d", len(entries)) + } + rollback := entries[0] + if rollback.Status != StatusSuccess { + t.Fatalf("expected rollback entry success, got %q", rollback.Status) + } + if rollback.VersionTo != "6.0.4" { + t.Fatalf("expected rollback target version 6.0.4, got %q", rollback.VersionTo) + } + if rollback.RelatedEventID != sourceEventID { + t.Fatalf("expected rollback to reference the source update entry") + } + + status := manager.GetStatus() + if status.Status != "completed" { + t.Fatalf("expected completed status after rollback, got %q", status.Status) + } +} + +func TestRollbackToBackupRejectsBadRequests(t *testing.T) { + t.Setenv("PULSE_ALLOW_DOCKER_UPDATES", "true") + dataDir := t.TempDir() + t.Setenv("PULSE_DATA_DIR", dataDir) + t.Setenv("PULSE_INSTALL_DIR", t.TempDir()) + + history, err := NewUpdateHistory(t.TempDir()) + if err != nil { + t.Fatalf("NewUpdateHistory: %v", err) + } + manager := &Manager{history: history} + + t.Run("no history sink", func(t *testing.T) { + bare := &Manager{} + if err := bare.RollbackToBackup(context.Background(), RollbackRequest{EventID: "x"}); err == nil { + t.Fatal("expected error without history") + } + }) + + t.Run("empty event ID", func(t *testing.T) { + err := manager.RollbackToBackup(context.Background(), RollbackRequest{EventID: " "}) + if err == nil || !strings.Contains(err.Error(), "event ID is required") { + t.Fatalf("expected event ID error, got %v", err) + } + }) + + t.Run("unknown entry", func(t *testing.T) { + err := manager.RollbackToBackup(context.Background(), RollbackRequest{EventID: "does-not-exist"}) + if err == nil || !strings.Contains(err.Error(), "history entry not found") { + t.Fatalf("expected not-found error, got %v", err) + } + }) + + t.Run("entry without backup", func(t *testing.T) { + eventID, err := history.CreateEntry(context.Background(), UpdateHistoryEntry{ + Action: "update", + Status: StatusSuccess, + }) + if err != nil { + t.Fatalf("CreateEntry: %v", err) + } + rollbackErr := manager.RollbackToBackup(context.Background(), RollbackRequest{EventID: eventID}) + if rollbackErr == nil || !strings.Contains(rollbackErr.Error(), "no retained backup") { + t.Fatalf("expected no-backup error, got %v", rollbackErr) + } + }) + + t.Run("pruned backup directory", func(t *testing.T) { + eventID, err := history.CreateEntry(context.Background(), UpdateHistoryEntry{ + Action: "update", + Status: StatusSuccess, + BackupPath: filepath.Join(dataDir, "backup-20200101-000000"), + }) + if err != nil { + t.Fatalf("CreateEntry: %v", err) + } + rollbackErr := manager.RollbackToBackup(context.Background(), RollbackRequest{EventID: eventID}) + if rollbackErr == nil || !strings.Contains(rollbackErr.Error(), "no longer exists") { + t.Fatalf("expected missing-backup error, got %v", rollbackErr) + } + }) + + t.Run("concurrent update in flight", func(t *testing.T) { + backupDir := filepath.Join(dataDir, "backup-20260710-030303") + if err := os.MkdirAll(backupDir, 0755); err != nil { + t.Fatalf("mkdir backup: %v", err) + } + eventID, err := history.CreateEntry(context.Background(), UpdateHistoryEntry{ + Action: "update", + Status: StatusSuccess, + BackupPath: backupDir, + }) + if err != nil { + t.Fatalf("CreateEntry: %v", err) + } + + manager.updateMu.Lock() + manager.updateInFlight = true + manager.updateMu.Unlock() + t.Cleanup(func() { + manager.updateMu.Lock() + manager.updateInFlight = false + manager.updateMu.Unlock() + }) + + rollbackErr := manager.RollbackToBackup(context.Background(), RollbackRequest{EventID: eventID}) + if rollbackErr == nil || !strings.Contains(rollbackErr.Error(), "already in progress") { + t.Fatalf("expected in-progress error, got %v", rollbackErr) + } + }) +}