From 4f31b336c56b1043d896139ec14aa8acb458f23f Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Mon, 13 Jul 2026 23:07:47 +1000 Subject: [PATCH] fix(vm): retry QMP dial on connection-reset during greeting (#525) The crash-supervisor auto-restart path (qmpHeartbeat -> HandleCrash -> MaybeRestart -> startQEMU -> newQMPClientWithHandshake -> dialQMPWithRetry) already unlinks the stale QMP socket and retries a refused/absent connect. But a freshly relaunched QEMU can accept the connect then reset the connection mid-greeting while it initialises, surfacing 'connection reset by peer' immediately after 'QEMU started successfully' as a ServerInternal. Add ECONNRESET and EPIPE to isTransientDialError so the dial is retried until QEMU services the monitor, closing the auto-restart QMP-connect race. --- spinifex/vm/lifecycle.go | 15 ++++++++++----- spinifex/vm/lifecycle_test.go | 10 ++++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/spinifex/vm/lifecycle.go b/spinifex/vm/lifecycle.go index 06916824f..265788e03 100644 --- a/spinifex/vm/lifecycle.go +++ b/spinifex/vm/lifecycle.go @@ -709,9 +709,10 @@ func removeStaleQMPSocket(path string) error { // dialQMPWithRetry redials the QMP socket until the connect+greeting succeeds or // the deadline passes. A just-relaunched QEMU can present the socket inode a beat // before it listen()s, and a stale inode from a SIGKILLed predecessor refuses the -// first connect; a single dial then loses the race. Only transient connect -// errors are retried — a greeting/handshake failure means we reached a listener, -// so it surfaces immediately. +// first connect; a fresh QEMU also accepts the connect then resets mid-greeting +// while it initialises. A single dial then loses the race. Only transient connect +// or reset errors are retried — a decode/handshake failure from a settled QEMU +// surfaces immediately. func dialQMPWithRetry(path string, greetingTimeout time.Duration) (*qmp.QMPClient, error) { deadline := time.Now().Add(qmpDialTimeout) for { @@ -728,9 +729,13 @@ func dialQMPWithRetry(path string, greetingTimeout time.Duration) (*qmp.QMPClien // isTransientDialError reports whether a QMP connect failed because the listener // is not up yet: connection refused (bound but pre-listen, or a stale dead -// inode) or the socket momentarily absent between unlink and rebind. +// inode), the socket momentarily absent between unlink and rebind, or the peer +// tearing the connection down mid-greeting (reset/broken pipe) — a fresh QEMU +// that accepts then resets before it can service the monitor. Each clears once +// QEMU finishes initialising, so the dial is retried rather than surfaced. func isTransientDialError(err error) bool { - return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT) + return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT) || + errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) } // newQMPClientWithHandshake dials the QMP socket and runs the qmp_capabilities diff --git a/spinifex/vm/lifecycle_test.go b/spinifex/vm/lifecycle_test.go index 8bc2f780e..b88243fc1 100644 --- a/spinifex/vm/lifecycle_test.go +++ b/spinifex/vm/lifecycle_test.go @@ -1156,11 +1156,17 @@ func TestRemoveStaleQMPSocket(t *testing.T) { } // TestIsTransientDialError pins which connect failures the QMP dial retries: a -// listener-not-up-yet (refused) or a socket briefly absent (enoent) are -// transient; a greeting/handshake error is not — it means a listener answered. +// listener-not-up-yet (refused), a socket briefly absent (enoent), or a peer +// resetting mid-greeting (reset/broken pipe from a QEMU still initialising) are +// transient; a clean decode/handshake failure is not — it means a settled +// listener answered. func TestIsTransientDialError(t *testing.T) { assert.True(t, isTransientDialError(syscall.ECONNREFUSED), "pre-listen/stale refuses connect") assert.True(t, isTransientDialError(syscall.ENOENT), "socket absent between unlink and rebind") + assert.True(t, isTransientDialError(syscall.ECONNRESET), "fresh QEMU resets mid-greeting while initialising") + assert.True(t, isTransientDialError(syscall.EPIPE), "peer tears down the greeting connection") + assert.True(t, isTransientDialError(fmt.Errorf("waiting for QMP greeting: %w", syscall.ECONNRESET)), + "a wrapped connection-reset from the greeting read is still transient") assert.False(t, isTransientDialError(fmt.Errorf("waiting for QMP greeting: eof")), "a reached-listener handshake failure must not be retried") }