fix(vm): retry QMP dial on connection-reset during greeting (#525)

The crash-supervisor auto-restart path (qmpHeartbeat -> HandleCrash ->
MaybeRestart -> startQEMU -> newQMPClientWithHandshake -> dialQMPWithRetry)
already unlinks the stale QMP socket and retries a refused/absent connect. But
a freshly relaunched QEMU can accept the connect then reset the connection
mid-greeting while it initialises, surfacing 'connection reset by peer'
immediately after 'QEMU started successfully' as a ServerInternal.

Add ECONNRESET and EPIPE to isTransientDialError so the dial is retried until
QEMU services the monitor, closing the auto-restart QMP-connect race.
This commit is contained in:
Josh Siviour
2026-07-13 23:07:47 +10:00
committed by GitHub
parent 119081fb95
commit 4f31b336c5
2 changed files with 18 additions and 7 deletions
+10 -5
View File
@@ -709,9 +709,10 @@ func removeStaleQMPSocket(path string) error {
// dialQMPWithRetry redials the QMP socket until the connect+greeting succeeds or
// the deadline passes. A just-relaunched QEMU can present the socket inode a beat
// before it listen()s, and a stale inode from a SIGKILLed predecessor refuses the
// first connect; a single dial then loses the race. Only transient connect
// errors are retried — a greeting/handshake failure means we reached a listener,
// so it surfaces immediately.
// first connect; a fresh QEMU also accepts the connect then resets mid-greeting
// while it initialises. A single dial then loses the race. Only transient connect
// or reset errors are retried — a decode/handshake failure from a settled QEMU
// surfaces immediately.
func dialQMPWithRetry(path string, greetingTimeout time.Duration) (*qmp.QMPClient, error) {
deadline := time.Now().Add(qmpDialTimeout)
for {
@@ -728,9 +729,13 @@ func dialQMPWithRetry(path string, greetingTimeout time.Duration) (*qmp.QMPClien
// isTransientDialError reports whether a QMP connect failed because the listener
// is not up yet: connection refused (bound but pre-listen, or a stale dead
// inode) or the socket momentarily absent between unlink and rebind.
// inode), the socket momentarily absent between unlink and rebind, or the peer
// tearing the connection down mid-greeting (reset/broken pipe) — a fresh QEMU
// that accepts then resets before it can service the monitor. Each clears once
// QEMU finishes initialising, so the dial is retried rather than surfaced.
func isTransientDialError(err error) bool {
return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT)
return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT) ||
errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE)
}
// newQMPClientWithHandshake dials the QMP socket and runs the qmp_capabilities
+8 -2
View File
@@ -1156,11 +1156,17 @@ func TestRemoveStaleQMPSocket(t *testing.T) {
}
// TestIsTransientDialError pins which connect failures the QMP dial retries: a
// listener-not-up-yet (refused) or a socket briefly absent (enoent) are
// transient; a greeting/handshake error is not — it means a listener answered.
// listener-not-up-yet (refused), a socket briefly absent (enoent), or a peer
// resetting mid-greeting (reset/broken pipe from a QEMU still initialising) are
// transient; a clean decode/handshake failure is not — it means a settled
// listener answered.
func TestIsTransientDialError(t *testing.T) {
assert.True(t, isTransientDialError(syscall.ECONNREFUSED), "pre-listen/stale refuses connect")
assert.True(t, isTransientDialError(syscall.ENOENT), "socket absent between unlink and rebind")
assert.True(t, isTransientDialError(syscall.ECONNRESET), "fresh QEMU resets mid-greeting while initialising")
assert.True(t, isTransientDialError(syscall.EPIPE), "peer tears down the greeting connection")
assert.True(t, isTransientDialError(fmt.Errorf("waiting for QMP greeting: %w", syscall.ECONNRESET)),
"a wrapped connection-reset from the greeting read is still transient")
assert.False(t, isTransientDialError(fmt.Errorf("waiting for QMP greeting: eof")),
"a reached-listener handshake failure must not be retried")
}