mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-20 11:21:49 +00:00
fix(vm): retry QMP dial on connection-reset during greeting (#525)
The crash-supervisor auto-restart path (qmpHeartbeat -> HandleCrash -> MaybeRestart -> startQEMU -> newQMPClientWithHandshake -> dialQMPWithRetry) already unlinks the stale QMP socket and retries a refused/absent connect. But a freshly relaunched QEMU can accept the connect then reset the connection mid-greeting while it initialises, surfacing 'connection reset by peer' immediately after 'QEMU started successfully' as a ServerInternal. Add ECONNRESET and EPIPE to isTransientDialError so the dial is retried until QEMU services the monitor, closing the auto-restart QMP-connect race.
This commit is contained in:
@@ -709,9 +709,10 @@ func removeStaleQMPSocket(path string) error {
|
||||
// dialQMPWithRetry redials the QMP socket until the connect+greeting succeeds or
|
||||
// the deadline passes. A just-relaunched QEMU can present the socket inode a beat
|
||||
// before it listen()s, and a stale inode from a SIGKILLed predecessor refuses the
|
||||
// first connect; a single dial then loses the race. Only transient connect
|
||||
// errors are retried — a greeting/handshake failure means we reached a listener,
|
||||
// so it surfaces immediately.
|
||||
// first connect; a fresh QEMU also accepts the connect then resets mid-greeting
|
||||
// while it initialises. A single dial then loses the race. Only transient connect
|
||||
// or reset errors are retried — a decode/handshake failure from a settled QEMU
|
||||
// surfaces immediately.
|
||||
func dialQMPWithRetry(path string, greetingTimeout time.Duration) (*qmp.QMPClient, error) {
|
||||
deadline := time.Now().Add(qmpDialTimeout)
|
||||
for {
|
||||
@@ -728,9 +729,13 @@ func dialQMPWithRetry(path string, greetingTimeout time.Duration) (*qmp.QMPClien
|
||||
|
||||
// isTransientDialError reports whether a QMP connect failed because the listener
|
||||
// is not up yet: connection refused (bound but pre-listen, or a stale dead
|
||||
// inode) or the socket momentarily absent between unlink and rebind.
|
||||
// inode), the socket momentarily absent between unlink and rebind, or the peer
|
||||
// tearing the connection down mid-greeting (reset/broken pipe) — a fresh QEMU
|
||||
// that accepts then resets before it can service the monitor. Each clears once
|
||||
// QEMU finishes initialising, so the dial is retried rather than surfaced.
|
||||
func isTransientDialError(err error) bool {
|
||||
return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT)
|
||||
return errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ENOENT) ||
|
||||
errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE)
|
||||
}
|
||||
|
||||
// newQMPClientWithHandshake dials the QMP socket and runs the qmp_capabilities
|
||||
|
||||
@@ -1156,11 +1156,17 @@ func TestRemoveStaleQMPSocket(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestIsTransientDialError pins which connect failures the QMP dial retries: a
|
||||
// listener-not-up-yet (refused) or a socket briefly absent (enoent) are
|
||||
// transient; a greeting/handshake error is not — it means a listener answered.
|
||||
// listener-not-up-yet (refused), a socket briefly absent (enoent), or a peer
|
||||
// resetting mid-greeting (reset/broken pipe from a QEMU still initialising) are
|
||||
// transient; a clean decode/handshake failure is not — it means a settled
|
||||
// listener answered.
|
||||
func TestIsTransientDialError(t *testing.T) {
|
||||
assert.True(t, isTransientDialError(syscall.ECONNREFUSED), "pre-listen/stale refuses connect")
|
||||
assert.True(t, isTransientDialError(syscall.ENOENT), "socket absent between unlink and rebind")
|
||||
assert.True(t, isTransientDialError(syscall.ECONNRESET), "fresh QEMU resets mid-greeting while initialising")
|
||||
assert.True(t, isTransientDialError(syscall.EPIPE), "peer tears down the greeting connection")
|
||||
assert.True(t, isTransientDialError(fmt.Errorf("waiting for QMP greeting: %w", syscall.ECONNRESET)),
|
||||
"a wrapped connection-reset from the greeting read is still transient")
|
||||
assert.False(t, isTransientDialError(fmt.Errorf("waiting for QMP greeting: eof")),
|
||||
"a reached-listener handshake failure must not be retried")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user