mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-26 21:47:20 +00:00
pulse-triage[bot]
03db748782
Bind the UI/API listener early so port conflicts fail fast, but srv.Serve only starts near the end of Run. Every synchronous initialization step in between therefore runs with a live listening socket whose accept queue fills while nothing accepts: the reported #2129 shape (bound port, non-zero Recv-Q, no HTTP response). A stall there is invisible because no log line is emitted while it runs. Track the last completed startup phase and arm an independent watchdog that fires if serving has not begun within two minutes, logging the last phase and every goroutine stack. This makes a recurrence diagnosable from the field log without an operator sending SIGQUIT. The watchdog is stopped when serving begins or Run exits and is safe to stop more than once. Change-source: pulse-maintainer
85 lines
2.6 KiB
Go
85 lines
2.6 KiB
Go
package server
|
|
|
|
import (
|
|
"runtime"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/rs/zerolog"
|
|
)
|
|
|
|
// startupWatchdogBudget bounds how long Run may spend between binding the
|
|
// UI/API listener and actually serving it.
|
|
//
|
|
// The listener is bound early so a port conflict fails fast, but nothing
|
|
// accepts connections until srv.Serve starts. Every synchronous initialization
|
|
// step in between (monitor start, router construction, optional AI/relay/
|
|
// telemetry/config-watcher startup) therefore runs with a live listening
|
|
// socket whose accept queue is filling and no reader. A normal startup
|
|
// completes in seconds; this budget only trips when one of those steps stalls,
|
|
// which is otherwise invisible because no log line is emitted while it runs.
|
|
const startupWatchdogBudget = 2 * time.Minute
|
|
|
|
// startupWatchdogStackBytes is the buffer reserved for the goroutine dump. The
|
|
// default 64KiB used elsewhere truncates under a large goroutine set; a stalled
|
|
// startup is exactly when the full stack matters.
|
|
const startupWatchdogStackBytes = 256 * 1024
|
|
|
|
// startupPhase records the last completed initialization step so a watchdog
|
|
// dump names where Run was when it stalled instead of only showing the stacks.
|
|
type startupPhase struct {
|
|
mu sync.Mutex
|
|
phase string
|
|
}
|
|
|
|
func (s *startupPhase) mark(phase string) {
|
|
if s == nil {
|
|
return
|
|
}
|
|
s.mu.Lock()
|
|
s.phase = phase
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
func (s *startupPhase) current() string {
|
|
if s == nil {
|
|
return "unknown"
|
|
}
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
if s.phase == "" {
|
|
return "unknown"
|
|
}
|
|
return s.phase
|
|
}
|
|
|
|
// startStartupWatchdog arms an independent timer that fires only if the server
|
|
// has not started serving within budget. It logs the last completed startup
|
|
// phase and every goroutine stack so a recurrence of the reported
|
|
// bound-but-not-serving startup stall (issue #2129) is diagnosable from the
|
|
// field log alone, without waiting for an operator to send SIGQUIT. The
|
|
// returned stop function must be called once serving begins or Run exits; it is
|
|
// safe to call more than once.
|
|
func startStartupWatchdog(logger zerolog.Logger, phase *startupPhase, budget time.Duration) (stop func()) {
|
|
done := make(chan struct{})
|
|
var once sync.Once
|
|
go func() {
|
|
timer := time.NewTimer(budget)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-done:
|
|
case <-timer.C:
|
|
buf := make([]byte, startupWatchdogStackBytes)
|
|
n := runtime.Stack(buf, true)
|
|
logger.Error().
|
|
Dur("budget", budget).
|
|
Str("last_phase", phase.current()).
|
|
Str("goroutines", string(buf[:n])).
|
|
Msg("Pulse has not started serving its UI/API listener within the startup watchdog budget; startup is stalled")
|
|
}
|
|
}()
|
|
return func() {
|
|
once.Do(func() { close(done) })
|
|
}
|
|
}
|