package cli import ( "fmt" "log/slog" "net" "net/http" "os" "os/exec" "strconv" "time" "github.com/PerpetualSoftware/pad/internal/config" ) // EnsureServer checks if the pad server is running; if not, starts it in the background. func EnsureServer(cfg *config.Config) error { // Only an explicitly configured local client should auto-manage a local // background process. Unconfigured or external clients should connect only // to their configured target. if !cfg.ManagesLocalServer() { return nil } if serverResponds(cfg.Host, cfg.Port) { return nil } // A refresh may have stopped the server on purpose and be about to // restart it (BUG-3215). Its window is its own: wait for it rather than // spawn a server with this caller's binary, flags and cwd over it. if err := awaitRefresh(cfg); err != nil { return err } if serverResponds(cfg.Host, cfg.Port) { return nil } // Something holds the port but sent no response: a server still starting // (it binds before it migrates, TASK-3620). Spawning over it only makes a // second start that loses the bind. Wait for it to serve instead, so the // caller's request does not sit in the backlog past its own timeout. if portHeld(cfg.Host, cfg.Port) { return awaitServing(cfg) } return spawnServer(cfg) } // startupWait bounds awaitServing. Generous, because a start that runs // migrations holds the port for as long as they take; a variable so a test can // shorten it. var startupWait = 2 * time.Minute // awaitServing waits for the server holding the port to answer HTTP. func awaitServing(cfg *config.Config) error { deadline := time.Now().Add(startupWait) for time.Now().Before(deadline) { if serverResponds(cfg.Host, cfg.Port) { return nil } if !portHeld(cfg.Host, cfg.Port) { // The holder went away without serving. Leave the port to the // next call's ordinary auto-start rather than race for it here. return fmt.Errorf("the process holding %s exited before it served; retry", cfg.Addr()) } time.Sleep(refreshPoll) } return fmt.Errorf("a process holds %s but has not answered HTTP within %s (a server still migrating?); "+ "not starting another over it. Check %s", cfg.Addr(), startupWait, cfg.LogFile()) } // spawnServer starts ` server start` in the background and waits // for it to answer. A variable so the tests can count spawns: the defect // BUG-3215 guards against is a spawn, and only a counter observes one that // did not happen. var spawnServer = startBackgroundServer func startBackgroundServer(cfg *config.Config) error { // Start server as background process exePath, err := os.Executable() if err != nil { return fmt.Errorf("find executable: %w", err) } cmd := exec.Command(exePath, "server", "start") setSysProcAttr(cmd) // Redirect stdout/stderr to log file logFile, err := os.OpenFile(cfg.LogFile(), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err != nil { return fmt.Errorf("open log file: %w", err) } cmd.Stdout = logFile cmd.Stderr = logFile if err := cmd.Start(); err != nil { logFile.Close() return fmt.Errorf("start server: %w", err) } // The PID file is deliberately NOT written here (BUG-2965, codex round 3). // The child claims it AFTER it binds the port, so the file always names the // process that actually owns the address. Writing it from the parent, at // spawn time, records a process that may never bind — and when the port is // held by an existing server that is briefly unhealthy, that write // overwrites the running server's own entry with a pid that is about to // exit, leaving a healthy server unaddressable by `pad server stop`. The // health wait below is what tells us the child got there. // Release the process so it doesn't become a zombie cmd.Process.Release() logFile.Close() // Wait for server to become healthy for i := 0; i < 30; i++ { time.Sleep(100 * time.Millisecond) if serverResponds(cfg.Host, cfg.Port) { return nil } } // The child binds before it migrates (TASK-3620), so a held port after // three seconds is a server still starting: wait for it as EnsureServer // does, rather than fail a call whose server is about to serve. if portHeld(cfg.Host, cfg.Port) { return awaitServing(cfg) } return fmt.Errorf("server failed to start within 3 seconds. Check %s for errors", cfg.LogFile()) } // StopServer stops the server this CLI's config points at — or explains why it // will not, which is the harder half. // // The PID file is a HANDLE, not evidence (BUG-2965), and the pid inside it is // not proof of identity (BUG-2969). Measured before this changed: a `sleep 600` // whose pid had been written into the file was SIGTERMed, and stop printed // "Server stopped." — os.FindProcess succeeds for any pid on Unix, nothing // asked whether the pid was ours, and the confirmation loop polled the PORT, // which is unhealthy from the first poll when nothing was ever serving. The // success check was satisfied by the failure case. // // So the order is: read the record, ask the PLATFORM whether a live pad server // owns it (an advisory lock on Unix, the process creation time on Windows), // and signal only on a yes. Stale and unprovable both mean signal nothing. func StopServer(cfg *config.Config) error { path := cfg.PIDFile() // One read, under the ownership probe: pidFileOwner returns the record it // verified, so the pid signalled below is the pid whose ownership was // established. An earlier separate read left a window for a successor to // claim the file between the two (codex round 1). rec, owner := pidFileOwner(path) if owner == pidFileStale && rec.PID == 0 { // No file at all. A server started outside this CLI still holds the // port, and saying "not running" about it is the BUG-2965 failure. if portHeld(cfg.Host, cfg.Port) { return fmt.Errorf( "a server is answering on %s but this CLI did not start it (no PID file at %s) — "+ "stop it where it was started, or signal the process listening on that address", cfg.Addr(), path) } return fmt.Errorf("server not running (no PID file)") } switch owner { case pidFileStale: // The writer is gone. The pid may since have been reused by something // unrelated, so it does not get signalled on the strength of a file // the dead process left behind. // The file is already gone: pidFileOwner removed it while holding the // lock, which is the only point at which no replacement can have // claimed the path (codex round 2). if portHeld(cfg.Host, cfg.Port) { // Deliberately NOT "the process is gone": the recorded pid may // well be alive — that is the dangerous case, since it is a // stranger that inherited the number — and a reader who checks // will find it running. What is true is that it does not own this // file, so it is not the server answering on that address. // // The message does not claim the file was removed: Unix drops it // under the ownership lock, Windows leaves it for the next start // to overwrite (see pidfile_windows.go), and a message that // asserted a cleanup the platform did not do would be the same // class of lie this whole item is about. return fmt.Errorf( "a server is answering on %s, but the PID file does not belong to it (%s) — "+ "nothing was signalled; stop that server where it was started", cfg.Addr(), describePIDRecord(rec)) } return fmt.Errorf("server not running (stale PID file: it named %s, which does not own it)", describePIDRecord(rec)) case pidFileUnprovable: return fmt.Errorf( "cannot confirm that %s belongs to this pad server, so it was not signalled — "+ "stop it where it was started, or remove %s if you know the process is gone", describePIDRecord(rec), path) case pidFileOurs: // Fall through to the stop. default: return fmt.Errorf("unrecognised PID file ownership state %q", owner) } process, err := os.FindProcess(rec.PID) if err != nil { return fmt.Errorf("process not found") } if err := stopProcess(process); err != nil { return fmt.Errorf("failed to stop server: %w", err) } // Wait for OUR PROCESS to exit — not for the port to go quiet. A port that // nothing was serving is quiet immediately, which is how the old loop // confirmed a kill that never touched a pad server. for i := 0; i < 60; i++ { if processIsGone(rec.PID) { // Deliberately NOT removing the file here (codex round 1): the // server removes its own on the way down, and by the time we // observe its exit a successor may already have bound the port and // claimed the path. Deleting then would unaddress the live server. // A file left behind by a crash is handled where it belongs — the // next stop reads it as stale and removes it there. return nil } time.Sleep(100 * time.Millisecond) } // Still there after six seconds. The file stays: the process it names is // alive, so the record is still true and still the handle for a retry. return fmt.Errorf("signalled %s but it has not exited after 6s — it may be draining connections", describePIDRecord(rec)) } // IsServerRunning reports whether a server answers on the configured address. func IsServerRunning(cfg *config.Config) bool { return portHeld(cfg.Host, cfg.Port) } // Two questions, two probes (TASK-3620). The old single probe was an HTTP GET // of /health that required a 200, and it was wrong both ways: // // - The probe shares one per-address rate-limit bucket across every CLI on // the machine, so a parallel fan-out drew 429s that read as "not running": // 206 `pad server start` spawns in two days, each opening the live // database before losing the bind. // - `pad server start` binds before it migrates, so a starting server holds // the port and answers nothing until startup is done. // // portHeld answers "may I start a server here, or touch the database under // it?": whether a TCP connect succeeds. serverResponds answers "is it serving // yet?": whether an HTTP request gets any response, whatever the status, since // /health itself only answers 200 and anything else is middleware in front of // a running server. // portHeld reports whether something holds host:port. func portHeld(host string, port int) bool { conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), time.Second) if err != nil { return false } _ = conn.Close() return true } // serverResponds reports whether an HTTP request to host:port gets a response. func serverResponds(host string, port int) bool { client := &http.Client{Timeout: 1 * time.Second} resp, err := client.Get(fmt.Sprintf("http://%s:%d/api/v1/health", host, port)) if err != nil { return false } _ = resp.Body.Close() return true } // ClaimPIDFile records this process in the PID file and holds the platform's // ownership marker for as long as it runs. The returned release drops that // marker and removes the file. // // CALL IT ONLY AFTER BINDING THE PORT. That ordering is the whole ownership // story and it took two codex rounds to arrive at (BUG-2965): whoever holds the // address is the process `stop` must signal, so the claim follows the bind // rather than racing it. A write before the bind lets a start that LOSES the // race record itself and then, on its way out, remove the file the winner // relies on. // // On Unix the marker is an advisory lock held on the file itself, which is what // lets `stop` distinguish this server from a stranger that inherited the pid // (BUG-2969). On Windows there is no such lock, and the recorded creation time // is the discriminator instead. // // Failure is NOT fatal: the server is what the caller asked for, and a missing // or unlocked PID file degrades `stop` to its port-probe message rather than // breaking anything. The returned release is always safe to call, and safe to // call twice. func ClaimPIDFile(path string) func() { release, err := holdPIDFile(path) if err != nil { slog.Warn("could not claim the PID file; `pad server stop` will not be able to address this process", "path", path, "error", err) return func() {} } rec := currentPIDRecord() if err := writePIDRecord(path, rec); err != nil { slog.Warn("could not write PID file; `pad server stop` will not be able to address this process", "path", path, "error", err) release() return func() {} } return func() { // Remove BEFORE dropping the lock: while we still hold it, no other // server can have claimed this path, so there is nothing of anyone // else's to delete. Dropping first would reopen the read-then-remove // race that codex round 4 found in the previous shape. if current, ok := readPIDRecord(path); ok && current.PID == rec.PID { if err := os.Remove(path); err != nil && !os.IsNotExist(err) { slog.Warn("could not remove PID file on shutdown", "path", path, "error", err) } } release() } }