From 702fc5aeb5fe5809bcfdac36f265b42051063c52 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Sun, 17 May 2026 09:54:11 +0100 Subject: [PATCH] Stabilize hot-dev backend startup recovery --- .../subsystems/deployment-installability.md | 9 ++- scripts/hot-dev.sh | 65 +++++++++++++++++-- scripts/tests/test-hot-dev-bg.sh | 11 +++- 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index f7c4db2cf..86d635f8f 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -935,7 +935,9 @@ That same dev-runtime orchestration boundary also owns watcher stability for the managed local stack: `scripts/hot-dev.sh` may only rebuild the backend for runtime Go sources, not `*_test.go` churn, and it must suppress `pulse` binary change events produced by its own successful managed rebuilds, managed backend -restarts, or startup build. +restarts, or startup build through shared watcher-state markers rather than +per-subshell timing alone. Parallel watcher streams must not start duplicate +managed rebuilds for the same backend artifact change. That same boundary also owns backend-liveness recovery, not just process- existence. The managed health monitor in `scripts/hot-dev.sh` must probe `http://127.0.0.1:${PULSE_DEV_API_PORT}/api/health` in addition to checking @@ -943,7 +945,10 @@ that a `./pulse` process exists, so an alive-but-unresponsive backend (hung goroutine, panic-recovery loop, port-bind failure with the process still running) is detected and restarted instead of leaving the dev frontend talking to a dead listener. Two consecutive missed health probes must trigger -a managed kill and restart of the unresponsive process. +a managed kill and restart of the unresponsive process only after the managed +backend startup/restart grace has elapsed; the monitor must not kill a backend +merely because the server has bound its listener before the HTTP health route +is ready. That same watcher boundary also owns backend-served demo coherence: `internal/api/frontend-modern/dist` changes must trigger a managed backend rebuild so the `go:embed` frontend on `:7655` cannot drift behind a freshly diff --git a/scripts/hot-dev.sh b/scripts/hot-dev.sh index cf4886a6a..db941b246 100755 --- a/scripts/hot-dev.sh +++ b/scripts/hot-dev.sh @@ -17,6 +17,10 @@ # PULSE_DATA_DIR=/path Override data directory # PULSE_DEV_API_PORT=7655 Backend API port (default: 7655) # FRONTEND_DEV_PORT=5173 Frontend dev server port (default: 5173) +# HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS=180 +# Backend /api/health grace after starts/restarts +# HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD=2 +# Consecutive failed /api/health probes before restart # # Pro Features Mode: # When pulse-enterprise repo exists and HOT_DEV_USE_PRO is not "false", @@ -243,6 +247,7 @@ EXTRA_CLEANUP_PORT=$((PULSE_DEV_API_PORT + 1)) HOT_DEV_RESTART_SENTINEL="${ROOT_DIR}/tmp/hot-dev.restart" HOT_DEV_VERIFY_LOCK="${HOT_DEV_VERIFY_LOCK_FILE:-${ROOT_DIR}/tmp/hot-dev.verify.lock}" HOT_DEV_BUILD_LOCK="${HOT_DEV_BUILD_LOCK_FILE:-${ROOT_DIR}/tmp/hot-dev.build.lock}" +HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE="${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE:-${ROOT_DIR}/tmp/hot-dev.self-build-ignore-until}" HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS="${HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS:-5}" EMBEDDED_FRONTEND_DIR="${ROOT_DIR}/internal/api/frontend-modern" EMBEDDED_FRONTEND_DIST_DIR="${EMBEDDED_FRONTEND_DIR}/dist" @@ -326,6 +331,8 @@ kill_port "${EXTRA_CLEANUP_PORT}" # Truncate debug log mkdir -p "$(dirname "${BACKEND_DEBUG_LOG}")" :> "${BACKEND_DEBUG_LOG}" +BACKEND_STARTED_AT_FILE="${HOT_DEV_BACKEND_STARTED_AT_FILE:-${ROOT_DIR}/tmp/hot-dev.backend.started-at}" +mkdir -p "$(dirname "${BACKEND_STARTED_AT_FILE}")" sleep 2 @@ -491,7 +498,12 @@ if [[ ${PRO_BUILD_SUCCESS:-false} == "true" ]]; then fi STARTED_BACKEND_PID="" +mark_backend_startup_grace() { + date +%s > "${BACKEND_STARTED_AT_FILE}" +} + start_backend_process() { + mark_backend_startup_grace LOG_LEVEL="${LOG_LEVEL:-debug}" \ FRONTEND_PORT="${PULSE_DEV_API_PORT:-7655}" \ PORT="${PULSE_DEV_API_PORT:-7655}" \ @@ -522,22 +534,44 @@ fi # Restarts Pulse if it dies unexpectedly (not from file watcher rebuild), # enforces single-instance, and detects the alive-but-unresponsive state # (process exists but /api/health is not serving) which a process-only check -# blind-spots. Two consecutive 5s misses on /api/health -> kill and restart. +# blind-spots. After backend startup grace, consecutive misses on /api/health +# -> kill and restart. log_info "Starting backend health monitor..." ( UNHEALTHY_STREAK=0 - UNHEALTHY_THRESHOLD=2 + UNHEALTHY_THRESHOLD="${HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD:-2}" + BACKEND_HEALTH_STARTUP_GRACE_SECONDS="${HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS:-180}" + + [[ "${UNHEALTHY_THRESHOLD}" =~ ^[1-9][0-9]*$ ]] || UNHEALTHY_THRESHOLD=2 + [[ "${BACKEND_HEALTH_STARTUP_GRACE_SECONDS}" =~ ^[0-9]+$ ]] || BACKEND_HEALTH_STARTUP_GRACE_SECONDS=180 backend_serving() { curl -sf -o /dev/null --max-time 3 \ "http://127.0.0.1:${PULSE_DEV_API_PORT:-7655}/api/health" 2>/dev/null } + backend_in_startup_grace() { + local backend_started_at + local now + + [[ -r "${BACKEND_STARTED_AT_FILE}" ]] || return 1 + backend_started_at="$(<"${BACKEND_STARTED_AT_FILE}")" + [[ "${backend_started_at}" =~ ^[0-9]+$ ]] || return 1 + + now="$(date +%s)" + (( now - backend_started_at < BACKEND_HEALTH_STARTUP_GRACE_SECONDS )) + } + while true; do sleep 5 PULSE_COUNT="$(hot_dev_pulse_process_count)" if [[ "$PULSE_COUNT" -eq 0 ]]; then + if backend_in_startup_grace; then + UNHEALTHY_STREAK=0 + log_warn "⚠️ Pulse process not running yet during backend startup grace (${BACKEND_HEALTH_STARTUP_GRACE_SECONDS}s)" + continue + fi log_warn "⚠️ Pulse died unexpectedly, restarting..." start_backend_process NEW_PID="${STARTED_BACKEND_PID}" @@ -562,6 +596,11 @@ log_info "Starting backend health monitor..." fi UNHEALTHY_STREAK=0 elif ! backend_serving; then + if backend_in_startup_grace; then + UNHEALTHY_STREAK=0 + log_warn "⚠️ Pulse /api/health not serving yet during backend startup grace (${BACKEND_HEALTH_STARTUP_GRACE_SECONDS}s)" + continue + fi UNHEALTHY_STREAK=$((UNHEALTHY_STREAK + 1)) log_warn "⚠️ Pulse alive but /api/health unresponsive (streak ${UNHEALTHY_STREAK}/${UNHEALTHY_THRESHOLD})" if [[ "$UNHEALTHY_STREAK" -ge "$UNHEALTHY_THRESHOLD" ]]; then @@ -607,15 +646,27 @@ log_info "Starting backend file watcher..." local now now=$(date +%s) SELF_BUILD_IGNORE_UNTIL=$((now + 5)) + mkdir -p "$(dirname "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}")" + printf '%s\n' "${SELF_BUILD_IGNORE_UNTIL}" > "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}" } manual_build_event_suppressed() { + local now + local shared_ignore_until + if build_lock_active; then return 0 fi - local now now=$(date +%s) + shared_ignore_until="" + if [[ -r "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}" ]]; then + shared_ignore_until="$(<"${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}")" + fi + if [[ "${shared_ignore_until}" =~ ^[0-9]+$ ]] && (( now < shared_ignore_until )); then + return 0 + fi + (( now < SELF_BUILD_IGNORE_UNTIL )) } @@ -701,6 +752,7 @@ log_info "Starting backend file watcher..." return fi LAST_RESTART_TIME=$now + mark_backend_startup_grace mark_self_build_output log_info "Restarting backend..." @@ -740,6 +792,11 @@ log_info "Starting backend file watcher..." rebuild_backend() { local changed_file=$1 + if build_lock_active; then + log_info "Managed build already in progress; skipping duplicate rebuild event." + return + fi + # Debounce: skip if we rebuilt less than 2 seconds ago (batch saves) local now now=$(date +%s) @@ -760,8 +817,8 @@ log_info "Starting backend file watcher..." # Use the same build logic as the initial build. if build_backend_binary; then mark_self_build_output - clear_build_lock restart_backend + clear_build_lock else clear_build_lock log_error "✗ Build failed; keeping the current backend process running." diff --git a/scripts/tests/test-hot-dev-bg.sh b/scripts/tests/test-hot-dev-bg.sh index 1d29335bf..a318a1857 100755 --- a/scripts/tests/test-hot-dev-bg.sh +++ b/scripts/tests/test-hot-dev-bg.sh @@ -651,7 +651,7 @@ test_makefile_routes_managed_runtime_through_npm() { test_hot_dev_script_advertises_foreground_escape_hatch() { local output - output="$(sed -n '1,30p' "${HOT_DEV}")" + output="$(sed -n '1,40p' "${HOT_DEV}")" assert_contains "hot-dev header identifies foreground escape hatch" "${output}" "hot-dev.sh - Foreground Pulse dev runtime escape hatch" assert_contains "hot-dev usage points to managed runtime first" "${output}" "npm run dev # Canonical managed dev runtime" @@ -669,7 +669,9 @@ test_hot_dev_script_ignores_test_only_backend_churn() { assert_contains "hot-dev fswatch covers the embedded frontend parent dir" "${output}" '"${ROOT_DIR}/pulse" "${HOT_DEV_RESTART_SENTINEL}" "${EMBEDDED_FRONTEND_DIR}"' assert_contains "hot-dev fswatch only treats the pulse binary path as a manual build trigger" "${output}" 'elif [[ "$changed_file" == "${ROOT_DIR}/pulse" ]]; then' assert_contains "hot-dev watcher declares a managed build lock path" "${output}" 'HOT_DEV_BUILD_LOCK="${HOT_DEV_BUILD_LOCK_FILE:-${ROOT_DIR}/tmp/hot-dev.build.lock}"' + assert_contains "hot-dev watcher declares shared self-build suppression marker" "${output}" 'HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE="${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE:-${ROOT_DIR}/tmp/hot-dev.self-build-ignore-until}"' assert_contains "hot-dev watcher suppresses manual binary restarts while a managed build is active" "${output}" 'if build_lock_active; then' + assert_contains "hot-dev watcher shares self-build suppression across watcher streams" "${output}" 'printf '\''%s\n'\'' "${SELF_BUILD_IGNORE_UNTIL}" > "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}"' assert_contains "hot-dev watcher suppresses self-build binary restart loops" "${output}" 'if manual_build_event_suppressed; then' assert_contains "hot-dev watcher suppresses startup self-build pulse events" "${output}" 'SELF_BUILD_IGNORE_UNTIL=$((WATCHER_READY_AT + HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS + 5))' assert_contains "hot-dev watcher seeds the startup pulse marker" "${output}" 'LAST_PULSE_BINARY_MARKER="$(file_event_marker "${ROOT_DIR}/pulse" || true)"' @@ -680,7 +682,11 @@ test_hot_dev_health_monitor_probes_api_health() { local output output="$(cat "${HOT_DEV}")" - assert_contains "hot-dev health monitor declares an unhealthy streak threshold" "${output}" 'UNHEALTHY_THRESHOLD=2' + assert_contains "hot-dev health monitor declares a configurable unhealthy streak threshold" "${output}" 'UNHEALTHY_THRESHOLD="${HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD:-2}"' + assert_contains "hot-dev health monitor declares backend startup grace" "${output}" 'BACKEND_HEALTH_STARTUP_GRACE_SECONDS="${HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS:-180}"' + assert_contains "hot-dev health monitor tracks backend restart time through a shared marker" "${output}" 'BACKEND_STARTED_AT_FILE="${HOT_DEV_BACKEND_STARTED_AT_FILE:-${ROOT_DIR}/tmp/hot-dev.backend.started-at}"' + assert_contains "hot-dev managed restarts mark backend startup grace before killing" "${output}" 'mark_backend_startup_grace' + assert_contains "hot-dev health monitor checks startup grace before killing a live backend" "${output}" 'backend_in_startup_grace' assert_contains "hot-dev health monitor probes /api/health on the dev backend port" "${output}" '"http://127.0.0.1:${PULSE_DEV_API_PORT:-7655}/api/health"' assert_contains "hot-dev health monitor restarts on alive-but-unresponsive state" "${output}" 'elif ! backend_serving; then' assert_contains "hot-dev health monitor kills unresponsive Pulse before restart" "${output}" 'Killing unresponsive Pulse and restarting' @@ -698,6 +704,7 @@ test_hot_dev_script_marks_managed_rebuild_output_before_build() { assert_contains "hot-dev rebuild path suppresses self-build binary churn" "${rebuild_block}" 'mark_self_build_output' assert_contains "hot-dev rebuild path raises the managed build lock" "${rebuild_block}" 'set_build_lock' + assert_contains "hot-dev rebuild path skips duplicate watcher rebuilds" "${rebuild_block}" 'if build_lock_active; then' assert_contains "hot-dev rebuild path clears the managed build lock" "${rebuild_block}" 'clear_build_lock' mark_line="$(printf '%s\n' "${rebuild_block}" | awk '/mark_self_build_output/ { print NR; exit }')"