gsadmin/pulse
1
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-09-10 02:25:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Stabilize hot-dev backend startup recovery

Browse Source
This commit is contained in:
rcourtman
2026-05-17 09:54:11 +01:00
parent 829d43a0d2
commit 702fc5aeb5
3 changed files with 77 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/release-control/v6/internal/subsystems/deployment-installability.md
+7 -2
View File
@@ -935,7 +935,9 @@ That same dev-runtime orchestration boundary also owns watcher stability for
the managed local stack: `scripts/hot-dev.sh` may only rebuild the backend for
runtime Go sources, not `*_test.go` churn, and it must suppress `pulse` binary
change events produced by its own successful managed rebuilds, managed backend
restarts, or startup build.
restarts, or startup build through shared watcher-state markers rather than
per-subshell timing alone. Parallel watcher streams must not start duplicate
managed rebuilds for the same backend artifact change.
That same boundary also owns backend-liveness recovery, not just process-
existence. The managed health monitor in `scripts/hot-dev.sh` must probe
`http://127.0.0.1:${PULSE_DEV_API_PORT}/api/health` in addition to checking
@@ -943,7 +945,10 @@ that a `./pulse` process exists, so an alive-but-unresponsive backend (hung
goroutine, panic-recovery loop, port-bind failure with the process still
running) is detected and restarted instead of leaving the dev frontend
talking to a dead listener. Two consecutive missed health probes must trigger
a managed kill and restart of the unresponsive process.
a managed kill and restart of the unresponsive process only after the managed
backend startup/restart grace has elapsed; the monitor must not kill a backend
merely because the server has bound its listener before the HTTP health route
is ready.
That same watcher boundary also owns backend-served demo coherence:
`internal/api/frontend-modern/dist` changes must trigger a managed backend
rebuild so the `go:embed` frontend on `:7655` cannot drift behind a freshly
scripts/hot-dev.sh
+61 -4
View File
@@ -17,6 +17,10 @@
# PULSE_DATA_DIR=/path Override data directory
# PULSE_DEV_API_PORT=7655 Backend API port (default: 7655)
# FRONTEND_DEV_PORT=5173 Frontend dev server port (default: 5173)
# HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS=180
# Backend /api/health grace after starts/restarts
# HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD=2
# Consecutive failed /api/health probes before restart
#
# Pro Features Mode:
# When pulse-enterprise repo exists and HOT_DEV_USE_PRO is not "false",
@@ -243,6 +247,7 @@ EXTRA_CLEANUP_PORT=$((PULSE_DEV_API_PORT + 1))
HOT_DEV_RESTART_SENTINEL="${ROOT_DIR}/tmp/hot-dev.restart"
HOT_DEV_VERIFY_LOCK="${HOT_DEV_VERIFY_LOCK_FILE:-${ROOT_DIR}/tmp/hot-dev.verify.lock}"
HOT_DEV_BUILD_LOCK="${HOT_DEV_BUILD_LOCK_FILE:-${ROOT_DIR}/tmp/hot-dev.build.lock}"
HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE="${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE:-${ROOT_DIR}/tmp/hot-dev.self-build-ignore-until}"
HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS="${HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS:-5}"
EMBEDDED_FRONTEND_DIR="${ROOT_DIR}/internal/api/frontend-modern"
EMBEDDED_FRONTEND_DIST_DIR="${EMBEDDED_FRONTEND_DIR}/dist"
@@ -326,6 +331,8 @@ kill_port "${EXTRA_CLEANUP_PORT}"
# Truncate debug log
mkdir -p "$(dirname "${BACKEND_DEBUG_LOG}")"
:> "${BACKEND_DEBUG_LOG}"
BACKEND_STARTED_AT_FILE="${HOT_DEV_BACKEND_STARTED_AT_FILE:-${ROOT_DIR}/tmp/hot-dev.backend.started-at}"
mkdir -p "$(dirname "${BACKEND_STARTED_AT_FILE}")"
sleep 2
@@ -491,7 +498,12 @@ if [[ ${PRO_BUILD_SUCCESS:-false} == "true" ]]; then
fi
STARTED_BACKEND_PID=""
mark_backend_startup_grace() {
date +%s > "${BACKEND_STARTED_AT_FILE}"
}
start_backend_process() {
mark_backend_startup_grace
LOG_LEVEL="${LOG_LEVEL:-debug}" \
FRONTEND_PORT="${PULSE_DEV_API_PORT:-7655}" \
PORT="${PULSE_DEV_API_PORT:-7655}" \
@@ -522,22 +534,44 @@ fi
# Restarts Pulse if it dies unexpectedly (not from file watcher rebuild),
# enforces single-instance, and detects the alive-but-unresponsive state
# (process exists but /api/health is not serving) which a process-only check
# blind-spots. Two consecutive 5s misses on /api/health -> kill and restart.
# blind-spots. After backend startup grace, consecutive misses on /api/health
# -> kill and restart.
log_info "Starting backend health monitor..."
(
UNHEALTHY_STREAK=0
UNHEALTHY_THRESHOLD=2
UNHEALTHY_THRESHOLD="${HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD:-2}"
BACKEND_HEALTH_STARTUP_GRACE_SECONDS="${HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS:-180}"
[[ "${UNHEALTHY_THRESHOLD}" =~ ^[1-9][0-9]*$ ]] || UNHEALTHY_THRESHOLD=2
[[ "${BACKEND_HEALTH_STARTUP_GRACE_SECONDS}" =~ ^[0-9]+$ ]] || BACKEND_HEALTH_STARTUP_GRACE_SECONDS=180
backend_serving() {
curl -sf -o /dev/null --max-time 3 \
"http://127.0.0.1:${PULSE_DEV_API_PORT:-7655}/api/health" 2>/dev/null
}
backend_in_startup_grace() {
local backend_started_at
local now
[[ -r "${BACKEND_STARTED_AT_FILE}" ]] || return 1
backend_started_at="$(<"${BACKEND_STARTED_AT_FILE}")"
[[ "${backend_started_at}" =~ ^[0-9]+$ ]] || return 1
now="$(date +%s)"
(( now - backend_started_at < BACKEND_HEALTH_STARTUP_GRACE_SECONDS ))
}
while true; do
sleep 5
PULSE_COUNT="$(hot_dev_pulse_process_count)"
if [[ "$PULSE_COUNT" -eq 0 ]]; then
if backend_in_startup_grace; then
UNHEALTHY_STREAK=0
log_warn "⚠️ Pulse process not running yet during backend startup grace (${BACKEND_HEALTH_STARTUP_GRACE_SECONDS}s)"
continue
fi
log_warn "⚠️ Pulse died unexpectedly, restarting..."
start_backend_process
NEW_PID="${STARTED_BACKEND_PID}"
@@ -562,6 +596,11 @@ log_info "Starting backend health monitor..."
fi
UNHEALTHY_STREAK=0
elif ! backend_serving; then
if backend_in_startup_grace; then
UNHEALTHY_STREAK=0
log_warn "⚠️ Pulse /api/health not serving yet during backend startup grace (${BACKEND_HEALTH_STARTUP_GRACE_SECONDS}s)"
continue
fi
UNHEALTHY_STREAK=$((UNHEALTHY_STREAK + 1))
log_warn "⚠️ Pulse alive but /api/health unresponsive (streak ${UNHEALTHY_STREAK}/${UNHEALTHY_THRESHOLD})"
if [[ "$UNHEALTHY_STREAK" -ge "$UNHEALTHY_THRESHOLD" ]]; then
@@ -607,15 +646,27 @@ log_info "Starting backend file watcher..."
local now
now=$(date +%s)
SELF_BUILD_IGNORE_UNTIL=$((now + 5))
mkdir -p "$(dirname "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}")"
printf '%s\n' "${SELF_BUILD_IGNORE_UNTIL}" > "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}"
}
manual_build_event_suppressed() {
local now
local shared_ignore_until
if build_lock_active; then
return 0
fi
local now
now=$(date +%s)
shared_ignore_until=""
if [[ -r "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}" ]]; then
shared_ignore_until="$(<"${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}")"
fi
if [[ "${shared_ignore_until}" =~ ^[0-9]+$ ]] && (( now < shared_ignore_until )); then
return 0
fi
(( now < SELF_BUILD_IGNORE_UNTIL ))
}
@@ -701,6 +752,7 @@ log_info "Starting backend file watcher..."
return
fi
LAST_RESTART_TIME=$now
mark_backend_startup_grace
mark_self_build_output
log_info "Restarting backend..."
@@ -740,6 +792,11 @@ log_info "Starting backend file watcher..."
rebuild_backend() {
local changed_file=$1
if build_lock_active; then
log_info "Managed build already in progress; skipping duplicate rebuild event."
return
fi
# Debounce: skip if we rebuilt less than 2 seconds ago (batch saves)
local now
now=$(date +%s)
@@ -760,8 +817,8 @@ log_info "Starting backend file watcher..."
# Use the same build logic as the initial build.
if build_backend_binary; then
mark_self_build_output
clear_build_lock
restart_backend
clear_build_lock
else
clear_build_lock
log_error "✗ Build failed; keeping the current backend process running."
scripts/tests/test-hot-dev-bg.sh
+9 -2
View File
@@ -651,7 +651,7 @@ test_makefile_routes_managed_runtime_through_npm() {
test_hot_dev_script_advertises_foreground_escape_hatch() {
local output
output="$(sed -n '1,30p' "${HOT_DEV}")"
output="$(sed -n '1,40p' "${HOT_DEV}")"
assert_contains "hot-dev header identifies foreground escape hatch" "${output}" "hot-dev.sh - Foreground Pulse dev runtime escape hatch"
assert_contains "hot-dev usage points to managed runtime first" "${output}" "npm run dev # Canonical managed dev runtime"
@@ -669,7 +669,9 @@ test_hot_dev_script_ignores_test_only_backend_churn() {
assert_contains "hot-dev fswatch covers the embedded frontend parent dir" "${output}" '"${ROOT_DIR}/pulse" "${HOT_DEV_RESTART_SENTINEL}" "${EMBEDDED_FRONTEND_DIR}"'
assert_contains "hot-dev fswatch only treats the pulse binary path as a manual build trigger" "${output}" 'elif [[ "$changed_file" == "${ROOT_DIR}/pulse" ]]; then'
assert_contains "hot-dev watcher declares a managed build lock path" "${output}" 'HOT_DEV_BUILD_LOCK="${HOT_DEV_BUILD_LOCK_FILE:-${ROOT_DIR}/tmp/hot-dev.build.lock}"'
assert_contains "hot-dev watcher declares shared self-build suppression marker" "${output}" 'HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE="${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE:-${ROOT_DIR}/tmp/hot-dev.self-build-ignore-until}"'
assert_contains "hot-dev watcher suppresses manual binary restarts while a managed build is active" "${output}" 'if build_lock_active; then'
assert_contains "hot-dev watcher shares self-build suppression across watcher streams" "${output}" 'printf '\''%s\n'\'' "${SELF_BUILD_IGNORE_UNTIL}" > "${HOT_DEV_SELF_BUILD_IGNORE_UNTIL_FILE}"'
assert_contains "hot-dev watcher suppresses self-build binary restart loops" "${output}" 'if manual_build_event_suppressed; then'
assert_contains "hot-dev watcher suppresses startup self-build pulse events" "${output}" 'SELF_BUILD_IGNORE_UNTIL=$((WATCHER_READY_AT + HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS + 5))'
assert_contains "hot-dev watcher seeds the startup pulse marker" "${output}" 'LAST_PULSE_BINARY_MARKER="$(file_event_marker "${ROOT_DIR}/pulse" || true)"'
@@ -680,7 +682,11 @@ test_hot_dev_health_monitor_probes_api_health() {
local output
output="$(cat "${HOT_DEV}")"
assert_contains "hot-dev health monitor declares an unhealthy streak threshold" "${output}" 'UNHEALTHY_THRESHOLD=2'
assert_contains "hot-dev health monitor declares a configurable unhealthy streak threshold" "${output}" 'UNHEALTHY_THRESHOLD="${HOT_DEV_BACKEND_UNHEALTHY_THRESHOLD:-2}"'
assert_contains "hot-dev health monitor declares backend startup grace" "${output}" 'BACKEND_HEALTH_STARTUP_GRACE_SECONDS="${HOT_DEV_BACKEND_HEALTH_STARTUP_GRACE_SECONDS:-180}"'
assert_contains "hot-dev health monitor tracks backend restart time through a shared marker" "${output}" 'BACKEND_STARTED_AT_FILE="${HOT_DEV_BACKEND_STARTED_AT_FILE:-${ROOT_DIR}/tmp/hot-dev.backend.started-at}"'
assert_contains "hot-dev managed restarts mark backend startup grace before killing" "${output}" 'mark_backend_startup_grace'
assert_contains "hot-dev health monitor checks startup grace before killing a live backend" "${output}" 'backend_in_startup_grace'
assert_contains "hot-dev health monitor probes /api/health on the dev backend port" "${output}" '"http://127.0.0.1:${PULSE_DEV_API_PORT:-7655}/api/health"'
assert_contains "hot-dev health monitor restarts on alive-but-unresponsive state" "${output}" 'elif ! backend_serving; then'
assert_contains "hot-dev health monitor kills unresponsive Pulse before restart" "${output}" 'Killing unresponsive Pulse and restarting'
@@ -698,6 +704,7 @@ test_hot_dev_script_marks_managed_rebuild_output_before_build() {
assert_contains "hot-dev rebuild path suppresses self-build binary churn" "${rebuild_block}" 'mark_self_build_output'
assert_contains "hot-dev rebuild path raises the managed build lock" "${rebuild_block}" 'set_build_lock'
assert_contains "hot-dev rebuild path skips duplicate watcher rebuilds" "${rebuild_block}" 'if build_lock_active; then'
assert_contains "hot-dev rebuild path clears the managed build lock" "${rebuild_block}" 'clear_build_lock'
mark_line="$(printf '%s\n' "${rebuild_block}" | awk '/mark_self_build_output/ { print NR; exit }')"