fix(otel): change log level in place so the gateway keeps shipping to the sink (#616)

Init bolts the OTLP bridge onto whatever slog default is installed at the
time. SetupRoutes then called SetDefaultJSONLogger purely to set a level,
and that bare slog.SetDefault replaced the fanout with a stdout-only
handler, so the gateway went dark to the sink for the rest of the process.
One nightly cell shipped 9 app-log docs, all within 30ms of startup, while
its journal held 574 request lines.

Backs the stdout handler with a LevelVar and adds SetLevel, so verbosity
changes no longer touch the handler chain.

spinifex/gateway/gateway.go

@@ -171,7 +171,9 @@ func (gw *GatewayConfig) SetupRoutes() http.Handler {
 		logLevel = slog.LevelInfo
 	}
 
-	otelsetup.SetDefaultJSONLogger(logLevel)
+	// Adjust the level only. Reinstalling the default logger here would drop the OTLP
+	// bridge Init fanned on at startup, blinding the sink to every line after this.
+	otelsetup.SetLevel(logLevel)
 
 	if gw.RateLimiter == nil {
 		gw.RateLimiter = NewAuthRateLimiter()

spinifex/otelsetup/slog.go

@@ -9,14 +9,28 @@ import (
 	"go.opentelemetry.io/otel/trace"
 )
 
+// defaultLevel backs the stdout handler's level. Held in a LevelVar so verbosity can
+// change without reinstalling the handler, which would discard the OTLP bridge Init
+// fans onto whatever default is installed at the time.
+var defaultLevel = new(slog.LevelVar)
+
 // SetDefaultJSONLogger installs the process-wide slog default: JSON to
 // stdout (journald) at the given level, with trace_id/span_id stamping.
+// Call once, before Init; use SetLevel to change verbosity afterwards.
 func SetDefaultJSONLogger(level slog.Level) {
+	defaultLevel.Set(level)
 	slog.SetDefault(slog.New(NewSlogHandler(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
-		Level: level,
+		Level: defaultLevel,
 	}))))
 }
 
+// SetLevel changes the stdout handler's level in place, leaving the handler chain
+// intact. Callers that only need to adjust verbosity must use this: calling
+// SetDefaultJSONLogger again replaces the default and silently unbolts OTLP export.
+func SetLevel(level slog.Level) {
+	defaultLevel.Set(level)
+}
+
 var _ slog.Handler = (*traceHandler)(nil)
 
 // traceHandler stamps trace_id/span_id from the record's context onto every

spinifex/otelsetup/slog_level_test.go

@@ -0,0 +1,66 @@
+package otelsetup
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"testing"
+)
+
+// recordingHandler stands in for the OTLP bridge: it only needs to prove it still
+// receives records after a level change.
+type recordingHandler struct{ msgs *[]string }
+
+func (recordingHandler) Enabled(context.Context, slog.Level) bool { return true }
+func (h recordingHandler) Handle(_ context.Context, r slog.Record) error {
+	*h.msgs = append(*h.msgs, r.Message)
+	return nil
+}
+func (h recordingHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
+func (h recordingHandler) WithGroup(string) slog.Handler      { return h }
+
+// A gateway that reinstalled the default logger to change verbosity silently detached
+// the OTLP bridge, so only startup lines ever reached the sink.
+func TestSetLevelKeepsFanoutAttached(t *testing.T) {
+	prev := slog.Default()
+	t.Cleanup(func() { slog.SetDefault(prev) })
+
+	var got []string
+	SetDefaultJSONLogger(slog.LevelInfo)
+	addFanoutHandler(recordingHandler{msgs: &got})
+
+	SetLevel(slog.LevelDebug)
+	slog.Info("after level change")
+
+	if len(got) != 1 || got[0] != "after level change" {
+		t.Fatalf("fanout handler lost after SetLevel, got %v", got)
+	}
+
+	// The hazard SetLevel exists to avoid: reinstalling the default drops the fanout,
+	// so nothing logged afterwards reaches the bridge.
+	SetDefaultJSONLogger(slog.LevelInfo)
+	slog.Info("after reinstall")
+	if len(got) != 1 {
+		t.Fatalf("expected reinstall to detach the fanout, got %v", got)
+	}
+}
+
+func TestSetLevelChangesVerbosityInPlace(t *testing.T) {
+	prev := slog.Default()
+	t.Cleanup(func() { slog.SetDefault(prev) })
+
+	var buf bytes.Buffer
+	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: defaultLevel})))
+
+	SetLevel(slog.LevelError)
+	slog.Info("suppressed")
+	if buf.Len() != 0 {
+		t.Fatalf("info logged at error level: %s", buf.String())
+	}
+
+	SetLevel(slog.LevelDebug)
+	slog.Debug("emitted")
+	if !bytes.Contains(buf.Bytes(), []byte("emitted")) {
+		t.Fatalf("debug not logged after level lowered: %s", buf.String())
+	}
+}