fix: log every duration as integer milliseconds under a _ms key (#827)

slog renders a bare time.Duration as an int64 nanosecond count and names
the unit nowhere, so every duration field in the fleet read like
"elapsed":135000000000. The repo idiom of .Round(time.Second) changed
the value without changing the shape and truncated every sub-second
measurement to zero.

otelsetup.Millis converts a duration for logging under a snake_case key
ending in _ms, and TestSlogDurationConvention type-checks ./spinifex/...
and ./cmd/... to fail the build when a raw Duration reaches an slog call
or a Millis value is logged under a key without the suffix. It found 55
sites across 34 files, well past the sites a grep for time.Since shows,
because most durations come from a config field or a package constant.

A handler-level ReplaceAttr would fix every site at once but can only
emit d.String(), which Elasticsearch maps as a keyword and nothing can
aggregate over.
This commit is contained in:
Josh Siviour
2026-08-18 20:38:13 +10:00
committed by GitHub
parent a4a644c558
commit ea4e7e3fb0
39 changed files with 271 additions and 61 deletions
+3 -2
View File
@@ -15,6 +15,7 @@ import (
"github.com/mulgadc/spinifex/internal/gwsign"
"github.com/mulgadc/spinifex/internal/rdsgw"
handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
"github.com/mulgadc/spinifex/spinifex/otelsetup"
)
// Overridable via -ldflags "-X main.version=...".
@@ -406,10 +407,10 @@ func retryObserved(ctx context.Context, what string, fn func(context.Context) er
}
if attempt >= retryErrorAttempt {
slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
"attempt", attempt, "retryIn", delay, "err", err)
"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
} else {
slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
"attempt", attempt, "retryIn", delay, "err", err)
"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
}
select {
+2 -1
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"github.com/mulgadc/spinifex/spinifex/otelsetup"
"log/slog"
"time"
@@ -66,7 +67,7 @@ func (g *paramGuard) Run(ctx context.Context) {
}
message := handlers_rds.ParameterRollbackMessage
slog.Error("rds-agent: "+message, "after", g.after)
slog.Error("rds-agent: "+message, "after_ms", otelsetup.Millis(g.after))
if g.cp != nil {
if _, err := g.cp.SubmitState(ctx, g.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
slog.Error("rds-agent: reporting the parameter rollback failed", "err", err)
+2 -1
View File
@@ -6,6 +6,7 @@ import (
"context"
"errors"
"fmt"
"github.com/mulgadc/spinifex/spinifex/otelsetup"
"io"
"log/slog"
"os/exec"
@@ -72,7 +73,7 @@ func (q *quiesceState) beginHoldLocked(label string, session engineSession, hold
session: session,
expiry: time.AfterFunc(hold, func() { q.expire(label) }),
}
slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold_ms", otelsetup.Millis(hold))
}
// Takes the hold off the engine, if one is still there, and disarms its
+2 -1
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"github.com/mulgadc/spinifex/spinifex/otelsetup"
"log/slog"
"time"
)
@@ -20,7 +21,7 @@ func (a *Agent) register(ctx context.Context) error {
a.hb.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
slog.Info("rds-agent: registered",
"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat", a.hb.interval)
"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat_ms", otelsetup.Millis(a.hb.interval))
return nil
})
}
+2 -1
View File
@@ -12,6 +12,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/daemon"
"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
"github.com/mulgadc/spinifex/spinifex/otelsetup"
"github.com/mulgadc/spinifex/spinifex/types"
"github.com/nats-io/nats.go"
"github.com/spf13/cobra"
@@ -333,7 +334,7 @@ func runNodeDrainLocal(cmd *cobra.Command, args []string) {
// `systemctl --failed`) without blocking the target's teardown;
// exiting 0 would hide guests left running as storage vanishes.
slog.Error("drain phase failed: no ACK from local daemon before timeout; guests may be left running while storage is about to be torn down",
"phase", phase, "node", node, "timeout", timeout, "error", err)
"phase", phase, "node", node, "timeout_ms", otelsetup.Millis(timeout), "error", err)
reportGuestsLeftRunning(true, fmt.Sprintf("[%s] drain failed: guests left running with storage about to be torn down", strings.ToUpper(phase)))
os.Exit(1)
}