From 08fc5e4c58f35df77023770bef7946c6353ed807 Mon Sep 17 00:00:00 2001
From: Josh Siviour <mrk@mulgadc.com>
Date: Mon, 20 Jul 2026 13:22:27 +1000
Subject: [PATCH] fix(northstar): initialise OTel SDK and source the telemetry
 environment (#555)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Northstar carried working northstar.dns.* instruments but shipped no
telemetry, so its dashboard was empty and the only northstar data in
Elasticsearch was journald scraped under the systemd unit name.

Two gaps, both outside the instrumentation. northstarStartCmd never called
initTelemetry, unlike every other service command, so the instruments
recorded into a no-op provider. The unit also did not source
/etc/spinifex/telemetry.env, which every other application unit reads for
OTEL_EXPORTER_OTLP_ENDPOINT, MULGA_ENV and MULGA_SOURCE — so even an
initialised SDK would have had no endpoint and no env label.

Add an invariant covering the second gap: any unit launching an spx service
must source the telemetry drop-in. A service missing it starts healthy and
silently reports nothing, which only surfaces when someone notices an empty
dashboard. Verified the test fails against the unpatched unit.
---
 build/systemd/spinifex-northstar.service  |  1 +
 cmd/spinifex/cmd/service.go               |  3 ++
 spinifex/systemd/units_invariants_test.go | 34 +++++++++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/build/systemd/spinifex-northstar.service b/build/systemd/spinifex-northstar.service
index 3e80e82e7..ff43360a0 100644
--- a/build/systemd/spinifex-northstar.service
+++ b/build/systemd/spinifex-northstar.service
@@ -5,6 +5,7 @@ After=spinifex-nats.service spinifex-predastore.service
 Wants=spinifex-predastore.service
 
 [Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-northstar
diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go
index 6b85aae4c..76cfba5e0 100644
--- a/cmd/spinifex/cmd/service.go
+++ b/cmd/spinifex/cmd/service.go
@@ -885,6 +885,9 @@ var northstarStartCmd = &cobra.Command{
 	SilenceUsage:  true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		fmt.Println("Starting northstar service...")
+
+		defer initTelemetry("northstar", false)()
+
 		return runNorthstarStart(northstarStartOptions{
 			configFile:      viper.GetString("config"),
 			configOverride:  viper.GetString("northstar-config"),
diff --git a/spinifex/systemd/units_invariants_test.go b/spinifex/systemd/units_invariants_test.go
index e30411164..e373fe039 100644
--- a/spinifex/systemd/units_invariants_test.go
+++ b/spinifex/systemd/units_invariants_test.go
@@ -261,3 +261,37 @@ func TestRG11_LeanUnits(t *testing.T) {
 		}
 	}
 }
+
+// TestApplicationUnitsExportTelemetry asserts every unit that runs an spx
+// application service sources the telemetry drop-in. That file carries
+// OTEL_EXPORTER_OTLP_ENDPOINT, MULGA_ENV and MULGA_SOURCE, so a unit missing it
+// starts a service whose instruments record into a no-op provider — the process
+// runs healthy and simply reports nothing, which is invisible until someone
+// notices an empty dashboard.
+//
+// Units that host an agent or a one-shot rather than an spx service are exempt:
+// they either export on their own (the collectors) or have nothing to export.
+func TestApplicationUnitsExportTelemetry(t *testing.T) {
+	dir := unitsDir(t)
+	const telemetry = "EnvironmentFile=-/etc/spinifex/telemetry.env"
+
+	exempt := []string{
+		"spinifex-nats-watchdog.service", // periodic health probe, no OTel SDK
+		"spinifex-shutdown.service",      // one-shot drain on halt
+		"regenerate-ssh-host-keys.service",
+	}
+
+	for _, name := range unitFiles(t, dir) {
+		if !strings.HasSuffix(name, ".service") || slices.Contains(exempt, name) {
+			continue
+		}
+		u := readUnit(t, dir, name)
+		// Only units that actually launch an spx service can emit telemetry.
+		if !strings.Contains(u, "/usr/local/bin/spx service ") {
+			continue
+		}
+		if !hasDirective(u, telemetry) {
+			t.Errorf("%s runs an spx service but does not source %s; its telemetry would be silently dropped", name, telemetry)
+		}
+	}
+}