feat(ci): stamp per-cell scoping labels on e2e telemetry (#564)

The nightly matrix runs every cell under a single ci.run_id, and each cell
names its hosts node1/node2/node3, so telemetry from all cells collapses into
one undifferentiated bucket in the ES sink. Root-causing a single failed cell
means aggregating across every other cell in the run.

Stamp ci.cell and ci.matrix alongside the existing ci.run_id. Workflow-level
env cannot reference the matrix context, so the attrs are overridden in each
matrixed job's own env block, leaving the workflow-level value as the
non-matrix fallback.

The numeric GitHub job id is not exposed in any context (github.job is the
YAML key, not the id), so resolve it best-effort via the jobs API and append
it if found. The lookup never fails the job.

e2e-suites can only distinguish single from multi: daemon telemetry is baked
into the VM at provision time, before the suite matrix exists.

Baremetal shipped no telemetry at all — neither the endpoint nor the attrs
crossed the workflow_call boundary. Pass them through as an explicit input so
the cell reaches the sink for the first time.

.github/workflows/e2e-baremetal.yml

@@ -9,6 +9,11 @@ on:
         required: false
         type: string
         default: ""
+      otel-attrs:
+        description: "OTEL_RESOURCE_ATTRIBUTES value for this run, e.g. ci.run_id=123,ci.cell=20 (caller-specific, so it's an input rather than a workflow-level default)"
+        required: false
+        type: string
+        default: ""
 
 permissions:
   contents: read
@@ -16,6 +21,10 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  # Same central ES/APM sink as the other four e2e workflows. Bare-metal had
+  # no default here before — this cell's telemetry never reached the sink at
+  # all (not just missing ci.cell), see bm-bootstrap.sh's telemetry.env step.
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
 
 jobs:
   e2e_baremetal:
@@ -28,6 +37,9 @@ jobs:
     env:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
+      # Caller-supplied (workflow_call input); empty for a manual
+      # workflow_dispatch run, which bm-bootstrap.sh treats as "no attrs".
+      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs }}
     steps:
       - name: Clean Workspace
         run: |

.github/workflows/e2e-nightly.yml

@@ -11,6 +11,10 @@ on:
 
 permissions:
   contents: read
+  # actions: read lets the "Resolve CI job id" step below look up this job's
+  # numeric id via the Actions API — github.job is only the YAML key, no
+  # context exposes the id itself.
+  actions: read
 
 # Scheduled runs are loaded from the default branch (main), so the schedule
 # trigger must stay in sync on main for cron to fire.
@@ -46,6 +50,11 @@ jobs:
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
+      # Job-level env wins over the workflow-level default above, so telemetry
+      # from this cell carries ci.cell/ci.matrix. Nine cells run under one
+      # run id and every cell names its hosts node1/node2/node3, so without
+      # this every cell's telemetry collapses into one unattributable bucket.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
 
     steps:
       # blank or scheduled run = full matrix.
@@ -64,6 +73,29 @@ jobs:
             echo "cell ${CELL} not in '${CELLS}' — subsequent steps will skip"
           fi
 
+      # Best-effort: github.job is this job's YAML key ("matrix"), not its
+      # numeric id, and no context exposes the id directly, so look it up by
+      # rendered job name via the Actions API before Bootstrap bakes
+      # SPINIFEX_OTEL_ATTRS into the VM's telemetry.env. Must never fail the
+      # job — on any lookup error or name mismatch, ci.job_id is simply
+      # omitted and the run proceeds with ci.cell alone.
+      - name: Resolve CI job id (best-effort)
+        if: steps.gate.outputs.selected == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          JOB_NAME: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
+        run: |
+          set +e
+          JOB_ID=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" --paginate \
+            --jq ".jobs[] | select(.name == \"${JOB_NAME}\") | .id" 2>/dev/null | head -n1)
+          if [ -n "$JOB_ID" ]; then
+            echo "SPINIFEX_OTEL_ATTRS=${SPINIFEX_OTEL_ATTRS},ci.job_id=${JOB_ID}" >> "$GITHUB_ENV"
+            echo "resolved ci.job_id=${JOB_ID} for job '${JOB_NAME}'"
+          else
+            echo "::warning::could not resolve numeric job id for '${JOB_NAME}' — continuing without ci.job_id"
+          fi
+          exit 0
+
       - name: Clean Workspace
         if: steps.gate.outputs.selected == 'true'
         run: |
@@ -373,4 +405,8 @@ jobs:
     uses: ./.github/workflows/e2e-baremetal.yml
     with:
       ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      # A reusable-workflow call gets none of this workflow's context (env:
+      # above, matrix.cell) for free — cell-20 needs its own attrs string.
+      # There's no matrix leg here, just the one fixed baremetal cell.
+      otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
     secrets: inherit

.github/workflows/e2e-suites.yml

@@ -64,6 +64,13 @@ jobs:
       matrix: ${{ steps.suites.outputs.matrix }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
+      # This job (not single_suite) is where "Bootstrap via Binary Install"
+      # bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
+      # suite matrix even exists — single_suite's legs share this one
+      # already-booted VM, so daemon telemetry can't be scoped any finer
+      # than "single env" per suite leg. ci.cell=single distinguishes it
+      # from multi_provision's VMs within the same run id.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -499,6 +506,9 @@ jobs:
       nodes_csv: ${{ steps.cluster.outputs.nodes_csv }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
+      # See single_provision above — daemon telemetry is baked in here, at
+      # provision time, before the multi_suite matrix exists.
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=multi
     steps:
       - name: Reap Orphan Procs
         run: |