spinifex/.github/workflows/e2e.yml

name: Spinifex E2E Tests

on:
  push:
    branches:
      - main
      - dev
    paths:
      - "**/*.go"
      - go.mod
      - go.sum
      - Makefile
      - tests/e2e/**
      - .github/workflows/e2e.yml
  workflow_dispatch:

env:
  GIT_BRANCH: ${{ github.ref_name }}

jobs:
  e2e_single:
    name: Single-Node E2E
    runs-on: [self-hosted, ci-single]
    timeout-minutes: 25
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
    steps:
      - name: Clean workspace
        run: rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          app-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v6
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - name: Clone and build
        working-directory: mulga/scripts/tofu-cluster
        run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"

      - name: Provision VM
        working-directory: mulga/scripts/tofu-cluster
        run: |
          export TF_VAR_git_branch="${GIT_BRANCH}"
          tofu init -input=false >> "$TOFU_LOG" 2>&1
          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
          tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1

      - name: Bootstrap Cluster
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_PREBUILT=true \
          SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
          ./bootstrap.sh env3 >> "$TOFU_LOG" 2>&1

      - name: Run E2E Tests
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh'

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true

  e2e_pseudo_multi:
    name: Pseudo Multi-Node E2E
    runs-on: [self-hosted, ci-pseudo]
    timeout-minutes: 25
    env:
      TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
    steps:
      - name: Clean workspace
        run: rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          app-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v6
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - name: Clone and build
        working-directory: mulga/scripts/tofu-cluster
        run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"

      - name: Provision VM
        working-directory: mulga/scripts/tofu-cluster
        run: |
          export TF_VAR_git_branch="${GIT_BRANCH}"
          tofu init -input=false >> "$TOFU_LOG" 2>&1
          tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
          ./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true
          tofu apply -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1

      - name: Bootstrap Cluster
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_PREBUILT=true \
          SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
          ./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1

      - name: Run E2E Tests
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-pseudo-multinode-e2e.sh'

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true

  e2e_real_multi:
    name: Real Multi-Node E2E
    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 25
    env:
      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
    steps:
      - name: Clean workspace
        run: rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          app-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v6
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - name: Clone and build
        working-directory: mulga/scripts/tofu-cluster
        run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"

      - name: Provision VMs
        working-directory: mulga/scripts/tofu-cluster
        run: |
          export TF_VAR_git_branch="${GIT_BRANCH}"
          tofu init -input=false >> "$TOFU_LOG" 2>&1
          tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
          ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
          tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1

      - name: Bootstrap Cluster
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_PREBUILT=true \
          SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
          ./bootstrap.sh env2 >> "$TOFU_LOG" 2>&1

      - name: Run E2E Tests
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

          # Build node IP list
          NODE_IPS=""
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
          done

          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
            "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS}"

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true