spinifex/.github/workflows/e2e.yml

name: Spinifex E2E Tests

on:
  push:
    branches:
      - main
    paths:
      - "**/*.go"
      - go.mod
      - go.sum
      - Makefile
      - tests/e2e/**
      - .github/workflows/e2e.yml
  workflow_dispatch:
    inputs:
      sha:
        description: "PR head SHA to report status on"
        required: false

permissions:
  statuses: write

env:
  GIT_BRANCH: ${{ github.ref_name }}

jobs:
  e2e_single:
    name: Single-Node E2E
    runs-on: [self-hosted, ci-single]
    timeout-minutes: 30
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
        run: rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v6
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - name: Build install artifacts
        working-directory: mulga/scripts/tofu-cluster
        run: |
          ./build-install-artifacts.sh "${GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Provision VM
        working-directory: mulga/scripts/tofu-cluster
        run: |
          export TF_VAR_git_branch="${GIT_BRANCH}"
          tofu init -input=false >> "$TOFU_LOG" 2>&1
          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
          tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1

      - name: Bootstrap via Binary Install
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_EXTERNAL_POOL="192.168.1.230-192.168.1.240" \
          ./bootstrap-install.sh env3 \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Run E2E Tests
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            'cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh  && ./run-lb-e2e.sh'

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true

  e2e_pseudo_multi:
    name: Pseudo Multi-Node E2E
    runs-on: [self-hosted, ci-pseudo]
    timeout-minutes: 30
    env:
      TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
        run: rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v6
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - name: Clone and build
        working-directory: mulga/scripts/tofu-cluster
        run: ./build-artifacts.sh "${GIT_BRANCH}" "${RUNNER_TEMP}/spinifex-artifacts.tar"

      - name: Provision VM
        working-directory: mulga/scripts/tofu-cluster
        run: |
          export TF_VAR_git_branch="${GIT_BRANCH}"
          tofu init -input=false >> "$TOFU_LOG" 2>&1
          tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
          ./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true
          tofu apply -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1

      - name: Bootstrap Cluster
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_PREBUILT=true \
          SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
          ./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1

      - name: Run E2E Tests
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            'cd ~/Development/mulga/spinifex && SPX_EXTERNAL_POOL="192.168.1.220-192.168.1.229" ./tests/e2e/run-pseudo-multinode-e2e.sh'

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true

  e2e_real_multi:
    name: Real Multi-Node E2E
    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 35
    env:
      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
    steps:
      - name: Clean Workspace
        run: rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v6
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - name: Build Install Artifacts
        working-directory: mulga/scripts/tofu-cluster
        run: |
          ./build-install-artifacts.sh "${GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Provision VMs
        working-directory: mulga/scripts/tofu-cluster
        run: |
          export TF_VAR_git_branch="${GIT_BRANCH}"
          tofu init -input=false >> "$TOFU_LOG" 2>&1
          tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
          ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
          tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1

      - name: Bootstrap via Binary Install
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_EXTERNAL_POOL="192.168.1.240-192.168.1.250" \
          ./bootstrap-install.sh env2 \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Run E2E Tests
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

          # Build node IP list
          NODE_IPS=""
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
          done

          NODE2_IP=$(echo "$INVENTORY" | jq -r '.nodes[1].wan_ip')

          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
            "cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true

  e2e_status:
    name: Report E2E Status
    if: always() && inputs.sha
    needs: [e2e_single, e2e_pseudo_multi, e2e_real_multi]
    runs-on: [self-hosted, ci-single]
    timeout-minutes: 5
    steps:
      - name: Report status to PR
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          if [[ "${{ needs.e2e_single.result }}" == "success" && \
                "${{ needs.e2e_pseudo_multi.result }}" == "success" && \
                "${{ needs.e2e_real_multi.result }}" == "success" ]]; then
            STATE="success"
            DESC="All E2E suites passed"
          else
            STATE="failure"
            DESC="E2E failed: single=${{ needs.e2e_single.result }} pseudo=${{ needs.e2e_pseudo_multi.result }} multi=${{ needs.e2e_real_multi.result }}"
          fi
          gh api "repos/${{ github.repository }}/statuses/${{ inputs.sha }}" \
            -f state="$STATE" \
            -f context=e2e \
            -f description="$DESC" \
            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"