From 92c17af8e37a0608c97ffad14ac051e4d0d10167 Mon Sep 17 00:00:00 2001 From: RustFS Date: Sun, 6 Sep 2026 14:13:21 +0800 Subject: [PATCH] ci(e2e): run distributed e2e on ubuntu-latest (#7253) --- .config/e2e-distributed-selection.txt | 2 +- .github/workflows/e2e-distributed.yml | 32 ++++++++++++++++++--------- docs/testing/distributed-e2e.md | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.config/e2e-distributed-selection.txt b/.config/e2e-distributed-selection.txt index 7291d5e03..d99f12976 100644 --- a/.config/e2e-distributed-selection.txt +++ b/.config/e2e-distributed-selection.txt @@ -1,2 +1,2 @@ -sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07 +sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2 sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07 diff --git a/.github/workflows/e2e-distributed.yml b/.github/workflows/e2e-distributed.yml index c3a5ca0e1..091d15c98 100644 --- a/.github/workflows/e2e-distributed.yml +++ b/.github/workflows/e2e-distributed.yml @@ -22,11 +22,13 @@ # Upgrade cases download the same pinned previous release as e2e-upgrade.yml. # # Isolated pool filesystems: expand/decommission/rebalance cases require -# independent `statfs` capacity. `sm-standard-4` is an ARC pod -# (`scripts/ci/check_runner_ephemerality.sh`) and usually has no -# `/dev/loop-control`, so `mount -o loop` fails with ENOENT ("mount failed: -# No such file or directory"). The prepare step therefore mounts four 1 GiB -# tmpfs instances and exports them as `RUSTFS_E2E_POOL_ROOTS`. +# independent `statfs` capacity. This job runs on GitHub-hosted +# `ubuntu-latest` because the self-hosted `sm-standard-4` ARC pods cannot +# create filesystems: `mount -o loop` fails with ENOENT (no +# `/dev/loop-control`), and `mount -t tmpfs` fails with "cannot mount tmpfs +# read-only" (no `CAP_SYS_ADMIN` in the initial namespace). The same reason +# `uring-integration` and `e2e-s3tests.yml` left that label. The prepare +# step mounts four 1 GiB tmpfs instances and exports `RUSTFS_E2E_POOL_ROOTS`. name: e2e-distributed @@ -76,7 +78,9 @@ concurrency: jobs: distributed: name: Distributed 4-node 4-disk e2e - runs-on: sm-standard-4 + # GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an + # ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only"). + runs-on: ubuntu-latest timeout-minutes: 180 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" @@ -97,7 +101,9 @@ jobs: uses: ./.github/actions/setup with: rust-version: stable - cache-shared-key: ci-e2e-distributed + # Dedicated key: ubuntu-latest and sm-standard-4 share runner.os, so + # a shared key would mix VM and ARC pod target/ artifacts. + cache-shared-key: ci-e2e-distributed-hosted cache-save-if: ${{ github.ref == 'refs/heads/main' }} install-build-packaging-tools: 'false' @@ -110,10 +116,14 @@ jobs: for pool in 0 1 2 3; do mountpoint="${mount_base}/pool-${pool}" mkdir -p "${mountpoint}" - # sm-standard-4 is an ARC pod without usable loop devices, so - # `mount -o loop` fails with ENOENT. Sized tmpfs still reports a - # distinct st_dev and independent 1G statfs capacity. - sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}" + # Sized tmpfs reports a distinct st_dev and independent 1G + # statfs capacity. Requires a VM runner (ubuntu-latest). + if ! sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"; then + echo "tmpfs mount failed on $(uname -a)" >&2 + findmnt || true + grep Cap /proc/self/status || true + exit 1 + fi sudo chmod 1777 "${mountpoint}" roots+=("${mountpoint}") done diff --git a/docs/testing/distributed-e2e.md b/docs/testing/distributed-e2e.md index 10d882f9b..5d997ddbf 100644 --- a/docs/testing/distributed-e2e.md +++ b/docs/testing/distributed-e2e.md @@ -17,7 +17,7 @@ A multi-pool layout in which any pool spans several localhost ports is not expre Data-movement cases fail closed. A decommission or rebalance test must observe a successful start response, an active state, a clean terminal state, non-zero movement counters, and post-operation object integrity. An unsupported response, HTTP 5xx, missing status fields, cleanup warning, or zero-progress terminal response fails the case; pre/post S3 availability alone is not evidence that movement ran. -The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated 1 GiB tmpfs filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use ext4 loop devices: the `sm-standard-4` ARC pods have no `/dev/loop-control`, so `mount -o loop` fails with `No such file or directory`. Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata. +The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job runs on GitHub-hosted `ubuntu-latest` and mounts four isolated 1 GiB tmpfs filesystems, then exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use the self-hosted `sm-standard-4` ARC pods: those cannot create filesystems (`mount -o loop` fails with `No such file or directory`, and `mount -t tmpfs` fails with `cannot mount tmpfs read-only`). Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata. The expansion fixture is an all-current-binary fleet, so it initializes pool metadata with the documented V3 write and fleet-confirmation gates. Decommission cases write their baseline objects, version history, and multipart data into pool 0 before adding pools 1–3, then retire pool 0. This makes a passing result evidence of user-data movement rather than merely an internal-metadata counter changing.