ci: isolate s3 test runner state (#3180)

2026-07-26 08:18:18 +00:00 · 2026-06-02 22:40:37 +08:00
parent a55ead42e7
commit 7cc730d9c0
6 changed files with 71 additions and 73 deletions
@@ -11,8 +11,8 @@
 # host set for RustFS - will be substituted via envsubst
 host = ${S3_HOST}

-# port for RustFS
-port = 9000
+# port for RustFS - will be substituted via envsubst
+port = ${S3_PORT}

 ## say "False" to disable TLS
 is_secure = False
@@ -182,11 +182,6 @@ jobs:
      - name: Checkout repository
        uses: actions/checkout@v6

-      - name: Clean up previous test run
-        run: |
-          rm -rf /tmp/rustfs
-          rm -f /tmp/rustfs.log
-
      - name: Download debug binary
        uses: actions/download-artifact@v7
        with:
@@ -209,14 +204,19 @@ jobs:
      - name: Run end-to-end tests
        run: |
          s3s-e2e --version
-          ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
+          RUN_ROOT="${RUNNER_TEMP}/rustfs-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
+          mkdir -p "${RUN_ROOT}"
+          RUSTFS_TEST_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
+          RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT}" \
+          RUSTFS_TEST_LOG="${RUN_ROOT}/rustfs.log" \
+          ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"

      - name: Upload test logs
        if: failure()
        uses: actions/upload-artifact@v6
        with:
          name: e2e-test-logs-${{ github.run_number }}
-          path: /tmp/rustfs.log
+          path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
          retention-days: 3

  s3-implemented-tests:
@@ -240,10 +240,16 @@ jobs:

      - name: Run implemented s3-tests
        run: |
+          RUN_ROOT="${RUNNER_TEMP}/rustfs-s3tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
+          mkdir -p "${RUN_ROOT}"
+          S3_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
          DEPLOY_MODE=binary \
          RUSTFS_BINARY=./target/debug/rustfs \
          TEST_MODE=single \
          MAXFAIL=1 \
+          S3_PORT="${S3_PORT}" \
+          DATA_ROOT="${RUN_ROOT}" \
+          S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
          ./scripts/s3-tests/run.sh

      - name: Upload s3 test artifacts
@@ -16,24 +16,36 @@ set -ex

 BIN=$1
 VOLUME=$2
+RUSTFS_PID=""
+RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT:-9000}"
+RUSTFS_TEST_LOG="${RUSTFS_TEST_LOG:-${VOLUME}.log}"

-chmod +x $BIN
-mkdir -p $VOLUME
+cleanup() {
+    if [ -n "${RUSTFS_PID}" ] && kill -0 "${RUSTFS_PID}" 2>/dev/null; then
+        kill "${RUSTFS_PID}" 2>/dev/null || true
+        wait "${RUSTFS_PID}" 2>/dev/null || true
+    fi
+}
+
+trap cleanup EXIT INT TERM
+
+chmod +x "$BIN"
+mkdir -p "$VOLUME"
+mkdir -p "$(dirname "${RUSTFS_TEST_LOG}")"

 export RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
 export RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
 export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug"
 export RUST_BACKTRACE=full
-$BIN $VOLUME > /tmp/rustfs.log 2>&1 &
+"$BIN" --address "127.0.0.1:${RUSTFS_TEST_PORT}" "$VOLUME" > "${RUSTFS_TEST_LOG}" 2>&1 &
+RUSTFS_PID=$!

 sleep 10

 export AWS_ACCESS_KEY_ID="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
 export AWS_SECRET_ACCESS_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
 export AWS_REGION=us-east-1
-export AWS_ENDPOINT_URL=http://localhost:9000
+export AWS_ENDPOINT_URL="http://localhost:${RUSTFS_TEST_PORT}"
 export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug"
 export RUST_BACKTRACE=full
 s3s-e2e
-
-killall $BIN
@@ -335,17 +335,17 @@ The test script automatically cleans up processes and containers when it exits.
 A dedicated cleanup script is available to clean up test resources:

 ```bash
-# Clean up port 9000 and test data directory
+# Report port 9000 usage and clean the test data directory
 ./scripts/s3-tests/cleanup.sh

-# Use custom port and host
-S3_PORT=9001 S3_HOST=127.0.0.1 ./scripts/s3-tests/cleanup.sh
+# Use custom port
+S3_PORT=9001 ./scripts/s3-tests/cleanup.sh
 ```

 The cleanup script will:
- Kill any process using the specified port (default: 9000)
+- Report any process using the specified port (default: 9000)
 - Clean test data directory at `target/test-data/rustfs-single/`
- Verify port is released
+- Leave unrelated processes running

 ### Manual Cleanup

@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-# Cleanup script for port 9000 and test data directory
-# This script kills any process using port 9000 and cleans the test data directory
+# Cleanup script for the local s3-tests data directory.
+# It reports port usage but does not kill unrelated processes.

 set -euo pipefail

@@ -22,7 +22,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
 DATA_DIR="${PROJECT_ROOT}/target/test-data/rustfs-single"
 PORT="${S3_PORT:-9000}"
-HOST="${S3_HOST:-127.0.0.1}"

 log_info() {
    echo "[INFO] $*"
@@ -32,17 +31,16 @@ log_warn() {
    echo "[WARN] $*"
 }

-cleanup_port_9000() {
+check_port_usage() {
    log_info "Checking for processes using port ${PORT}..."
-    
-    # Try to find process using port 9000
+
    PID=""
-    
+
    # Try lsof first (works on macOS and Linux)
    if command -v lsof >/dev/null 2>&1; then
        PID=$(lsof -ti:${PORT} 2>/dev/null || true)
    fi
-    
+
    # Fallback: try using netstat or ss
    if [ -z "$PID" ]; then
        if command -v netstat >/dev/null 2>&1; then
@@ -51,49 +49,13 @@ cleanup_port_9000() {
            PID=$(ss -tuln 2>/dev/null | grep ":${PORT} " | awk '{print $6}' | cut -d',' -f2 | cut -d'=' -f2 | head -1 || true)
        fi
    fi
-    
-    # If we found a PID, kill it
+
    if [ -n "$PID" ] && [ "$PID" != "-" ]; then
-        log_info "Found process ${PID} using port ${PORT}, killing it..."
-        kill -9 "$PID" 2>/dev/null || true
-        sleep 2
-        
-        # Verify the process is gone
-        if kill -0 "$PID" 2>/dev/null; then
-            log_warn "Process ${PID} is still running, trying force kill..."
-            kill -9 "$PID" 2>/dev/null || true
-            sleep 1
-        fi
+        log_warn "Port ${PORT} is in use by PID(s): ${PID}"
+        log_warn "Not killing unrelated processes; stop the owner manually or use another S3_PORT."
    else
        log_info "No process found using port ${PORT}"
    fi
-    
-    # Also try to kill any rustfs processes (more aggressive cleanup)
-    if pgrep -f "rustfs.*${PORT}" >/dev/null 2>&1; then
-        log_info "Killing any remaining rustfs processes using port ${PORT}..."
-        pkill -f "rustfs.*${PORT}" 2>/dev/null || true
-        sleep 1
-    fi
-    
-    # Verify port is released
-    log_info "Verifying port ${PORT} is released..."
-    for i in {1..10}; do
-        if command -v nc >/dev/null 2>&1; then
-            if ! nc -z "${HOST}" "${PORT}" 2>/dev/null; then
-                log_info "Port ${PORT} is now available"
-                break
-            fi
-        elif timeout 1 bash -c "cat < /dev/null > /dev/tcp/${HOST}/${PORT}" 2>/dev/null; then
-            # Port is still in use
-            if [ $i -eq 10 ]; then
-                log_warn "Port ${PORT} may still be in use after cleanup attempts"
-            fi
-        else
-            log_info "Port ${PORT} is now available"
-            break
-        fi
-        sleep 1
-    done
 }

 cleanup_data_directory() {
@@ -120,7 +82,7 @@ cleanup_data_directory() {

 main() {
    log_info "Starting cleanup..."
-    cleanup_port_9000
+    check_port_usage
    cleanup_data_directory
    log_info "Cleanup completed"
 }
@@ -196,7 +196,12 @@ ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts/s3tests-${TEST_MODE}"
 CONTAINER_NAME="rustfs-${TEST_MODE}"
 NETWORK_NAME="rustfs-net"
 DATA_ROOT="${DATA_ROOT:-target}"
-DATA_DIR="${PROJECT_ROOT}/${DATA_ROOT}/test-data/${CONTAINER_NAME}"
+if [[ "${DATA_ROOT}" = /* ]]; then
+    DATA_BASE="${DATA_ROOT}"
+else
+    DATA_BASE="${PROJECT_ROOT}/${DATA_ROOT}"
+fi
+DATA_DIR="${DATA_BASE}/test-data/${CONTAINER_NAME}"
 RUSTFS_PID=""

 show_usage() {
@@ -655,8 +660,16 @@ log_info "Generating s3tests config..."
 mkdir -p "${ARTIFACTS_DIR}"

 # Resolve template and output paths (relative to PROJECT_ROOT)
-TEMPLATE_PATH="${PROJECT_ROOT}/${S3TESTS_CONF_TEMPLATE}"
-CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}"
+if [[ "${S3TESTS_CONF_TEMPLATE}" = /* ]]; then
+    TEMPLATE_PATH="${S3TESTS_CONF_TEMPLATE}"
+else
+    TEMPLATE_PATH="${PROJECT_ROOT}/${S3TESTS_CONF_TEMPLATE}"
+fi
+if [[ "${S3TESTS_CONF}" = /* ]]; then
+    CONF_OUTPUT_PATH="${S3TESTS_CONF}"
+else
+    CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}"
+fi

 # Check if template exists
 if [ ! -f "${TEMPLATE_PATH}" ]; then
@@ -681,9 +694,10 @@ fi

 log_info "Using template: ${TEMPLATE_PATH}"
 log_info "Generating config: ${CONF_OUTPUT_PATH}"
+mkdir -p "$(dirname "${CONF_OUTPUT_PATH}")"

 # Export all required variables for envsubst
-export S3_HOST S3_ACCESS_KEY S3_SECRET_KEY S3_ALT_ACCESS_KEY S3_ALT_SECRET_KEY
+export S3_HOST S3_PORT S3_ACCESS_KEY S3_SECRET_KEY S3_ALT_ACCESS_KEY S3_ALT_SECRET_KEY
 envsubst < "${TEMPLATE_PATH}" > "${CONF_OUTPUT_PATH}" || {
    log_error "Failed to generate s3tests config"
    exit 1
@@ -814,7 +828,11 @@ if [ "${XDIST}" != "0" ]; then
 fi

 # Resolve config path (absolute path for tox)
-CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}"
+if [[ "${S3TESTS_CONF}" = /* ]]; then
+    CONF_OUTPUT_PATH="${S3TESTS_CONF}"
+else
+    CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}"
+fi

 PYTEST_SELECTION_ARGS=()
 if [[ "${USE_FILE_TEST_NODES}" == "true" ]]; then