ci: isolate s3 test runner state (#3180)

.github/s3tests/s3tests.conf

@@ -11,8 +11,8 @@
 # host set for RustFS - will be substituted via envsubst
 host = ${S3_HOST}
 
-# port for RustFS
+# port for RustFS - will be substituted via envsubst
-port = 9000
+port = ${S3_PORT}
 
 ## say "False" to disable TLS
 is_secure = False

.github/workflows/ci.yml

@@ -182,11 +182,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v6
 
-      - name: Clean up previous test run
-        run: |
-          rm -rf /tmp/rustfs
-          rm -f /tmp/rustfs.log
-
       - name: Download debug binary
         uses: actions/download-artifact@v7
         with:
@@ -209,14 +204,19 @@ jobs:
       - name: Run end-to-end tests
         run: |
           s3s-e2e --version
-          ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
+          RUN_ROOT="${RUNNER_TEMP}/rustfs-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
+          mkdir -p "${RUN_ROOT}"
+          RUSTFS_TEST_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
+          RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT}" \
+          RUSTFS_TEST_LOG="${RUN_ROOT}/rustfs.log" \
+          ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
 
       - name: Upload test logs
         if: failure()
         uses: actions/upload-artifact@v6
         with:
           name: e2e-test-logs-${{ github.run_number }}
-          path: /tmp/rustfs.log
+          path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
           retention-days: 3
 
   s3-implemented-tests:
@@ -240,10 +240,16 @@ jobs:
 
       - name: Run implemented s3-tests
         run: |
+          RUN_ROOT="${RUNNER_TEMP}/rustfs-s3tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
+          mkdir -p "${RUN_ROOT}"
+          S3_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
           DEPLOY_MODE=binary \
           RUSTFS_BINARY=./target/debug/rustfs \
           TEST_MODE=single \
           MAXFAIL=1 \
+          S3_PORT="${S3_PORT}" \
+          DATA_ROOT="${RUN_ROOT}" \
+          S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
           ./scripts/s3-tests/run.sh
 
       - name: Upload s3 test artifacts

scripts/e2e-run.sh

@@ -16,24 +16,36 @@ set -ex
 
 BIN=$1
 VOLUME=$2
+RUSTFS_PID=""
+RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT:-9000}"
+RUSTFS_TEST_LOG="${RUSTFS_TEST_LOG:-${VOLUME}.log}"
 
-chmod +x $BIN
+cleanup() {
-mkdir -p $VOLUME
+    if [ -n "${RUSTFS_PID}" ] && kill -0 "${RUSTFS_PID}" 2>/dev/null; then
+        kill "${RUSTFS_PID}" 2>/dev/null || true
+        wait "${RUSTFS_PID}" 2>/dev/null || true
+    fi
+}
+
+trap cleanup EXIT INT TERM
+
+chmod +x "$BIN"
+mkdir -p "$VOLUME"
+mkdir -p "$(dirname "${RUSTFS_TEST_LOG}")"
 
 export RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
 export RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
 export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug"
 export RUST_BACKTRACE=full
-$BIN $VOLUME > /tmp/rustfs.log 2>&1 &
+"$BIN" --address "127.0.0.1:${RUSTFS_TEST_PORT}" "$VOLUME" > "${RUSTFS_TEST_LOG}" 2>&1 &
+RUSTFS_PID=$!
 
 sleep 10
 
 export AWS_ACCESS_KEY_ID="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
 export AWS_SECRET_ACCESS_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
 export AWS_REGION=us-east-1
-export AWS_ENDPOINT_URL=http://localhost:9000
+export AWS_ENDPOINT_URL="http://localhost:${RUSTFS_TEST_PORT}"
 export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug"
 export RUST_BACKTRACE=full
 s3s-e2e
-
-killall $BIN

scripts/s3-tests/README.md

@@ -335,17 +335,17 @@ The test script automatically cleans up processes and containers when it exits.
 A dedicated cleanup script is available to clean up test resources:
 
 ```bash
-# Clean up port 9000 and test data directory
+# Report port 9000 usage and clean the test data directory
 ./scripts/s3-tests/cleanup.sh
 
-# Use custom port and host
+# Use custom port
-S3_PORT=9001 S3_HOST=127.0.0.1 ./scripts/s3-tests/cleanup.sh
+S3_PORT=9001 ./scripts/s3-tests/cleanup.sh
 ```
 
 The cleanup script will:
-- Kill any process using the specified port (default: 9000)
+- Report any process using the specified port (default: 9000)
 - Clean test data directory at `target/test-data/rustfs-single/`
-- Verify port is released
+- Leave unrelated processes running
 
 ### Manual Cleanup
 

scripts/s3-tests/cleanup.sh

@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Cleanup script for port 9000 and test data directory
+# Cleanup script for the local s3-tests data directory.
-# This script kills any process using port 9000 and cleans the test data directory
+# It reports port usage but does not kill unrelated processes.
 
 set -euo pipefail
 
@@ -22,7 +22,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
 DATA_DIR="${PROJECT_ROOT}/target/test-data/rustfs-single"
 PORT="${S3_PORT:-9000}"
-HOST="${S3_HOST:-127.0.0.1}"
 
 log_info() {
     echo "[INFO] $*"
@@ -32,10 +31,9 @@ log_warn() {
     echo "[WARN] $*"
 }
 
-cleanup_port_9000() {
+check_port_usage() {
     log_info "Checking for processes using port ${PORT}..."
 
-    # Try to find process using port 9000
     PID=""
 
     # Try lsof first (works on macOS and Linux)
@@ -52,48 +50,12 @@ cleanup_port_9000() {
         fi
     fi
 
-    # If we found a PID, kill it
     if [ -n "$PID" ] && [ "$PID" != "-" ]; then
-        log_info "Found process ${PID} using port ${PORT}, killing it..."
+        log_warn "Port ${PORT} is in use by PID(s): ${PID}"
-        kill -9 "$PID" 2>/dev/null || true
+        log_warn "Not killing unrelated processes; stop the owner manually or use another S3_PORT."
-        sleep 2
-        
-        # Verify the process is gone
-        if kill -0 "$PID" 2>/dev/null; then
-            log_warn "Process ${PID} is still running, trying force kill..."
-            kill -9 "$PID" 2>/dev/null || true
-            sleep 1
-        fi
     else
         log_info "No process found using port ${PORT}"
     fi
-    
-    # Also try to kill any rustfs processes (more aggressive cleanup)
-    if pgrep -f "rustfs.*${PORT}" >/dev/null 2>&1; then
-        log_info "Killing any remaining rustfs processes using port ${PORT}..."
-        pkill -f "rustfs.*${PORT}" 2>/dev/null || true
-        sleep 1
-    fi
-    
-    # Verify port is released
-    log_info "Verifying port ${PORT} is released..."
-    for i in {1..10}; do
-        if command -v nc >/dev/null 2>&1; then
-            if ! nc -z "${HOST}" "${PORT}" 2>/dev/null; then
-                log_info "Port ${PORT} is now available"
-                break
-            fi
-        elif timeout 1 bash -c "cat < /dev/null > /dev/tcp/${HOST}/${PORT}" 2>/dev/null; then
-            # Port is still in use
-            if [ $i -eq 10 ]; then
-                log_warn "Port ${PORT} may still be in use after cleanup attempts"
-            fi
-        else
-            log_info "Port ${PORT} is now available"
-            break
-        fi
-        sleep 1
-    done
 }
 
 cleanup_data_directory() {
@@ -120,7 +82,7 @@ cleanup_data_directory() {
 
 main() {
     log_info "Starting cleanup..."
-    cleanup_port_9000
+    check_port_usage
     cleanup_data_directory
     log_info "Cleanup completed"
 }

scripts/s3-tests/run.sh

@@ -196,7 +196,12 @@ ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts/s3tests-${TEST_MODE}"
 CONTAINER_NAME="rustfs-${TEST_MODE}"
 NETWORK_NAME="rustfs-net"
 DATA_ROOT="${DATA_ROOT:-target}"
-DATA_DIR="${PROJECT_ROOT}/${DATA_ROOT}/test-data/${CONTAINER_NAME}"
+if [[ "${DATA_ROOT}" = /* ]]; then
+    DATA_BASE="${DATA_ROOT}"
+else
+    DATA_BASE="${PROJECT_ROOT}/${DATA_ROOT}"
+fi
+DATA_DIR="${DATA_BASE}/test-data/${CONTAINER_NAME}"
 RUSTFS_PID=""
 
 show_usage() {
@@ -655,8 +660,16 @@ log_info "Generating s3tests config..."
 mkdir -p "${ARTIFACTS_DIR}"
 
 # Resolve template and output paths (relative to PROJECT_ROOT)
-TEMPLATE_PATH="${PROJECT_ROOT}/${S3TESTS_CONF_TEMPLATE}"
+if [[ "${S3TESTS_CONF_TEMPLATE}" = /* ]]; then
-CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}"
+    TEMPLATE_PATH="${S3TESTS_CONF_TEMPLATE}"
+else
+    TEMPLATE_PATH="${PROJECT_ROOT}/${S3TESTS_CONF_TEMPLATE}"
+fi
+if [[ "${S3TESTS_CONF}" = /* ]]; then
+    CONF_OUTPUT_PATH="${S3TESTS_CONF}"
+else
+    CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}"
+fi
 
 # Check if template exists
 if [ ! -f "${TEMPLATE_PATH}" ]; then
@@ -681,9 +694,10 @@ fi
 
 log_info "Using template: ${TEMPLATE_PATH}"
 log_info "Generating config: ${CONF_OUTPUT_PATH}"
+mkdir -p "$(dirname "${CONF_OUTPUT_PATH}")"
 
 # Export all required variables for envsubst
-export S3_HOST S3_ACCESS_KEY S3_SECRET_KEY S3_ALT_ACCESS_KEY S3_ALT_SECRET_KEY
+export S3_HOST S3_PORT S3_ACCESS_KEY S3_SECRET_KEY S3_ALT_ACCESS_KEY S3_ALT_SECRET_KEY
 envsubst < "${TEMPLATE_PATH}" > "${CONF_OUTPUT_PATH}" || {
     log_error "Failed to generate s3tests config"
     exit 1
@@ -814,7 +828,11 @@ if [ "${XDIST}" != "0" ]; then
 fi
 
 # Resolve config path (absolute path for tox)
-CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}"
+if [[ "${S3TESTS_CONF}" = /* ]]; then
+    CONF_OUTPUT_PATH="${S3TESTS_CONF}"
+else
+    CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}"
+fi
 
 PYTEST_SELECTION_ARGS=()
 if [[ "${USE_FILE_TEST_NODES}" == "true" ]]; then