diff --git a/.github/s3tests/s3tests.conf b/.github/s3tests/s3tests.conf index fa2c4d57c..cc7b51274 100644 --- a/.github/s3tests/s3tests.conf +++ b/.github/s3tests/s3tests.conf @@ -11,8 +11,8 @@ # host set for RustFS - will be substituted via envsubst host = ${S3_HOST} -# port for RustFS -port = 9000 +# port for RustFS - will be substituted via envsubst +port = ${S3_PORT} ## say "False" to disable TLS is_secure = False diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efd868a72..e44cb7b6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,11 +182,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Clean up previous test run - run: | - rm -rf /tmp/rustfs - rm -f /tmp/rustfs.log - - name: Download debug binary uses: actions/download-artifact@v7 with: @@ -209,14 +204,19 @@ jobs: - name: Run end-to-end tests run: | s3s-e2e --version - ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs + RUN_ROOT="${RUNNER_TEMP}/rustfs-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}" + mkdir -p "${RUN_ROOT}" + RUSTFS_TEST_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')" + RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT}" \ + RUSTFS_TEST_LOG="${RUN_ROOT}/rustfs.log" \ + ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data" - name: Upload test logs if: failure() uses: actions/upload-artifact@v6 with: name: e2e-test-logs-${{ github.run_number }} - path: /tmp/rustfs.log + path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log retention-days: 3 s3-implemented-tests: @@ -240,10 +240,16 @@ jobs: - name: Run implemented s3-tests run: | + RUN_ROOT="${RUNNER_TEMP}/rustfs-s3tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}" + mkdir -p "${RUN_ROOT}" + S3_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')" DEPLOY_MODE=binary \ RUSTFS_BINARY=./target/debug/rustfs \ TEST_MODE=single \ MAXFAIL=1 \ + S3_PORT="${S3_PORT}" \ + DATA_ROOT="${RUN_ROOT}" \ + S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \ ./scripts/s3-tests/run.sh - name: Upload s3 test artifacts diff --git a/scripts/e2e-run.sh b/scripts/e2e-run.sh index 632f89412..ed66c1aed 100755 --- a/scripts/e2e-run.sh +++ b/scripts/e2e-run.sh @@ -16,24 +16,36 @@ set -ex BIN=$1 VOLUME=$2 +RUSTFS_PID="" +RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT:-9000}" +RUSTFS_TEST_LOG="${RUSTFS_TEST_LOG:-${VOLUME}.log}" -chmod +x $BIN -mkdir -p $VOLUME +cleanup() { + if [ -n "${RUSTFS_PID}" ] && kill -0 "${RUSTFS_PID}" 2>/dev/null; then + kill "${RUSTFS_PID}" 2>/dev/null || true + wait "${RUSTFS_PID}" 2>/dev/null || true + fi +} + +trap cleanup EXIT INT TERM + +chmod +x "$BIN" +mkdir -p "$VOLUME" +mkdir -p "$(dirname "${RUSTFS_TEST_LOG}")" export RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}" export RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}" export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug" export RUST_BACKTRACE=full -$BIN $VOLUME > /tmp/rustfs.log 2>&1 & +"$BIN" --address "127.0.0.1:${RUSTFS_TEST_PORT}" "$VOLUME" > "${RUSTFS_TEST_LOG}" 2>&1 & +RUSTFS_PID=$! sleep 10 export AWS_ACCESS_KEY_ID="${RUSTFS_ACCESS_KEY:-rustfsadmin}" export AWS_SECRET_ACCESS_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}" export AWS_REGION=us-east-1 -export AWS_ENDPOINT_URL=http://localhost:9000 +export AWS_ENDPOINT_URL="http://localhost:${RUSTFS_TEST_PORT}" export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug" export RUST_BACKTRACE=full s3s-e2e - -killall $BIN diff --git a/scripts/s3-tests/README.md b/scripts/s3-tests/README.md index b6a7683fe..6643f5b83 100644 --- a/scripts/s3-tests/README.md +++ b/scripts/s3-tests/README.md @@ -335,17 +335,17 @@ The test script automatically cleans up processes and containers when it exits. A dedicated cleanup script is available to clean up test resources: ```bash -# Clean up port 9000 and test data directory +# Report port 9000 usage and clean the test data directory ./scripts/s3-tests/cleanup.sh -# Use custom port and host -S3_PORT=9001 S3_HOST=127.0.0.1 ./scripts/s3-tests/cleanup.sh +# Use custom port +S3_PORT=9001 ./scripts/s3-tests/cleanup.sh ``` The cleanup script will: -- Kill any process using the specified port (default: 9000) +- Report any process using the specified port (default: 9000) - Clean test data directory at `target/test-data/rustfs-single/` -- Verify port is released +- Leave unrelated processes running ### Manual Cleanup diff --git a/scripts/s3-tests/cleanup.sh b/scripts/s3-tests/cleanup.sh index 91a0c0273..879ab4387 100755 --- a/scripts/s3-tests/cleanup.sh +++ b/scripts/s3-tests/cleanup.sh @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Cleanup script for port 9000 and test data directory -# This script kills any process using port 9000 and cleans the test data directory +# Cleanup script for the local s3-tests data directory. +# It reports port usage but does not kill unrelated processes. set -euo pipefail @@ -22,7 +22,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" DATA_DIR="${PROJECT_ROOT}/target/test-data/rustfs-single" PORT="${S3_PORT:-9000}" -HOST="${S3_HOST:-127.0.0.1}" log_info() { echo "[INFO] $*" @@ -32,17 +31,16 @@ log_warn() { echo "[WARN] $*" } -cleanup_port_9000() { +check_port_usage() { log_info "Checking for processes using port ${PORT}..." - - # Try to find process using port 9000 + PID="" - + # Try lsof first (works on macOS and Linux) if command -v lsof >/dev/null 2>&1; then PID=$(lsof -ti:${PORT} 2>/dev/null || true) fi - + # Fallback: try using netstat or ss if [ -z "$PID" ]; then if command -v netstat >/dev/null 2>&1; then @@ -51,49 +49,13 @@ cleanup_port_9000() { PID=$(ss -tuln 2>/dev/null | grep ":${PORT} " | awk '{print $6}' | cut -d',' -f2 | cut -d'=' -f2 | head -1 || true) fi fi - - # If we found a PID, kill it + if [ -n "$PID" ] && [ "$PID" != "-" ]; then - log_info "Found process ${PID} using port ${PORT}, killing it..." - kill -9 "$PID" 2>/dev/null || true - sleep 2 - - # Verify the process is gone - if kill -0 "$PID" 2>/dev/null; then - log_warn "Process ${PID} is still running, trying force kill..." - kill -9 "$PID" 2>/dev/null || true - sleep 1 - fi + log_warn "Port ${PORT} is in use by PID(s): ${PID}" + log_warn "Not killing unrelated processes; stop the owner manually or use another S3_PORT." else log_info "No process found using port ${PORT}" fi - - # Also try to kill any rustfs processes (more aggressive cleanup) - if pgrep -f "rustfs.*${PORT}" >/dev/null 2>&1; then - log_info "Killing any remaining rustfs processes using port ${PORT}..." - pkill -f "rustfs.*${PORT}" 2>/dev/null || true - sleep 1 - fi - - # Verify port is released - log_info "Verifying port ${PORT} is released..." - for i in {1..10}; do - if command -v nc >/dev/null 2>&1; then - if ! nc -z "${HOST}" "${PORT}" 2>/dev/null; then - log_info "Port ${PORT} is now available" - break - fi - elif timeout 1 bash -c "cat < /dev/null > /dev/tcp/${HOST}/${PORT}" 2>/dev/null; then - # Port is still in use - if [ $i -eq 10 ]; then - log_warn "Port ${PORT} may still be in use after cleanup attempts" - fi - else - log_info "Port ${PORT} is now available" - break - fi - sleep 1 - done } cleanup_data_directory() { @@ -120,7 +82,7 @@ cleanup_data_directory() { main() { log_info "Starting cleanup..." - cleanup_port_9000 + check_port_usage cleanup_data_directory log_info "Cleanup completed" } diff --git a/scripts/s3-tests/run.sh b/scripts/s3-tests/run.sh index fd19bd9a3..4ff7cca59 100755 --- a/scripts/s3-tests/run.sh +++ b/scripts/s3-tests/run.sh @@ -196,7 +196,12 @@ ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts/s3tests-${TEST_MODE}" CONTAINER_NAME="rustfs-${TEST_MODE}" NETWORK_NAME="rustfs-net" DATA_ROOT="${DATA_ROOT:-target}" -DATA_DIR="${PROJECT_ROOT}/${DATA_ROOT}/test-data/${CONTAINER_NAME}" +if [[ "${DATA_ROOT}" = /* ]]; then + DATA_BASE="${DATA_ROOT}" +else + DATA_BASE="${PROJECT_ROOT}/${DATA_ROOT}" +fi +DATA_DIR="${DATA_BASE}/test-data/${CONTAINER_NAME}" RUSTFS_PID="" show_usage() { @@ -655,8 +660,16 @@ log_info "Generating s3tests config..." mkdir -p "${ARTIFACTS_DIR}" # Resolve template and output paths (relative to PROJECT_ROOT) -TEMPLATE_PATH="${PROJECT_ROOT}/${S3TESTS_CONF_TEMPLATE}" -CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}" +if [[ "${S3TESTS_CONF_TEMPLATE}" = /* ]]; then + TEMPLATE_PATH="${S3TESTS_CONF_TEMPLATE}" +else + TEMPLATE_PATH="${PROJECT_ROOT}/${S3TESTS_CONF_TEMPLATE}" +fi +if [[ "${S3TESTS_CONF}" = /* ]]; then + CONF_OUTPUT_PATH="${S3TESTS_CONF}" +else + CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}" +fi # Check if template exists if [ ! -f "${TEMPLATE_PATH}" ]; then @@ -681,9 +694,10 @@ fi log_info "Using template: ${TEMPLATE_PATH}" log_info "Generating config: ${CONF_OUTPUT_PATH}" +mkdir -p "$(dirname "${CONF_OUTPUT_PATH}")" # Export all required variables for envsubst -export S3_HOST S3_ACCESS_KEY S3_SECRET_KEY S3_ALT_ACCESS_KEY S3_ALT_SECRET_KEY +export S3_HOST S3_PORT S3_ACCESS_KEY S3_SECRET_KEY S3_ALT_ACCESS_KEY S3_ALT_SECRET_KEY envsubst < "${TEMPLATE_PATH}" > "${CONF_OUTPUT_PATH}" || { log_error "Failed to generate s3tests config" exit 1 @@ -814,7 +828,11 @@ if [ "${XDIST}" != "0" ]; then fi # Resolve config path (absolute path for tox) -CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}" +if [[ "${S3TESTS_CONF}" = /* ]]; then + CONF_OUTPUT_PATH="${S3TESTS_CONF}" +else + CONF_OUTPUT_PATH="${PROJECT_ROOT}/${S3TESTS_CONF}" +fi PYTEST_SELECTION_ARGS=() if [[ "${USE_FILE_TEST_NODES}" == "true" ]]; then