mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
Zhengchao An
a9894843d9
Speeds up the CI pipeline without changing what is tested: - Drop the global CARGO_BUILD_JOBS=2 and per-job --jobs 2 flags, which halved compiler parallelism on the 4-core runners. - Stop embedding hashFiles(Cargo.lock) in rust-cache shared keys. The action already fingerprints lockfiles internally; putting the hash in the key prefix meant any PR that touched Cargo.lock started from a completely cold cache instead of reusing unchanged dependencies. - Give the e2e-tests job the full setup action with dependency caching. Its migration-proof step compiles the e2e_test crate (which pulls in most of the workspace) and previously did so cold on every run with no cache, on a 2-core runner. - Set profile.dev debug = "line-tables-only": keeps usable backtraces while cutting compile/link time, target-dir size (faster cache save/restore), and debug-binary artifact upload/download. - Split fmt + repo-script checks into a compile-free quick-checks job on ubuntu-latest so contributors get feedback in ~1 minute instead of after the full test run. - Collapse the five per-filter migration-proof cargo test reruns into one filtered nextest invocation; the same tests already run in the full nextest pass. - Remove the touch rustfs/build.rs + forced rebuild from the debug binary jobs (version stamping is irrelevant for e2e binaries) and the unreachable Windows cross-compile branch in build.yml. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
446 lines
15 KiB
YAML
446 lines
15 KiB
YAML
# Copyright 2024 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
name: Continuous Integration
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
paths-ignore:
|
|
- "**.md"
|
|
- "docs/**"
|
|
- "deploy/**"
|
|
- "scripts/dev_*.sh"
|
|
- "scripts/probe.sh"
|
|
- "LICENSE*"
|
|
- ".gitignore"
|
|
- ".dockerignore"
|
|
- "README*"
|
|
- "**/*.png"
|
|
- "**/*.jpg"
|
|
- "**/*.svg"
|
|
- ".github/workflows/build.yml"
|
|
- ".github/workflows/docker.yml"
|
|
- ".github/workflows/audit.yml"
|
|
- ".github/workflows/performance.yml"
|
|
pull_request:
|
|
branches: [ main ]
|
|
paths-ignore:
|
|
- "**.md"
|
|
- "docs/**"
|
|
- "deploy/**"
|
|
- "scripts/dev_*.sh"
|
|
- "scripts/probe.sh"
|
|
- "LICENSE*"
|
|
- ".gitignore"
|
|
- ".dockerignore"
|
|
- "README*"
|
|
- "**/*.png"
|
|
- "**/*.jpg"
|
|
- "**/*.svg"
|
|
- ".github/workflows/build.yml"
|
|
- ".github/workflows/docker.yml"
|
|
- ".github/workflows/audit.yml"
|
|
- ".github/workflows/performance.yml"
|
|
merge_group:
|
|
types: [ checks_requested ]
|
|
schedule:
|
|
- cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
RUST_BACKTRACE: 1
|
|
|
|
jobs:
|
|
|
|
skip-check:
|
|
name: Skip Duplicate Actions
|
|
permissions:
|
|
actions: write
|
|
contents: read
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
should_skip: ${{ steps.skip_check.outputs.should_skip }}
|
|
steps:
|
|
- name: Skip duplicate actions
|
|
id: skip_check
|
|
uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5
|
|
with:
|
|
concurrent_skipping: "same_content_newer"
|
|
cancel_others: true
|
|
paths_ignore: '["*.md", "docs/**", "deploy/**"]'
|
|
do_not_skip: '["workflow_dispatch", "schedule", "merge_group", "release", "push"]'
|
|
|
|
typos:
|
|
name: Typos
|
|
needs: skip-check
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
|
- name: Typos check with custom config file
|
|
uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
|
|
|
|
# Fast, compile-free checks that fail early so contributors get feedback in
|
|
# ~1 minute instead of waiting for the full test job.
|
|
quick-checks:
|
|
name: Quick Checks
|
|
needs: skip-check
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Install ripgrep
|
|
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
|
|
|
- name: Install Rust toolchain
|
|
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
|
with:
|
|
components: rustfmt
|
|
|
|
- name: Check code formatting
|
|
run: cargo fmt --all --check
|
|
|
|
- name: Check unsafe code allowances
|
|
run: ./scripts/check_unsafe_code_allowances.sh
|
|
|
|
- name: Check layered dependencies
|
|
run: ./scripts/check_layer_dependencies.sh
|
|
|
|
- name: Check architecture migration rules
|
|
run: ./scripts/check_architecture_migration_rules.sh
|
|
|
|
test-and-lint:
|
|
name: Test and Lint
|
|
needs: skip-check
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: ubicloud-standard-4
|
|
timeout-minutes: 60
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: stable
|
|
cache-shared-key: ci-test
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- name: Run tests
|
|
run: |
|
|
cargo nextest run --all --exclude e2e_test
|
|
cargo test --all --doc
|
|
|
|
# Explicit gate for migration-critical suites. These tests already ran in
|
|
# the full nextest pass above; a single filtered nextest invocation keeps
|
|
# the named gate without rebuilding or re-running them one package at a time.
|
|
- name: Run rebalance/decommission migration proofs
|
|
run: |
|
|
cargo nextest run -p rustfs-ecstore --lib \
|
|
-E 'test(data_movement) or test(rebalance) or test(decommission) or test(source_cleanup) or test(delete_marker)'
|
|
|
|
- name: Run clippy lints
|
|
run: cargo clippy --all-targets -- -D warnings
|
|
|
|
test-and-lint-rio-v2:
|
|
name: Test and Lint (rio-v2)
|
|
needs: skip-check
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: ubicloud-standard-4
|
|
timeout-minutes: 60
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: stable
|
|
cache-shared-key: ci-test-rio-v2
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- name: Run rio-v2 feature tests
|
|
run: |
|
|
cargo nextest run -p rustfs -p rustfs-ecstore --features rio-v2
|
|
cargo test -p rustfs --doc --features rio-v2
|
|
|
|
- name: Run rio-v2 clippy lints
|
|
run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
|
|
|
|
test-and-lint-protocols:
|
|
name: "Test and Lint (${{ matrix.features.name }})"
|
|
needs: skip-check
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: ubicloud-standard-4
|
|
timeout-minutes: 60
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
features:
|
|
- name: swift
|
|
flags: "--features swift"
|
|
- name: sftp
|
|
flags: "--features sftp"
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: stable
|
|
cache-shared-key: ci-test-${{ matrix.features.name }}
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- name: Run tests with ${{ matrix.features.name }}
|
|
run: |
|
|
cargo nextest run -p rustfs -p rustfs-protocols ${{ matrix.features.flags }}
|
|
|
|
- name: Run clippy with ${{ matrix.features.name }}
|
|
run: |
|
|
cargo clippy -p rustfs -p rustfs-protocols --all-targets ${{ matrix.features.flags }} -- -D warnings
|
|
|
|
build-rustfs-debug-binary:
|
|
name: Build RustFS Debug Binary
|
|
needs: skip-check
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: sm-standard-4
|
|
timeout-minutes: 30
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: stable
|
|
cache-shared-key: ci-rustfs-debug-binary
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Build debug binary
|
|
run: cargo build -p rustfs --bins
|
|
|
|
- name: Upload debug binary
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-debug-binary
|
|
path: target/debug/rustfs
|
|
if-no-files-found: error
|
|
retention-days: 1
|
|
|
|
build-rustfs-debug-binary-rio-v2:
|
|
name: Build RustFS Debug Binary (rio-v2)
|
|
needs: skip-check
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: sm-standard-4
|
|
timeout-minutes: 30
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: stable
|
|
cache-shared-key: ci-rustfs-debug-binary-rio-v2
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Build debug binary with rio-v2
|
|
run: cargo build -p rustfs --bins --features rio-v2
|
|
|
|
- name: Upload debug binary
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: rustfs-debug-binary-rio-v2
|
|
path: target/debug/rustfs
|
|
if-no-files-found: error
|
|
retention-days: 1
|
|
|
|
e2e-tests:
|
|
name: End-to-End Tests
|
|
needs: [ skip-check, build-rustfs-debug-binary ]
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: sm-standard-2
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
# Full setup with dependency caching: the migration-proof step below
|
|
# compiles the e2e_test crate, which pulls in most of the workspace.
|
|
# Without a restored cache this was a cold multi-GB build on every run.
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: stable
|
|
cache-shared-key: ci-e2e
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
# Download after the cache restore so the freshly built binary from the
|
|
# build job always wins over anything restored into target/debug.
|
|
- name: Download debug binary
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
|
with:
|
|
name: rustfs-debug-binary
|
|
path: target/debug
|
|
|
|
- name: Make binary executable
|
|
run: chmod +x ./target/debug/rustfs
|
|
|
|
- name: Run delete-marker migration proof
|
|
run: cargo test -p e2e_test delete_marker_migration_semantics -- --nocapture --test-threads=1
|
|
|
|
- name: Install s3s-e2e test tool
|
|
uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
|
|
with:
|
|
tool: s3s-e2e
|
|
git: https://github.com/s3s-project/s3s.git
|
|
rev: 62cb4a71dd759a6ec56b64c4c42fcc183a2c6a52
|
|
|
|
- name: Run end-to-end tests
|
|
run: |
|
|
s3s-e2e --version
|
|
RUN_ROOT="${RUNNER_TEMP}/rustfs-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
|
|
mkdir -p "${RUN_ROOT}"
|
|
RUSTFS_TEST_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
|
|
RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT}" \
|
|
RUSTFS_TEST_LOG="${RUN_ROOT}/rustfs.log" \
|
|
./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
|
|
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: e2e-test-logs-${{ github.run_number }}
|
|
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
|
|
retention-days: 3
|
|
|
|
e2e-tests-rio-v2:
|
|
name: End-to-End Tests (rio-v2)
|
|
needs: [ skip-check, build-rustfs-debug-binary-rio-v2 ]
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: sm-standard-2
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Clean up previous test run
|
|
run: |
|
|
rm -rf /tmp/rustfs
|
|
rm -f /tmp/rustfs.log
|
|
|
|
- name: Download debug binary
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
|
with:
|
|
name: rustfs-debug-binary-rio-v2
|
|
path: target/debug
|
|
|
|
- name: Make binary executable
|
|
run: chmod +x ./target/debug/rustfs
|
|
|
|
- name: Setup Rust toolchain for s3s-e2e installation
|
|
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
|
|
|
- name: Install s3s-e2e test tool
|
|
uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
|
|
with:
|
|
tool: s3s-e2e
|
|
git: https://github.com/s3s-project/s3s.git
|
|
rev: 62cb4a71dd759a6ec56b64c4c42fcc183a2c6a52
|
|
|
|
- name: Run end-to-end tests
|
|
run: |
|
|
s3s-e2e --version
|
|
./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
|
|
|
|
- name: Upload test logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: e2e-test-logs-rio-v2-${{ github.run_number }}
|
|
path: /tmp/rustfs.log
|
|
retention-days: 3
|
|
|
|
s3-implemented-tests:
|
|
name: S3 Implemented Tests
|
|
needs: [ skip-check, build-rustfs-debug-binary, e2e-tests ]
|
|
if: needs.skip-check.outputs.should_skip != 'true'
|
|
runs-on: ubicloud-standard-4
|
|
timeout-minutes: 60
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Download debug binary
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
|
with:
|
|
name: rustfs-debug-binary
|
|
path: target/debug
|
|
|
|
- name: Make binary executable
|
|
run: chmod +x ./target/debug/rustfs
|
|
|
|
- name: Run implemented s3-tests
|
|
run: |
|
|
RUN_ROOT="${RUNNER_TEMP}/rustfs-s3tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
|
|
mkdir -p "${RUN_ROOT}"
|
|
S3_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
|
|
DEPLOY_MODE=binary \
|
|
RUSTFS_BINARY=./target/debug/rustfs \
|
|
TEST_MODE=single \
|
|
MAXFAIL=1 \
|
|
S3_PORT="${S3_PORT}" \
|
|
DATA_ROOT="${RUN_ROOT}" \
|
|
S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
|
|
./scripts/s3-tests/run.sh
|
|
|
|
- name: Upload s3 test artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: s3tests-implemented-${{ github.run_number }}
|
|
path: artifacts/s3tests-single/**
|
|
if-no-files-found: ignore
|
|
retention-days: 3
|