diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a645c769e..fe0f57b7d 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -40,28 +40,43 @@ inputs: description: "Target architecture to add" required: false default: "" - github-token: - description: "GitHub token for API access" + install-build-packaging-tools: + description: >- + Install musl-tools/zip/unzip, needed for musl linking and release + packaging. Off for CI test lanes, which use none of them. required: false - default: "" + default: "true" + install-test-tools: + description: >- + Install cargo-nextest and the rustfmt/clippy components. Off for release + and audit lanes, which run no tests and no lints. + required: false + default: "true" runs: using: "composite" steps: + # protobuf-compiler is deliberately absent: the setup-protoc step below + # installs 34.1 into the tool cache and prepends it to PATH, so the apt + # build (older, and never version-matched) was shadowed on every run and + # simply never used. - name: Install system dependencies (Ubuntu) if: runner.os == 'Linux' shell: bash run: | sudo apt-get update sudo apt-get install -y \ - musl-tools \ build-essential \ pkg-config \ libssl-dev \ - ripgrep \ - unzip \ - zip \ - protobuf-compiler + ripgrep + + # musl-gcc is needed by the native musl release leg, and zip/unzip by the + # release packaging steps. No CI test lane touches any of them. + - name: Install packaging and cross-linking dependencies (Ubuntu) + if: runner.os == 'Linux' && inputs.install-build-packaging-tools == 'true' + shell: bash + run: sudo apt-get install -y musl-tools zip unzip - name: Install protoc uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1 @@ -79,7 +94,7 @@ runs: with: toolchain: ${{ inputs.rust-version }} targets: ${{ inputs.target }} - components: rustfmt, clippy + components: ${{ inputs.install-test-tools == 'true' && 'rustfmt, clippy' || '' }} - name: Install Zig if: inputs.install-cross-tools == 'true' @@ -90,6 +105,7 @@ runs: uses: taiki-e/install-action@a21ae4029b089b9ddc45704028756f51ab8abe48 # cargo-zigbuild - name: Install cargo-nextest + if: inputs.install-test-tools == 'true' uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest - name: Setup Rust cache diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5f39fe992..f744e4bee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -265,7 +265,6 @@ jobs: rust-version: stable target: ${{ matrix.target }} cache-shared-key: build-${{ matrix.target }} - github-token: ${{ secrets.GITHUB_TOKEN }} # main only. A cache saved on refs/tags/X is scoped to that tag: no # other tag, no main run and no PR can restore it, so every release # cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag, @@ -276,6 +275,7 @@ jobs: # finds that tag's own warm cache and falls back to main's. cache-save-if: ${{ github.ref == 'refs/heads/main' }} install-cross-tools: ${{ matrix.cross }} + install-test-tools: 'false' - name: Download static console assets shell: bash diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml index 4688a08f7..d80e03f42 100644 --- a/.github/workflows/cache-warm.yml +++ b/.github/workflows/cache-warm.yml @@ -96,8 +96,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-dev - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'true' + install-build-packaging-tools: 'false' # --all-targets covers the test binaries nextest builds, including # e2e_test, which test-and-lint's own run excludes. The second build adds @@ -131,8 +131,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-feat-rio - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'true' + install-build-packaging-tools: 'false' - name: Build ci-feat-rio superset run: | @@ -159,8 +159,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-feat-proto - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'true' + install-build-packaging-tools: 'false' - name: Build ci-feat-proto superset run: | @@ -186,8 +186,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-uring - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'true' + install-build-packaging-tools: 'false' - name: Install build dependencies run: sudo apt-get update && sudo apt-get install -y protobuf-compiler diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45f462b0d..335fe949c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,8 +185,8 @@ jobs: # rust-cache's post step, so writing from here saved nothing (12 of 15 # consecutive main-push runs were cancelled). See rustfs/backlog#1600. cache-shared-key: ci-dev - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'false' + install-build-packaging-tools: 'false' - name: Prepare test evidence run: | @@ -359,8 +359,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-dev - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'false' + install-build-packaging-tools: 'false' # test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303: # its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded @@ -397,8 +397,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-feat-rio - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'false' + install-build-packaging-tools: 'false' - name: Run rio-v2 clippy lints run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings @@ -439,8 +439,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-feat-proto - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'false' + install-build-packaging-tools: 'false' - name: Run clippy with ${{ matrix.features.name }} run: | @@ -468,7 +468,7 @@ jobs: rust-version: stable cache-shared-key: ci-dev cache-save-if: 'false' - github-token: ${{ secrets.GITHUB_TOKEN }} + install-build-packaging-tools: 'false' - name: Build debug binary run: cargo build -p rustfs --bins --features e2e-test-hooks @@ -499,7 +499,7 @@ jobs: rust-version: stable cache-shared-key: ci-feat-rio cache-save-if: 'false' - github-token: ${{ secrets.GITHUB_TOKEN }} + install-build-packaging-tools: 'false' - name: Build debug binary with rio-v2 run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks @@ -546,11 +546,8 @@ jobs: # instead of this lane's ~1.3GB. cache-warm.yml warms this key on # ubuntu-latest for the same reason. cache-shared-key: ci-uring - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'false' - - - name: Install build dependencies - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + install-build-packaging-tools: 'false' # ext4 supports O_DIRECT; the runner's default TMPDIR may sit on tmpfs or # overlayfs, where open(O_DIRECT) returns EINVAL/EOPNOTSUPP and the native @@ -606,8 +603,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-dev - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'false' + install-build-packaging-tools: 'false' # Download after the cache restore so the freshly built binary from the # build job always wins over anything restored into target/debug. @@ -687,8 +684,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-dev - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: 'false' + install-build-packaging-tools: 'false' # Download after the cache restore so the freshly built binary from the # build job always wins over anything restored into target/debug. diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index b7e1fb25d..fb8a18b6f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -68,8 +68,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-coverage - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: ${{ github.ref == 'refs/heads/main' }} + install-build-packaging-tools: 'false' - name: Install cargo-llvm-cov uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ebb955da8..3a8ef675b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -340,32 +340,28 @@ jobs: CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}" VARIANT_SUFFIX="${{ matrix.suffix }}" - # Convert version format for Dockerfile compatibility + # Convert version format for Dockerfile compatibility. The former + # DOCKER_CHANNEL was "release" down every branch and was passed as a + # build-arg no Dockerfile declares, so it is gone. case "$VERSION" in "latest") - # For stable latest, use RELEASE=latest + release CHANNEL DOCKER_RELEASE="latest" - DOCKER_CHANNEL="release" ;; v*) # For versioned releases (v1.0.0), remove 'v' prefix for Dockerfile DOCKER_RELEASE="${VERSION#v}" - DOCKER_CHANNEL="release" ;; *) # For other versions, pass as-is DOCKER_RELEASE="${VERSION}" - DOCKER_CHANNEL="release" ;; esac echo "docker_release=$DOCKER_RELEASE" >> "$GITHUB_OUTPUT" - echo "docker_channel=$DOCKER_CHANNEL" >> "$GITHUB_OUTPUT" echo "🐳 Docker build parameters:" echo " - Original version: $VERSION" echo " - Docker RELEASE: $DOCKER_RELEASE" - echo " - Docker CHANNEL: $DOCKER_CHANNEL" # Generate tags based on build type # Only support release and prerelease builds (no development builds) @@ -419,18 +415,24 @@ jobs: push: ${{ needs.build-check.outputs.should_push == 'true' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: | - type=gha,scope=docker-${{ matrix.variant }} - cache-to: | - type=gha,mode=max,scope=docker-${{ matrix.variant }} + # No layer cache. This build compiles nothing — it downloads a + # release zip and runs apk/apt — so the cache could only save the + # minute or two those take, while creating a correctness problem: with + # RELEASE=latest the binary URL is resolved by curl *inside* a RUN + # layer, and the layer key does not include what that resolved to. A + # rebuild at the same RELEASE value (dispatch with version=latest, or + # a re-run of the same version) would hit the old layer and ship the + # previous release's binary. mode=max also consumed the same 10GB + # Actions cache quota the Rust lanes are fighting over. + # + # Only RELEASE is passed: it is the sole build-arg the Dockerfiles + # declare besides TARGETARCH. BUILDTIME, VERSION, BUILD_TYPE, REVISION + # and CHANNEL were never read by any stage (and BUILDTIME's $(date ...) + # was a literal here, not a shell substitution). BUILD_DATE and VCS_REF + # are declared by the Dockerfiles but deliberately left unset — + # supplying them would change the published image labels. build-args: | - BUILDTIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - VERSION=${{ needs.build-check.outputs.version }} - BUILD_TYPE=${{ needs.build-check.outputs.build_type }} - REVISION=${{ github.sha }} RELEASE=${{ steps.meta.outputs.docker_release }} - CHANNEL=${{ steps.meta.outputs.docker_channel }} - BUILDKIT_INLINE_CACHE=1 provenance: true sbom: true # Add retry mechanism by splitting the build process diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml index 1d94753e5..2a0a3f532 100644 --- a/.github/workflows/e2e-replication-nightly.yml +++ b/.github/workflows/e2e-replication-nightly.yml @@ -70,8 +70,8 @@ jobs: with: rust-version: stable cache-shared-key: ci-e2e-repl - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: ${{ github.ref == 'refs/heads/main' }} + install-build-packaging-tools: 'false' # awscurl lets the STS dual-node test actually exercise its path. Without # it the test skips gracefully with a visible log line diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 1082da1de..fe8e683b4 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -93,7 +93,6 @@ jobs: with: rust-version: nightly cache-shared-key: fuzz-${{ hashFiles('fuzz/Cargo.lock') }} - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }} - name: Install cargo-fuzz diff --git a/.github/workflows/minio-interop.yml b/.github/workflows/minio-interop.yml index b043b84a1..6de00067a 100644 --- a/.github/workflows/minio-interop.yml +++ b/.github/workflows/minio-interop.yml @@ -64,7 +64,6 @@ jobs: with: rust-version: stable cache-shared-key: ci-minio-interop - github-token: ${{ secrets.GITHUB_TOKEN }} cache-save-if: ${{ github.ref == 'refs/heads/main' }} - name: Generate real MinIO fixtures via Docker diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml index ad3e6f1c6..ebab72c8d 100644 --- a/.github/workflows/performance-ab.yml +++ b/.github/workflows/performance-ab.yml @@ -106,7 +106,6 @@ jobs: rust-version: stable cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }} cache-save-if: ${{ github.ref == 'refs/heads/main' }} - github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build release rustfs run: cargo build --release --bin rustfs @@ -157,7 +156,6 @@ jobs: rust-version: stable cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }} cache-save-if: ${{ github.ref == 'refs/heads/main' }} - github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install warp run: |