mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-03 20:07:42 +00:00
Zhengchao An
6c99d4fe22
nix-flake-update opens a PR every Sunday that changes flake.lock and nothing else. flake.lock is consumed only by Nix packaging — cargo never reads it — yet it was in no paths filter, so each of those PRs ran the full Continuous Integration pipeline and the merge then ran Build and Release too. Added to all four lists that have to agree: ci.yml's push and pull_request paths-ignore, build.yml's push paths-ignore, and ci-docs-only.yml's paths. The cron stays. flake.lock still has consumers — anyone running `nix build` or `nix develop` — so stopping the updates would remove the workflow's output, not just its CI cost. Those four lists drifting is a silent failure, so scripts/check_ci_paths_sync.sh now asserts the pair that matters: ci.yml's pull_request paths-ignore must equal ci-docs-only.yml's paths. An entry present only in the first means a PR touching those files triggers neither workflow, nobody reports "Test and Lint" or "Quick Checks", and the PR waits on a required check forever. It also asserts both companion job names still exist, since renaming one produces the same hang. The push list is not compared: no required check is reported for push events. Also in this cleanup: - nix-flake-update's GITHUB_TOKEN drops to contents: read. The branch push and the pull request are both made by update-flake-lock with the FLAKE_UPDATE_TOKEN PAT, so the write scopes were an unused repo-write credential on an unattended weekly job. - build.yml's build_docker dispatch input is documented as advisory. docker.yml triggers on workflow_run and requires the triggering event to be a tag push, so a manual dispatch never reaches it whatever this input says. - The nine disabled workflow files get a banner saying so. Their disabled_manually state lives in GitHub's UI and is invisible when reading the file, which has already misled one audit into treating dead workflows as live. Refs: rustfs/backlog#1598, rustfs/backlog#1603
1029 lines
39 KiB
YAML
1029 lines
39 KiB
YAML
# Copyright 2024 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Build and Release Workflow
|
|
#
|
|
# This workflow builds RustFS binaries and automatically triggers Docker image builds.
|
|
#
|
|
# Flow:
|
|
# 1. Build binaries for multiple platforms
|
|
# 2. Upload binaries to OSS storage
|
|
# 3. Trigger docker.yml to build and push images using the uploaded binaries
|
|
#
|
|
# Platform scope:
|
|
# - Pushes to main (development builds) build the Linux targets only — they
|
|
# feed the dev download channel and Docker dev images. Tags, the weekly
|
|
# schedule and manual dispatch build the full platform matrix.
|
|
#
|
|
# Manual Parameters:
|
|
# - build_docker: Build and push Docker images (default: true)
|
|
# - platforms: Comma-separated platform IDs or 'all' (default: all)
|
|
|
|
name: Build and Release
|
|
|
|
on:
|
|
push:
|
|
tags: [ "*.*.*" ]
|
|
branches: [ main ]
|
|
paths-ignore:
|
|
- "**.md"
|
|
- "**.txt"
|
|
- ".github/**"
|
|
- "docs/**"
|
|
- "deploy/**"
|
|
- "scripts/dev_*.sh"
|
|
- "LICENSE*"
|
|
- "README*"
|
|
- "**/*.png"
|
|
- "**/*.jpg"
|
|
- "**/*.svg"
|
|
- ".gitignore"
|
|
- ".dockerignore"
|
|
- "flake.lock"
|
|
schedule:
|
|
- cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
|
|
workflow_dispatch:
|
|
inputs:
|
|
build_docker:
|
|
# Advisory only. docker.yml triggers on workflow_run and its job-level
|
|
# condition requires the triggering event to be a tag push, so a manual
|
|
# dispatch of this workflow never produces images regardless of this
|
|
# value. Kept because the summary step reports it; wiring it up would
|
|
# mean teaching docker.yml's version parser a second event shape.
|
|
description: "Build and push Docker images after binary build (ignored: dispatch runs never reach docker.yml)"
|
|
required: false
|
|
default: true
|
|
type: boolean
|
|
platforms:
|
|
description: "Comma-separated targets or 'all' (e.g. linux-x86_64-musl,macos-aarch64)"
|
|
required: false
|
|
default: "all"
|
|
type: string
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.ref || github.run_id }}
|
|
cancel-in-progress: ${{ github.event_name == 'push' }}
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
RUST_BACKTRACE: 1
|
|
# Optimize build performance
|
|
CARGO_INCREMENTAL: 0
|
|
|
|
jobs:
|
|
# Build strategy check - determine build type based on trigger
|
|
build-check:
|
|
name: Build Strategy Check
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
outputs:
|
|
should_build: ${{ steps.check.outputs.should_build }}
|
|
build_type: ${{ steps.check.outputs.build_type }}
|
|
version: ${{ steps.check.outputs.version }}
|
|
short_sha: ${{ steps.check.outputs.short_sha }}
|
|
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Determine build strategy
|
|
id: check
|
|
run: |
|
|
should_build=false
|
|
build_type="none"
|
|
version=""
|
|
short_sha=""
|
|
is_prerelease=false
|
|
|
|
# Get short SHA for all builds
|
|
short_sha=$(git rev-parse --short HEAD)
|
|
|
|
# Determine build type based on trigger
|
|
if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
|
|
# Tag push - preview, release, or prerelease
|
|
should_build=true
|
|
tag_name="${GITHUB_REF#refs/tags/}"
|
|
version="${tag_name}"
|
|
|
|
# Preview tags publish a GitHub prerelease for validation, but
|
|
# must not update any latest channel.
|
|
if [[ "$tag_name" =~ -preview\.[0-9]+$ ]]; then
|
|
build_type="preview"
|
|
is_prerelease=true
|
|
echo "🔍 Preview build detected: $tag_name"
|
|
elif [[ "$tag_name" == *"-preview"* ]]; then
|
|
echo "❌ Invalid preview tag: $tag_name (expected suffix: -preview.<number>)" >&2
|
|
exit 1
|
|
elif [[ "$tag_name" == *"alpha"* ]] || [[ "$tag_name" == *"beta"* ]] || [[ "$tag_name" == *"rc"* ]]; then
|
|
build_type="prerelease"
|
|
is_prerelease=true
|
|
echo "🚀 Prerelease build detected: $tag_name"
|
|
else
|
|
build_type="release"
|
|
echo "📦 Release build detected: $tag_name"
|
|
fi
|
|
elif [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
|
|
# Main branch push - development build
|
|
should_build=true
|
|
build_type="development"
|
|
version="dev-${short_sha}"
|
|
echo "🛠️ Development build detected"
|
|
elif [[ "${{ github.event_name }}" == "schedule" ]] || \
|
|
[[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
|
# Scheduled or manual build
|
|
should_build=true
|
|
build_type="development"
|
|
version="dev-${short_sha}"
|
|
echo "⚡ Manual/scheduled build detected"
|
|
fi
|
|
|
|
{
|
|
echo "should_build=$should_build"
|
|
echo "build_type=$build_type"
|
|
echo "version=$version"
|
|
echo "short_sha=$short_sha"
|
|
echo "is_prerelease=$is_prerelease"
|
|
} >> "$GITHUB_OUTPUT"
|
|
|
|
echo "📊 Build Summary:"
|
|
echo " - Should build: $should_build"
|
|
echo " - Build type: $build_type"
|
|
echo " - Version: $version"
|
|
echo " - Short SHA: $short_sha"
|
|
echo " - Is prerelease: $is_prerelease"
|
|
|
|
# Build RustFS binaries
|
|
prepare-platform-matrix:
|
|
name: Prepare Platform Matrix
|
|
needs: build-check
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
outputs:
|
|
matrix: ${{ steps.select.outputs.matrix }}
|
|
selected: ${{ steps.select.outputs.selected }}
|
|
steps:
|
|
- name: Select target platforms
|
|
id: select
|
|
shell: bash
|
|
env:
|
|
# via env, not interpolation: a dispatch input is free-form text and
|
|
# would otherwise be pasted into the script for bash to evaluate.
|
|
RAW_PLATFORMS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.platforms || 'all' }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
selected="$RAW_PLATFORMS"
|
|
selected="$(echo "${selected}" | tr -d '[:space:]')"
|
|
if [[ -z "${selected}" ]]; then
|
|
selected="all"
|
|
fi
|
|
|
|
# Per-merge development builds only feed the dev download channel
|
|
# and the Docker dev images, which consume the Linux binaries; at
|
|
# the usual merge cadence most per-merge builds are cancelled by the
|
|
# next merge anyway. macOS and Windows stay covered by tag builds,
|
|
# the weekly scheduled build and manual dispatch.
|
|
if [[ "${selected}" == "all" \
|
|
&& "${{ github.event_name }}" == "push" \
|
|
&& "${{ needs.build-check.outputs.build_type }}" == "development" ]]; then
|
|
selected="linux-x86_64-musl,linux-aarch64-musl,linux-x86_64-gnu,linux-aarch64-gnu"
|
|
echo "Development (main push) build: restricting to Linux targets"
|
|
fi
|
|
|
|
all='{"include":[
|
|
{"target_id":"linux-x86_64-musl","os":"sm-standard-2","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":""},
|
|
{"target_id":"linux-aarch64-musl","os":"sm-standard-2","target":"aarch64-unknown-linux-musl","cross":true,"platform":"linux","rustflags":""},
|
|
{"target_id":"linux-x86_64-gnu","os":"sm-standard-2","target":"x86_64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":""},
|
|
{"target_id":"linux-aarch64-gnu","os":"sm-standard-2","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":""},
|
|
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":""},
|
|
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":""}
|
|
]}'
|
|
|
|
if [[ "${selected}" == "all" ]]; then
|
|
matrix="$(jq -c . <<<"${all}")"
|
|
else
|
|
unknown="$(jq -rn --arg selected "${selected}" --argjson all "${all}" '
|
|
($selected | split(",") | map(select(length > 0))) as $req
|
|
| ($all.include | map(.target_id)) as $known
|
|
| [$req[] | select(( $known | index(.) ) == null)]
|
|
')"
|
|
if [[ "$(jq 'length' <<<"${unknown}")" -gt 0 ]]; then
|
|
echo "Unknown platforms: $(jq -r 'join(\",\")' <<<"${unknown}")" >&2
|
|
echo "Allowed: $(jq -r '.include[].target_id' <<<"${all}" | paste -sd ',' -)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
matrix="$(jq -c --arg selected "${selected}" '
|
|
($selected | split(",") | map(select(length > 0))) as $req
|
|
| .include |= map(select(.target_id as $id | ($req | index($id))))
|
|
' <<<"${all}")"
|
|
fi
|
|
|
|
echo "selected=${selected}" >> "$GITHUB_OUTPUT"
|
|
echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
|
|
echo "Selected platforms: ${selected}"
|
|
|
|
build-rustfs:
|
|
name: Build RustFS
|
|
needs: [ build-check, prepare-platform-matrix ]
|
|
if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
|
|
runs-on: ${{ matrix.os }}
|
|
timeout-minutes: 150
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
# Release binaries ship without dial9 telemetry and therefore do not need
|
|
# --cfg tokio_unstable. Telemetry builds opt in explicitly (see
|
|
# `make build-profiling`); crates/obs/build.rs fails the compile if the
|
|
# `dial9` feature is enabled without the flag.
|
|
RUSTFLAGS: "${{ matrix.rustflags }}"
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJson(needs.prepare-platform-matrix.outputs.matrix) }}
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: stable
|
|
target: ${{ matrix.target }}
|
|
cache-shared-key: build-${{ matrix.target }}
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
# main only. A cache saved on refs/tags/X is scoped to that tag: no
|
|
# other tag, no main run and no PR can restore it, so every release
|
|
# cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
|
|
# six legs each) that nobody could read, evicting the hot lanes from
|
|
# the repo-wide 10GB quota. Tag builds still restore the main-scoped
|
|
# cache, since default-branch caches are readable from every ref.
|
|
# The one real cost: re-running a failed leg of the same tag no longer
|
|
# finds that tag's own warm cache and falls back to main's.
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
install-cross-tools: ${{ matrix.cross }}
|
|
|
|
- name: Download static console assets
|
|
shell: bash
|
|
env:
|
|
GITHUB_TOKEN: ${{ github.token }}
|
|
run: |
|
|
mkdir -p ./rustfs/static
|
|
|
|
verify_sha256() {
|
|
local expected="$1"
|
|
local file="$2"
|
|
local actual=""
|
|
local actual_line=""
|
|
|
|
if command -v sha256sum >/dev/null 2>&1; then
|
|
actual_line=$(sha256sum -- "$file") || return 1
|
|
elif command -v shasum >/dev/null 2>&1; then
|
|
actual_line=$(shasum -a 256 -- "$file") || return 1
|
|
else
|
|
echo "No SHA-256 verification tool found" >&2
|
|
return 1
|
|
fi
|
|
actual="${actual_line%%[[:space:]]*}"
|
|
|
|
if [ "$actual" = "$expected" ]; then
|
|
echo "SHA256 verified OK: $actual"
|
|
return 0
|
|
else
|
|
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
download_console_assets() {
|
|
local curl_bin="curl"
|
|
local python_bin="python3"
|
|
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
|
|
local console_json="console-release.json"
|
|
local console_url
|
|
local console_sha256
|
|
local curl_auth_args=()
|
|
console_tag=""
|
|
|
|
if [[ "${{ matrix.platform }}" == "windows" ]]; then
|
|
curl_bin="curl.exe"
|
|
else
|
|
chmod +w ./rustfs/static/LICENSE || true
|
|
fi
|
|
|
|
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
|
python_bin="python"
|
|
fi
|
|
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
|
echo "No Python interpreter found for release metadata parsing" >&2
|
|
return 1
|
|
fi
|
|
|
|
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
|
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
|
|
fi
|
|
|
|
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
|
|
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
|
|
|
|
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
|
|
import json
|
|
import re
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
release = json.load(handle)
|
|
|
|
tag = release.get("tag_name", "") or "unknown"
|
|
|
|
for asset in release.get("assets", []):
|
|
name = asset.get("name", "")
|
|
digest = asset.get("digest", "")
|
|
url = asset.get("browser_download_url", "")
|
|
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
|
|
sha256 = digest.split(":", 1)[1]
|
|
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
|
|
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
|
|
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
|
|
break
|
|
else:
|
|
raise SystemExit("no console zip asset with sha256 digest found")
|
|
PY
|
|
) || return 1
|
|
|
|
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
|
|
echo "Console release metadata is missing URL or SHA-256 digest" >&2
|
|
return 1
|
|
fi
|
|
|
|
echo "Console release: ${console_tag}"
|
|
echo "Downloading console asset: ${console_url}"
|
|
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
|
|
verify_sha256 "$console_sha256" console.zip || return 2
|
|
unzip -o console.zip -d ./rustfs/static || return 2
|
|
}
|
|
|
|
if download_console_assets; then
|
|
rm -f console.zip console-release.json
|
|
else
|
|
status=$?
|
|
rm -f console.zip console-release.json
|
|
if [[ "$status" -eq 2 ]]; then
|
|
echo "Console asset integrity verification failed" >&2
|
|
fi
|
|
echo "Failed to download verified console assets" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -s ./rustfs/static/index.html ]]; then
|
|
echo "Console asset archive is missing static/index.html" >&2
|
|
exit 1
|
|
fi
|
|
|
|
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
|
|
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
|
|
|
|
- name: Build RustFS
|
|
shell: bash
|
|
run: |
|
|
# Force rebuild by touching build.rs
|
|
touch rustfs/build.rs
|
|
|
|
if [[ "${{ matrix.cross }}" == "true" ]]; then
|
|
# All cross targets in the matrix are Linux; zigbuild handles them.
|
|
cargo zigbuild --release --target ${{ matrix.target }} -p rustfs --bins
|
|
else
|
|
cargo build --release --target ${{ matrix.target }} -p rustfs --bins
|
|
fi
|
|
|
|
- name: Create release package
|
|
id: package
|
|
shell: bash
|
|
run: |
|
|
BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
|
|
VERSION="${{ needs.build-check.outputs.version }}"
|
|
SHORT_SHA="${{ needs.build-check.outputs.short_sha }}"
|
|
|
|
# Extract platform and arch from target
|
|
TARGET="${{ matrix.target }}"
|
|
PLATFORM="${{ matrix.platform }}"
|
|
# Map target to architecture and variant
|
|
case "$TARGET" in
|
|
*x86_64*musl*)
|
|
ARCH="x86_64"
|
|
VARIANT="musl"
|
|
;;
|
|
*x86_64*gnu*)
|
|
ARCH="x86_64"
|
|
VARIANT="gnu"
|
|
;;
|
|
*x86_64*)
|
|
ARCH="x86_64"
|
|
VARIANT=""
|
|
;;
|
|
*aarch64*musl*|*arm64*musl*)
|
|
ARCH="aarch64"
|
|
VARIANT="musl"
|
|
;;
|
|
*aarch64*gnu*|*arm64*gnu*)
|
|
ARCH="aarch64"
|
|
VARIANT="gnu"
|
|
;;
|
|
*aarch64*|*arm64*)
|
|
ARCH="aarch64"
|
|
VARIANT=""
|
|
;;
|
|
*armv7*)
|
|
ARCH="armv7"
|
|
VARIANT=""
|
|
;;
|
|
*)
|
|
ARCH="unknown"
|
|
VARIANT=""
|
|
;;
|
|
esac
|
|
|
|
# Normalize version used for package filenames
|
|
PACKAGE_VERSION="${VERSION}"
|
|
if [[ "$PACKAGE_VERSION" == v* ]]; then
|
|
PACKAGE_VERSION="${PACKAGE_VERSION#v}"
|
|
fi
|
|
|
|
# Generate package name based on build type
|
|
if [[ -n "$VARIANT" ]]; then
|
|
ARCH_WITH_VARIANT="${ARCH}-${VARIANT}"
|
|
else
|
|
ARCH_WITH_VARIANT="${ARCH}"
|
|
fi
|
|
PACKAGE_BASENAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}"
|
|
|
|
if [[ "$BUILD_TYPE" == "development" ]]; then
|
|
# Development build: rustfs-${platform}-${arch}-${variant}-dev-${short_sha}.zip
|
|
PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}-dev-${SHORT_SHA}"
|
|
else
|
|
# Release/Prerelease build: rustfs-${platform}-${arch}-${variant}-v${version}.zip
|
|
PACKAGE_NAME="${PACKAGE_BASENAME}-v${PACKAGE_VERSION}"
|
|
fi
|
|
|
|
# Create zip packages for all platforms
|
|
# Ensure zip is available
|
|
if ! command -v zip &> /dev/null; then
|
|
if [[ "${{ matrix.os }}" == "ubuntu-latest" || "${{ matrix.platform }}" == "linux" ]]; then
|
|
sudo apt-get update && sudo apt-get install -y zip
|
|
fi
|
|
fi
|
|
|
|
cd target/${{ matrix.target }}/release
|
|
# Determine the binary name based on platform
|
|
if [[ "${{ matrix.platform }}" == "windows" ]]; then
|
|
BINARY_NAME="rustfs.exe"
|
|
else
|
|
BINARY_NAME="rustfs"
|
|
fi
|
|
|
|
# Verify the binary exists before packaging
|
|
if [[ ! -f "$BINARY_NAME" ]]; then
|
|
echo "❌ Binary $BINARY_NAME not found in $(pwd)"
|
|
if [[ "${{ matrix.platform }}" == "windows" ]]; then
|
|
dir
|
|
else
|
|
ls -la
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
# Universal packaging function
|
|
package_zip() {
|
|
local src=$1
|
|
local dst=$2
|
|
if [[ "${{ matrix.platform }}" == "windows" ]]; then
|
|
# Windows uses PowerShell Compress-Archive
|
|
powershell -Command "Compress-Archive -Path '$src' -DestinationPath '$dst' -Force"
|
|
elif command -v zip &> /dev/null; then
|
|
# Unix systems use zip command
|
|
zip "$dst" "$src"
|
|
else
|
|
echo "❌ No zip utility available"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Create the zip package
|
|
echo "Start packaging: $BINARY_NAME -> ../../../${PACKAGE_NAME}.zip"
|
|
package_zip "$BINARY_NAME" "../../../${PACKAGE_NAME}.zip"
|
|
|
|
cd ../../..
|
|
|
|
# Verify the package was created
|
|
if [[ -f "${PACKAGE_NAME}.zip" ]]; then
|
|
echo "✅ Package created successfully: ${PACKAGE_NAME}.zip"
|
|
if [[ "${{ matrix.platform }}" == "windows" ]]; then
|
|
dir
|
|
else
|
|
ls -lh "${PACKAGE_NAME}.zip"
|
|
fi
|
|
else
|
|
echo "❌ Failed to create package: ${PACKAGE_NAME}.zip"
|
|
exit 1
|
|
fi
|
|
|
|
# Create latest version files right after the main package
|
|
LATEST_FILES=""
|
|
if [[ "$BUILD_TYPE" == "release" ]] || [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
|
# Create latest version filename
|
|
# Convert from rustfs-linux-x86_64-musl-v1.0.0 to rustfs-linux-x86_64-musl-latest
|
|
LATEST_FILE="${PACKAGE_BASENAME}-latest.zip"
|
|
|
|
echo "🔄 Creating latest version: ${PACKAGE_NAME}.zip -> $LATEST_FILE"
|
|
cp "${PACKAGE_NAME}.zip" "$LATEST_FILE"
|
|
|
|
if [[ -f "$LATEST_FILE" ]]; then
|
|
echo "✅ Latest version created: $LATEST_FILE"
|
|
LATEST_FILES="$LATEST_FILE"
|
|
fi
|
|
elif [[ "$BUILD_TYPE" == "development" ]]; then
|
|
# Development builds (only main branch triggers development builds)
|
|
# Create main-latest version filename
|
|
# Convert from rustfs-linux-x86_64-dev-abc123 to rustfs-linux-x86_64-main-latest
|
|
MAIN_LATEST_FILE="${PACKAGE_BASENAME}-main-latest.zip"
|
|
|
|
echo "🔄 Creating main-latest version: ${PACKAGE_NAME}.zip -> $MAIN_LATEST_FILE"
|
|
cp "${PACKAGE_NAME}.zip" "$MAIN_LATEST_FILE"
|
|
|
|
if [[ -f "$MAIN_LATEST_FILE" ]]; then
|
|
echo "✅ Main-latest version created: $MAIN_LATEST_FILE"
|
|
LATEST_FILES="$MAIN_LATEST_FILE"
|
|
|
|
# Also create a generic main-latest for Docker builds (Linux only)
|
|
if [[ "${{ matrix.platform }}" == "linux" ]]; then
|
|
DOCKER_MAIN_LATEST_FILE="rustfs-linux-${ARCH_WITH_VARIANT}-main-latest.zip"
|
|
|
|
echo "🔄 Creating Docker main-latest version: ${PACKAGE_NAME}.zip -> $DOCKER_MAIN_LATEST_FILE"
|
|
cp "${PACKAGE_NAME}.zip" "$DOCKER_MAIN_LATEST_FILE"
|
|
|
|
if [[ -f "$DOCKER_MAIN_LATEST_FILE" ]]; then
|
|
echo "✅ Docker main-latest version created: $DOCKER_MAIN_LATEST_FILE"
|
|
LATEST_FILES="$LATEST_FILES $DOCKER_MAIN_LATEST_FILE"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
{
|
|
echo "package_name=${PACKAGE_NAME}"
|
|
echo "package_file=${PACKAGE_NAME}.zip"
|
|
echo "latest_files=${LATEST_FILES}"
|
|
echo "build_type=${BUILD_TYPE}"
|
|
echo "version=${VERSION}"
|
|
} >> "$GITHUB_OUTPUT"
|
|
|
|
echo "📦 Package created: ${PACKAGE_NAME}.zip"
|
|
if [[ -n "$LATEST_FILES" ]]; then
|
|
echo "📦 Latest files created: $LATEST_FILES"
|
|
fi
|
|
echo "🔧 Build type: ${BUILD_TYPE}"
|
|
echo "📊 Version: ${VERSION}"
|
|
|
|
- name: Verify packaged console
|
|
if: matrix.target == 'x86_64-unknown-linux-gnu'
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
find_free_port() {
|
|
python3 - <<'PY'
|
|
import socket
|
|
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
print(sock.getsockname()[1])
|
|
PY
|
|
}
|
|
|
|
package_dir="$(mktemp -d)"
|
|
data_dir="$(mktemp -d)"
|
|
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
|
server_pid=""
|
|
trap '
|
|
if [[ -n "${server_pid:-}" ]]; then
|
|
kill "$server_pid" 2>/dev/null || true
|
|
wait "$server_pid" 2>/dev/null || true
|
|
fi
|
|
rm -rf "$package_dir" "$data_dir"
|
|
' EXIT
|
|
|
|
unzip -q "${{ steps.package.outputs.package_file }}" -d "$package_dir"
|
|
api_port="$(find_free_port)"
|
|
console_port="$(find_free_port)"
|
|
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
|
|
|
"$package_dir/rustfs" server \
|
|
--address "127.0.0.1:${api_port}" \
|
|
--console-enable \
|
|
--console-address "127.0.0.1:${console_port}" \
|
|
--access-key console-smoke \
|
|
--secret-key console-smoke-secret \
|
|
"$data_dir" >"$log_file" 2>&1 &
|
|
server_pid=$!
|
|
|
|
for _ in {1..40}; do
|
|
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
|
|
if [[ "$response" == 200\ text/html* ]]; then
|
|
exit 0
|
|
fi
|
|
sleep 0.25
|
|
done
|
|
|
|
echo "Console endpoint did not return 200 text/html: $response" >&2
|
|
cat "$log_file"
|
|
exit 1
|
|
|
|
- name: Upload to GitHub artifacts
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: ${{ steps.package.outputs.package_name }}
|
|
path: "rustfs-*.zip"
|
|
retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 7 }}
|
|
|
|
- name: Upload to Cloudflare R2
|
|
if: env.R2_ACCESS_KEY_ID != '' && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease' || needs.build-check.outputs.build_type == 'development')
|
|
env:
|
|
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
|
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
|
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
|
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
|
AWS_EC2_METADATA_DISABLED: true
|
|
shell: bash
|
|
run: |
|
|
BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
|
|
|
|
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
|
echo "⚠️ R2 credentials or endpoint missing, skipping artifact upload"
|
|
exit 0
|
|
fi
|
|
|
|
# The self-hosted Linux runners do not ship the aws CLI (GitHub-hosted
|
|
# images did). Install it on demand so R2 uploads survive a fresh runner
|
|
# instead of hard-failing here.
|
|
if ! command -v aws >/dev/null 2>&1; then
|
|
echo "aws CLI not found on runner; installing..."
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
sudo apt-get update && sudo apt-get install -y awscli
|
|
elif command -v brew >/dev/null 2>&1; then
|
|
brew install awscli
|
|
else
|
|
echo "❌ aws CLI missing and no apt-get/brew to install it; cannot upload to R2"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
|
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
|
export AWS_DEFAULT_REGION="auto"
|
|
|
|
# Determine upload path based on build type
|
|
if [[ "$BUILD_TYPE" == "development" ]]; then
|
|
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
|
|
echo "📤 Uploading development build to R2 dev directory"
|
|
else
|
|
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
|
echo "📤 Uploading release build to R2 release directory"
|
|
fi
|
|
|
|
# Upload all rustfs zip files to R2 using S3-compatible API
|
|
echo "📤 Uploading all rustfs-*.zip files to $R2_PATH..."
|
|
for zip_file in rustfs-*.zip; do
|
|
if [[ -f "$zip_file" ]]; then
|
|
echo "Uploading: $zip_file to $R2_PATH..."
|
|
aws s3 cp "$zip_file" "$R2_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
|
echo "✅ Uploaded: $zip_file"
|
|
fi
|
|
done
|
|
|
|
echo "✅ Upload completed successfully"
|
|
|
|
# Build summary
|
|
build-summary:
|
|
name: Build Summary
|
|
needs: [ build-check, build-rustfs ]
|
|
if: always() && needs.build-check.outputs.should_build == 'true'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Build completion summary
|
|
shell: bash
|
|
env:
|
|
# dispatch input via env: free-form text must not be pasted into the
|
|
# script for bash to evaluate.
|
|
INPUT_BUILD_DOCKER: ${{ github.event.inputs.build_docker }}
|
|
run: |
|
|
BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
|
|
VERSION="${{ needs.build-check.outputs.version }}"
|
|
|
|
echo "🎉 Build completed successfully!"
|
|
echo "📦 Build type: $BUILD_TYPE"
|
|
echo "🔢 Version: $VERSION"
|
|
echo ""
|
|
|
|
# Check build status
|
|
BUILD_STATUS="${{ needs.build-rustfs.result }}"
|
|
|
|
echo "📊 Build Results:"
|
|
echo " 📦 All platforms: $BUILD_STATUS"
|
|
echo ""
|
|
|
|
case "$BUILD_TYPE" in
|
|
"preview")
|
|
echo "🔍 Preview artifacts are published in a GitHub prerelease"
|
|
echo "⏭️ Preview releases do not update latest channels"
|
|
;;
|
|
"development")
|
|
echo "🛠️ Development build artifacts have been uploaded to OSS dev directory"
|
|
echo "⚠️ This is a development build - not suitable for production use"
|
|
;;
|
|
"release")
|
|
echo "🚀 Release build artifacts have been uploaded to OSS release directory"
|
|
echo "✅ This build is ready for production use"
|
|
echo "🏷️ GitHub Release will be created in this workflow"
|
|
;;
|
|
"prerelease")
|
|
echo "🧪 Prerelease build artifacts have been uploaded to OSS release directory"
|
|
echo "⚠️ This is a prerelease build - use with caution"
|
|
echo "🏷️ GitHub Release will be created in this workflow"
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
echo "🐳 Docker Images:"
|
|
if [[ "$BUILD_TYPE" == "preview" ]]; then
|
|
echo "⏭️ Preview tags do not publish Docker images"
|
|
elif [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
|
|
echo "⏭️ Docker image build was skipped (binary only build)"
|
|
elif [[ "$BUILD_STATUS" == "success" ]]; then
|
|
echo "🔄 Docker images will be built and pushed automatically via workflow_run event"
|
|
else
|
|
echo "❌ Docker image build will be skipped due to build failure"
|
|
fi
|
|
|
|
# Create GitHub Release for every valid release tag, including previews
|
|
create-release:
|
|
name: Create GitHub Release
|
|
needs: [ build-check, build-rustfs ]
|
|
if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: write
|
|
outputs:
|
|
release_id: ${{ steps.create.outputs.release_id }}
|
|
release_url: ${{ steps.create.outputs.release_url }}
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Create GitHub Release
|
|
id: create
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
shell: bash
|
|
run: |
|
|
TAG="${{ needs.build-check.outputs.version }}"
|
|
VERSION="${{ needs.build-check.outputs.version }}"
|
|
IS_PRERELEASE="${{ needs.build-check.outputs.is_prerelease }}"
|
|
BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
|
|
TARGET_COMMITISH=$(git rev-parse --verify "refs/tags/${TAG}^{commit}")
|
|
|
|
# Determine release type for title
|
|
if [[ "$BUILD_TYPE" == "preview" ]]; then
|
|
RELEASE_TYPE="preview"
|
|
elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
|
if [[ "$TAG" == *"alpha"* ]]; then
|
|
RELEASE_TYPE="alpha"
|
|
elif [[ "$TAG" == *"beta"* ]]; then
|
|
RELEASE_TYPE="beta"
|
|
elif [[ "$TAG" == *"rc"* ]]; then
|
|
RELEASE_TYPE="rc"
|
|
else
|
|
RELEASE_TYPE="prerelease"
|
|
fi
|
|
else
|
|
RELEASE_TYPE="release"
|
|
fi
|
|
|
|
# Create release title
|
|
if [[ "$IS_PRERELEASE" == "true" ]]; then
|
|
TITLE="RustFS $VERSION (${RELEASE_TYPE})"
|
|
else
|
|
TITLE="RustFS $VERSION"
|
|
fi
|
|
|
|
./scripts/release/create_or_update_release.sh \
|
|
"$TAG" \
|
|
"$TARGET_COMMITISH" \
|
|
"$TITLE" \
|
|
"$IS_PRERELEASE"
|
|
|
|
# Prepare and upload release assets
|
|
upload-release-assets:
|
|
name: Upload Release Assets
|
|
needs: [ build-check, build-rustfs, create-release ]
|
|
if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: write
|
|
actions: read
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Download all build artifacts
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
|
with:
|
|
path: ./artifacts
|
|
pattern: rustfs-*
|
|
merge-multiple: true
|
|
|
|
- name: Prepare release assets
|
|
id: prepare
|
|
shell: bash
|
|
run: |
|
|
VERSION="${{ needs.build-check.outputs.version }}"
|
|
TAG="${{ needs.build-check.outputs.version }}"
|
|
|
|
mkdir -p ./release-assets
|
|
|
|
# Copy and verify artifacts (including latest files created during build)
|
|
ASSETS_COUNT=0
|
|
for file in ./artifacts/*.zip; do
|
|
if [[ -f "$file" ]]; then
|
|
cp "$file" ./release-assets/
|
|
ASSETS_COUNT=$((ASSETS_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $ASSETS_COUNT -eq 0 ]]; then
|
|
echo "❌ No artifacts found!"
|
|
exit 1
|
|
fi
|
|
|
|
cd ./release-assets
|
|
|
|
# Generate checksums for all files (including latest versions)
|
|
if compgen -G "*.zip" >/dev/null; then
|
|
sha256sum -- *.zip > SHA256SUMS
|
|
sha512sum -- *.zip > SHA512SUMS
|
|
fi
|
|
|
|
cd ..
|
|
python3 scripts/security/generate_release_supply_chain_assets.py \
|
|
--asset-dir ./release-assets \
|
|
--version "$VERSION" \
|
|
--tag "$TAG" \
|
|
--build-type "${{ needs.build-check.outputs.build_type }}" \
|
|
--repository "${{ github.repository }}" \
|
|
--ref "${{ github.ref }}" \
|
|
--sha "${{ github.sha }}" \
|
|
--run-id "${{ github.run_id }}" \
|
|
--run-attempt "${{ github.run_attempt }}"
|
|
cd ./release-assets
|
|
|
|
echo "📦 Prepared assets:"
|
|
ls -la
|
|
|
|
echo "🔢 Total asset count: $ASSETS_COUNT"
|
|
|
|
- name: Upload to GitHub Release
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
shell: bash
|
|
run: |
|
|
TAG="${{ needs.build-check.outputs.version }}"
|
|
|
|
cd ./release-assets
|
|
|
|
# Upload all files
|
|
for file in *; do
|
|
if [[ -f "$file" ]]; then
|
|
echo "📤 Uploading $file..."
|
|
gh release upload "$TAG" "$file" --clobber
|
|
fi
|
|
done
|
|
|
|
echo "✅ All assets uploaded successfully"
|
|
|
|
# Update latest.json for every release tag (stable and prerelease): the
|
|
# project currently ships prerelease tags only, so gating this to stable
|
|
# left the version pointer permanently stale. release_type records whether
|
|
# the pointed-to version is a prerelease.
|
|
update-latest-version:
|
|
name: Update Latest Version
|
|
needs: [ build-check, publish-release ]
|
|
if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Update latest.json
|
|
env:
|
|
OSS_ACCESS_KEY_ID: ${{ secrets.ALICLOUDOSS_KEY_ID }}
|
|
OSS_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
|
|
OSS_REGION: cn-beijing
|
|
OSS_ENDPOINT: https://oss-cn-beijing.aliyuncs.com
|
|
shell: bash
|
|
run: |
|
|
if [[ -z "$OSS_ACCESS_KEY_ID" ]]; then
|
|
echo "⚠️ OSS credentials not available, skipping latest.json update"
|
|
exit 0
|
|
fi
|
|
|
|
VERSION="${{ needs.build-check.outputs.version }}"
|
|
TAG="${{ needs.build-check.outputs.version }}"
|
|
|
|
if [[ "${{ needs.build-check.outputs.build_type }}" == "prerelease" ]]; then
|
|
RELEASE_TYPE="prerelease"
|
|
else
|
|
RELEASE_TYPE="stable"
|
|
fi
|
|
|
|
# Install ossutil
|
|
OSSUTIL_VERSION="2.1.1"
|
|
OSSUTIL_ZIP="ossutil-${OSSUTIL_VERSION}-linux-amd64.zip"
|
|
OSSUTIL_DIR="ossutil-${OSSUTIL_VERSION}-linux-amd64"
|
|
OSSUTIL_SHA256="60f3a808cbbdfd4b5269b8f967c6a66f564c9dacff52d93a5d21a588976ab5a2"
|
|
|
|
curl --fail -L -o "$OSSUTIL_ZIP" "https://gosspublic.alicdn.com/ossutil/v2/${OSSUTIL_VERSION}/${OSSUTIL_ZIP}"
|
|
printf '%s %s\n' "$OSSUTIL_SHA256" "$OSSUTIL_ZIP" | sha256sum -c -
|
|
unzip "$OSSUTIL_ZIP"
|
|
OSSUTIL_BIN="${RUNNER_TEMP}/ossutil"
|
|
mv "${OSSUTIL_DIR}/ossutil" "$OSSUTIL_BIN"
|
|
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
|
|
chmod +x "$OSSUTIL_BIN"
|
|
|
|
# Create latest.json
|
|
cat > latest.json << EOF
|
|
{
|
|
"version": "${VERSION}",
|
|
"tag": "${TAG}",
|
|
"release_date": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"release_type": "${RELEASE_TYPE}",
|
|
"download_url": "https://github.com/${{ github.repository }}/releases/tag/${TAG}"
|
|
}
|
|
EOF
|
|
|
|
# Upload to OSS
|
|
"$OSSUTIL_BIN" cp latest.json oss://rustfs-version/latest.json --force
|
|
|
|
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
|
|
|
|
# Publish release (remove draft status)
|
|
publish-release:
|
|
name: Publish Release
|
|
needs: [ build-check, create-release, upload-release-assets ]
|
|
if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: write
|
|
steps:
|
|
- name: Publish release
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
shell: bash
|
|
run: |
|
|
TAG="${{ needs.build-check.outputs.version }}"
|
|
BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
|
|
RELEASE_ID="${{ needs.create-release.outputs.release_id }}"
|
|
|
|
# Publish the release and correct its channel state on retries.
|
|
# Only a stable final release may become GitHub Latest.
|
|
if [[ "$BUILD_TYPE" == "release" ]]; then
|
|
gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \
|
|
-F draft=false \
|
|
-F prerelease=false \
|
|
-f make_latest=true >/dev/null
|
|
else
|
|
gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}" \
|
|
-F draft=false \
|
|
-F prerelease=true \
|
|
-f make_latest=false >/dev/null
|
|
fi
|
|
|
|
echo "🎉 Released $TAG successfully!"
|
|
echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
|