Compare commits

...

1 Commits

Author SHA1 Message Date
hector dbf9597c95 ci: add manual packaging of the origin/main tip
Dispatch-only workflow that compiles the current main commit with a
caller-supplied version and uploads zip, deb, and rpm artifacts.

Co-authored-by: hector <majinghe@users.noreply.github.com>
2026-11-11 02:16:02 +00:00
6 changed files with 1120 additions and 1 deletions
+20
View File
@@ -994,6 +994,26 @@
"policy_reason": "Preserve operator-disabled state; enabling requires a separate policy decision.",
"origin": "baseline"
},
{
"workflow": ".github/workflows/manual-main-packages.yml",
"name": "Manual main packages",
"roles": [
"manual"
],
"events": [
"workflow_dispatch"
],
"cadence": "event",
"timezone": "UTC",
"schedule": [],
"owner": "@overtrue",
"policy": "active",
"monitor": "none",
"result_owner": "manual-main-packages.yml",
"disposition": "Packages the origin/main tip as workflow artifacts only. No GitHub Release or download-channel writes.",
"monitor_reason": "Operator-requested packaging has no periodic freshness obligation.",
"origin": "manual-main-packages"
},
{
"workflow": ".github/workflows/minio-interop.yml",
"name": "minio-interop",
+415
View File
@@ -0,0 +1,415 @@
# Copyright 2026 RustFS Team
# SPDX-License-Identifier: Apache-2.0
# Manual packaging of the current origin/main tip. Workflow artifacts only.
# Dispatch uses this file from the selected branch. The build fetches
# refs/heads/main and checks out that commit, not the dispatch ref or a tag.
name: Manual main packages
on:
workflow_dispatch:
inputs:
architecture:
description: "Linux targets: x86 (x86_64 musl+gnu), arm (aarch64 musl+gnu), or all"
required: true
type: choice
options:
- x86
- arm
- all
default: all
version:
description: "Binary and package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v."
required: true
type: string
permissions:
contents: read
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
CARGO_INCREMENTAL: 0
jobs:
prepare:
name: Resolve main tip and version
if: github.repository == 'rustfs/rustfs'
runs-on: ubuntu-latest
timeout-minutes: 15
outputs:
source_sha: ${{ steps.pin.outputs.source_sha }}
version: ${{ steps.version.outputs.version }}
toolchain: ${{ steps.inputs.outputs.toolchain }}
console: ${{ steps.inputs.outputs.console }}
matrix: ${{ steps.matrix.outputs.matrix }}
steps:
- name: Checkout main
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
ref: main
fetch-depth: 0
persist-credentials: false
- name: Pin the tip of origin/main
id: pin
shell: bash
run: |
set -euo pipefail
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
sha=$(git rev-parse FETCH_HEAD)
git checkout --detach "$sha"
test "$(git rev-parse HEAD)" = "$sha"
printf 'source_sha=%s\n' "$sha" >> "$GITHUB_OUTPUT"
printf 'Pinned origin/main %s\n' "$sha"
- name: Validate package version
id: version
shell: bash
env:
PACKAGE_VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
bash scripts/release/package_manual_artifacts.sh validate \
--version "$PACKAGE_VERSION" \
--arch x86_64 \
--libc gnu | tee "$RUNNER_TEMP/package-version.txt"
printf 'version=%s\n' "$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
- name: Install stable Rust
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
toolchain: stable
- name: Resolve console assets and toolchain
id: inputs
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
toolchain=$(rustc --version | awk '{print $2}')
[[ "$toolchain" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
gh api repos/rustfs/console/releases/latest > console-release.json
TOOLCHAIN="$toolchain" python3 - <<'PY'
import json, os, sys
sys.path.insert(0, "scripts/release")
import candidate
console = candidate.console_asset(candidate.read_json("console-release.json"))
candidate.emit_output("console", json.dumps(console, separators=(",", ":")))
candidate.emit_output("toolchain", os.environ["TOOLCHAIN"])
print(f"toolchain={os.environ['TOOLCHAIN']}")
print(f"console={console['tag']} {console['name']}")
PY
- name: Select Linux targets
id: matrix
shell: bash
env:
ARCHITECTURE: ${{ inputs.architecture }}
run: |
set -euo pipefail
matrix=$(bash scripts/release/package_manual_artifacts.sh matrix "$ARCHITECTURE")
count=$(python3 -c 'import json,sys; print(len(json.loads(sys.argv[1])["include"]))' "$matrix")
[[ "$count" -gt 0 ]]
printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT"
printf 'Selected %s target(s) for %s\n' "$count" "$ARCHITECTURE"
package:
name: Package RustFS (${{ matrix.target_id }})
needs: prepare
if: github.repository == 'rustfs/rustfs'
# x86_64 uses sm-standard-4, the official release compile pool: the workspace
# release build needs that disk (placement rule: disk) and reads build.yml's
# cache without writing it. aarch64 uses ubicloud-standard-16-arm so the
# packaged binary and console smoke execute natively (placement rule: hardware).
runs-on: ${{ matrix.os }}
timeout-minutes: 210
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
RUSTUP_TOOLCHAIN: ${{ needs.prepare.outputs.toolchain }}
RUSTFLAGS: "${{ matrix.rustflags }}"
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
steps:
- name: Checkout the pinned origin/main commit
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
ref: ${{ needs.prepare.outputs.source_sha }}
fetch-depth: 0
persist-credentials: false
- name: Confirm the checkout is that main commit
shell: bash
env:
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
git merge-base --is-ancestor "$SOURCE_SHA" FETCH_HEAD
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: ${{ needs.prepare.outputs.toolchain }}
target: ${{ matrix.target }}
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
# build.yml owns this key. This job only restores it.
cache-save-if: 'false'
install-cross-tools: ${{ matrix.cross }}
install-build-packaging-tools: 'true'
install-test-tools: 'false'
- name: Download static console assets
shell: bash
env:
GH_TOKEN: ${{ github.token }}
CONSOLE_IDENTITY: ${{ needs.prepare.outputs.console }}
run: |
set -euo pipefail
python3 - <<'PYCONSOLE'
import json, os, pathlib, subprocess, zipfile
value = json.loads(os.environ["CONSOLE_IDENTITY"])
subprocess.run(["python3", ".github/actions/setup/download_release_asset.py",
"--repository", "rustfs/console", "--tag", value["tag"], "--asset", value["name"],
"--sha256", value["sha256"], "--output", "console.zip"], check=True)
destination = pathlib.Path("rustfs/static")
destination.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile("console.zip") as archive:
for member in archive.infolist():
path = pathlib.PurePosixPath(member.filename)
if path.is_absolute() or ".." in path.parts or "\\" in member.filename:
raise SystemExit("unsafe Console archive member")
archive.extractall(destination)
if not (destination / "index.html").is_file():
raise SystemExit("Console archive has no index.html")
PYCONSOLE
rm -f console.zip
- name: Build RustFS
shell: bash
env:
RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}
BUILD_TARGET: ${{ matrix.target }}
BUILD_OS: ${{ matrix.build_os }}
BUILD_FEATURES: ${{ matrix.features }}
BUILD_GLIBC: ${{ matrix.glibc }}
BUILD_CROSS: ${{ matrix.cross }}
run: |
set -euo pipefail
touch rustfs/build.rs
feature_args=()
if [[ -n "$BUILD_FEATURES" ]]; then
feature_args=(--features "$BUILD_FEATURES")
fi
# gnu targets pin glibc 2.31 through cargo-zigbuild (<target>.<glibc>).
target_arg="$BUILD_TARGET"
if [[ -n "$BUILD_GLIBC" ]]; then
target_arg="${target_arg}.${BUILD_GLIBC}"
fi
profile_args=()
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
profile_args=(--config 'profile.release.strip="none"')
fi
case "$BUILD_CROSS" in
true)
cargo zigbuild --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
;;
false)
cargo build --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
;;
*)
printf 'unexpected cross flag: %q\n' "$BUILD_CROSS" >&2
exit 1
;;
esac
bin="target/${BUILD_TARGET}/release/rustfs"
version_output=$("$bin" --version)
first=${version_output%%$'\n'*}
if [[ "$first" != "rustfs ${RUSTFS_BUILD_VERSION}" ]]; then
printf 'binary version line is %q, expected %q\n' "$first" "rustfs ${RUSTFS_BUILD_VERSION}" >&2
printf '%s\n' "$version_output" >&2
exit 1
fi
if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
! grep -Fxq "build os : ${BUILD_OS}" <<< "$version_output" || \
! grep -Fxq "rust channel : stable-${BUILD_TARGET}" <<< "$version_output"; then
printf 'release binary reports an unexpected compile target\n' >&2
printf '%s\n' "$version_output" >&2
exit 1
fi
if [[ -n "$BUILD_GLIBC" ]]; then
python3 scripts/release/check_glibc.py "$BUILD_GLIBC" "$bin"
fi
- name: Generate the CPU symbol catalog
if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
shell: bash
env:
BUILD_TARGET: ${{ matrix.target }}
run: |
set -euo pipefail
cargo metadata --locked --format-version 1 --filter-platform "$BUILD_TARGET" \
--features rustfs/pyroscope > "$RUNNER_TEMP/cpu-catalog-metadata.json"
features=$(jq -ce '
(.packages[] | select(.name == "rustfs") | .id) as $id |
.resolve.nodes[] | select(.id == $id) | .features | map(select(. != "default")) | sort
' "$RUNNER_TEMP/cpu-catalog-metadata.json")
release_dir="target/${BUILD_TARGET}/release"
rm -f "$release_dir/rustfs.cpu-symbol-catalog.json" "$release_dir/rustfs.cpu-symbol-catalog.sha256"
cargo run --locked -p rustfs-connect-symbol-catalog -- \
"$release_dir/rustfs" "$(git rev-parse HEAD)" "$BUILD_TARGET" "$features" \
"$release_dir/rustfs.cpu-symbol-catalog.json" > "$release_dir/rustfs.cpu-symbol-catalog.sha256"
- name: Install DEB and RPM tools
shell: bash
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y ruby ruby-dev build-essential rpm fakeroot cpio
sudo gem install fpm --version 1.17.0 --no-document
- name: Package zip, deb, and rpm
shell: bash
env:
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
ARCH: ${{ matrix.arch }}
LIBC: ${{ matrix.libc }}
BUILD_TARGET: ${{ matrix.target }}
BUILD_FEATURES: ${{ matrix.features }}
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
run: |
set -euo pipefail
binary="target/${BUILD_TARGET}/release/rustfs"
extra=()
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
extra=(
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.json"
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.sha256"
)
fi
bash scripts/release/package_manual_artifacts.sh package \
--version "$PACKAGE_VERSION" \
--arch "$ARCH" \
--libc "$LIBC" \
--binary "$binary" \
--output dist \
"${extra[@]}"
printf 'Packaged %s from origin/main %s\n' "$PACKAGE_VERSION" "$SOURCE_SHA"
- name: Verify the packaged binary
shell: bash
env:
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
ARCH: ${{ matrix.arch }}
LIBC: ${{ matrix.libc }}
BUILD_TARGET: ${{ matrix.target }}
BUILD_OS: ${{ matrix.build_os }}
BUILD_FEATURES: ${{ matrix.features }}
run: |
set -euo pipefail
stem="rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}"
package_file="dist/${stem}.zip"
[[ -f "$package_file" && -f "dist/${stem}.deb" && -f "dist/${stem}.rpm" ]]
[[ "$(find dist -maxdepth 1 -type f | wc -l)" -eq 3 ]]
archive_members=$(unzip -Z1 "$package_file" | sort)
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
expected_members=$(printf '%s\n' rustfs rustfs.cpu-symbol-catalog.json rustfs.cpu-symbol-catalog.sha256 | sort)
else
expected_members=$(printf '%s\n' rustfs)
fi
if [[ "$archive_members" != "$expected_members" ]]; then
printf 'Linux package does not contain the expected files\n' >&2
printf '%s\n' "$archive_members" >&2
exit 1
fi
binary_dir=$(mktemp -d)
trap 'rm -rf "$binary_dir"' EXIT
unzip -q "$package_file" rustfs -d "$binary_dir"
version_output=$("$binary_dir/rustfs" --version)
first=${version_output%%$'\n'*}
if [[ "$first" != "rustfs ${PACKAGE_VERSION}" ]]; then
printf 'packaged binary version line is %q, expected %q\n' "$first" "rustfs ${PACKAGE_VERSION}" >&2
exit 1
fi
if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
! grep -Fxq "build os : ${BUILD_OS}" <<< "$version_output"; then
printf 'packaged binary reports an unexpected compile target\n' >&2
exit 1
fi
- name: Verify packaged console
if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'aarch64-unknown-linux-gnu'
shell: bash
env:
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
ARCH: ${{ matrix.arch }}
LIBC: ${{ matrix.libc }}
run: |
set -euo pipefail
package_file="dist/rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}.zip"
find_free_port() {
python3 - <<'PY'
import socket
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
print(sock.getsockname()[1])
PY
}
package_dir=$(mktemp -d)
data_dir=$(mktemp -d)
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
server_pid=""
trap '
if [[ -n "${server_pid:-}" ]]; then
kill "$server_pid" 2>/dev/null || true
wait "$server_pid" 2>/dev/null || true
fi
rm -rf "$package_dir" "$data_dir"
' EXIT
unzip -q "$package_file" -d "$package_dir"
api_port=$(find_free_port)
console_port=$(find_free_port)
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
"$package_dir/rustfs" server \
--address "127.0.0.1:${api_port}" \
--console-enable \
--console-address "127.0.0.1:${console_port}" \
--access-key console-smoke \
--secret-key console-smoke-secret \
"$data_dir" >"$log_file" 2>&1 &
server_pid=$!
for _ in {1..40}; do
response=$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)
if [[ "$response" == 200\ text/html* ]]; then
exit 0
fi
sleep 0.25
done
printf 'Console endpoint did not return 200 text/html: %s\n' "$response" >&2
cat "$log_file"
exit 1
- name: Upload packages
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: packages-${{ matrix.target_id }}-v${{ needs.prepare.outputs.version }}
path: dist/*
if-no-files-found: error
compression-level: 0
retention-days: 30
+1 -1
View File
@@ -161,7 +161,7 @@ their issue closes.
| Entry | Status | Purpose | Wiring / docs |
|---|---|---|---|
| `fuzz/` | ci-gate | Unified cargo-fuzz runner and helpers for the `fuzz/` sub-workspace | fuzz.yml; `fuzz/README.md` |
| `release/` | ci-gate | Release creation and DEB/RPM version-normalization helpers | build.yml; package.yml |
| `release/` | ci-gate | Release creation and DEB/RPM version-normalization helpers | build.yml; package.yml; manual-main-packages.yml |
| `s3-tests/` | ci-gate | ceph/s3-tests compatibility harness (allow-lists, patches, report tooling) | ci.yml; e2e-s3tests.yml; `scripts/s3-tests/README.md` |
| `security/` | ci-gate | Workflow-pin enforcement and release supply-chain asset generation | audit.yml; build.yml |
| `table-catalog/` | dev-tool | S3-Tables / pyiceberg validation suite | `docs/architecture/s3-tables-support-matrix.md` |
+406
View File
@@ -0,0 +1,406 @@
#!/usr/bin/env bash
# Copyright 2026 RustFS Team
# SPDX-License-Identifier: Apache-2.0
# Manual zip/deb/rpm packaging for one Linux target.
# Version rules and deb/rpm metadata come from package_versions.sh.
# deb/rpm payloads are built by build_deb.sh and build_rpm.sh.
# Dependency strings match scripts/release/deliver.py package().
set -euo pipefail
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
helper="${repo_root}/scripts/release/package_versions.sh"
die() {
printf 'package_manual_artifacts: %s\n' "$1" >&2
exit 1
}
usage() {
cat <<'EOF' >&2
usage: package_manual_artifacts.sh matrix <x86|arm|all>
package_manual_artifacts.sh validate --version VER --arch ARCH --libc LIBC
package_manual_artifacts.sh package --version VER --arch ARCH --libc LIBC --binary PATH --output DIR [--zip-extra FILE]...
EOF
exit 2
}
require_value() {
local flag=$1
# An empty value is a version/path error, not a missing argument. Flag-shaped
# values mean the next option was consumed by mistake.
if [[ $# -lt 2 || ${2:-} == --* ]]; then
die "missing value for ${flag}"
fi
}
resolve_version() {
local version=$1 arch=$2 libc=$3
local release_err prerelease_err release_out prerelease_out
release_err=$(mktemp)
prerelease_err=$(mktemp)
if release_out=$(bash "$helper" release "$version" "" "$arch" "$libc" 2>"$release_err"); then
printf 'release\n%s\n' "$release_out"
rm -f "$release_err" "$prerelease_err"
return 0
fi
if prerelease_out=$(bash "$helper" prerelease "$version" "" "$arch" "$libc" 2>"$prerelease_err"); then
printf 'prerelease\n%s\n' "$prerelease_out"
rm -f "$release_err" "$prerelease_err"
return 0
fi
printf 'invalid package version: %s\n' "$version" >&2
printf 'not a release: %s\n' "$(<"$release_err")" >&2
printf 'not a prerelease: %s\n' "$(<"$prerelease_err")" >&2
rm -f "$release_err" "$prerelease_err"
return 1
}
load_versions() {
local resolved line key value
resolved=$(resolve_version "$version" "$arch" "$libc") || exit 1
build_type=${resolved%%$'\n'*}
local rest=${resolved#*$'\n'}
deb_version=''
rpm_version=''
rpm_release=''
deb_file=''
rpm_file=''
while IFS= read -r line || [[ -n "$line" ]]; do
[[ -n "$line" ]] || continue
key=${line%%=*}
value=${line#*=}
case "$key" in
deb_version) deb_version=$value ;;
rpm_version) rpm_version=$value ;;
rpm_release) rpm_release=$value ;;
deb_file) deb_file=$value ;;
rpm_file) rpm_file=$value ;;
*) die "unexpected package_versions field: ${key}" ;;
esac
done <<< "$rest"
[[ -n "$deb_version" && -n "$rpm_version" && -n "$rpm_release" && -n "$deb_file" && -n "$rpm_file" ]] ||
die "incomplete package version output"
local stem=${deb_file%.deb}
[[ "$stem" == "${rpm_file%.rpm}" ]] || die "deb and rpm file names use different stems"
[[ "$stem" == "rustfs-linux-${arch}-${libc}-v${version}" ]] ||
die "package stem does not match rustfs-linux-${arch}-${libc}-v${version}"
zip_file="${stem}.zip"
case "$build_type" in
release | prerelease) ;;
*) die "unsupported build type: ${build_type}" ;;
esac
}
emit_resolved() {
printf 'build_type=%s\n' "$build_type"
printf 'deb_version=%s\n' "$deb_version"
printf 'rpm_version=%s\n' "$rpm_version"
printf 'rpm_release=%s\n' "$rpm_release"
printf 'deb_file=%s\n' "$deb_file"
printf 'rpm_file=%s\n' "$rpm_file"
printf 'zip_file=%s\n' "$zip_file"
}
emit_matrix() {
local choice=$1
command -v python3 >/dev/null 2>&1 || die "python3 is required"
python3 - "$choice" <<'PY'
import json
import sys
choice = sys.argv[1]
# Linux rows match the release matrix in .github/workflows/build.yml.
# arch, libc, and build_os are the packaging names for those same rows.
rows = [
{"target_id": "linux-x86_64-musl", "os": "sm-standard-4", "target": "x86_64-unknown-linux-musl", "cross": False, "platform": "linux", "rustflags": "", "features": "", "glibc": "", "arch": "x86_64", "libc": "musl", "build_os": "linux-x86_64"},
{"target_id": "linux-aarch64-musl", "os": "ubicloud-standard-16-arm", "target": "aarch64-unknown-linux-musl", "cross": False, "platform": "linux", "rustflags": "", "features": "", "glibc": "", "arch": "aarch64", "libc": "musl", "build_os": "linux-aarch64"},
{"target_id": "linux-x86_64-gnu", "os": "sm-standard-4", "target": "x86_64-unknown-linux-gnu", "cross": True, "platform": "linux", "rustflags": "", "features": "pyroscope", "glibc": "2.31", "arch": "x86_64", "libc": "gnu", "build_os": "linux-x86_64"},
{"target_id": "linux-aarch64-gnu", "os": "ubicloud-standard-16-arm", "target": "aarch64-unknown-linux-gnu", "cross": True, "platform": "linux", "rustflags": "", "features": "pyroscope", "glibc": "2.31", "arch": "aarch64", "libc": "gnu", "build_os": "linux-aarch64"},
]
if choice == "all":
selected = rows
elif choice == "x86":
selected = [row for row in rows if row["arch"] == "x86_64"]
elif choice == "arm":
selected = [row for row in rows if row["arch"] == "aarch64"]
else:
raise SystemExit(
"package_manual_artifacts: unsupported architecture "
f"(expected x86, arm, or all): {choice}"
)
if len(selected) == 0:
raise SystemExit("package_manual_artifacts: no targets selected")
json.dump({"include": selected}, sys.stdout, separators=(",", ":"))
sys.stdout.write("\n")
PY
}
parse_identity() {
version=''
arch=''
libc=''
local saw_version=0 saw_arch=0 saw_libc=0
while [[ $# -gt 0 ]]; do
case "$1" in
--version)
require_value "$@"
saw_version=1
version=$2
shift 2
;;
--arch)
require_value "$@"
saw_arch=1
arch=$2
shift 2
;;
--libc)
require_value "$@"
saw_libc=1
libc=$2
shift 2
;;
*)
die "unknown argument: $1"
;;
esac
done
[[ "$saw_version" == 1 && "$saw_arch" == 1 && "$saw_libc" == 1 ]] ||
die "validate requires --version, --arch, and --libc"
}
abspath() {
local path=$1
local dir base
dir=$(cd "$(dirname "$path")" && pwd)
base=$(basename "$path")
printf '%s/%s' "$dir" "$base"
}
sha256_file() {
sha256sum "$1" | awk 'NR==1 { print $1 }'
}
package_binary() {
local binary='' output=''
local -a zip_extras=()
local saw_version=0 saw_arch=0 saw_libc=0
version=''
arch=''
libc=''
while [[ $# -gt 0 ]]; do
case "$1" in
--version)
require_value "$@"
saw_version=1
version=$2
shift 2
;;
--arch)
require_value "$@"
saw_arch=1
arch=$2
shift 2
;;
--libc)
require_value "$@"
saw_libc=1
libc=$2
shift 2
;;
--binary)
require_value "$@"
binary=$2
shift 2
;;
--output)
require_value "$@"
output=$2
shift 2
;;
--zip-extra)
require_value "$@"
zip_extras+=("$2")
shift 2
;;
*)
die "unknown argument: $1"
;;
esac
done
[[ "$saw_version" == 1 && "$saw_arch" == 1 && "$saw_libc" == 1 && -n "$binary" && -n "$output" ]] ||
die "package requires --version, --arch, --libc, --binary, and --output"
[[ -f "$binary" && ! -L "$binary" ]] || die "binary must be a regular file"
load_versions
local -a extra_names=()
local extra base seen=' '
local has_json=0 has_sha=0
for extra in "${zip_extras[@]}"; do
base=$(basename "$extra")
case "$base" in
rustfs.cpu-symbol-catalog.json) has_json=1 ;;
rustfs.cpu-symbol-catalog.sha256) has_sha=1 ;;
*) die "unsupported zip member: ${base}" ;;
esac
[[ "$seen" != *" ${base} "* ]] || die "duplicate zip member: ${base}"
seen+=" ${base} "
[[ -f "$extra" && ! -L "$extra" ]] || die "zip extra must be a regular file: ${base}"
extra_names+=("$base")
done
[[ "$has_json" == "$has_sha" ]] ||
die "cpu symbol catalog json and sha256 must be packaged together"
local tool
for tool in zip unzip sha256sum dpkg-deb fakeroot rpm rpm2cpio cpio fpm; do
command -v "$tool" >/dev/null 2>&1 || die "required tool not found: ${tool}"
done
binary=$(abspath "$binary")
local -a absolute_extras=()
for extra in "${zip_extras[@]}"; do
absolute_extras+=("$(abspath "$extra")")
done
mkdir -p "$output"
output=$(cd "$output" && pwd)
local produced
for produced in "$zip_file" "$deb_file" "$rpm_file"; do
[[ ! -e "$output/$produced" ]] || die "refusing to overwrite ${output}/${produced}"
done
# Globals so the EXIT trap can still see them after this function returns.
work=$(mktemp -d)
stage=$(mktemp -d)
cleanup() {
rm -rf "${work:-}" "${stage:-}"
}
trap cleanup EXIT
local zip_dir="$work/zip"
mkdir -p "$zip_dir"
cp "$binary" "$zip_dir/rustfs"
chmod 755 "$zip_dir/rustfs"
local index=0
for extra in "${absolute_extras[@]}"; do
cp "$extra" "$zip_dir/${extra_names[$index]}"
index=$((index + 1))
done
(
cd "$zip_dir"
zip "$stage/$zip_file" rustfs "${extra_names[@]}"
)
local members expected
members=$(unzip -Z1 "$stage/$zip_file" | sort)
expected=$(printf '%s\n' rustfs "${extra_names[@]}" | sort)
[[ "$members" == "$expected" ]] || die "zip members do not match the release archive"
[[ "$(unzip -p "$stage/$zip_file" rustfs | sha256sum | awk '{print $1}')" == "$(sha256_file "$binary")" ]] ||
die "zip binary differs from the packaged executable"
ln -s "$repo_root/scripts" "$work/scripts"
ln -s "$repo_root/deploy" "$work/deploy"
cp "$repo_root/LICENSE" "$repo_root/README.md" "$work/"
mkdir -p "$work/bin"
cp "$binary" "$work/bin/rustfs"
chmod 755 "$work/bin/rustfs"
local deb_arch deb_depends rpm_depends
case "$arch" in
x86_64) deb_arch=amd64 ;;
aarch64) deb_arch=arm64 ;;
*) die "unsupported architecture: ${arch}" ;;
esac
case "$libc" in
gnu)
deb_depends='libc6 (>= 2.31)'
rpm_depends='glibc >= 2.31'
;;
musl)
deb_depends=''
rpm_depends=''
;;
*) die "unsupported libc: ${libc}" ;;
esac
(
cd "$work"
GITHUB_OUTPUT="$work/outputs" \
DEB_VERSION="$deb_version" \
RPM_VERSION="$rpm_version" \
RPM_RELEASE="$rpm_release" \
DEB_FILE="$deb_file" \
RPM_FILE="$rpm_file" \
DEB_ARCH="$deb_arch" \
RPM_ARCH="$arch" \
DEB_DEPENDS="$deb_depends" \
RPM_DEPENDS="$rpm_depends" \
LIBC="$libc" \
bash "$repo_root/scripts/release/build_deb.sh"
GITHUB_OUTPUT="$work/outputs" \
DEB_VERSION="$deb_version" \
RPM_VERSION="$rpm_version" \
RPM_RELEASE="$rpm_release" \
DEB_FILE="$deb_file" \
RPM_FILE="$rpm_file" \
DEB_ARCH="$deb_arch" \
RPM_ARCH="$arch" \
DEB_DEPENDS="$deb_depends" \
RPM_DEPENDS="$rpm_depends" \
LIBC="$libc" \
bash "$repo_root/scripts/release/build_rpm.sh"
)
cp "$work/$deb_file" "$stage/$deb_file"
cp "$work/$rpm_file" "$stage/$rpm_file"
local unpacked expected_hash actual_hash
expected_hash=$(sha256_file "$binary")
unpacked="$work/unpacked-deb"
mkdir -p "$unpacked"
dpkg-deb --extract "$stage/$deb_file" "$unpacked"
actual_hash=$(sha256_file "$unpacked/usr/bin/rustfs")
[[ "$actual_hash" == "$expected_hash" ]] || die "deb binary differs from the packaged executable"
[[ "$(dpkg-deb -f "$stage/$deb_file" Package)" == rustfs ]] || die "deb package name mismatch"
[[ "$(dpkg-deb -f "$stage/$deb_file" Version)" == "$deb_version" ]] || die "deb version mismatch"
[[ "$(dpkg-deb -f "$stage/$deb_file" Architecture)" == "$deb_arch" ]] || die "deb architecture mismatch"
unpacked="$work/unpacked-rpm"
mkdir -p "$unpacked"
rpm2cpio "$stage/$rpm_file" | (cd "$unpacked" && cpio --extract --make-directories --quiet ./usr/bin/rustfs)
actual_hash=$(sha256_file "$unpacked/usr/bin/rustfs")
[[ "$actual_hash" == "$expected_hash" ]] || die "rpm binary differs from the packaged executable"
local rpm_metadata expected_metadata
rpm_metadata=$(rpm -qp --qf '%{NAME}\n%{VERSION}\n%{RELEASE}\n%{ARCH}\n' "$stage/$rpm_file")
expected_metadata=$(printf 'rustfs\n%s\n%s\n%s' "$rpm_version" "$rpm_release" "$arch")
[[ "$rpm_metadata" == "$expected_metadata" ]] || die "rpm metadata mismatch"
mv "$stage/$zip_file" "$output/$zip_file"
mv "$stage/$deb_file" "$output/$deb_file"
mv "$stage/$rpm_file" "$output/$rpm_file"
emit_resolved
}
main() {
local command=${1:-}
case "$command" in
matrix)
[[ $# -eq 2 ]] || usage
emit_matrix "$2"
;;
validate)
shift
parse_identity "$@"
load_versions
emit_resolved
;;
package)
shift
package_binary "$@"
;;
*)
usage
;;
esac
}
main "$@"
@@ -0,0 +1,226 @@
#!/usr/bin/env python3
# Copyright 2026 RustFS Team
# SPDX-License-Identifier: Apache-2.0
"""Contract tests for manual origin/main packaging."""
from __future__ import annotations
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import sys
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "release" / "package_manual_artifacts.sh"
WORKFLOW = ROOT / ".github" / "workflows" / "manual-main-packages.yml"
sys.path.insert(0, str(ROOT / "scripts"))
from ci_gate import workflow_document # noqa: E402
import validation_registry as registry # noqa: E402
def run_script(*args, check=False):
return subprocess.run(["bash", str(SCRIPT), *args], cwd=ROOT, text=True,
capture_output=True, timeout=60, check=check)
def trigger(text):
match = re.search(r"^on:\n(.*?)(?=^[^ #\n]|\Z)", text, re.M | re.S)
if match is None:
raise AssertionError("workflow has no event declaration")
return re.findall(r"^ (\w+):", match.group(1), re.M)
class VersionAndMatrixTests(unittest.TestCase):
def test_release_and_prerelease_names_come_from_package_versions(self):
cases = {
("1.2.3", "x86_64", "gnu"): """\
build_type=release
deb_version=1.2.3
rpm_version=1.2.3
rpm_release=1
deb_file=rustfs-linux-x86_64-gnu-v1.2.3.deb
rpm_file=rustfs-linux-x86_64-gnu-v1.2.3.rpm
zip_file=rustfs-linux-x86_64-gnu-v1.2.3.zip
""",
("1.2.3-alpha.1", "aarch64", "gnu"): """\
build_type=prerelease
deb_version=1.2.3~alpha.1
rpm_version=1.2.3~alpha.1
rpm_release=1
deb_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.deb
rpm_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.rpm
zip_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.zip
""",
("1.2.3-beta.2", "x86_64", "musl"): """\
build_type=prerelease
deb_version=1.2.3~beta.2
rpm_version=1.2.3~beta.2
rpm_release=1
deb_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.deb
rpm_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.rpm
zip_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.zip
""",
("1.2.3-rc.4", "aarch64", "musl"): """\
build_type=prerelease
deb_version=1.2.3~rc.4
rpm_version=1.2.3~rc.4
rpm_release=1
deb_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.deb
rpm_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.rpm
zip_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.zip
""",
}
for (version, arch, libc), expected in cases.items():
with self.subTest(version=version, arch=arch, libc=libc):
result = run_script("validate", "--version", version, "--arch", arch, "--libc", libc)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout, expected)
def test_invalid_versions_fail_without_stdout(self):
for version in ("", "v1.2.3", "1.2", "01.2.3", "1.2.3-preview.1", "1.2.3-alpha",
"1.2.3-rc.01", "1.2.3-rc.1-extra", "1.2.3 ", "1.2.3$(id)", "dev-abcdef"):
with self.subTest(version=version):
result = run_script("validate", "--version", version, "--arch", "x86_64", "--libc", "gnu")
self.assertNotEqual(result.returncode, 0)
self.assertEqual(result.stdout, "")
self.assertIn("invalid package version", result.stderr)
self.assertIn("not a release:", result.stderr)
self.assertIn("not a prerelease:", result.stderr)
def test_invalid_version_does_not_create_packages(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
binary = root / "rustfs"
binary.write_bytes(b"not packaged")
output = root / "dist"
result = run_script("package", "--version", "v1.2.3", "--arch", "x86_64", "--libc", "gnu",
"--binary", str(binary), "--output", str(output))
self.assertNotEqual(result.returncode, 0)
self.assertEqual(result.stdout, "")
self.assertFalse(output.exists())
def test_catalog_members_must_be_the_official_pair(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
binary = root / "rustfs"
binary.write_bytes(b"not packaged")
catalog = root / "rustfs.cpu-symbol-catalog.json"
catalog.write_text("{}\n", encoding="utf-8")
result = run_script("package", "--version", "1.2.3", "--arch", "x86_64", "--libc", "gnu",
"--binary", str(binary), "--output", str(root / "dist"),
"--zip-extra", str(catalog))
self.assertNotEqual(result.returncode, 0)
self.assertIn("packaged together", result.stderr)
self.assertFalse((root / "dist").exists())
other = root / "notes.txt"
other.write_text("nope\n", encoding="utf-8")
rejected = run_script("package", "--version", "1.2.3", "--arch", "x86_64", "--libc", "musl",
"--binary", str(binary), "--output", str(root / "dist"),
"--zip-extra", str(other))
self.assertNotEqual(rejected.returncode, 0)
self.assertIn("unsupported zip member", rejected.stderr)
def test_architecture_matrix_matches_official_linux_rows(self):
source = (ROOT / ".github" / "workflows" / "build.yml").read_text(encoding="utf-8")
official = json.loads(re.search(r"all='(\{.*?\})'", source, re.S).group(1))["include"]
linux = [row for row in official if row["platform"] == "linux"]
def derived(row):
arch = "aarch64" if row["target"].startswith("aarch64-") else "x86_64"
libc = row["target_id"].rsplit("-", 1)[1]
return {**row, "arch": arch, "libc": libc, "build_os": f"linux-{arch}"}
expected = {
"all": [derived(row) for row in linux],
"x86": [derived(row) for row in linux if row["target"].startswith("x86_64-")],
"arm": [derived(row) for row in linux if row["target"].startswith("aarch64-")],
}
self.assertEqual([row["target_id"] for row in expected["x86"]],
["linux-x86_64-musl", "linux-x86_64-gnu"])
self.assertEqual([row["target_id"] for row in expected["arm"]],
["linux-aarch64-musl", "linux-aarch64-gnu"])
for choice, rows in expected.items():
with self.subTest(choice=choice):
result = run_script("matrix", choice, check=True)
self.assertEqual(json.loads(result.stdout), {"include": rows})
for choice in ("", "ALL", "linux", "x86_64", "x86,arm"):
with self.subTest(choice=choice):
result = run_script("matrix", choice)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(result.stdout, "")
self.assertIn("unsupported architecture", result.stderr)
class WorkflowContractTests(unittest.TestCase):
def test_dispatch_builds_origin_main_and_only_uploads_artifacts(self):
text = WORKFLOW.read_text(encoding="utf-8")
self.assertEqual(trigger(text), ["workflow_dispatch"])
document = workflow_document(text)
events = document.get("on", document.get(True))
self.assertEqual(set(events), {"workflow_dispatch"})
inputs = events["workflow_dispatch"]["inputs"]
self.assertEqual(inputs["architecture"]["type"], "choice")
self.assertEqual(inputs["architecture"]["options"], ["x86", "arm", "all"])
self.assertTrue(inputs["architecture"]["required"])
self.assertEqual(inputs["version"]["type"], "string")
self.assertTrue(inputs["version"]["required"])
self.assertEqual(document["permissions"], {"contents": "read"})
self.assertEqual(re.findall(r"^\s+ref:\s*(.+)$", text, re.M),
["main", "${{ needs.prepare.outputs.source_sha }}"])
self.assertIn("refs/heads/main", text)
self.assertNotIn("github.sha", text)
self.assertIn("package_manual_artifacts.sh validate", text)
self.assertIn("package_manual_artifacts.sh matrix", text)
self.assertIn("package_manual_artifacts.sh package", text)
self.assertIn("RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}", text)
self.assertIn("cargo zigbuild --release --locked", text)
self.assertIn("cargo build --release --locked", text)
self.assertIn("scripts/release/check_glibc.py", text)
self.assertIn('cache-save-if: \'false\'', text)
self.assertIn("actions/upload-artifact@", text)
self.assertIn("dist/*", text)
for forbidden in ("contents: write", "aws s3", "gh release", "softprops/action-gh-release",
"docker push", "R2_ACCESS_KEY_ID", "workflow_call:"):
self.assertNotIn(forbidden, text)
self.assertEqual(text.count("persist-credentials: false"), text.count("actions/checkout@"))
item = registry.workflow_entry(".github/workflows/manual-main-packages.yml")
self.assertEqual(item["name"], "Manual main packages")
self.assertEqual(item["events"], ["workflow_dispatch"])
self.assertEqual(item["schedule"], [])
self.assertEqual(item["monitor"], "none")
self.assertNotEqual(item["origin"], "baseline")
def test_registry_accepts_the_added_workflow(self):
self.assertTrue(registry.check_wiring(registry.load_registry()))
class PackageToolTests(unittest.TestCase):
def test_zip_deb_and_rpm_wrap_the_same_binary_when_tools_exist(self):
required = ("dpkg-deb", "fakeroot", "rpm", "rpm2cpio", "cpio", "fpm", "zip", "unzip")
missing = [tool for tool in required if shutil.which(tool) is None]
if missing or os.name != "posix":
self.skipTest("packaging tools not installed: " + ", ".join(missing))
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
payload = b"manual main package fixture\n"
binary = root / "rustfs"
binary.write_bytes(payload)
result = run_script("package", "--version", "9.9.9-rc.8", "--arch", "x86_64", "--libc", "gnu",
"--binary", str(binary), "--output", str(root / "dist"))
self.assertEqual(result.returncode, 0, result.stderr)
dist = root / "dist"
self.assertEqual(sorted(path.name for path in dist.iterdir()), [
"rustfs-linux-x86_64-gnu-v9.9.9-rc.8.deb",
"rustfs-linux-x86_64-gnu-v9.9.9-rc.8.rpm",
"rustfs-linux-x86_64-gnu-v9.9.9-rc.8.zip",
])
self.assertIn("deb_version=9.9.9~rc.8", result.stdout)
self.assertIn("zip_file=rustfs-linux-x86_64-gnu-v9.9.9-rc.8.zip", result.stdout)
if __name__ == "__main__":
unittest.main()
+52
View File
@@ -11,6 +11,7 @@ import subprocess
import sys
import tempfile
import unittest
import zipfile
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parent / "release"))
@@ -63,6 +64,57 @@ class PackageIntegrationTests(unittest.TestCase):
deliver.package(fixtures.inputs(), "linux-x86_64-musl", root / "output")
self.assertFalse(list((root / "output").glob("packages-*.json")))
def test_manual_packages_match_release_names_and_bytes(self):
script = Path(__file__).resolve().parent / "release" / "package_manual_artifacts.sh"
cases = (
("9.9.9-rc.8", "x86_64", "gnu", True),
("9.9.9", "aarch64", "musl", False),
)
for version, arch, libc, with_catalog in cases:
with self.subTest(version=version, arch=arch, libc=libc), tempfile.TemporaryDirectory() as temp:
root = Path(temp)
payload = f"manual-package:{version}:{arch}:{libc}\n".encode()
binary = root / "rustfs"
binary.write_bytes(payload)
command = ["bash", str(script), "package", "--version", version, "--arch", arch, "--libc", libc,
"--binary", str(binary), "--output", str(root / "dist")]
if with_catalog:
for name in ("rustfs.cpu-symbol-catalog.json", "rustfs.cpu-symbol-catalog.sha256"):
(root / name).write_text(name + "\n", encoding="utf-8")
command.extend(["--zip-extra", str(root / name)])
subprocess.run(command, check=True, text=True, timeout=180)
stem = f"rustfs-linux-{arch}-{libc}-v{version}"
dist = root / "dist"
self.assertEqual(sorted(path.name for path in dist.iterdir()),
[f"{stem}.deb", f"{stem}.rpm", f"{stem}.zip"])
expected_hash = hashlib.sha256(payload).hexdigest()
with zipfile.ZipFile(dist / f"{stem}.zip") as archive:
names = sorted(archive.namelist())
self.assertEqual(hashlib.sha256(archive.read("rustfs")).hexdigest(), expected_hash)
if with_catalog:
self.assertEqual(names, ["rustfs", "rustfs.cpu-symbol-catalog.json", "rustfs.cpu-symbol-catalog.sha256"])
else:
self.assertEqual(names, ["rustfs"])
deb = dist / f"{stem}.deb"
self.assertEqual(subprocess.check_output(["dpkg-deb", "-f", deb, "Version"], text=True).strip(),
version.replace("-", "~"))
self.assertEqual(subprocess.check_output(["dpkg-deb", "-f", deb, "Architecture"], text=True).strip(),
"amd64" if arch == "x86_64" else "arm64")
depends = subprocess.check_output(["dpkg-deb", "-f", deb, "Depends"], text=True).strip()
self.assertEqual(depends, "libc6 (>= 2.31)" if libc == "gnu" else "")
unpacked = root / "deb"
subprocess.run(["dpkg-deb", "--extract", str(deb), str(unpacked)], check=True)
self.assertEqual(hashlib.sha256((unpacked / "usr/bin/rustfs").read_bytes()).hexdigest(), expected_hash)
rpm = dist / f"{stem}.rpm"
identity = subprocess.check_output(
["rpm", "-qp", "--qf", "%{NAME} %{VERSION} %{RELEASE} %{ARCH}", str(rpm)], text=True)
self.assertEqual(identity, f"rustfs {version.replace('-', '~')} 1 {arch}")
requires = subprocess.check_output(["rpm", "-qp", "--requires", str(rpm)], text=True)
if libc == "gnu":
self.assertIn("glibc >= 2.31", requires)
else:
self.assertNotIn("glibc", requires)
if __name__ == "__main__":
unittest.main()