gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-27 08:38:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/rustfs/s3-rustfs into dev_issue_233

Browse Source
This commit is contained in:
DamonXue
2025-03-02 19:54:06 +08:00
parent 145bb6ac7c 03c3d4abfe
commit 69b6c0f660
12 changed files with 128 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cargo/config.toml
+2
View File
@@ -0,0 +1,2 @@
[target.x86_64-unknown-linux-gnu]
rustflags = ["-Clink-arg=-fuse-ld=lld"]
.github/actions/setup/action.yml
+1
View File
@@ -21,6 +21,7 @@ runs:
run: |
sudo apt update
sudo apt install -y \
lld \
libdbus-1-dev \
libwayland-dev \
libwebkit2gtk-4.1-dev \
.github/workflows/build.yml
+40 -6
View File
@@ -15,9 +15,9 @@ jobs:
strategy:
matrix:
variant:
- { profile: dev, target: x86_64-unknown-linux-gnu, glibc: "default" }
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" }
- { profile: dev, target: x86_64-unknown-linux-gnu, glibc: "default" }
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" }
steps:
- uses: actions/checkout@v4
@@ -25,6 +25,15 @@ jobs:
with:
cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }}
- name: Download and Extract Static Assets
run: |
url="https://dl.rustfs.com/console/rustfs-console-latest.zip"
mkdir -p static
curl -L -o static_assets.zip "$url"
unzip -o static_assets.zip -d ./rustfs/static
rm static_assets.zip
ls -la ./rustfs/static
- name: Build
run: |
./scripts/build.py \
@@ -32,10 +41,34 @@ jobs:
--target ${{ matrix.variant.target }} \
--glibc ${{ matrix.variant.glibc }}
- name: Package Binary and Static Assets
id: package
run: |
# Create artifact filename
ARTIFACT_NAME="rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}"
if [ "${{ matrix.variant.glibc }}" != "default" ]; then
ARTIFACT_NAME="${ARTIFACT_NAME}-glibc${{ matrix.variant.glibc }}"
fi
echo "artifact_name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT
# Determine binary path
bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.bin"
if [ -f "target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin" ]; then
bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin"
fi
# Create package
mkdir -p ${ARTIFACT_NAME}
cp "$bin_path" ${ARTIFACT_NAME}/rustfs
# cp -r static ${ARTIFACT_NAME}/
zip -r ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME}
ls -la
- uses: actions/upload-artifact@v4
with:
name: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }}
path: ./target/artifacts/*
name: ${{ steps.package.outputs.artifact_name }}
path: ${{ steps.package.outputs.artifact_name }}.zip
retention-days: 7
merge:
runs-on: ubuntu-latest
@@ -43,5 +76,6 @@ jobs:
steps:
- uses: actions/upload-artifact/merge@v4
with:
name: rustfs
name: rustfs-packages
pattern: 'rustfs-*'
delete-merged: true
.github/workflows/ci.yml
+10 -86
View File
@@ -37,13 +37,6 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup
with:
cache-shared-key: "develop"
- name: Install s3s-e2e
run: |
cargo install s3s-e2e --git https://github.com/Nugine/s3s.git
s3s-e2e --version
- name: Format
run: cargo fmt --all --check
@@ -52,41 +45,24 @@ jobs:
run: cargo check --all-targets
# TODO: cargo clippy
- name: Test
run: cargo test --all --exclude e2e_test
- name: Build debug
run: |
touch rustfs/build.rs
cargo build -p rustfs --bins
# - name: Build release
# run: |
# touch rustfs/build.rs
# cargo build -p rustfs --bins --release
- run: |
- name: Pack artifacts
run: |
mkdir -p ./target/artifacts
cp target/debug/rustfs ./target/artifacts/rustfs-debug
# cp target/release/rustfs ./target/artifacts/rustfs-release
- uses: actions/upload-artifact@v4
with:
name: rustfs
path: ./target/artifacts/*
test:
name: Test
needs:
- skip-check
- develop
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup
with:
cache-shared-key: "develop"
cache-save-if: "false"
- run: cargo test --all --exclude e2e_test
s3s-e2e:
name: E2E (s3s-e2e)
needs:
@@ -96,10 +72,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-shared-key: "develop"
cache-save-if: false
cache-on-failure: true
cache-all-crates: true
- name: Install s3s-e2e
run: |
@@ -111,65 +88,12 @@ jobs:
name: rustfs
path: ./target/artifacts
- name: Run rustfs
run: |
./scripts/e2e-run.sh ./target/artifacts/rustfs-debug /data/rustfs
sleep 10
- name: Run s3s-e2e
timeout-minutes: 10
run: |
export AWS_ACCESS_KEY_ID=rustfsadmin
export AWS_SECRET_ACCESS_KEY=rustfsadmin
export AWS_REGION=us-east-1
export AWS_ENDPOINT_URL=http://localhost:9000
export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug"
export RUST_BACKTRACE=full
s3s-e2e
sudo killall rustfs-debug
./scripts/e2e-run.sh ./target/artifacts/rustfs-debug /tmp/rustfs
- uses: actions/upload-artifact@v4
with:
name: s3s-e2e.logs
path: /tmp/rustfs.log
# mint:
# name: E2E (mint)
# needs:
# - skip-check
# - develop
# if: needs.skip-check.outputs.should_skip != 'true'
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
# - uses: ./.github/actions/setup
# with:
# cache-shared-key: "develop"
# cache-save-if: false
# - uses: actions/download-artifact@v4
# with:
# name: rustfs
# path: ./target/artifacts
# - name: Run rustfs
# run: |
# ./scripts/e2e-run.sh ./target/artifacts/rustfs-release /data/rustfs
# sleep 10
# - name: Run mint
# timeout-minutes: 10
# run: |
# docker run \
# -e "SERVER_ENDPOINT=localhost:9000" \
# -e "ACCESS_KEY=rustfsadmin" \
# -e "SECRET_KEY=rustfsadmin" \
# -e "ENABLE_HTTPS=0" \
# --network host \
# minio/mint:edge
# sudo killall rustfs-release
# - uses: actions/upload-artifact@v4
# with:
# name: mint.logs
# path: /tmp/rustfs.log
Cargo.lock
Generated
+4 -4
View File
@@ -6154,18 +6154,18 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "strum"
version = "0.26.3"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32"
dependencies = [
"strum_macros",
]
[[package]]
name = "strum_macros"
version = "0.26.4"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8"
dependencies = [
"heck 0.5.0",
"proc-macro2",
Cargo.toml
+2 -2
View File
@@ -36,7 +36,7 @@ backon = "1.3.0"
bytes = "1.9.0"
bytesize = "1.3.0"
chrono = { version = "0.4.40", features = ["serde"] }
clap = { version = "4.5.27", features = ["derive", "env"] }
clap = { version = "4.5.31", features = ["derive", "env"] }
dioxus = { version = "0.6.3", features = ["router"] }
dirs = "6.0.0"
ecstore = { path = "./ecstore" }
@@ -99,7 +99,7 @@ tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] }
tracing-appender = "0.2.3"
transform-stream = "0.3.1"
url = "2.5.4"
uuid = { version = "1.12.1", features = [
uuid = { version = "1.15.1", features = [
"v4",
"fast-rng",
"macro-diagnostics",
README.md
+50 -1
View File
@@ -1 +1,50 @@
# s3-rustfs
# How to compile RustFS
| Must package | Version |
|--------------|---------|
| Rust | 1.8.5 |
| protoc | 27.0 |
| flatc | 24.0+ |
Download Links:
https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip
https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip
Or use Docker:
- uses: arduino/setup-protoc@v3
with:
version: "27.0"
- uses: Nugine/setup-flatc@v1
with:
version: "24.3.25"
# How to add Console web
1. wget [http://dl.rustfs.com/console/console.latest.tar.gz](https://dl.rustfs.com/console/rustfs-console-latest.zip)
2. mkdir in this repos folder `./rustfs/static`
3. Compile RustFS
# Star RustFS
Add Env infomation:
```
export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug"
export RUSTFS_VOLUMES="./target/volume/test"
export RUSTFS_ADDRESS="0.0.0.0:9000"
export RUSTFS_CONSOLE_ENABLE=true
export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001"
export RUSTFS_SERVER_ENDPOINT="http://127.0.0.1:9000"
```
You need replace your real data folder:
```
./rustfs /data/rustfs
```
ecstore/Cargo.toml
+1 -1
View File
@@ -32,7 +32,7 @@ s3s.workspace = true
http.workspace = true
highway = "1.3.0"
url.workspace = true
uuid = { version = "1.12.1", features = ["v4", "fast-rng", "serde"] }
uuid = { version = "1.15.1", features = ["v4", "fast-rng", "serde"] }
reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] }
transform-stream = "0.3.1"
lazy_static.workspace = true
iam/Cargo.toml
+1 -1
View File
@@ -18,7 +18,7 @@ ecstore = { path = "../ecstore" }
serde_json.workspace = true
async-trait.workspace = true
thiserror.workspace = true
strum = { version = "0.26.3", features = ["derive"] }
strum = { version = "0.27.1", features = ["derive"] }
arc-swap = "1.7.1"
crypto = { path = "../crypto" }
ipnetwork = { version = "0.21.1", features = ["serde"] }
rustfs/Cargo.toml
+3 -3
View File
@@ -65,7 +65,7 @@ tower.workspace = true
tracing-error.workspace = true
tracing-subscriber.workspace = true
transform-stream.workspace = true
uuid = "1.12.1"
uuid = "1.15.1"
url.workspace = true
admin = { path = "../api/admin" }
axum.workspace = true
@@ -79,7 +79,7 @@ iam = { path = "../iam" }
jsonwebtoken = "9.3.0"
tower-http = { version = "0.6.2", features = ["cors"] }
mime_guess = "2.0.5"
rust-embed = { version = "8.5.0", features = ["interpolate-folder-path"] }
rust-embed = { version = "8.6.0", features = ["interpolate-folder-path"] }
local-ip-address = "0.6.3"
chrono = "0.4"
# 添加proc-macro所需依赖
@@ -97,7 +97,7 @@ futures-util.workspace = true
# uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] }
ecstore = { path = "../ecstore" }
s3s.workspace = true
clap = { version = "4.5.27", features = ["derive","env"] }
clap = { version = "4.5.31", features = ["derive","env"] }
tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] }
hyper-util = { version = "0.1.10", features = [
"tokio",
scripts/e2e-run.sh
+13 -2
View File
@@ -3,9 +3,20 @@ BIN=$1
VOLUME=$2
chmod +x $BIN
sudo mkdir -p $VOLUME
mkdir -p $VOLUME
export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug"
export RUST_BACKTRACE=full
$BIN $VOLUME > /tmp/rustfs.log 2>&1 &
sudo nohup $BIN $VOLUME > /tmp/rustfs.log 2>&1 &
sleep 10
export AWS_ACCESS_KEY_ID=rustfsadmin
export AWS_SECRET_ACCESS_KEY=rustfsadmin
export AWS_REGION=us-east-1
export AWS_ENDPOINT_URL=http://localhost:9000
export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug"
export RUST_BACKTRACE=full
s3s-e2e
killall $BIN
scripts/static.sh
Executable
+1
View File
@@ -0,0 +1 @@
curl -L "https://dl.rustfs.com/console/rustfs-console-latest.zip" -o tempfile.zip && unzip -o tempfile.zip -d ./rustfs/static && rm tempfile.zip