diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..624d27249 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.x86_64-unknown-linux-gnu] +rustflags = ["-Clink-arg=-fuse-ld=lld"] diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index ae177cb9b..4c7afb30e 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -21,6 +21,7 @@ runs: run: | sudo apt update sudo apt install -y \ + lld \ libdbus-1-dev \ libwayland-dev \ libwebkit2gtk-4.1-dev \ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b85c3f800..b830105f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,9 +15,9 @@ jobs: strategy: matrix: variant: - - { profile: dev, target: x86_64-unknown-linux-gnu, glibc: "default" } - - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" } + - { profile: dev, target: x86_64-unknown-linux-gnu, glibc: "default" } + - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } + - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" } steps: - uses: actions/checkout@v4 @@ -25,6 +25,15 @@ jobs: with: cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }} + - name: Download and Extract Static Assets + run: | + url="https://dl.rustfs.com/console/rustfs-console-latest.zip" + mkdir -p static + curl -L -o static_assets.zip "$url" + unzip -o static_assets.zip -d ./rustfs/static + rm static_assets.zip + ls -la ./rustfs/static + - name: Build run: | ./scripts/build.py \ @@ -32,10 +41,34 @@ jobs: --target ${{ matrix.variant.target }} \ --glibc ${{ matrix.variant.glibc }} + - name: Package Binary and Static Assets + id: package + run: | + # Create artifact filename + ARTIFACT_NAME="rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}" + if [ "${{ matrix.variant.glibc }}" != "default" ]; then + ARTIFACT_NAME="${ARTIFACT_NAME}-glibc${{ matrix.variant.glibc }}" + fi + echo "artifact_name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT + + # Determine binary path + bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.bin" + if [ -f "target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin" ]; then + bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin" + fi + + # Create package + mkdir -p ${ARTIFACT_NAME} + cp "$bin_path" ${ARTIFACT_NAME}/rustfs + # cp -r static ${ARTIFACT_NAME}/ + zip -r ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME} + ls -la + - uses: actions/upload-artifact@v4 with: - name: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }} - path: ./target/artifacts/* + name: ${{ steps.package.outputs.artifact_name }} + path: ${{ steps.package.outputs.artifact_name }}.zip + retention-days: 7 merge: runs-on: ubuntu-latest @@ -43,5 +76,6 @@ jobs: steps: - uses: actions/upload-artifact/merge@v4 with: - name: rustfs + name: rustfs-packages + pattern: 'rustfs-*' delete-merged: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09c54587d..783f20ff3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,13 +37,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - with: - cache-shared-key: "develop" - - - name: Install s3s-e2e - run: | - cargo install s3s-e2e --git https://github.com/Nugine/s3s.git - s3s-e2e --version - name: Format run: cargo fmt --all --check @@ -52,41 +45,24 @@ jobs: run: cargo check --all-targets # TODO: cargo clippy + - name: Test + run: cargo test --all --exclude e2e_test + - name: Build debug run: | touch rustfs/build.rs cargo build -p rustfs --bins - - # - name: Build release - # run: | - # touch rustfs/build.rs - # cargo build -p rustfs --bins --release - - run: | + - name: Pack artifacts + run: | mkdir -p ./target/artifacts cp target/debug/rustfs ./target/artifacts/rustfs-debug - # cp target/release/rustfs ./target/artifacts/rustfs-release - uses: actions/upload-artifact@v4 with: name: rustfs path: ./target/artifacts/* - test: - name: Test - needs: - - skip-check - - develop - if: needs.skip-check.outputs.should_skip != 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup - with: - cache-shared-key: "develop" - cache-save-if: "false" - - run: cargo test --all --exclude e2e_test - s3s-e2e: name: E2E (s3s-e2e) needs: @@ -96,10 +72,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/setup + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 with: - cache-shared-key: "develop" - cache-save-if: false + cache-on-failure: true + cache-all-crates: true - name: Install s3s-e2e run: | @@ -111,65 +88,12 @@ jobs: name: rustfs path: ./target/artifacts - - name: Run rustfs - run: | - ./scripts/e2e-run.sh ./target/artifacts/rustfs-debug /data/rustfs - sleep 10 - - name: Run s3s-e2e timeout-minutes: 10 run: | - export AWS_ACCESS_KEY_ID=rustfsadmin - export AWS_SECRET_ACCESS_KEY=rustfsadmin - export AWS_REGION=us-east-1 - export AWS_ENDPOINT_URL=http://localhost:9000 - export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug" - export RUST_BACKTRACE=full - s3s-e2e - sudo killall rustfs-debug + ./scripts/e2e-run.sh ./target/artifacts/rustfs-debug /tmp/rustfs - uses: actions/upload-artifact@v4 with: name: s3s-e2e.logs path: /tmp/rustfs.log - - # mint: - # name: E2E (mint) - # needs: - # - skip-check - # - develop - # if: needs.skip-check.outputs.should_skip != 'true' - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - uses: ./.github/actions/setup - # with: - # cache-shared-key: "develop" - # cache-save-if: false - - # - uses: actions/download-artifact@v4 - # with: - # name: rustfs - # path: ./target/artifacts - - # - name: Run rustfs - # run: | - # ./scripts/e2e-run.sh ./target/artifacts/rustfs-release /data/rustfs - # sleep 10 - - # - name: Run mint - # timeout-minutes: 10 - # run: | - # docker run \ - # -e "SERVER_ENDPOINT=localhost:9000" \ - # -e "ACCESS_KEY=rustfsadmin" \ - # -e "SECRET_KEY=rustfsadmin" \ - # -e "ENABLE_HTTPS=0" \ - # --network host \ - # minio/mint:edge - # sudo killall rustfs-release - - # - uses: actions/upload-artifact@v4 - # with: - # name: mint.logs - # path: /tmp/rustfs.log diff --git a/Cargo.lock b/Cargo.lock index ab06299a2..9c85b971c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6154,18 +6154,18 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" dependencies = [ "strum_macros", ] [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" dependencies = [ "heck 0.5.0", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index a092a687c..64774f099 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ backon = "1.3.0" bytes = "1.9.0" bytesize = "1.3.0" chrono = { version = "0.4.40", features = ["serde"] } -clap = { version = "4.5.27", features = ["derive", "env"] } +clap = { version = "4.5.31", features = ["derive", "env"] } dioxus = { version = "0.6.3", features = ["router"] } dirs = "6.0.0" ecstore = { path = "./ecstore" } @@ -99,7 +99,7 @@ tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] } tracing-appender = "0.2.3" transform-stream = "0.3.1" url = "2.5.4" -uuid = { version = "1.12.1", features = [ +uuid = { version = "1.15.1", features = [ "v4", "fast-rng", "macro-diagnostics", diff --git a/README.md b/README.md index 3bfa6b7a2..1e4f16b5f 100644 --- a/README.md +++ b/README.md @@ -1 +1,50 @@ -# s3-rustfs \ No newline at end of file +# How to compile RustFS + +| Must package | Version | +|--------------|---------| +| Rust | 1.8.5 | +| protoc | 27.0 | +| flatc | 24.0+ | + +Download Links: + +https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip + +https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip + +Or use Docker: + +- uses: arduino/setup-protoc@v3 + with: + version: "27.0" + +- uses: Nugine/setup-flatc@v1 + with: + version: "24.3.25" + +# How to add Console web + +1. wget [http://dl.rustfs.com/console/console.latest.tar.gz](https://dl.rustfs.com/console/rustfs-console-latest.zip) + +2. mkdir in this repos folder `./rustfs/static` + +3. Compile RustFS + +# Star RustFS + +Add Env infomation: + +``` +export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug" +export RUSTFS_VOLUMES="./target/volume/test" +export RUSTFS_ADDRESS="0.0.0.0:9000" +export RUSTFS_CONSOLE_ENABLE=true +export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001" +export RUSTFS_SERVER_ENDPOINT="http://127.0.0.1:9000" +``` + +You need replace your real data folder: + +``` +./rustfs /data/rustfs +``` diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index d332c9043..cacafa49d 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -32,7 +32,7 @@ s3s.workspace = true http.workspace = true highway = "1.3.0" url.workspace = true -uuid = { version = "1.12.1", features = ["v4", "fast-rng", "serde"] } +uuid = { version = "1.15.1", features = ["v4", "fast-rng", "serde"] } reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] } transform-stream = "0.3.1" lazy_static.workspace = true diff --git a/iam/Cargo.toml b/iam/Cargo.toml index e6e47259a..6c66d8f6e 100644 --- a/iam/Cargo.toml +++ b/iam/Cargo.toml @@ -18,7 +18,7 @@ ecstore = { path = "../ecstore" } serde_json.workspace = true async-trait.workspace = true thiserror.workspace = true -strum = { version = "0.26.3", features = ["derive"] } +strum = { version = "0.27.1", features = ["derive"] } arc-swap = "1.7.1" crypto = { path = "../crypto" } ipnetwork = { version = "0.21.1", features = ["serde"] } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 7e7159388..8ba8809cd 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -65,7 +65,7 @@ tower.workspace = true tracing-error.workspace = true tracing-subscriber.workspace = true transform-stream.workspace = true -uuid = "1.12.1" +uuid = "1.15.1" url.workspace = true admin = { path = "../api/admin" } axum.workspace = true @@ -79,7 +79,7 @@ iam = { path = "../iam" } jsonwebtoken = "9.3.0" tower-http = { version = "0.6.2", features = ["cors"] } mime_guess = "2.0.5" -rust-embed = { version = "8.5.0", features = ["interpolate-folder-path"] } +rust-embed = { version = "8.6.0", features = ["interpolate-folder-path"] } local-ip-address = "0.6.3" chrono = "0.4" # 添加proc-macro所需依赖 @@ -97,7 +97,7 @@ futures-util.workspace = true # uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] } ecstore = { path = "../ecstore" } s3s.workspace = true -clap = { version = "4.5.27", features = ["derive","env"] } +clap = { version = "4.5.31", features = ["derive","env"] } tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] } hyper-util = { version = "0.1.10", features = [ "tokio", diff --git a/scripts/e2e-run.sh b/scripts/e2e-run.sh index 26e3d9c69..9b4392e85 100755 --- a/scripts/e2e-run.sh +++ b/scripts/e2e-run.sh @@ -3,9 +3,20 @@ BIN=$1 VOLUME=$2 chmod +x $BIN -sudo mkdir -p $VOLUME +mkdir -p $VOLUME export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug" export RUST_BACKTRACE=full +$BIN $VOLUME > /tmp/rustfs.log 2>&1 & -sudo nohup $BIN $VOLUME > /tmp/rustfs.log 2>&1 & +sleep 10 + +export AWS_ACCESS_KEY_ID=rustfsadmin +export AWS_SECRET_ACCESS_KEY=rustfsadmin +export AWS_REGION=us-east-1 +export AWS_ENDPOINT_URL=http://localhost:9000 +export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug" +export RUST_BACKTRACE=full +s3s-e2e + +killall $BIN diff --git a/scripts/static.sh b/scripts/static.sh new file mode 100755 index 000000000..9fd7469d8 --- /dev/null +++ b/scripts/static.sh @@ -0,0 +1 @@ +curl -L "https://dl.rustfs.com/console/rustfs-console-latest.zip" -o tempfile.zip && unzip -o tempfile.zip -d ./rustfs/static && rm tempfile.zip \ No newline at end of file