diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index a1c92ecf3..000000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[target.x86_64-unknown-linux-gnu] -rustflags = [ - "-C", "link-arg=-fuse-ld=bfd" -] - -[target.x86_64-unknown-linux-musl] -linker = "x86_64-linux-musl-gcc" \ No newline at end of file diff --git a/.docker/observability/README_ZH.md b/.docker/observability/README_ZH.md index 7ba5342bf..53b6db0bc 100644 --- a/.docker/observability/README_ZH.md +++ b/.docker/observability/README_ZH.md @@ -9,7 +9,7 @@ Jaeger、Prometheus 等）而需要运行和维护多个代理/收集器的必 2. 执行以下命令启动服务： ```bash -docker compose up -d -f docker-compose.yml +docker compose -f docker-compose.yml up -d ``` ### 访问监控面板 @@ -34,7 +34,7 @@ docker compose up -d -f docker-compose.yml | service_name | 服务名称 | rustfs | | service_version | 服务版本 | 1.0.0 | | environment | 运行环境 | production | -| meter_interval | 指标导出间隔 (秒) | 30 | +| meter_interval | 指标导出间隔 (秒) | 30 | | sample_ratio | 采样率 | 1.0 | | use_stdout | 是否输出到控制台 | true/false | | logger_level | 日志级别 | info | diff --git a/.docker/observability/config/obs-multi.toml b/.docker/observability/config/obs-multi.toml new file mode 100644 index 000000000..2637a4012 --- /dev/null +++ b/.docker/observability/config/obs-multi.toml @@ -0,0 +1,34 @@ +[observability] +endpoint = "http://otel-collector:4317" # Default is "http://localhost:4317" if not specified +use_stdout = false # Output with stdout, true output, false no output +sample_ratio = 2.0 +meter_interval = 30 +service_name = "rustfs" +service_version = "0.1.0" +environments = "production" +logger_level = "debug" +local_logging_enabled = true + +#[[sinks]] +#type = "Kafka" +#brokers = "localhost:9092" +#topic = "logs" +#batch_size = 100 # Default is 100 if not specified +#batch_timeout_ms = 1000 # Default is 1000ms if not specified +# +#[[sinks]] +#type = "Webhook" +#endpoint = "http://localhost:8080/webhook" +#auth_token = "" +#batch_size = 100 # Default is 3 if not specified +#batch_timeout_ms = 1000 # Default is 100ms if not specified + +[[sinks]] +type = "File" +path = "/root/data/logs/rustfs.log" +buffer_size = 100 # Default is 8192 bytes if not specified +flush_interval_ms = 1000 +flush_threshold = 100 + +[logger] +queue_capacity = 10 \ No newline at end of file diff --git a/.docker/observability/config/obs.toml b/.docker/observability/config/obs.toml index e4ea037b1..58069fc5c 100644 --- a/.docker/observability/config/obs.toml +++ b/.docker/observability/config/obs.toml @@ -1,5 +1,5 @@ [observability] -endpoint = "http://otel-collector:4317" # Default is "http://localhost:4317" if not specified +endpoint = "http://localhost:4317" # Default is "http://localhost:4317" if not specified use_stdout = false # Output with stdout, true output, false no output sample_ratio = 2.0 meter_interval = 30 @@ -9,26 +9,26 @@ environments = "production" logger_level = "debug" local_logging_enabled = true -[sinks] -[sinks.kafka] # Kafka sink is disabled by default -enabled = false -bootstrap_servers = "localhost:9092" -topic = "logs" -batch_size = 100 # Default is 100 if not specified -batch_timeout_ms = 1000 # Default is 1000ms if not specified +#[[sinks]] +#type = "Kafka" +#brokers = "localhost:9092" +#topic = "logs" +#batch_size = 100 # Default is 100 if not specified +#batch_timeout_ms = 1000 # Default is 1000ms if not specified +# +#[[sinks]] +#type = "Webhook" +#endpoint = "http://localhost:8080/webhook" +#auth_token = "" +#batch_size = 100 # Default is 3 if not specified +#batch_timeout_ms = 1000 # Default is 100ms if not specified -[sinks.webhook] -enabled = false -endpoint = "http://localhost:8080/webhook" -auth_token = "" -batch_size = 100 # Default is 3 if not specified -batch_timeout_ms = 1000 # Default is 100ms if not specified - -[sinks.file] -enabled = true -path = "/root/data/logs/app.log" -batch_size = 10 -batch_timeout_ms = 1000 # Default is 8192 bytes if not specified +[[sinks]] +type = "File" +path = "/root/data/logs/rustfs.log" +buffer_size = 100 # Default is 8192 bytes if not specified +flush_interval_ms = 1000 +flush_threshold = 100 [logger] queue_capacity = 10 \ No newline at end of file diff --git a/.docker/observability/docker-compose.yml b/.docker/observability/docker-compose.yml index f6714bb26..55e4f84c8 100644 --- a/.docker/observability/docker-compose.yml +++ b/.docker/observability/docker-compose.yml @@ -1,6 +1,6 @@ services: otel-collector: - image: otel/opentelemetry-collector-contrib:0.120.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.124.0 environment: - TZ=Asia/Shanghai volumes: @@ -16,7 +16,7 @@ services: networks: - otel-network jaeger: - image: jaegertracing/jaeger:2.4.0 + image: jaegertracing/jaeger:2.5.0 environment: - TZ=Asia/Shanghai ports: @@ -26,7 +26,7 @@ services: networks: - otel-network prometheus: - image: prom/prometheus:v3.2.1 + image: prom/prometheus:v3.3.0 environment: - TZ=Asia/Shanghai volumes: @@ -36,7 +36,7 @@ services: networks: - otel-network loki: - image: grafana/loki:3.4.2 + image: grafana/loki:3.5.0 environment: - TZ=Asia/Shanghai volumes: @@ -47,7 +47,7 @@ services: networks: - otel-network grafana: - image: grafana/grafana:11.6.0 + image: grafana/grafana:11.6.1 ports: - "3000:3000" # Web UI environment: diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index f74dc6fbe..345eec131 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -6,17 +6,25 @@ inputs: rust-version: required: true default: "stable" + description: "Rust version to use" cache-shared-key: required: true default: "" + description: "Cache key for shared cache" cache-save-if: required: true default: true + description: "Cache save condition" + run-os: + required: true + default: "ubuntu-latest" + description: "Running system" runs: using: "composite" steps: - name: Install system dependencies + if: inputs.run-os == 'ubuntu-latest' shell: bash run: | sudo apt update @@ -42,5 +50,5 @@ runs: shared-key: ${{ inputs.cache-shared-key }} save-if: ${{ inputs.cache-save-if }} - - uses: mlugg/setup-zig@v1 + - uses: mlugg/setup-zig@v2 - uses: taiki-e/install-action@cargo-zigbuild diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 01e49f6c5..bf729d21c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build +name: Build RustFS And GUI on: workflow_dispatch: @@ -11,66 +11,262 @@ on: jobs: build-rustfs: - runs-on: ubuntu-latest - + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] variant: - # - { profile: dev, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } - # - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" } + - { profile: release, target: aarch64-apple-darwin, glibc: "default" } + #- { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } + #- { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } + #- { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" } + exclude: + # Linux targets on non-Linux systems + - os: macos-latest + variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } + - os: macos-latest + variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } + - os: macos-latest + variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } + - os: macos-latest + variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" } + + # Apple targets on non-macOS systems + - os: ubuntu-latest + variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" } + - os: windows-latest + variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" } + + # Windows targets on non-Windows systems + - os: ubuntu-latest + variant: { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" } + - os: macos-latest + variant: { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" } steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup + - name: Checkout repository + uses: actions/checkout@v4.2.2 with: - cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }} + fetch-depth: 0 + # Installation system dependencies + - name: Install system dependencies (Ubuntu) + if: runner.os == 'Linux' + run: | + sudo apt update + sudo apt install -y musl-tools build-essential lld libdbus-1-dev libwayland-dev libwebkit2gtk-4.1-dev libxdo-dev + shell: bash + + #Install Rust using dtolnay/rust-toolchain + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + targets: ${{ matrix.variant.target }} + components: rustfmt, clippy + + # Install system dependencies + - name: Cache Protoc + id: cache-protoc + uses: actions/cache@v4.2.3 + with: + path: /Users/runner/hostedtoolcache/protoc + key: protoc-${{ runner.os }}-30.2 + restore-keys: | + protoc-${{ runner.os }}- + + - name: Install Protoc + if: steps.cache-protoc.outputs.cache-hit != 'true' + uses: arduino/setup-protoc@v3 + with: + version: '30.2' + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Flatc + uses: Nugine/setup-flatc@v1 + with: + version: "25.2.10" + + # Cache Cargo dependencies + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + cache-all-crates: true + shared-key: rustfs-${{ matrix.os }}-${{ matrix.variant.profile }}-${{ matrix.variant.target }}-${{ matrix.variant.glibc }}-${{ hashFiles('**/Cargo.lock') }} + save-if: ${{ github.event_name != 'pull_request' }} + + # Set up Zig for cross-compilation + - uses: mlugg/setup-zig@v2 + if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'linux') + + - uses: taiki-e/install-action@cargo-zigbuild + if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'linux') + + # Download static resources - name: Download and Extract Static Assets run: | url="https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip" - mkdir -p static - curl -L -o static_assets.zip "$url" - unzip -o static_assets.zip -d ./rustfs/static - rm static_assets.zip - ls -la ./rustfs/static - - name: Build + # Create a static resource directory + mkdir -p ./rustfs/static + + # Download static resources + echo "::group::Downloading static assets" + curl -L -o static_assets.zip "$url" --retry 3 + + # Unzip static resources + echo "::group::Extracting static assets" + if [ "${{ runner.os }}" = "Windows" ]; then + 7z x static_assets.zip -o./rustfs/static + del static_assets.zip + else + unzip -o static_assets.zip -d ./rustfs/static + rm static_assets.zip + fi + + echo "::group::Static assets content" + ls -la ./rustfs/static + shell: bash + + # Build rustfs + - name: Build rustfs + id: build + shell: bash run: | - ./scripts/build.py \ - --profile ${{ matrix.variant.profile }} \ - --target ${{ matrix.variant.target }} \ - --glibc ${{ matrix.variant.glibc }} + echo "::group::Setting up build parameters" + PROFILE="${{ matrix.variant.profile }}" + TARGET="${{ matrix.variant.target }}" + GLIBC="${{ matrix.variant.glibc }}" + + # Determine whether to use zigbuild + USE_ZIGBUILD=false + if [[ "$GLIBC" != "default" || "$TARGET" == *"linux"* ]]; then + USE_ZIGBUILD=true + echo "Using zigbuild for cross-compilation" + fi + + # Determine the target parameters + TARGET_ARG="$TARGET" + if [[ "$GLIBC" != "default" ]]; then + TARGET_ARG="${TARGET}.${GLIBC}" + echo "Using custom glibc target: $TARGET_ARG" + fi + + # Confirm the profile directory name + if [[ "$PROFILE" == "dev" ]]; then + PROFILE_DIR="debug" + else + PROFILE_DIR="$PROFILE" + fi + + # Determine the binary suffix + BIN_SUFFIX="" + if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then + BIN_SUFFIX=".exe" + fi + + # Determine the binary name - Use the appropriate extension for Windows + BIN_NAME="rustfs.${PROFILE}.${TARGET}" + if [[ "$GLIBC" != "default" ]]; then + BIN_NAME="${BIN_NAME}.glibc${GLIBC}" + fi + + # Windows systems use exe suffix, and other systems do not have suffix + if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then + BIN_NAME="${BIN_NAME}.exe" + else + BIN_NAME="${BIN_NAME}.bin" + fi + + echo "Binary name will be: $BIN_NAME" + + echo "::group::Building rustfs" + # Refresh build information + touch rustfs/build.rs + + # Identify the build command and execute it + if [[ "$USE_ZIGBUILD" == "true" ]]; then + echo "Build command: cargo zigbuild --profile $PROFILE --target $TARGET_ARG -p rustfs --bins" + cargo zigbuild --profile $PROFILE --target $TARGET_ARG -p rustfs --bins + else + echo "Build command: cargo build --profile $PROFILE --target $TARGET_ARG -p rustfs --bins" + cargo build --profile $PROFILE --target $TARGET_ARG -p rustfs --bins + fi + + # Determine the binary path and output path + BIN_PATH="target/${TARGET_ARG}/${PROFILE_DIR}/rustfs${BIN_SUFFIX}" + OUT_PATH="target/artifacts/${BIN_NAME}" + + # Create a target directory + mkdir -p target/artifacts + + echo "Copying binary from ${BIN_PATH} to ${OUT_PATH}" + cp "${BIN_PATH}" "${OUT_PATH}" + + # Record the output path for use in the next steps + echo "bin_path=${OUT_PATH}" >> $GITHUB_OUTPUT + echo "bin_name=${BIN_NAME}" >> $GITHUB_OUTPUT - name: Package Binary and Static Assets id: package run: | - # Create artifact filename + # Create component file name ARTIFACT_NAME="rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}" if [ "${{ matrix.variant.glibc }}" != "default" ]; then ARTIFACT_NAME="${ARTIFACT_NAME}-glibc${{ matrix.variant.glibc }}" fi echo "artifact_name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT - # Determine binary path - bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.bin" - if [ -f "target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin" ]; then - bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin" + # Get the binary path + BIN_PATH="${{ steps.build.outputs.bin_path }}" + + # Create a packaged directory structure - only contains bin and docs directories + mkdir -p ${ARTIFACT_NAME}/{bin,docs} + + # Copy binary files (note the difference between Windows and other systems) + if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then + cp "${BIN_PATH}" ${ARTIFACT_NAME}/bin/rustfs.exe + else + cp "${BIN_PATH}" ${ARTIFACT_NAME}/bin/rustfs fi - # Create package - mkdir -p ${ARTIFACT_NAME} - cp "$bin_path" ${ARTIFACT_NAME}/rustfs - zip -r ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME} - ls -la + # copy documents and licenses + if [ -f "LICENSE" ]; then + cp LICENSE ${ARTIFACT_NAME}/docs/ + fi + if [ -f "README.md" ]; then + cp README.md ${ARTIFACT_NAME}/docs/ + fi + + # Packaged as zip + if [ "${{ runner.os }}" = "Windows" ]; then + 7z a ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME} + else + zip -r ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME} + fi + + echo "Created artifact: ${ARTIFACT_NAME}.zip" + ls -la ${ARTIFACT_NAME}.zip + shell: bash - uses: actions/upload-artifact@v4 with: name: ${{ steps.package.outputs.artifact_name }} path: ${{ steps.package.outputs.artifact_name }}.zip retention-days: 7 + - name: Upload to Aliyun OSS + if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' uses: JohnGuan/oss-upload-action@main with: key-id: ${{ secrets.ALICLOUDOSS_KEY_ID }} @@ -79,100 +275,124 @@ jobs: bucket: rustfs-artifacts assets: | ${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.zip - ${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.latest.zip + ${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.latest.zip - build-rustfs-gui: - runs-on: ubuntu-latest - needs: build-rustfs - - strategy: - matrix: - variant: - - { profile: release, target: x86_64-unknown-linux-gnu } - # - { profile: release, target: x86_64-apple-darwin } - if: startsWith(github.ref, 'refs/tags/') - steps: - - uses: actions/checkout@v4 - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: "rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}" - - name: Display structure of downloaded files + # Determine whether to perform GUI construction based on conditions + - name: Prepare for GUI build + if: startsWith(github.ref, 'refs/tags/') + id: prepare_gui run: | - ls -R - unzip -o -j "rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}.zip" -d ./cli/rustfs-gui/embedded-rustfs/ - ls -la cli/rustfs-gui/embedded-rustfs - # - name: Cache dioxus-cli - # uses: actions/cache@v4 - # with: - # path: ~/.cargo/bin/dx - # key: ${{ runner.os }}-dioxus-cli-${{ hashFiles('**/Cargo.lock') }} - # restore-keys: | - # ${{ runner.os }}-dioxus-cli- - # - # - name: Install dioxus-cli - # run: | - # if [ ! -f ~/.cargo/bin/dx ]; then - # cargo install dioxus-cli - # fi + # Create a target directory + mkdir -p ./cli/rustfs-gui/embedded-rustfs/ + + # Copy the currently built binary to the embedded-rustfs directory + if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then + cp "${{ steps.build.outputs.bin_path }}" ./cli/rustfs-gui/embedded-rustfs/rustfs.exe + else + cp "${{ steps.build.outputs.bin_path }}" ./cli/rustfs-gui/embedded-rustfs/rustfs + fi + + echo "Copied binary to embedded-rustfs directory" + ls -la ./cli/rustfs-gui/embedded-rustfs/ + shell: bash + + #Install the dioxus-cli tool - uses: taiki-e/cache-cargo-install-action@v2 + if: startsWith(github.ref, 'refs/tags/') with: tool: dioxus-cli + + # Build and package GUI applications - name: Build and Bundle rustfs-gui + if: startsWith(github.ref, 'refs/tags/') + id: build_gui + shell: bash run: | - ls -la + echo "::group::Setting up build parameters for GUI" + PROFILE="${{ matrix.variant.profile }}" + TARGET="${{ matrix.variant.target }}" + GLIBC="${{ matrix.variant.glibc }}" + RELEASE_PATH="target/artifacts/$TARGET" - release_path="target/${{ matrix.variant.target }}" - mkdir -p ${release_path} - cd cli/rustfs-gui - ls -la embedded-rustfs + # Make sure the output directory exists + mkdir -p ${RELEASE_PATH} - # Configure the linker based on the target - case "${{ matrix.target }}" in + # Configure the target platform linker + echo "::group::Configuring linker for $TARGET" + case "$TARGET" in "x86_64-unknown-linux-gnu") - # Default gcc - export CC_x86_64_unknown_linux_gnu=gcc - export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc - ;; + export CC_x86_64_unknown_linux_gnu=gcc + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc + ;; + "x86_64-unknown-linux-musl") + export CC_x86_64_unknown_linux_musl=musl-gcc + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=musl-gcc + ;; "aarch64-unknown-linux-gnu") - # AArch64 Cross-compiler - export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc - ;; - "x86_64-apple-darwin") - # macOS default clang - export CC_x86_64_apple_darwin=clang - export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=clang - ;; + export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc + ;; + "aarch64-unknown-linux-musl") + export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-musl-gcc + ;; "aarch64-apple-darwin") - # macOS ARM64 used clang - export CC_aarch64_apple_darwin=clang - export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=clang - ;; + export CC_aarch64_apple_darwin=clang + export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=clang + ;; + "x86_64-pc-windows-msvc") + export CC_x86_64_pc_windows_msvc=cl + export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER=link + ;; esac - # Validating Environment Variables (for Debugging) - echo "CC for ${{ matrix.target }}: $CC_${{ matrix.target }}" - echo "Linker for ${{ matrix.target }}: $CARGO_TARGET_${{ matrix.target }}_LINKER" - if [[ "${{ matrix.variant.target }}" == *"apple-darwin"* ]]; then - dx bundle --platform macos --package-types "macos" --package-types "dmg" --package-types "ios" --release --profile release --out-dir ../../${release_path} - elif [[ "${{ matrix.variant.target }}" == *"windows-msvc"* ]]; then - dx bundle --platform windows --package-types "msi" --release --profile release --out-dir ../../${release_path} - elif [[ "${{ matrix.variant.target }}" == *"unknown-linux-gnu"* ]]; then - dx bundle --platform linux --package-types "deb" --package-types "rpm" --package-types "appimage" --release --profile release --out-dir ../../${release_path} + echo "::group::Building GUI application" + cd cli/rustfs-gui + + # Building according to the target platform + if [[ "$TARGET" == *"apple-darwin"* ]]; then + echo "Building for macOS" + dx bundle --platform macos --package-types "macos" --package-types "dmg" --release --profile ${PROFILE} --out-dir ../../${RELEASE_PATH} + elif [[ "$TARGET" == *"windows-msvc"* ]]; then + echo "Building for Windows" + dx bundle --platform windows --package-types "msi" --release --profile ${PROFILE} --out-dir ../../${RELEASE_PATH} + elif [[ "$TARGET" == *"linux"* ]]; then + echo "Building for Linux" + dx bundle --platform linux --package-types "deb" --package-types "rpm" --package-types "appimage" --release --profile ${PROFILE} --out-dir ../../${RELEASE_PATH} fi - cd ../.. - GUI_ARTIFACT_NAME="rustfs-gui-${{ matrix.variant.profile }}-${{ matrix.variant.target }}" - zip -r ${GUI_ARTIFACT_NAME}.zip ${release_path}/* - echo "gui_artifact_name=${GUI_ARTIFACT_NAME}" >> $GITHUB_OUTPUT - ls -la ${release_path} + cd ../.. + + # Create component name + GUI_ARTIFACT_NAME="rustfs-gui-${PROFILE}-${TARGET}" + + if [ "$GLIBC" != "default" ]; then + GUI_ARTIFACT_NAME="${GUI_ARTIFACT_NAME}-glibc${GLIBC}" + fi + + echo "::group::Packaging GUI application" + # Select packaging method according to the operating system + if [ "${{ runner.os }}" = "Windows" ]; then + 7z a ${GUI_ARTIFACT_NAME}.zip ${RELEASE_PATH}/* + else + zip -r ${GUI_ARTIFACT_NAME}.zip ${RELEASE_PATH}/* + fi + + echo "gui_artifact_name=${GUI_ARTIFACT_NAME}" >> $GITHUB_OUTPUT + echo "Created GUI artifact: ${GUI_ARTIFACT_NAME}.zip" + ls -la ${GUI_ARTIFACT_NAME}.zip + + # Upload GUI components - uses: actions/upload-artifact@v4 + if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ steps.package.outputs.gui_artifact_name }} - path: ${{ steps.package.outputs.gui_artifact_name }}.zip + name: ${{ steps.build_gui.outputs.gui_artifact_name }} + path: ${{ steps.build_gui.outputs.gui_artifact_name }}.zip retention-days: 7 - - name: Upload to Aliyun OSS + + # Upload GUI to Alibaba Cloud OSS + - name: Upload GUI to Aliyun OSS + if: startsWith(github.ref, 'refs/tags/') uses: JohnGuan/oss-upload-action@main with: key-id: ${{ secrets.ALICLOUDOSS_KEY_ID }} @@ -180,15 +400,17 @@ jobs: region: oss-cn-beijing bucket: rustfs-artifacts assets: | - ${{ steps.package.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.gui_artifact_name }}.zip - ${{ steps.package.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.gui_artifact_name }}.latest.zip + ${{ steps.build_gui.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.zip + ${{ steps.build_gui.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.latest.zip + merge: runs-on: ubuntu-latest - needs: [ build-rustfs, build-rustfs-gui ] + needs: [ build-rustfs ] + if: startsWith(github.ref, 'refs/tags/') steps: - uses: actions/upload-artifact/merge@v4 with: name: rustfs-packages pattern: "rustfs-*" - delete-merged: true + delete-merged: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8f19d5fca..0a1501ce9 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ cli/rustfs-gui/embedded-rustfs/rustfs deploy/config/obs.toml *.log deploy/certs/* -*jsonl \ No newline at end of file +*jsonl +.env +.rustfs.sys \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 81d6b4c3e..fa09fb754 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,12 +185,14 @@ dependencies = [ "async-trait", "bytes", "chrono", + "common", "datafusion", "ecstore", "futures", "futures-core", "http", "object_store", + "pin-project-lite", "s3s", "snafu", "tokio", @@ -711,9 +713,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ "axum-core", "bytes", @@ -799,7 +801,7 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile", "rustls-pki-types", "tokio", @@ -1204,9 +1206,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1430,7 +1432,7 @@ dependencies = [ "lazy_static", "scopeguard", "tokio", - "tonic 0.13.0", + "tonic 0.13.1", "tracing-error", ] @@ -1744,7 +1746,7 @@ dependencies = [ "pbkdf2", "rand 0.8.5", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "test-case", "thiserror 2.0.12", "time", @@ -2139,7 +2141,7 @@ dependencies = [ "md-5", "rand 0.8.5", "regex", - "sha2 0.10.8", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -3018,6 +3020,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dpi" version = "0.1.1" @@ -3061,7 +3069,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tonic 0.13.0", + "tonic 0.13.1", "tower 0.5.2", "url", ] @@ -3091,7 +3099,7 @@ dependencies = [ "madmin", "md-5", "netif", - "nix", + "nix 0.30.1", "num", "num_cpus", "path-absolutize", @@ -3105,6 +3113,7 @@ dependencies = [ "reqwest", "rmp", "rmp-serde", + "rustfs-config", "s3s", "s3s-policy", "serde", @@ -3119,7 +3128,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", - "tonic 0.13.0", + "tonic 0.13.1", "tower 0.5.2", "tracing", "tracing-error", @@ -3267,6 +3276,18 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "fixedbitset" version = "0.5.7" @@ -4129,7 +4150,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", @@ -4776,22 +4797,23 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.0", "libc", + "redox_syscall 0.5.11", ] [[package]] name = "libsystemd" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b85fe9dc49de659d05829fdf72b5770c0a5952d1055c34a39f6d4e932bce175d" +checksum = "19c97a761fc86953c5b885422b22c891dbf5bcb9dcc99d0110d6ce4c052759f0" dependencies = [ "hmac 0.12.1", "libc", "log", - "nix", + "nix 0.29.0", "nom 8.0.0", "once_cell", "serde", - "sha2 0.10.8", + "sha2 0.10.9", "thiserror 2.0.12", "uuid", ] @@ -4847,13 +4869,13 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" [[package]] name = "local-ip-address" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3669cf5561f8d27e8fc84cc15e58350e70f557d4d65f70e3154e54cd2f8e1782" +checksum = "656b3b27f8893f7bbf9485148ff9a65f019e3f33bd5cdc87c83cab16b3fd9ec8" dependencies = [ "libc", "neli", - "thiserror 1.0.69", + "thiserror 2.0.12", "windows-sys 0.59.0", ] @@ -4870,7 +4892,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tonic 0.13.0", + "tonic 0.13.1", "tracing", "tracing-error", "url", @@ -5225,6 +5247,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nodrop" version = "0.1.14" @@ -6092,7 +6126,7 @@ checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" dependencies = [ "once_cell", "pest", - "sha2 0.10.8", + "sha2 0.10.9", ] [[package]] @@ -6587,7 +6621,7 @@ dependencies = [ "prost-build", "protobuf", "tokio", - "tonic 0.13.0", + "tonic 0.13.1", "tonic-build", "tower 0.5.2", ] @@ -6641,7 +6675,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.26", + "rustls 0.23.27", "socket2", "thiserror 2.0.12", "tokio", @@ -6660,7 +6694,7 @@ dependencies = [ "rand 0.9.1", "ring", "rustc-hash 2.1.1", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -6879,6 +6913,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.11" @@ -6986,7 +7029,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.26", + "rustls 0.23.27", "rustls-pemfile", "rustls-pki-types", "serde", @@ -7143,9 +7186,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "8.7.0" +version = "8.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5fbc0ee50fcb99af7cebb442e5df7b5b45e9460ffa3f8f549cd26b862bec49d" +checksum = "60e425e204264b144d4c929d126d0de524b40a961686414bab5040f7465c71be" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -7172,7 +7215,7 @@ version = "8.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d55b95147fe01265d06b3955db798bdaed52e60e2211c41137701b3aba8e21" dependencies = [ - "sha2 0.10.8", + "sha2 0.10.9", "walkdir", ] @@ -7244,7 +7287,6 @@ dependencies = [ "iam", "lazy_static", "libsystemd", - "local-ip-address", "lock", "madmin", "matchit", @@ -7259,29 +7301,42 @@ dependencies = [ "query", "rmp-serde", "rust-embed", + "rustfs-config", "rustfs-event-notifier", "rustfs-obs", - "rustls 0.23.26", - "rustls-pemfile", - "rustls-pki-types", + "rustfs-utils", + "rustls 0.23.27", "s3s", "serde", "serde_json", "serde_urlencoded", "shadow-rs", + "socket2", "tikv-jemallocator", "time", "tokio", "tokio-rustls 0.26.2", "tokio-stream", + "tokio-tar", "tokio-util", - "tonic 0.13.0", + "tonic 0.13.1", "tonic-build", "tower 0.5.2", "tower-http", "tracing", "transform-stream", "uuid", + "zip", +] + +[[package]] +name = "rustfs-config" +version = "0.0.1" +dependencies = [ + "config", + "const-str", + "serde", + "serde_json", ] [[package]] @@ -7291,6 +7346,7 @@ dependencies = [ "async-trait", "axum", "config", + "dotenvy", "http", "rdkafka", "reqwest", @@ -7322,7 +7378,7 @@ dependencies = [ "rust-embed", "serde", "serde_json", - "sha2 0.10.8", + "sha2 0.10.9", "tokio", "tracing-appender", "tracing-subscriber", @@ -7345,6 +7401,8 @@ dependencies = [ "opentelemetry_sdk", "rdkafka", "reqwest", + "rustfs-config", + "rustfs-utils", "serde", "serde_json", "smallvec", @@ -7358,6 +7416,18 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rustfs-utils" +version = "0.0.1" +dependencies = [ + "local-ip-address", + "rustfs-config", + "rustls 0.23.27", + "rustls-pemfile", + "rustls-pki-types", + "tracing", +] + [[package]] name = "rustix" version = "0.38.44" @@ -7400,16 +7470,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.26" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.1", + "rustls-webpki 0.103.2", "subtle", "zeroize", ] @@ -7458,9 +7528,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.1" +version = "0.103.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +checksum = "7149975849f1abb3832b246010ef62ccc80d3a76169517ada7188252b9cfb437" dependencies = [ "aws-lc-rs", "ring", @@ -7858,9 +7928,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -8596,9 +8666,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" dependencies = [ "backtrace", "bytes", @@ -8641,7 +8711,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.26", + "rustls 0.23.27", "tokio", ] @@ -8657,10 +8727,25 @@ dependencies = [ ] [[package]] -name = "tokio-util" -version = "0.7.14" +name = "tokio-tar" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -8755,9 +8840,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85839f0b32fd242bb3209262371d07feda6d780d16ee9d2bc88581b89da1549b" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" dependencies = [ "async-trait", "axum", @@ -8785,9 +8870,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85f0383fadd15609306383a90e85eaed44169f931a5d2be1b42c76ceff1825e" +checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" dependencies = [ "prettyplease", "proc-macro2", @@ -10142,7 +10227,7 @@ dependencies = [ "once_cell", "percent-encoding", "raw-window-handle 0.6.2", - "sha2 0.10.8", + "sha2 0.10.9", "soup3", "tao-macros", "thiserror 1.0.69", @@ -10176,6 +10261,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "xattr" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" +dependencies = [ + "libc", + "rustix 1.0.5", +] + [[package]] name = "xdg-home" version = "1.3.0" @@ -10252,7 +10347,7 @@ dependencies = [ "futures-sink", "futures-util", "hex", - "nix", + "nix 0.29.0", "ordered-stream", "rand 0.8.5", "serde", @@ -10283,7 +10378,7 @@ dependencies = [ "futures-core", "futures-lite", "hex", - "nix", + "nix 0.29.0", "ordered-stream", "serde", "serde_repr", @@ -10439,6 +10534,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "zip" +version = "0.0.1" +dependencies = [ + "async-compression", + "tokio", + "tokio-stream", + "tokio-tar", + "xz2", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index d129e86fb..809c13a78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,21 +1,24 @@ [workspace] members = [ - "madmin", # Management dashboard and admin API interface - "rustfs", # Core file system implementation - "ecstore", # Erasure coding storage implementation - "e2e_test", # End-to-end test suite + "appauth", # Application authentication and authorization + "cli/rustfs-gui", # Graphical user interface client "common/common", # Shared utilities and data structures "common/lock", # Distributed locking implementation "common/protos", # Protocol buffer definitions "common/workers", # Worker thread pools and task scheduling - "iam", # Identity and Access Management - "crypto", # Cryptography and security features - "cli/rustfs-gui", # Graphical user interface client - "crates/obs", # Observability utilities + "crates/config", # Configuration management "crates/event-notifier", # Event notification system + "crates/obs", # Observability utilities + "crates/utils", # Utility functions and helpers + "crypto", # Cryptography and security features + "ecstore", # Erasure coding storage implementation + "e2e_test", # End-to-end test suite + "iam", # Identity and Access Management + "madmin", # Management dashboard and admin API interface + "rustfs", # Core file system implementation "s3select/api", # S3 Select API interface "s3select/query", # S3 Select query engine - "appauth", # Application authentication and authorization + "crates/zip", ] resolver = "2" @@ -45,22 +48,28 @@ policy = { path = "./policy", version = "0.0.1" } protos = { path = "./common/protos", version = "0.0.1" } query = { path = "./s3select/query", version = "0.0.1" } rustfs = { path = "./rustfs", version = "0.0.1" } +zip = { path = "./crates/zip", version = "0.0.1" } +rustfs-config = { path = "./crates/config", version = "0.0.1" } rustfs-obs = { path = "crates/obs", version = "0.0.1" } rustfs-event-notifier = { path = "crates/event-notifier", version = "0.0.1" } +rustfs-utils = { path = "crates/utils", version = "0.0.1" } workers = { path = "./common/workers", version = "0.0.1" } +tokio-tar = "0.3.1" atoi = "2.0.0" async-recursion = "1.1.1" async-trait = "0.1.88" atomic_enum = "0.3.0" -axum = "0.8.3" +axum = "0.8.4" axum-extra = "0.10.1" axum-server = { version = "0.7.2", features = ["tls-rustls"] } backon = "1.5.0" +blake2 = "0.10.6" bytes = "1.10.1" bytesize = "2.0.1" -chrono = { version = "0.4.40", features = ["serde"] } +chrono = { version = "0.4.41", features = ["serde"] } clap = { version = "4.5.37", features = ["derive", "env"] } config = "0.15.11" +const-str = { version = "0.6.2", features = ["std", "proc"] } datafusion = "46.0.1" derive_builder = "0.20.2" dioxus = { version = "0.6.3", features = ["router"] } @@ -69,7 +78,9 @@ flatbuffers = "25.2.10" futures = "0.3.31" futures-core = "0.3.31" futures-util = "0.3.31" +glob = "0.3.2" hex = "0.4.3" +highway = { version = "1.3.0" } hyper = "1.6.0" hyper-util = { version = "0.1.11", features = [ "tokio", @@ -86,13 +97,15 @@ keyring = { version = "3.6.2", features = [ "sync-secret-service", ] } lazy_static = "1.5.0" -libsystemd = { version = "0.7.1" } -local-ip-address = "0.6.3" +libsystemd = { version = "0.7.2" } +local-ip-address = "0.6.5" matchit = "0.8.4" md-5 = "0.10.6" mime = "0.3.17" mime_guess = "2.0.5" netif = "0.1.6" +nix = { version = "0.30.1", features = ["fs"] } +num_cpus = { version = "1.16.0" } nvml-wrapper = "0.10.0" object_store = "0.11.2" opentelemetry = { version = "0.29.1" } @@ -103,7 +116,9 @@ opentelemetry-appender-tracing = { version = "0.29.1", features = [ opentelemetry_sdk = { version = "0.29.0" } opentelemetry-stdout = { version = "0.29.0" } opentelemetry-otlp = { version = "0.29.0" } -opentelemetry-semantic-conventions = { version = "0.29.0", features = ["semconv_experimental"] } +opentelemetry-semantic-conventions = { version = "0.29.0", features = [ + "semconv_experimental", +] } parking_lot = "0.12.3" pin-project-lite = "0.2.16" # pin-utils = "0.1.0" @@ -113,6 +128,8 @@ prost-types = "0.13.5" protobuf = "3.7" rand = "0.8.5" rdkafka = { version = "0.37.0", features = ["tokio"] } +reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] } +regex = { version = "1.11.1" } reqwest = { version = "0.12.15", default-features = false, features = [ "rustls-tls", "charset", @@ -129,8 +146,8 @@ rfd = { version = "0.15.3", default-features = false, features = [ rmp = "0.8.14" rmp-serde = "1.3.0" rumqttc = { version = "0.24" } -rust-embed = "8.7.0" -rustls = { version = "0.23.26" } +rust-embed = { version = "8.7.1" } +rustls = { version = "0.23.27" } rustls-pki-types = "1.11.0" rustls-pemfile = "2.2.0" s3s = { git = "https://github.com/Nugine/s3s.git", rev = "4733cdfb27b2713e832967232cbff413bb768c10" } @@ -140,9 +157,10 @@ serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" serde_urlencoded = "0.7.1" serde_with = "3.12.0" -sha2 = "0.10.8" +sha2 = "0.10.9" smallvec = { version = "1.15.0", features = ["serde"] } snafu = "0.8.5" +socket2 = "0.5.9" strum = { version = "0.27.1", features = ["derive"] } sysinfo = "0.34.2" tempfile = "3.19.1" @@ -155,12 +173,13 @@ time = { version = "0.3.41", features = [ "macros", "serde", ] } -tokio = { version = "1.44.2", features = ["fs", "rt-multi-thread"] } + +tokio = { version = "1.45.0", features = ["fs", "rt-multi-thread"] } tokio-rustls = { version = "0.26.2", default-features = false } -tokio-stream = "0.1.17" -tokio-util = { version = "0.7.14", features = ["io", "compat"] } -tonic = { version = "0.13.0", features = ["gzip"] } -tonic-build = "0.13.0" +tokio-stream = { version = "0.1.17" } +tokio-util = { version = "0.7.15", features = ["io", "compat"] } +tonic = { version = "0.13.1", features = ["gzip"] } +tonic-build = { version = "0.13.1" } tower = { version = "0.5.2", features = ["timeout"] } tower-http = { version = "0.6.2", features = ["cors"] } tracing = "0.1.41" @@ -176,6 +195,7 @@ uuid = { version = "1.16.0", features = [ "fast-rng", "macro-diagnostics", ] } +winapi = { version = "0.3.9" } [profile.wasm-dev] @@ -190,10 +210,10 @@ inherits = "dev" [profile.release] opt-level = 3 -lto = "fat" +lto = "thin" codegen-units = 1 -panic = "abort" # Optional, remove the panic expansion code -strip = true # strip symbol information to reduce binary size +panic = "abort" # Optional, remove the panic expansion code +strip = true # strip symbol information to reduce binary size [profile.production] inherits = "release" diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..e69de29bb diff --git a/README.md b/README.md index 79d9a342c..603f7c80f 100644 --- a/README.md +++ b/README.md @@ -59,30 +59,12 @@ export RUSTFS_ADDRESS="0.0.0.0:9000" export RUSTFS_CONSOLE_ENABLE=true export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001" -# Observability config (option 1: config file) +# Observability config export RUSTFS_OBS_CONFIG="./deploy/config/obs.toml" -# Observability config (option 2: environment variables) -export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 -export RUSTFS__OBSERVABILITY__USE_STDOUT=true -export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 -export RUSTFS__OBSERVABILITY__METER_INTERVAL=30 -export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs -export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 -export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop -export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info -export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true +# Event message configuration +#export RUSTFS_EVENT_CONFIG="./deploy/config/event.toml" -# Logging sinks -export RUSTFS__SINKS__FILE__ENABLED=true -export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log" -export RUSTFS__SINKS__WEBHOOK__ENABLED=false -export RUSTFS__SINKS__WEBHOOK__ENDPOINT="" -export RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN="" -export RUSTFS__SINKS__KAFKA__ENABLED=false -export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS="" -export RUSTFS__SINKS__KAFKA__TOPIC="" -export RUSTFS__LOGGER__QUEUE_CAPACITY=10 ``` #### Start the service @@ -102,7 +84,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10 2. Start the observability stack: ```bash - docker compose up -d -f docker-compose.yml + docker compose -f docker-compose.yml up -d ``` #### Access Monitoring Dashboards @@ -116,7 +98,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10 1. Copy the example configuration: ```bash cd deploy/config - cp obs.toml.example obs.toml + cp obs.example.toml obs.toml ``` 2. Edit `obs.toml` with the following parameters: diff --git a/README_ZH.md b/README_ZH.md index c2016fb7c..b0fd4816a 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -59,30 +59,11 @@ export RUSTFS_ADDRESS="0.0.0.0:9000" export RUSTFS_CONSOLE_ENABLE=true export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001" -# 可观测性配置（方式一：配置文件） +# 可观测性配置 export RUSTFS_OBS_CONFIG="./deploy/config/obs.toml" -# 可观测性配置（方式二：环境变量） -export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 -export RUSTFS__OBSERVABILITY__USE_STDOUT=true -export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 -export RUSTFS__OBSERVABILITY__METER_INTERVAL=30 -export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs -export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 -export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop -export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info -export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true - -# 日志接收器 -export RUSTFS__SINKS__FILE__ENABLED=true -export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log" -export RUSTFS__SINKS__WEBHOOK__ENABLED=false -export RUSTFS__SINKS__WEBHOOK__ENDPOINT="" -export RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN="" -export RUSTFS__SINKS__KAFKA__ENABLED=false -export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS="" -export RUSTFS__SINKS__KAFKA__TOPIC="" -export RUSTFS__LOGGER__QUEUE_CAPACITY=10 +# 事件消息配置 +#export RUSTFS_EVENT_CONFIG="./deploy/config/event.toml" ``` #### 启动服务 @@ -102,7 +83,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10 2. 启动可观测性系统： ```bash - docker compose up -d -f docker-compose.yml + docker compose -f docker-compose.yml up -d ``` #### 访问监控面板 @@ -116,7 +97,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10 1. 复制示例配置： ```bash cd deploy/config - cp obs.toml.example obs.toml + cp obs.example.toml obs.toml ``` 2. 编辑 `obs.toml` 配置文件，参数如下： diff --git a/common/common/src/lib.rs b/common/common/src/lib.rs index a8998fee0..90250c5eb 100644 --- a/common/common/src/lib.rs +++ b/common/common/src/lib.rs @@ -2,6 +2,9 @@ pub mod error; pub mod globals; pub mod last_minute; +// is ',' +pub static DEFAULT_DELIMITER: u8 = 44; + /// Defers evaluation of a block of code until the end of the scope. #[macro_export] macro_rules! defer { diff --git a/common/workers/src/workers.rs b/common/workers/src/workers.rs index bd12d9285..f6a6a94ef 100644 --- a/common/workers/src/workers.rs +++ b/common/workers/src/workers.rs @@ -3,13 +3,13 @@ use tokio::sync::{Mutex, Notify}; use tracing::info; pub struct Workers { - available: Mutex, // 可用的工作槽 - notify: Notify, // 用于通知等待的任务 - limit: usize, // 最大并发工作数 + available: Mutex, // Available working slots + notify: Notify, // Used to notify waiting tasks + limit: usize, // Maximum number of concurrent jobs } impl Workers { - // 创建 Workers 对象，允许最多 n 个作业并发执行 + // Create a Workers object that allows up to n jobs to execute concurrently pub fn new(n: usize) -> Result, &'static str> { if n == 0 { return Err("n must be > 0"); @@ -22,7 +22,7 @@ impl Workers { })) } - // 让一个作业获得执行的机会 + // Give a job a chance to be executed pub async fn take(&self) { loop { let mut available = self.available.lock().await; @@ -37,15 +37,15 @@ impl Workers { } } - // 让一个作业释放其机会 + // Release a job's slot pub async fn give(&self) { let mut available = self.available.lock().await; info!("worker give, {}", *available); - *available += 1; // 增加可用槽 - self.notify.notify_one(); // 通知一个等待的任务 + *available += 1; // Increase available slots + self.notify.notify_one(); // Notify a waiting task } - // 等待所有并发作业完成 + // Wait for all concurrent jobs to complete pub async fn wait(&self) { loop { { @@ -54,7 +54,7 @@ impl Workers { break; } } - // 等待直到所有槽都被释放 + // Wait until all slots are freed self.notify.notified().await; } info!("worker wait end"); diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml new file mode 100644 index 000000000..1a81e30d5 --- /dev/null +++ b/crates/config/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "rustfs-config" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +config = { workspace = true } +const-str = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } + + +[lints] +workspace = true diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs new file mode 100644 index 000000000..3d427f51c --- /dev/null +++ b/crates/config/src/config.rs @@ -0,0 +1,23 @@ +use crate::event::config::NotifierConfig; +use crate::ObservabilityConfig; + +/// RustFs configuration +pub struct RustFsConfig { + pub observability: ObservabilityConfig, + pub event: NotifierConfig, +} + +impl RustFsConfig { + pub fn new() -> Self { + Self { + observability: ObservabilityConfig::new(), + event: NotifierConfig::new(), + } + } +} + +impl Default for RustFsConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs new file mode 100644 index 000000000..c5ad125a7 --- /dev/null +++ b/crates/config/src/constants/app.rs @@ -0,0 +1,91 @@ +use const_str::concat; + +/// Application name +/// Default value: RustFs +/// Environment variable: RUSTFS_APP_NAME +pub const APP_NAME: &str = "RustFs"; +/// Application version +/// Default value: 1.0.0 +/// Environment variable: RUSTFS_VERSION +pub const VERSION: &str = "0.0.1"; + +/// Default configuration logger level +/// Default value: info +/// Environment variable: RUSTFS_LOG_LEVEL +pub const DEFAULT_LOG_LEVEL: &str = "info"; + +/// Default configuration use stdout +/// Default value: true +pub const USE_STDOUT: bool = true; + +/// Default configuration sample ratio +/// Default value: 1.0 +pub const SAMPLE_RATIO: f64 = 1.0; +/// Default configuration meter interval +/// Default value: 30 +pub const METER_INTERVAL: u64 = 30; + +/// Default configuration service version +/// Default value: 0.0.1 +pub const SERVICE_VERSION: &str = "0.0.1"; + +/// Default configuration environment +/// Default value: production +pub const ENVIRONMENT: &str = "production"; + +/// maximum number of connections +/// This is the maximum number of connections that the server will accept. +/// This is used to limit the number of connections to the server. +pub const MAX_CONNECTIONS: usize = 100; +/// timeout for connections +/// This is the timeout for connections to the server. +/// This is used to limit the time that a connection can be open. +pub const DEFAULT_TIMEOUT_MS: u64 = 3000; + +/// Default Access Key +/// Default value: rustfsadmin +/// Environment variable: RUSTFS_ACCESS_KEY +/// Command line argument: --access-key +/// Example: RUSTFS_ACCESS_KEY=rustfsadmin +/// Example: --access-key rustfsadmin +pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin"; +/// Default Secret Key +/// Default value: rustfsadmin +/// Environment variable: RUSTFS_SECRET_KEY +/// Command line argument: --secret-key +/// Example: RUSTFS_SECRET_KEY=rustfsadmin +/// Example: --secret-key rustfsadmin +pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; +/// Default configuration file for observability +/// Default value: config/obs.toml +/// Environment variable: RUSTFS_OBS_CONFIG +/// Command line argument: --obs-config +/// Example: RUSTFS_OBS_CONFIG=config/obs.toml +/// Example: --obs-config config/obs.toml +/// Example: --obs-config /etc/rustfs/obs.toml +pub const DEFAULT_OBS_CONFIG: &str = "./deploy/config/obs.toml"; + +/// Default TLS key for rustfs +/// This is the default key for TLS. +pub const RUSTFS_TLS_KEY: &str = "rustfs_key.pem"; + +/// Default TLS cert for rustfs +/// This is the default cert for TLS. +pub const RUSTFS_TLS_CERT: &str = "rustfs_cert.pem"; + +/// Default port for rustfs +/// This is the default port for rustfs. +/// This is used to bind the server to a specific port. +pub const DEFAULT_PORT: u16 = 9000; + +/// Default address for rustfs +/// This is the default address for rustfs. +pub const DEFAULT_ADDRESS: &str = concat!(":", DEFAULT_PORT); + +/// Default port for rustfs console +/// This is the default port for rustfs console. +pub const DEFAULT_CONSOLE_PORT: u16 = 9002; + +/// Default address for rustfs console +/// This is the default address for rustfs console. +pub const DEFAULT_CONSOLE_ADDRESS: &str = concat!(":", DEFAULT_CONSOLE_PORT); diff --git a/crates/config/src/constants/mod.rs b/crates/config/src/constants/mod.rs new file mode 100644 index 000000000..04023c887 --- /dev/null +++ b/crates/config/src/constants/mod.rs @@ -0,0 +1 @@ +pub(crate) mod app; diff --git a/crates/config/src/event/adapters.rs b/crates/config/src/event/adapters.rs new file mode 100644 index 000000000..d66bf19e5 --- /dev/null +++ b/crates/config/src/event/adapters.rs @@ -0,0 +1,27 @@ +use crate::event::kafka::KafkaAdapter; +use crate::event::mqtt::MqttAdapter; +use crate::event::webhook::WebhookAdapter; +use serde::{Deserialize, Serialize}; + +/// Configuration for the notification system. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum AdapterConfig { + Webhook(WebhookAdapter), + Kafka(KafkaAdapter), + Mqtt(MqttAdapter), +} + +impl AdapterConfig { + /// create a new configuration with default values + pub fn new() -> Self { + Self::Webhook(WebhookAdapter::new()) + } +} + +impl Default for AdapterConfig { + /// create a new configuration with default values + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/event/config.rs b/crates/config/src/event/config.rs new file mode 100644 index 000000000..ea364c9a0 --- /dev/null +++ b/crates/config/src/event/config.rs @@ -0,0 +1,43 @@ +use crate::event::adapters::AdapterConfig; +use serde::{Deserialize, Serialize}; +use std::env; + +#[allow(dead_code)] +const DEFAULT_CONFIG_FILE: &str = "event"; + +/// Configuration for the notification system. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NotifierConfig { + #[serde(default = "default_store_path")] + pub store_path: String, + #[serde(default = "default_channel_capacity")] + pub channel_capacity: usize, + pub adapters: Vec, +} + +impl Default for NotifierConfig { + fn default() -> Self { + Self::new() + } +} + +impl NotifierConfig { + /// create a new configuration with default values + pub fn new() -> Self { + Self { + store_path: default_store_path(), + channel_capacity: default_channel_capacity(), + adapters: vec![AdapterConfig::new()], + } + } +} + +/// Provide temporary directories as default storage paths +fn default_store_path() -> String { + env::temp_dir().join("event-notification").to_string_lossy().to_string() +} + +/// Provides the recommended default channel capacity for high concurrency systems +fn default_channel_capacity() -> usize { + 10000 // Reasonable default values for high concurrency systems +} diff --git a/crates/config/src/event/kafka.rs b/crates/config/src/event/kafka.rs new file mode 100644 index 000000000..164113742 --- /dev/null +++ b/crates/config/src/event/kafka.rs @@ -0,0 +1,29 @@ +use serde::{Deserialize, Serialize}; + +/// Configuration for the Kafka adapter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KafkaAdapter { + pub brokers: String, + pub topic: String, + pub max_retries: u32, + pub timeout: u64, +} + +impl KafkaAdapter { + /// create a new configuration with default values + pub fn new() -> Self { + Self { + brokers: "localhost:9092".to_string(), + topic: "kafka_topic".to_string(), + max_retries: 3, + timeout: 1000, + } + } +} + +impl Default for KafkaAdapter { + /// create a new configuration with default values + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/event/mod.rs b/crates/config/src/event/mod.rs new file mode 100644 index 000000000..80dfd45e5 --- /dev/null +++ b/crates/config/src/event/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod adapters; +pub(crate) mod config; +pub(crate) mod kafka; +pub(crate) mod mqtt; +pub(crate) mod webhook; diff --git a/crates/config/src/event/mqtt.rs b/crates/config/src/event/mqtt.rs new file mode 100644 index 000000000..ee9835323 --- /dev/null +++ b/crates/config/src/event/mqtt.rs @@ -0,0 +1,31 @@ +use serde::{Deserialize, Serialize}; + +/// Configuration for the MQTT adapter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MqttAdapter { + pub broker: String, + pub port: u16, + pub client_id: String, + pub topic: String, + pub max_retries: u32, +} + +impl MqttAdapter { + /// create a new configuration with default values + pub fn new() -> Self { + Self { + broker: "localhost".to_string(), + port: 1883, + client_id: "mqtt_client".to_string(), + topic: "mqtt_topic".to_string(), + max_retries: 3, + } + } +} + +impl Default for MqttAdapter { + /// create a new configuration with default values + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/event/webhook.rs b/crates/config/src/event/webhook.rs new file mode 100644 index 000000000..95b3adad1 --- /dev/null +++ b/crates/config/src/event/webhook.rs @@ -0,0 +1,51 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Configuration for the notification system. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebhookAdapter { + pub endpoint: String, + pub auth_token: Option, + pub custom_headers: Option>, + pub max_retries: u32, + pub timeout: u64, +} + +impl WebhookAdapter { + /// verify that the configuration is valid + pub fn validate(&self) -> Result<(), String> { + // verify that endpoint cannot be empty + if self.endpoint.trim().is_empty() { + return Err("Webhook endpoint cannot be empty".to_string()); + } + + // verification timeout must be reasonable + if self.timeout == 0 { + return Err("Webhook timeout must be greater than 0".to_string()); + } + + // Verify that the maximum number of retry is reasonable + if self.max_retries > 10 { + return Err("Maximum retry count cannot exceed 10".to_string()); + } + + Ok(()) + } + + /// Get the default configuration + pub fn new() -> Self { + Self { + endpoint: "".to_string(), + auth_token: None, + custom_headers: Some(HashMap::new()), + max_retries: 3, + timeout: 1000, + } + } +} + +impl Default for WebhookAdapter { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs new file mode 100644 index 000000000..44a9fe3c5 --- /dev/null +++ b/crates/config/src/lib.rs @@ -0,0 +1,11 @@ +use crate::observability::config::ObservabilityConfig; + +mod config; +mod constants; +mod event; +mod observability; + +pub use config::RustFsConfig; +pub use constants::app::*; + +pub use event::config::NotifierConfig; diff --git a/crates/config/src/observability/config.rs b/crates/config/src/observability/config.rs new file mode 100644 index 000000000..b43f3646b --- /dev/null +++ b/crates/config/src/observability/config.rs @@ -0,0 +1,28 @@ +use crate::observability::logger::LoggerConfig; +use crate::observability::otel::OtelConfig; +use crate::observability::sink::SinkConfig; +use serde::{Deserialize, Serialize}; + +/// Observability configuration +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct ObservabilityConfig { + pub otel: OtelConfig, + pub sinks: Vec, + pub logger: Option, +} + +impl ObservabilityConfig { + pub fn new() -> Self { + Self { + otel: OtelConfig::new(), + sinks: vec![SinkConfig::new()], + logger: Some(LoggerConfig::new()), + } + } +} + +impl Default for ObservabilityConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/observability/file.rs b/crates/config/src/observability/file.rs new file mode 100644 index 000000000..6ec74c8b3 --- /dev/null +++ b/crates/config/src/observability/file.rs @@ -0,0 +1,59 @@ +use serde::{Deserialize, Serialize}; +use std::env; + +/// File sink configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileSink { + pub path: String, + #[serde(default = "default_buffer_size")] + pub buffer_size: Option, + #[serde(default = "default_flush_interval_ms")] + pub flush_interval_ms: Option, + #[serde(default = "default_flush_threshold")] + pub flush_threshold: Option, +} + +impl FileSink { + pub fn new() -> Self { + Self { + path: env::var("RUSTFS_SINKS_FILE_PATH") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(default_path), + buffer_size: default_buffer_size(), + flush_interval_ms: default_flush_interval_ms(), + flush_threshold: default_flush_threshold(), + } + } +} + +impl Default for FileSink { + fn default() -> Self { + Self::new() + } +} + +fn default_buffer_size() -> Option { + Some(8192) +} +fn default_flush_interval_ms() -> Option { + Some(1000) +} +fn default_flush_threshold() -> Option { + Some(100) +} + +fn default_path() -> String { + let temp_dir = env::temp_dir().join("rustfs"); + + if let Err(e) = std::fs::create_dir_all(&temp_dir) { + eprintln!("Failed to create log directory: {}", e); + return "rustfs/rustfs.log".to_string(); + } + + temp_dir + .join("rustfs.log") + .to_str() + .unwrap_or("rustfs/rustfs.log") + .to_string() +} diff --git a/crates/config/src/observability/kafka.rs b/crates/config/src/observability/kafka.rs new file mode 100644 index 000000000..104ab1da4 --- /dev/null +++ b/crates/config/src/observability/kafka.rs @@ -0,0 +1,36 @@ +use serde::{Deserialize, Serialize}; + +/// Kafka sink configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KafkaSink { + pub brokers: String, + pub topic: String, + #[serde(default = "default_batch_size")] + pub batch_size: Option, + #[serde(default = "default_batch_timeout_ms")] + pub batch_timeout_ms: Option, +} + +impl KafkaSink { + pub fn new() -> Self { + Self { + brokers: "localhost:9092".to_string(), + topic: "rustfs".to_string(), + batch_size: default_batch_size(), + batch_timeout_ms: default_batch_timeout_ms(), + } + } +} + +impl Default for KafkaSink { + fn default() -> Self { + Self::new() + } +} + +fn default_batch_size() -> Option { + Some(100) +} +fn default_batch_timeout_ms() -> Option { + Some(1000) +} diff --git a/crates/config/src/observability/logger.rs b/crates/config/src/observability/logger.rs new file mode 100644 index 000000000..68a0bff45 --- /dev/null +++ b/crates/config/src/observability/logger.rs @@ -0,0 +1,21 @@ +use serde::{Deserialize, Serialize}; + +/// Logger configuration +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct LoggerConfig { + pub queue_capacity: Option, +} + +impl LoggerConfig { + pub fn new() -> Self { + Self { + queue_capacity: Some(10000), + } + } +} + +impl Default for LoggerConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs new file mode 100644 index 000000000..216c074e2 --- /dev/null +++ b/crates/config/src/observability/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod config; +pub(crate) mod file; +pub(crate) mod kafka; +pub(crate) mod logger; +pub(crate) mod otel; +pub(crate) mod sink; +pub(crate) mod webhook; diff --git a/crates/config/src/observability/otel.rs b/crates/config/src/observability/otel.rs new file mode 100644 index 000000000..77785c687 --- /dev/null +++ b/crates/config/src/observability/otel.rs @@ -0,0 +1,69 @@ +use crate::constants::app::{ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT}; +use crate::{APP_NAME, DEFAULT_LOG_LEVEL}; +use serde::{Deserialize, Serialize}; +use std::env; + +/// OpenTelemetry configuration +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct OtelConfig { + pub endpoint: String, // Endpoint for metric collection + pub use_stdout: Option, // Output to stdout + pub sample_ratio: Option, // Trace sampling ratio + pub meter_interval: Option, // Metric collection interval + pub service_name: Option, // Service name + pub service_version: Option, // Service version + pub environment: Option, // Environment + pub logger_level: Option, // Logger level + pub local_logging_enabled: Option, // Local logging enabled +} + +impl OtelConfig { + pub fn new() -> Self { + extract_otel_config_from_env() + } +} + +impl Default for OtelConfig { + fn default() -> Self { + Self::new() + } +} + +// Helper function: Extract observable configuration from environment variables +fn extract_otel_config_from_env() -> OtelConfig { + OtelConfig { + endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()), + use_stdout: env::var("RUSTFS_OBSERVABILITY_USE_STDOUT") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(USE_STDOUT)), + sample_ratio: env::var("RUSTFS_OBSERVABILITY_SAMPLE_RATIO") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(SAMPLE_RATIO)), + meter_interval: env::var("RUSTFS_OBSERVABILITY_METER_INTERVAL") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(METER_INTERVAL)), + service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(APP_NAME.to_string())), + service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(SERVICE_VERSION.to_string())), + environment: env::var("RUSTFS_OBSERVABILITY_ENVIRONMENT") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(ENVIRONMENT.to_string())), + logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(DEFAULT_LOG_LEVEL.to_string())), + local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(false)), + } +} diff --git a/crates/config/src/observability/sink.rs b/crates/config/src/observability/sink.rs new file mode 100644 index 000000000..9339e06ee --- /dev/null +++ b/crates/config/src/observability/sink.rs @@ -0,0 +1,25 @@ +use crate::observability::file::FileSink; +use crate::observability::kafka::KafkaSink; +use crate::observability::webhook::WebhookSink; +use serde::{Deserialize, Serialize}; + +/// Sink configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum SinkConfig { + Kafka(KafkaSink), + Webhook(WebhookSink), + File(FileSink), +} + +impl SinkConfig { + pub fn new() -> Self { + Self::File(FileSink::new()) + } +} + +impl Default for SinkConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/config/src/observability/webhook.rs b/crates/config/src/observability/webhook.rs new file mode 100644 index 000000000..8e1d32f82 --- /dev/null +++ b/crates/config/src/observability/webhook.rs @@ -0,0 +1,39 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Webhook sink configuration +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct WebhookSink { + pub endpoint: String, + pub auth_token: String, + pub headers: Option>, + #[serde(default = "default_max_retries")] + pub max_retries: Option, + #[serde(default = "default_retry_delay_ms")] + pub retry_delay_ms: Option, +} + +impl WebhookSink { + pub fn new() -> Self { + Self { + endpoint: "".to_string(), + auth_token: "".to_string(), + headers: Some(HashMap::new()), + max_retries: default_max_retries(), + retry_delay_ms: default_retry_delay_ms(), + } + } +} + +impl Default for WebhookSink { + fn default() -> Self { + Self::new() + } +} + +fn default_max_retries() -> Option { + Some(3) +} +fn default_retry_delay_ms() -> Option { + Some(100) +} diff --git a/crates/event-notifier/Cargo.toml b/crates/event-notifier/Cargo.toml index 5d1d33906..8d9acd9a0 100644 --- a/crates/event-notifier/Cargo.toml +++ b/crates/event-notifier/Cargo.toml @@ -9,13 +9,12 @@ version.workspace = true [features] default = ["webhook"] webhook = ["dep:reqwest"] -kafka = ["rdkafka"] mqtt = ["rumqttc"] +kafka = ["dep:rdkafka"] [dependencies] async-trait = { workspace = true } config = { workspace = true } -rdkafka = { workspace = true, features = ["tokio"], optional = true } reqwest = { workspace = true, optional = true } rumqttc = { workspace = true, optional = true } serde = { workspace = true } @@ -29,12 +28,16 @@ tokio = { workspace = true, features = ["sync", "net", "macros", "signal", "rt-m tokio-util = { workspace = true } uuid = { workspace = true, features = ["v4", "serde"] } +# Only enable kafka features and related dependencies on Linux +[target.'cfg(target_os = "linux")'.dependencies] +rdkafka = { workspace = true, features = ["tokio"], optional = true } [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } tracing-subscriber = { workspace = true } http = { workspace = true } axum = { workspace = true } +dotenvy = "0.15.7" [lints] workspace = true diff --git a/crates/event-notifier/examples/.env-zh.example b/crates/event-notifier/examples/.env-zh.example deleted file mode 100644 index 00228e162..000000000 --- a/crates/event-notifier/examples/.env-zh.example +++ /dev/null @@ -1,28 +0,0 @@ -# ===== 全局配置 ===== -NOTIFIER__STORE_PATH=/var/log/event-notification -NOTIFIER__CHANNEL_CAPACITY=5000 - -# ===== 适配器配置（数组格式） ===== -# Webhook 适配器（索引 0） -NOTIFIER__ADAPTERS_0__type=Webhook -NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook -NOTIFIER__ADAPTERS_0__auth_token=your-auth-token -NOTIFIER__ADAPTERS_0__max_retries=3 -NOTIFIER__ADAPTERS_0__timeout=50 -NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value -NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value - -# Kafka 适配器（索引 1） -NOTIFIER__ADAPTERS_1__type=Kafka -NOTIFIER__ADAPTERS_1__brokers=localhost:9092 -NOTIFIER__ADAPTERS_1__topic=notifications -NOTIFIER__ADAPTERS_1__max_retries=3 -NOTIFIER__ADAPTERS_1__timeout=60 - -# MQTT 适配器（索引 2） -NOTIFIER__ADAPTERS_2__type=Mqtt -NOTIFIER__ADAPTERS_2__broker=mqtt.example.com -NOTIFIER__ADAPTERS_2__port=1883 -NOTIFIER__ADAPTERS_2__client_id=event-notifier -NOTIFIER__ADAPTERS_2__topic=events -NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file diff --git a/crates/event-notifier/examples/.env.example b/crates/event-notifier/examples/.env.example index af343863d..c6d371428 100644 --- a/crates/event-notifier/examples/.env.example +++ b/crates/event-notifier/examples/.env.example @@ -1,28 +1,28 @@ -# ===== global configuration ===== -NOTIFIER__STORE_PATH=/var/log/event-notification -NOTIFIER__CHANNEL_CAPACITY=5000 - -# ===== adapter configuration array format ===== -# webhook adapter index 0 -NOTIFIER__ADAPTERS_0__type=Webhook -NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook -NOTIFIER__ADAPTERS_0__auth_token=your-auth-token -NOTIFIER__ADAPTERS_0__max_retries=3 -NOTIFIER__ADAPTERS_0__timeout=50 -NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value -NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value - -# kafka adapter index 1 -NOTIFIER__ADAPTERS_1__type=Kafka -NOTIFIER__ADAPTERS_1__brokers=localhost:9092 -NOTIFIER__ADAPTERS_1__topic=notifications -NOTIFIER__ADAPTERS_1__max_retries=3 -NOTIFIER__ADAPTERS_1__timeout=60 - -# mqtt adapter index 2 -NOTIFIER__ADAPTERS_2__type=Mqtt -NOTIFIER__ADAPTERS_2__broker=mqtt.example.com -NOTIFIER__ADAPTERS_2__port=1883 -NOTIFIER__ADAPTERS_2__client_id=event-notifier -NOTIFIER__ADAPTERS_2__topic=events -NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file +## ===== global configuration ===== +#NOTIFIER__STORE_PATH=/var/log/event-notification +#NOTIFIER__CHANNEL_CAPACITY=5000 +# +## ===== adapter configuration array format ===== +## webhook adapter index 0 +#NOTIFIER__ADAPTERS_0__type=Webhook +#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook +#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token +#NOTIFIER__ADAPTERS_0__max_retries=3 +#NOTIFIER__ADAPTERS_0__timeout=50 +#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value +#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value +# +## kafka adapter index 1 +#NOTIFIER__ADAPTERS_1__type=Kafka +#NOTIFIER__ADAPTERS_1__brokers=localhost:9092 +#NOTIFIER__ADAPTERS_1__topic=notifications +#NOTIFIER__ADAPTERS_1__max_retries=3 +#NOTIFIER__ADAPTERS_1__timeout=60 +# +## mqtt adapter index 2 +#NOTIFIER__ADAPTERS_2__type=Mqtt +#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com +#NOTIFIER__ADAPTERS_2__port=1883 +#NOTIFIER__ADAPTERS_2__client_id=event-notifier +#NOTIFIER__ADAPTERS_2__topic=events +#NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file diff --git a/crates/event-notifier/examples/.env.zh.example b/crates/event-notifier/examples/.env.zh.example new file mode 100644 index 000000000..47f543080 --- /dev/null +++ b/crates/event-notifier/examples/.env.zh.example @@ -0,0 +1,28 @@ +## ===== 全局配置 ===== +#NOTIFIER__STORE_PATH=/var/log/event-notification +#NOTIFIER__CHANNEL_CAPACITY=5000 +# +## ===== 适配器配置（数组格式） ===== +## Webhook 适配器（索引 0） +#NOTIFIER__ADAPTERS_0__type=Webhook +#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook +#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token +#NOTIFIER__ADAPTERS_0__max_retries=3 +#NOTIFIER__ADAPTERS_0__timeout=50 +#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value +#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value +# +## Kafka 适配器（索引 1） +#NOTIFIER__ADAPTERS_1__type=Kafka +#NOTIFIER__ADAPTERS_1__brokers=localhost:9092 +#NOTIFIER__ADAPTERS_1__topic=notifications +#NOTIFIER__ADAPTERS_1__max_retries=3 +#NOTIFIER__ADAPTERS_1__timeout=60 +# +## MQTT 适配器（索引 2） +#NOTIFIER__ADAPTERS_2__type=Mqtt +#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com +#NOTIFIER__ADAPTERS_2__port=1883 +#NOTIFIER__ADAPTERS_2__client_id=event-notifier +#NOTIFIER__ADAPTERS_2__topic=events +#NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file diff --git a/crates/event-notifier/examples/simple.rs b/crates/event-notifier/examples/simple.rs index 93005f0a7..27d422b06 100644 --- a/crates/event-notifier/examples/simple.rs +++ b/crates/event-notifier/examples/simple.rs @@ -33,7 +33,9 @@ async fn main() -> Result<(), Box> { // loading configuration from environment variables let _config = NotifierConfig::event_load_config(Some("./crates/event-notifier/examples/event.toml".to_string())); tracing::info!("event_load_config config: {:?} \n", _config); - + dotenvy::dotenv()?; + let _config = NotifierConfig::event_load_config(None); + tracing::info!("event_load_config config: {:?} \n", _config); let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?)); let adapters = create_adapters(&config.adapters)?; diff --git a/crates/event-notifier/examples/webhook.rs b/crates/event-notifier/examples/webhook.rs index c754f8223..4cdf02c66 100644 --- a/crates/event-notifier/examples/webhook.rs +++ b/crates/event-notifier/examples/webhook.rs @@ -37,7 +37,7 @@ async fn receive_webhook(Json(payload): Json) -> StatusCode { println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second); println!( "received a webhook request time:{} content:\n {}", - seconds.to_string(), + seconds, serde_json::to_string_pretty(&payload).unwrap() ); StatusCode::OK @@ -66,10 +66,10 @@ fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) { // calculate month let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; - for m in 0..12 { - if total_seconds >= days_in_month[m] * seconds_per_day { + for m in &days_in_month { + if total_seconds >= m * seconds_per_day { month += 1; - total_seconds -= days_in_month[m] * seconds_per_day; + total_seconds -= m * seconds_per_day; } else { break; } diff --git a/crates/event-notifier/src/adapter/mod.rs b/crates/event-notifier/src/adapter/mod.rs index fa12aa97b..426fd2d83 100644 --- a/crates/event-notifier/src/adapter/mod.rs +++ b/crates/event-notifier/src/adapter/mod.rs @@ -4,7 +4,7 @@ use crate::Event; use async_trait::async_trait; use std::sync::Arc; -#[cfg(feature = "kafka")] +#[cfg(all(feature = "kafka", target_os = "linux"))] pub(crate) mod kafka; #[cfg(feature = "mqtt")] pub(crate) mod mqtt; @@ -31,7 +31,7 @@ pub fn create_adapters(configs: &[AdapterConfig]) -> Result { adapters.push(Arc::new(kafka::KafkaAdapter::new(kafka_config)?)); } @@ -43,7 +43,7 @@ pub fn create_adapters(configs: &[AdapterConfig]) -> Result return Err(Error::FeatureDisabled("webhook")), - #[cfg(not(feature = "kafka"))] + #[cfg(any(not(feature = "kafka"), not(target_os = "linux")))] AdapterConfig::Kafka(_) => return Err(Error::FeatureDisabled("kafka")), #[cfg(not(feature = "mqtt"))] AdapterConfig::Mqtt(_) => return Err(Error::FeatureDisabled("mqtt")), diff --git a/crates/event-notifier/src/bus.rs b/crates/event-notifier/src/bus.rs index bd4b81c5d..5cabfc22e 100644 --- a/crates/event-notifier/src/bus.rs +++ b/crates/event-notifier/src/bus.rs @@ -7,11 +7,13 @@ use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; use tokio::time::Duration; use tokio_util::sync::CancellationToken; +use tracing::instrument; /// Handles incoming events from the producer. /// /// This function is responsible for receiving events from the producer and sending them to the appropriate adapters. /// It also handles the shutdown process and saves any pending logs to the event store. +#[instrument(skip_all)] pub async fn event_bus( mut rx: mpsc::Receiver, adapters: Vec>, diff --git a/crates/event-notifier/src/config.rs b/crates/event-notifier/src/config.rs index ab46d8e8f..3414f3fa6 100644 --- a/crates/event-notifier/src/config.rs +++ b/crates/event-notifier/src/config.rs @@ -1,4 +1,4 @@ -use config::{Config, Environment, File, FileFormat}; +use config::{Config, File, FileFormat}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::env; @@ -138,15 +138,6 @@ impl NotifierConfig { let app_config = Config::builder() .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false)) .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false)) - .add_source( - Environment::default() - .prefix("NOTIFIER") - .prefix_separator("__") - .separator("__") - .list_separator("_") - .with_list_parse_key("adapters") - .try_parsing(true), - ) .build() .unwrap_or_default(); match app_config.try_deserialize::() { @@ -162,7 +153,7 @@ impl NotifierConfig { } } -const DEFAULT_CONFIG_FILE: &str = "obs"; +const DEFAULT_CONFIG_FILE: &str = "event"; /// Provide temporary directories as default storage paths fn default_store_path() -> String { diff --git a/crates/event-notifier/src/error.rs b/crates/event-notifier/src/error.rs index e6c061de9..ebdaf899d 100644 --- a/crates/event-notifier/src/error.rs +++ b/crates/event-notifier/src/error.rs @@ -15,7 +15,7 @@ pub enum Error { Serde(#[from] serde_json::Error), #[error("HTTP error: {0}")] Http(#[from] reqwest::Error), - #[cfg(feature = "kafka")] + #[cfg(all(feature = "kafka", target_os = "linux"))] #[error("Kafka error: {0}")] Kafka(#[from] rdkafka::error::KafkaError), #[cfg(feature = "mqtt")] diff --git a/crates/event-notifier/src/event.rs b/crates/event-notifier/src/event.rs index ce32aecca..1bf100d75 100644 --- a/crates/event-notifier/src/event.rs +++ b/crates/event-notifier/src/event.rs @@ -15,6 +15,18 @@ pub struct Identity { pub principal_id: String, } +impl Identity { + /// Create a new Identity instance + pub fn new(principal_id: String) -> Self { + Self { principal_id } + } + + /// Set the principal ID + pub fn set_principal_id(&mut self, principal_id: String) { + self.principal_id = principal_id; + } +} + /// A struct representing the bucket information #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Bucket { @@ -24,6 +36,32 @@ pub struct Bucket { pub arn: String, } +impl Bucket { + /// Create a new Bucket instance + pub fn new(name: String, owner_identity: Identity, arn: String) -> Self { + Self { + name, + owner_identity, + arn, + } + } + + /// Set the name of the bucket + pub fn set_name(&mut self, name: String) { + self.name = name; + } + + /// Set the ARN of the bucket + pub fn set_arn(&mut self, arn: String) { + self.arn = arn; + } + + /// Set the owner identity of the bucket + pub fn set_owner_identity(&mut self, owner_identity: Identity) { + self.owner_identity = owner_identity; + } +} + /// A struct representing the object information #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Object { @@ -41,6 +79,64 @@ pub struct Object { pub sequencer: String, } +impl Object { + /// Create a new Object instance + pub fn new( + key: String, + size: Option, + etag: Option, + content_type: Option, + user_metadata: Option>, + version_id: Option, + sequencer: String, + ) -> Self { + Self { + key, + size, + etag, + content_type, + user_metadata, + version_id, + sequencer, + } + } + + /// Set the key + pub fn set_key(&mut self, key: String) { + self.key = key; + } + + /// Set the size + pub fn set_size(&mut self, size: Option) { + self.size = size; + } + + /// Set the etag + pub fn set_etag(&mut self, etag: Option) { + self.etag = etag; + } + + /// Set the content type + pub fn set_content_type(&mut self, content_type: Option) { + self.content_type = content_type; + } + + /// Set the user metadata + pub fn set_user_metadata(&mut self, user_metadata: Option>) { + self.user_metadata = user_metadata; + } + + /// Set the version ID + pub fn set_version_id(&mut self, version_id: Option) { + self.version_id = version_id; + } + + /// Set the sequencer + pub fn set_sequencer(&mut self, sequencer: String) { + self.sequencer = sequencer; + } +} + /// A struct representing the metadata of the event #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Metadata { @@ -52,6 +148,57 @@ pub struct Metadata { pub object: Object, } +impl Default for Metadata { + fn default() -> Self { + Self::new() + } +} +impl Metadata { + /// Create a new Metadata instance with default values + pub fn new() -> Self { + Self { + schema_version: "1.0".to_string(), + configuration_id: "default".to_string(), + bucket: Bucket::new( + "default".to_string(), + Identity::new("default".to_string()), + "arn:aws:s3:::default".to_string(), + ), + object: Object::new("default".to_string(), None, None, None, None, None, "default".to_string()), + } + } + + /// Create a new Metadata instance + pub fn create(schema_version: String, configuration_id: String, bucket: Bucket, object: Object) -> Self { + Self { + schema_version, + configuration_id, + bucket, + object, + } + } + + /// Set the schema version + pub fn set_schema_version(&mut self, schema_version: String) { + self.schema_version = schema_version; + } + + /// Set the configuration ID + pub fn set_configuration_id(&mut self, configuration_id: String) { + self.configuration_id = configuration_id; + } + + /// Set the bucket + pub fn set_bucket(&mut self, bucket: Bucket) { + self.bucket = bucket; + } + + /// Set the object + pub fn set_object(&mut self, object: Object) { + self.object = object; + } +} + /// A struct representing the source of the event #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Source { @@ -61,6 +208,28 @@ pub struct Source { pub user_agent: String, } +impl Source { + /// Create a new Source instance + pub fn new(host: String, port: String, user_agent: String) -> Self { + Self { host, port, user_agent } + } + + /// Set the host + pub fn set_host(&mut self, host: String) { + self.host = host; + } + + /// Set the port + pub fn set_port(&mut self, port: String) { + self.port = port; + } + + /// Set the user agent + pub fn set_user_agent(&mut self, user_agent: String) { + self.user_agent = user_agent; + } +} + /// Builder for creating an Event. /// /// This struct is used to build an Event object with various parameters. diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs index 25fadeeb4..0ffcb8b00 100644 --- a/crates/event-notifier/src/global.rs +++ b/crates/event-notifier/src/global.rs @@ -1,6 +1,7 @@ use crate::{create_adapters, Error, Event, NotifierConfig, NotifierSystem}; use std::sync::{atomic, Arc}; use tokio::sync::{Mutex, OnceCell}; +use tracing::instrument; static GLOBAL_SYSTEM: OnceCell>> = OnceCell::const_new(); static INITIALIZED: atomic::AtomicBool = atomic::AtomicBool::new(false); @@ -113,6 +114,7 @@ pub fn is_ready() -> bool { /// - The system is not initialized. /// - The system is not ready. /// - Sending the event fails. +#[instrument(fields(event))] pub async fn send_event(event: Event) -> Result<(), Error> { if !READY.load(atomic::Ordering::SeqCst) { return Err(Error::custom("Notification system not ready, please wait for initialization to complete")); @@ -124,6 +126,7 @@ pub async fn send_event(event: Event) -> Result<(), Error> { } /// Shuts down the notification system. +#[instrument] pub async fn shutdown() -> Result<(), Error> { if let Some(system) = GLOBAL_SYSTEM.get() { tracing::info!("Shutting down notification system start"); @@ -189,7 +192,7 @@ mod tests { let config = NotifierConfig::default(); let _ = initialize(config.clone()).await; // first initialization let result = initialize(config).await; // second initialization - assert!(!result.is_ok(), "Initialization should succeed"); + assert!(result.is_err(), "Initialization should succeed"); assert!(result.is_err(), "Re-initialization should fail"); } @@ -211,7 +214,7 @@ mod tests { ..Default::default() }; let result = initialize(config).await; - assert!(!result.is_err(), "Initialization with invalid config should fail"); + assert!(result.is_ok(), "Initialization with invalid config should fail"); assert!(is_initialized(), "System should not be marked as initialized after failure"); assert!(is_ready(), "System should not be marked as ready after failure"); } diff --git a/crates/event-notifier/src/lib.rs b/crates/event-notifier/src/lib.rs index 20ef935d2..fe2e5e3da 100644 --- a/crates/event-notifier/src/lib.rs +++ b/crates/event-notifier/src/lib.rs @@ -8,7 +8,7 @@ mod notifier; mod store; pub use adapter::create_adapters; -#[cfg(feature = "kafka")] +#[cfg(all(feature = "kafka", target_os = "linux"))] pub use adapter::kafka::KafkaAdapter; #[cfg(feature = "mqtt")] pub use adapter::mqtt::MqttAdapter; @@ -16,7 +16,7 @@ pub use adapter::mqtt::MqttAdapter; pub use adapter::webhook::WebhookAdapter; pub use adapter::ChannelAdapter; pub use bus::event_bus; -#[cfg(feature = "kafka")] +#[cfg(all(feature = "kafka", target_os = "linux"))] pub use config::KafkaConfig; #[cfg(feature = "mqtt")] pub use config::MqttConfig; diff --git a/crates/event-notifier/src/notifier.rs b/crates/event-notifier/src/notifier.rs index 0b17ddddd..5ab17d371 100644 --- a/crates/event-notifier/src/notifier.rs +++ b/crates/event-notifier/src/notifier.rs @@ -2,6 +2,7 @@ use crate::{event_bus, ChannelAdapter, Error, Event, EventStore, NotifierConfig} use std::sync::Arc; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; +use tracing::instrument; /// The `NotificationSystem` struct represents the notification system. /// It manages the event bus and the adapters. @@ -18,6 +19,7 @@ pub struct NotifierSystem { impl NotifierSystem { /// Creates a new `NotificationSystem` instance. + #[instrument(skip(config))] pub async fn new(config: NotifierConfig) -> Result { let (tx, rx) = mpsc::channel::(config.channel_capacity); let store = Arc::new(EventStore::new(&config.store_path).await?); @@ -44,6 +46,7 @@ impl NotifierSystem { /// Starts the notification system. /// It initializes the event bus and the producer. + #[instrument(skip_all)] pub async fn start(&mut self, adapters: Vec>) -> Result<(), Error> { if self.shutdown.is_cancelled() { let error = Error::custom("System is shutting down"); @@ -67,6 +70,7 @@ impl NotifierSystem { /// Sends an event to the notification system. /// This method is used to send events to the event bus. + #[instrument(skip(self))] pub async fn send_event(&self, event: Event) -> Result<(), Error> { self.log(tracing::Level::DEBUG, "send_event", &format!("Sending event: {:?}", event)); if self.shutdown.is_cancelled() { @@ -85,6 +89,7 @@ impl NotifierSystem { /// Shuts down the notification system. /// This method is used to cancel the event bus and producer tasks. + #[instrument(skip(self))] pub async fn shutdown(&mut self) -> Result<(), Error> { tracing::info!("Shutting down the notification system"); self.shutdown.cancel(); @@ -112,10 +117,13 @@ impl NotifierSystem { self.shutdown.is_cancelled() } - fn handle_error(&self, context: &str, error: &Error) { + #[instrument(skip(self))] + pub fn handle_error(&self, context: &str, error: &Error) { self.log(tracing::Level::ERROR, context, &format!("{:?}", error)); // TODO Can be extended to record to files or send to monitoring systems } + + #[instrument(skip(self))] fn log(&self, level: tracing::Level, context: &str, message: &str) { match level { tracing::Level::ERROR => tracing::error!("[{}] {}", context, message), diff --git a/crates/event-notifier/src/store.rs b/crates/event-notifier/src/store.rs index eca26674a..249116152 100644 --- a/crates/event-notifier/src/store.rs +++ b/crates/event-notifier/src/store.rs @@ -5,6 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use tokio::fs::{create_dir_all, File, OpenOptions}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter}; use tokio::sync::RwLock; +use tracing::instrument; /// `EventStore` is a struct that manages the storage of event logs. pub struct EventStore { @@ -21,6 +22,7 @@ impl EventStore { }) } + #[instrument(skip(self))] pub async fn save_logs(&self, logs: &[Log]) -> Result<(), Error> { let _guard = self.lock.write().await; let file_path = format!( diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 26c264335..12c9831ec 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -13,11 +13,11 @@ workspace = true default = ["file"] file = [] gpu = ["dep:nvml-wrapper"] -kafka = ["dep:rdkafka"] webhook = ["dep:reqwest"] -full = ["file", "gpu", "kafka", "webhook"] +kafka = ["dep:rdkafka"] [dependencies] +rustfs-config = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } config = { workspace = true } @@ -29,6 +29,7 @@ opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] } opentelemetry-stdout = { workspace = true } opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "gzip-tonic"] } opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } +rustfs-utils = { workspace = true, features = ["ip"] } serde = { workspace = true } smallvec = { workspace = true, features = ["serde"] } tracing = { workspace = true, features = ["std", "attributes"] } @@ -37,12 +38,14 @@ tracing-error = { workspace = true } tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] } tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] } -rdkafka = { workspace = true, features = ["tokio"], optional = true } reqwest = { workspace = true, optional = true, default-features = false } serde_json = { workspace = true } sysinfo = { workspace = true } thiserror = { workspace = true } +# Only enable kafka features and related dependencies on Linux +[target.'cfg(target_os = "linux")'.dependencies] +rdkafka = { workspace = true, features = ["tokio"], optional = true } [dev-dependencies] diff --git a/crates/obs/examples/config.toml b/crates/obs/examples/config.toml index 8fff21f24..3a398bc38 100644 --- a/crates/obs/examples/config.toml +++ b/crates/obs/examples/config.toml @@ -9,26 +9,27 @@ environments = "develop" logger_level = "debug" local_logging_enabled = true # Default is false if not specified -[sinks] -[sinks.kafka] -enabled = false -bootstrap_servers = "localhost:9092" -topic = "logs" -batch_size = 100 # Default is 100 if not specified -batch_timeout_ms = 1000 # Default is 1000ms if not specified -[sinks.webhook] -enabled = false -endpoint = "http://localhost:8080/webhook" -auth_token = "" -batch_size = 100 # Default is 3 if not specified -batch_timeout_ms = 1000 # Default is 100ms if not specified +#[[sinks]] +#type = "Kafka" +#bootstrap_servers = "localhost:9092" +#topic = "logs" +#batch_size = 100 # Default is 100 if not specified +#batch_timeout_ms = 100 # Default is 1000ms if not specified +# +#[[sinks]] +#type = "Webhook" +#endpoint = "http://localhost:8080/webhook" +#auth_token = "" +#batch_size = 100 # Default is 3 if not specified +#batch_timeout_ms = 100 # Default is 100ms if not specified -[sinks.file] -enabled = true -path = "deploy/logs/app.log" -batch_size = 100 -batch_timeout_ms = 1000 # Default is 8192 bytes if not specified +[[sinks]] +type = "File" +path = "deploy/logs/rustfs.log" +buffer_size = 102 # Default is 8192 bytes if not specified +flush_interval_ms = 1000 +flush_threshold = 100 [logger] queue_capacity = 10000 \ No newline at end of file diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index 737d2c0d2..770203de3 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -1,6 +1,6 @@ -use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT}; -use config::{Config, Environment, File, FileFormat}; -use serde::Deserialize; +use config::{Config, File, FileFormat}; +use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT}; +use serde::{Deserialize, Serialize}; use std::env; /// OpenTelemetry Configuration @@ -11,7 +11,7 @@ use std::env; /// Add use_stdout for output to stdout /// Add logger level for log level /// Add local_logging_enabled for local logging enabled -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct OtelConfig { pub endpoint: String, // Endpoint for metric collection pub use_stdout: Option, // Output to stdout @@ -24,7 +24,7 @@ pub struct OtelConfig { pub local_logging_enabled: Option, // Local logging enabled } -// Helper function: Extract observable configuration from environment variables +/// Helper function: Extract observable configuration from environment variables fn extract_otel_config_from_env() -> OtelConfig { OtelConfig { endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()), @@ -43,7 +43,7 @@ fn extract_otel_config_from_env() -> OtelConfig { service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME") .ok() .and_then(|v| v.parse().ok()) - .or(Some(SERVICE_NAME.to_string())), + .or(Some(APP_NAME.to_string())), service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION") .ok() .and_then(|v| v.parse().ok()) @@ -55,7 +55,7 @@ fn extract_otel_config_from_env() -> OtelConfig { logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL") .ok() .and_then(|v| v.parse().ok()) - .or(Some(LOGGER_LEVEL.to_string())), + .or(Some(DEFAULT_LOG_LEVEL.to_string())), local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED") .ok() .and_then(|v| v.parse().ok()) @@ -63,36 +63,89 @@ fn extract_otel_config_from_env() -> OtelConfig { } } -impl Default for OtelConfig { - fn default() -> Self { +impl OtelConfig { + /// Create a new instance of OtelConfig with default values + /// + /// # Returns + /// A new instance of OtelConfig + pub fn new() -> Self { extract_otel_config_from_env() } } +impl Default for OtelConfig { + fn default() -> Self { + Self::new() + } +} + /// Kafka Sink Configuration - Add batch parameters -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct KafkaSinkConfig { - pub enabled: bool, - pub bootstrap_servers: String, + pub brokers: String, pub topic: String, pub batch_size: Option, // Batch size, default 100 pub batch_timeout_ms: Option, // Batch timeout time, default 1000ms } +impl KafkaSinkConfig { + pub fn new() -> Self { + Self { + brokers: env::var("RUSTFS_SINKS_KAFKA_BROKERS") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "localhost:9092".to_string()), + topic: env::var("RUSTFS_SINKS_KAFKA_TOPIC") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "default_topic".to_string()), + batch_size: Some(100), + batch_timeout_ms: Some(1000), + } + } +} + +impl Default for KafkaSinkConfig { + fn default() -> Self { + Self::new() + } +} + /// Webhook Sink Configuration - Add Retry Parameters -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct WebhookSinkConfig { - pub enabled: bool, pub endpoint: String, pub auth_token: String, pub max_retries: Option, // Maximum number of retry times, default 3 pub retry_delay_ms: Option, // Retry the delay cardinality, default 100ms } +impl WebhookSinkConfig { + pub fn new() -> Self { + Self { + endpoint: env::var("RUSTFS_SINKS_WEBHOOK_ENDPOINT") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "http://localhost:8080".to_string()), + auth_token: env::var("RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "default_token".to_string()), + max_retries: Some(3), + retry_delay_ms: Some(100), + } + } +} + +impl Default for WebhookSinkConfig { + fn default() -> Self { + Self::new() + } +} + /// File Sink Configuration - Add buffering parameters -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct FileSinkConfig { - pub enabled: bool, pub path: String, pub buffer_size: Option, // Write buffer size, default 8192 pub flush_interval_ms: Option, // Refresh interval time, default 1000ms @@ -107,19 +160,15 @@ impl FileSinkConfig { eprintln!("Failed to create log directory: {}", e); return "rustfs/rustfs.log".to_string(); } - + println!("Using log directory: {:?}", temp_dir); temp_dir .join("rustfs.log") .to_str() .unwrap_or("rustfs/rustfs.log") .to_string() } -} - -impl Default for FileSinkConfig { - fn default() -> Self { - FileSinkConfig { - enabled: true, + pub fn new() -> Self { + Self { path: env::var("RUSTFS_SINKS_FILE_PATH") .ok() .filter(|s| !s.trim().is_empty()) @@ -131,38 +180,53 @@ impl Default for FileSinkConfig { } } +impl Default for FileSinkConfig { + fn default() -> Self { + Self::new() + } +} + /// Sink configuration collection -#[derive(Debug, Deserialize, Clone)] -pub struct SinkConfig { - pub kafka: Option, - pub webhook: Option, - pub file: Option, +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum SinkConfig { + File(FileSinkConfig), + Kafka(KafkaSinkConfig), + Webhook(WebhookSinkConfig), +} + +impl SinkConfig { + pub fn new() -> Self { + Self::File(FileSinkConfig::new()) + } } impl Default for SinkConfig { fn default() -> Self { - SinkConfig { - kafka: None, - webhook: None, - file: Some(FileSinkConfig::default()), - } + Self::new() } } ///Logger Configuration -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct LoggerConfig { pub queue_capacity: Option, } -impl Default for LoggerConfig { - fn default() -> Self { - LoggerConfig { +impl LoggerConfig { + pub fn new() -> Self { + Self { queue_capacity: Some(10000), } } } +impl Default for LoggerConfig { + fn default() -> Self { + Self::new() + } +} + /// Overall application configuration /// Add observability, sinks, and logger configuration /// @@ -180,7 +244,7 @@ impl Default for LoggerConfig { #[derive(Debug, Deserialize, Clone)] pub struct AppConfig { pub observability: OtelConfig, - pub sinks: SinkConfig, + pub sinks: Vec, pub logger: Option, } @@ -192,7 +256,7 @@ impl AppConfig { pub fn new() -> Self { Self { observability: OtelConfig::default(), - sinks: SinkConfig::default(), + sinks: vec![SinkConfig::default()], logger: Some(LoggerConfig::default()), } } @@ -258,14 +322,6 @@ pub fn load_config(config_dir: Option) -> AppConfig { let app_config = Config::builder() .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false)) .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false)) - .add_source( - Environment::default() - .prefix("RUSTFS") - .prefix_separator("__") - .separator("__") - .with_list_parse_key("volumes") - .try_parsing(true), - ) .build() .unwrap_or_default(); diff --git a/crates/obs/src/global.rs b/crates/obs/src/global.rs index 8d392ad94..07657fb41 100644 --- a/crates/obs/src/global.rs +++ b/crates/obs/src/global.rs @@ -3,14 +3,6 @@ use std::sync::{Arc, Mutex}; use tokio::sync::{OnceCell, SetError}; use tracing::{error, info}; -pub(crate) const USE_STDOUT: bool = true; -pub(crate) const SERVICE_NAME: &str = "RustFS"; -pub(crate) const SAMPLE_RATIO: f64 = 1.0; -pub(crate) const METER_INTERVAL: u64 = 60; -pub(crate) const SERVICE_VERSION: &str = "0.1.0"; -pub(crate) const ENVIRONMENT: &str = "production"; -pub(crate) const LOGGER_LEVEL: &str = "info"; - /// Global guard for OpenTelemetry tracing static GLOBAL_GUARD: OnceCell>> = OnceCell::const_new(); diff --git a/crates/obs/src/lib.rs b/crates/obs/src/lib.rs index d41d0e66d..6a8f6219a 100644 --- a/crates/obs/src/lib.rs +++ b/crates/obs/src/lib.rs @@ -32,20 +32,13 @@ mod config; mod entry; mod global; mod logger; -mod sink; +mod sinks; mod system; mod telemetry; -mod utils; mod worker; use crate::logger::InitLogStatus; pub use config::load_config; -#[cfg(feature = "file")] -pub use config::FileSinkConfig; -#[cfg(feature = "kafka")] -pub use config::KafkaSinkConfig; -#[cfg(feature = "webhook")] -pub use config::WebhookSinkConfig; pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig}; pub use entry::args::Args; pub use entry::audit::{ApiDetails, AuditLogEntry}; @@ -79,7 +72,7 @@ use tracing::{error, info}; /// ``` pub async fn init_obs(config: AppConfig) -> (Arc>, telemetry::OtelGuard) { let guard = init_telemetry(&config.observability); - let sinks = sink::create_sinks(&config).await; + let sinks = sinks::create_sinks(&config).await; let logger = init_global_logger(&config, sinks).await; let obs_config = config.observability.clone(); tokio::spawn(async move { diff --git a/crates/obs/src/logger.rs b/crates/obs/src/logger.rs index 6329ab8fc..02e0bf9b2 100644 --- a/crates/obs/src/logger.rs +++ b/crates/obs/src/logger.rs @@ -1,6 +1,6 @@ -use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION}; -use crate::sink::Sink; +use crate::sinks::Sink; use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry}; +use rustfs_config::{APP_NAME, ENVIRONMENT, SERVICE_VERSION}; use std::sync::Arc; use std::time::SystemTime; use tokio::sync::mpsc::{self, Receiver, Sender}; @@ -428,7 +428,7 @@ impl Default for InitLogStatus { fn default() -> Self { Self { timestamp: SystemTime::now(), - service_name: String::from(SERVICE_NAME), + service_name: String::from(APP_NAME), version: SERVICE_VERSION.to_string(), environment: ENVIRONMENT.to_string(), } @@ -442,7 +442,7 @@ impl InitLogStatus { let version = config.service_version.unwrap_or(SERVICE_VERSION.to_string()); Self { timestamp: SystemTime::now(), - service_name: String::from(SERVICE_NAME), + service_name: String::from(APP_NAME), version, environment, } diff --git a/crates/obs/src/sink.rs b/crates/obs/src/sink.rs deleted file mode 100644 index 4df212b31..000000000 --- a/crates/obs/src/sink.rs +++ /dev/null @@ -1,497 +0,0 @@ -use crate::{AppConfig, LogRecord, UnifiedLogEntry}; -use async_trait::async_trait; -use std::sync::Arc; -use tokio::fs::OpenOptions; -use tokio::io; -use tokio::io::AsyncWriteExt; - -/// Sink Trait definition, asynchronously write logs -#[async_trait] -pub trait Sink: Send + Sync { - async fn write(&self, entry: &UnifiedLogEntry); -} - -#[cfg(feature = "kafka")] -/// Kafka Sink Implementation -pub struct KafkaSink { - producer: rdkafka::producer::FutureProducer, - topic: String, - batch_size: usize, - batch_timeout_ms: u64, - entries: Arc>>, - last_flush: Arc, -} - -#[cfg(feature = "kafka")] -impl KafkaSink { - /// Create a new KafkaSink instance - pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self { - // Create Arc-wrapped values first - let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size))); - let last_flush = Arc::new(std::sync::atomic::AtomicU64::new( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64, - )); - let sink = KafkaSink { - producer: producer.clone(), - topic: topic.clone(), - batch_size, - batch_timeout_ms, - entries: entries.clone(), - last_flush: last_flush.clone(), - }; - - // Start background flusher - tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms)); - - sink - } - - /// Add a getter method to read the batch_timeout_ms field - #[allow(dead_code)] - pub fn batch_timeout(&self) -> u64 { - self.batch_timeout_ms - } - - /// Add a method to dynamically adjust the timeout if needed - #[allow(dead_code)] - pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) { - self.batch_timeout_ms = new_timeout_ms; - } - - async fn periodic_flush( - producer: rdkafka::producer::FutureProducer, - topic: String, - entries: Arc>>, - last_flush: Arc, - timeout_ms: u64, - ) { - loop { - tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await; - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - - let last = last_flush.load(std::sync::atomic::Ordering::Relaxed); - - if now - last >= timeout_ms { - let mut batch = entries.lock().await; - if !batch.is_empty() { - Self::send_batch(&producer, &topic, batch.drain(..).collect()).await; - last_flush.store(now, std::sync::atomic::Ordering::Relaxed); - } - } - } - } - - async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec) { - for entry in entries { - let payload = match serde_json::to_string(&entry) { - Ok(p) => p, - Err(e) => { - eprintln!("Failed to serialize log entry: {}", e); - continue; - } - }; - - let span_id = entry.get_timestamp().to_rfc3339(); - - let _ = producer - .send( - rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id), - std::time::Duration::from_secs(5), - ) - .await; - } - } -} - -#[cfg(feature = "kafka")] -#[async_trait] -impl Sink for KafkaSink { - async fn write(&self, entry: &UnifiedLogEntry) { - let mut batch = self.entries.lock().await; - batch.push(entry.clone()); - - let should_flush_by_size = batch.len() >= self.batch_size; - let should_flush_by_time = { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed); - now - last >= self.batch_timeout_ms - }; - - if should_flush_by_size || should_flush_by_time { - // Existing flush logic - let entries_to_send: Vec = batch.drain(..).collect(); - let producer = self.producer.clone(); - let topic = self.topic.clone(); - - self.last_flush.store( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64, - std::sync::atomic::Ordering::Relaxed, - ); - - tokio::spawn(async move { - KafkaSink::send_batch(&producer, &topic, entries_to_send).await; - }); - } - } -} - -#[cfg(feature = "kafka")] -impl Drop for KafkaSink { - fn drop(&mut self) { - // Perform any necessary cleanup here - // For example, you might want to flush any remaining entries - let producer = self.producer.clone(); - let topic = self.topic.clone(); - let entries = self.entries.clone(); - let last_flush = self.last_flush.clone(); - - tokio::spawn(async move { - let mut batch = entries.lock().await; - if !batch.is_empty() { - KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await; - last_flush.store( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64, - std::sync::atomic::Ordering::Relaxed, - ); - } - }); - - eprintln!("Dropping KafkaSink with topic: {}", self.topic); - } -} - -#[cfg(feature = "webhook")] -/// Webhook Sink Implementation -pub struct WebhookSink { - endpoint: String, - auth_token: String, - client: reqwest::Client, - max_retries: usize, - retry_delay_ms: u64, -} - -#[cfg(feature = "webhook")] -impl WebhookSink { - pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self { - WebhookSink { - endpoint, - auth_token, - client: reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(10)) - .build() - .unwrap_or_else(|_| reqwest::Client::new()), - max_retries, - retry_delay_ms, - } - } -} - -#[cfg(feature = "webhook")] -#[async_trait] -impl Sink for WebhookSink { - async fn write(&self, entry: &UnifiedLogEntry) { - let mut retries = 0; - let url = self.endpoint.clone(); - let entry_clone = entry.clone(); - let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap(); - while retries < self.max_retries { - match self - .client - .post(&url) - .header(reqwest::header::AUTHORIZATION, auth_value.clone()) - .json(&entry_clone) - .send() - .await - { - Ok(response) if response.status().is_success() => { - return; - } - _ => { - retries += 1; - if retries < self.max_retries { - tokio::time::sleep(tokio::time::Duration::from_millis( - self.retry_delay_ms * (1 << retries), // Exponential backoff - )) - .await; - } - } - } - } - - eprintln!("Failed to send log to webhook after {} retries", self.max_retries); - } -} - -#[cfg(feature = "webhook")] -impl Drop for WebhookSink { - fn drop(&mut self) { - // Perform any necessary cleanup here - // For example, you might want to log that the sink is being dropped - eprintln!("Dropping WebhookSink with URL: {}", self.endpoint); - } -} - -#[cfg(feature = "file")] -/// File Sink Implementation -pub struct FileSink { - path: String, - buffer_size: usize, - writer: Arc>>, - entry_count: std::sync::atomic::AtomicUsize, - last_flush: std::sync::atomic::AtomicU64, - flush_interval_ms: u64, // Time between flushes - flush_threshold: usize, // Number of entries before flush -} - -#[cfg(feature = "file")] -impl FileSink { - /// Create a new FileSink instance - pub async fn new( - path: String, - buffer_size: usize, - flush_interval_ms: u64, - flush_threshold: usize, - ) -> Result { - // check if the file exists - let file_exists = tokio::fs::metadata(&path).await.is_ok(); - // if the file not exists, create it - if !file_exists { - tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?; - tracing::debug!("File does not exist, creating it. Path: {:?}", path) - } - let file = if file_exists { - // If the file exists, open it in append mode - tracing::debug!("FileSink: File exists, opening in append mode."); - OpenOptions::new().append(true).create(true).open(&path).await? - } else { - // If the file does not exist, create it - tracing::debug!("FileSink: File does not exist, creating a new file."); - // Create the file and write a header or initial content if needed - OpenOptions::new().create(true).truncate(true).write(true).open(&path).await? - }; - let writer = io::BufWriter::with_capacity(buffer_size, file); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - Ok(FileSink { - path, - buffer_size, - writer: Arc::new(tokio::sync::Mutex::new(writer)), - entry_count: std::sync::atomic::AtomicUsize::new(0), - last_flush: std::sync::atomic::AtomicU64::new(now), - flush_interval_ms, - flush_threshold, - }) - } - - #[allow(dead_code)] - async fn initialize_writer(&mut self) -> io::Result<()> { - let file = tokio::fs::File::create(&self.path).await?; - - // Use buffer_size to create a buffer writer with a specified capacity - let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file); - - // Replace the original writer with the new Mutex - self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer)); - Ok(()) - } - - // Get the current buffer size - #[allow(dead_code)] - pub fn buffer_size(&self) -> usize { - self.buffer_size - } - - // How to dynamically adjust the buffer size - #[allow(dead_code)] - pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> { - if self.buffer_size != new_size { - self.buffer_size = new_size; - // Reinitialize the writer directly, without checking is_some() - self.initialize_writer().await?; - } - Ok(()) - } - - // Check if flushing is needed based on count or time - fn should_flush(&self) -> bool { - // Check entry count threshold - if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold { - return true; - } - - // Check time threshold - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - - let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed); - now - last >= self.flush_interval_ms - } -} - -#[cfg(feature = "file")] -#[async_trait] -impl Sink for FileSink { - async fn write(&self, entry: &UnifiedLogEntry) { - let line = format!("{:?}\n", entry); - let mut writer = self.writer.lock().await; - - if let Err(e) = writer.write_all(line.as_bytes()).await { - eprintln!( - "Failed to write log to file {}: {},entry timestamp:{:?}", - self.path, - e, - entry.get_timestamp() - ); - return; - } - - // Only flush periodically to improve performance - // Logic to determine when to flush could be added here - // Increment the entry count - self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - - // Check if we should flush - if self.should_flush() { - if let Err(e) = writer.flush().await { - eprintln!("Failed to flush log file {}: {}", self.path, e); - return; - } - - // Reset counters - self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed); - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as u64; - - self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed); - } - } -} - -#[cfg(feature = "file")] -impl Drop for FileSink { - fn drop(&mut self) { - let writer = self.writer.clone(); - let path = self.path.clone(); - - tokio::task::spawn_blocking(move || { - let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(async { - let mut writer = writer.lock().await; - if let Err(e) = writer.flush().await { - eprintln!("Failed to flush log file {}: {}", path, e); - } - }); - }); - } -} - -/// Create a list of Sink instances -pub async fn create_sinks(config: &AppConfig) -> Vec> { - let mut sinks: Vec> = Vec::new(); - - #[cfg(feature = "kafka")] - { - match &config.sinks.kafka { - Some(sink_kafka) => { - if sink_kafka.enabled { - match rdkafka::config::ClientConfig::new() - .set("bootstrap.servers", &sink_kafka.bootstrap_servers) - .set("message.timeout.ms", "5000") - .create() - { - Ok(producer) => { - sinks.push(Arc::new(KafkaSink::new( - producer, - sink_kafka.topic.clone(), - sink_kafka.batch_size.unwrap_or(100), - sink_kafka.batch_timeout_ms.unwrap_or(1000), - ))); - } - Err(e) => { - tracing::error!("Failed to create Kafka producer: {}", e); - } - } - } else { - tracing::info!("Kafka sink is disabled in the configuration"); - } - } - _ => { - tracing::info!("Kafka sink is not configured or disabled"); - } - } - } - #[cfg(feature = "webhook")] - { - match &config.sinks.webhook { - Some(sink_webhook) => { - if sink_webhook.enabled { - sinks.push(Arc::new(WebhookSink::new( - sink_webhook.endpoint.clone(), - sink_webhook.auth_token.clone(), - sink_webhook.max_retries.unwrap_or(3), - sink_webhook.retry_delay_ms.unwrap_or(100), - ))); - } else { - tracing::info!("Webhook sink is disabled in the configuration"); - } - } - _ => { - tracing::info!("Webhook sink is not configured or disabled"); - } - } - } - - #[cfg(feature = "file")] - { - // let config = config.clone(); - match &config.sinks.file { - Some(sink_file) => { - tracing::info!("File sink is enabled in the configuration"); - let path = if sink_file.enabled { - sink_file.path.clone() - } else { - "rustfs.log".to_string() - }; - tracing::debug!("FileSink: Using path: {}", path); - sinks.push(Arc::new( - FileSink::new( - path.clone(), - sink_file.buffer_size.unwrap_or(8192), - sink_file.flush_interval_ms.unwrap_or(1000), - sink_file.flush_threshold.unwrap_or(100), - ) - .await - .unwrap(), - )); - } - _ => { - tracing::info!("File sink is not configured or disabled"); - } - } - } - - sinks -} diff --git a/crates/obs/src/sinks/file.rs b/crates/obs/src/sinks/file.rs new file mode 100644 index 000000000..3e2b0db41 --- /dev/null +++ b/crates/obs/src/sinks/file.rs @@ -0,0 +1,164 @@ +use crate::sinks::Sink; +use crate::{LogRecord, UnifiedLogEntry}; +use async_trait::async_trait; +use std::sync::Arc; +use tokio::fs::OpenOptions; +use tokio::io; +use tokio::io::AsyncWriteExt; + +/// File Sink Implementation +pub struct FileSink { + path: String, + buffer_size: usize, + writer: Arc>>, + entry_count: std::sync::atomic::AtomicUsize, + last_flush: std::sync::atomic::AtomicU64, + flush_interval_ms: u64, // Time between flushes + flush_threshold: usize, // Number of entries before flush +} + +impl FileSink { + /// Create a new FileSink instance + pub async fn new( + path: String, + buffer_size: usize, + flush_interval_ms: u64, + flush_threshold: usize, + ) -> Result { + // check if the file exists + let file_exists = tokio::fs::metadata(&path).await.is_ok(); + // if the file not exists, create it + if !file_exists { + tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?; + tracing::debug!("File does not exist, creating it. Path: {:?}", path) + } + let file = if file_exists { + // If the file exists, open it in append mode + tracing::debug!("FileSink: File exists, opening in append mode."); + OpenOptions::new().append(true).create(true).open(&path).await? + } else { + // If the file does not exist, create it + tracing::debug!("FileSink: File does not exist, creating a new file."); + // Create the file and write a header or initial content if needed + OpenOptions::new().create(true).truncate(true).write(true).open(&path).await? + }; + let writer = io::BufWriter::with_capacity(buffer_size, file); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + Ok(FileSink { + path, + buffer_size, + writer: Arc::new(tokio::sync::Mutex::new(writer)), + entry_count: std::sync::atomic::AtomicUsize::new(0), + last_flush: std::sync::atomic::AtomicU64::new(now), + flush_interval_ms, + flush_threshold, + }) + } + + #[allow(dead_code)] + async fn initialize_writer(&mut self) -> io::Result<()> { + let file = tokio::fs::File::create(&self.path).await?; + + // Use buffer_size to create a buffer writer with a specified capacity + let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file); + + // Replace the original writer with the new Mutex + self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer)); + Ok(()) + } + + // Get the current buffer size + #[allow(dead_code)] + pub fn buffer_size(&self) -> usize { + self.buffer_size + } + + // How to dynamically adjust the buffer size + #[allow(dead_code)] + pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> { + if self.buffer_size != new_size { + self.buffer_size = new_size; + // Reinitialize the writer directly, without checking is_some() + self.initialize_writer().await?; + } + Ok(()) + } + + // Check if flushing is needed based on count or time + fn should_flush(&self) -> bool { + // Check entry count threshold + if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold { + return true; + } + + // Check time threshold + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + + let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed); + now - last >= self.flush_interval_ms + } +} + +#[async_trait] +impl Sink for FileSink { + async fn write(&self, entry: &UnifiedLogEntry) { + let line = format!("{:?}\n", entry); + let mut writer = self.writer.lock().await; + + if let Err(e) = writer.write_all(line.as_bytes()).await { + eprintln!( + "Failed to write log to file {}: {},entry timestamp:{:?}", + self.path, + e, + entry.get_timestamp() + ); + return; + } + + // Only flush periodically to improve performance + // Logic to determine when to flush could be added here + // Increment the entry count + self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + // Check if we should flush + if self.should_flush() { + if let Err(e) = writer.flush().await { + eprintln!("Failed to flush log file {}: {}", self.path, e); + return; + } + + // Reset counters + self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + + self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed); + } + } +} + +impl Drop for FileSink { + fn drop(&mut self) { + let writer = self.writer.clone(); + let path = self.path.clone(); + + tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let mut writer = writer.lock().await; + if let Err(e) = writer.flush().await { + eprintln!("Failed to flush log file {}: {}", path, e); + } + }); + }); + } +} diff --git a/crates/obs/src/sinks/kafka.rs b/crates/obs/src/sinks/kafka.rs new file mode 100644 index 000000000..e4ef34199 --- /dev/null +++ b/crates/obs/src/sinks/kafka.rs @@ -0,0 +1,165 @@ +use crate::sinks::Sink; +use crate::{LogRecord, UnifiedLogEntry}; +use async_trait::async_trait; +use std::sync::Arc; + +/// Kafka Sink Implementation +pub struct KafkaSink { + producer: rdkafka::producer::FutureProducer, + topic: String, + batch_size: usize, + batch_timeout_ms: u64, + entries: Arc>>, + last_flush: Arc, +} + +impl KafkaSink { + /// Create a new KafkaSink instance + pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self { + // Create Arc-wrapped values first + let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size))); + let last_flush = Arc::new(std::sync::atomic::AtomicU64::new( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64, + )); + let sink = KafkaSink { + producer: producer.clone(), + topic: topic.clone(), + batch_size, + batch_timeout_ms, + entries: entries.clone(), + last_flush: last_flush.clone(), + }; + + // Start background flusher + tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms)); + + sink + } + + /// Add a getter method to read the batch_timeout_ms field + #[allow(dead_code)] + pub fn batch_timeout(&self) -> u64 { + self.batch_timeout_ms + } + + /// Add a method to dynamically adjust the timeout if needed + #[allow(dead_code)] + pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) { + self.batch_timeout_ms = new_timeout_ms; + } + + async fn periodic_flush( + producer: rdkafka::producer::FutureProducer, + topic: String, + entries: Arc>>, + last_flush: Arc, + timeout_ms: u64, + ) { + loop { + tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await; + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + + let last = last_flush.load(std::sync::atomic::Ordering::Relaxed); + + if now - last >= timeout_ms { + let mut batch = entries.lock().await; + if !batch.is_empty() { + Self::send_batch(&producer, &topic, batch.drain(..).collect()).await; + last_flush.store(now, std::sync::atomic::Ordering::Relaxed); + } + } + } + } + + async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec) { + for entry in entries { + let payload = match serde_json::to_string(&entry) { + Ok(p) => p, + Err(e) => { + eprintln!("Failed to serialize log entry: {}", e); + continue; + } + }; + + let span_id = entry.get_timestamp().to_rfc3339(); + + let _ = producer + .send( + rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id), + std::time::Duration::from_secs(5), + ) + .await; + } + } +} + +#[async_trait] +impl Sink for KafkaSink { + async fn write(&self, entry: &UnifiedLogEntry) { + let mut batch = self.entries.lock().await; + batch.push(entry.clone()); + + let should_flush_by_size = batch.len() >= self.batch_size; + let should_flush_by_time = { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed); + now - last >= self.batch_timeout_ms + }; + + if should_flush_by_size || should_flush_by_time { + // Existing flush logic + let entries_to_send: Vec = batch.drain(..).collect(); + let producer = self.producer.clone(); + let topic = self.topic.clone(); + + self.last_flush.store( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64, + std::sync::atomic::Ordering::Relaxed, + ); + + tokio::spawn(async move { + KafkaSink::send_batch(&producer, &topic, entries_to_send).await; + }); + } + } +} + +impl Drop for KafkaSink { + fn drop(&mut self) { + // Perform any necessary cleanup here + // For example, you might want to flush any remaining entries + let producer = self.producer.clone(); + let topic = self.topic.clone(); + let entries = self.entries.clone(); + let last_flush = self.last_flush.clone(); + + tokio::spawn(async move { + let mut batch = entries.lock().await; + if !batch.is_empty() { + KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await; + last_flush.store( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64, + std::sync::atomic::Ordering::Relaxed, + ); + } + }); + + eprintln!("Dropping KafkaSink with topic: {}", self.topic); + } +} diff --git a/crates/obs/src/sinks/mod.rs b/crates/obs/src/sinks/mod.rs new file mode 100644 index 000000000..3abafd3fe --- /dev/null +++ b/crates/obs/src/sinks/mod.rs @@ -0,0 +1,92 @@ +use crate::{AppConfig, SinkConfig, UnifiedLogEntry}; +use async_trait::async_trait; +use std::sync::Arc; + +#[cfg(feature = "file")] +mod file; +#[cfg(all(feature = "kafka", target_os = "linux"))] +mod kafka; +#[cfg(feature = "webhook")] +mod webhook; + +/// Sink Trait definition, asynchronously write logs +#[async_trait] +pub trait Sink: Send + Sync { + async fn write(&self, entry: &UnifiedLogEntry); +} + +/// Create a list of Sink instances +pub async fn create_sinks(config: &AppConfig) -> Vec> { + let mut sinks: Vec> = Vec::new(); + + for sink_config in &config.sinks { + match sink_config { + #[cfg(all(feature = "kafka", target_os = "linux"))] + SinkConfig::Kafka(kafka_config) => { + match rdkafka::config::ClientConfig::new() + .set("bootstrap.servers", &kafka_config.brokers) + .set("message.timeout.ms", "5000") + .create() + { + Ok(producer) => { + sinks.push(Arc::new(kafka::KafkaSink::new( + producer, + kafka_config.topic.clone(), + kafka_config.batch_size.unwrap_or(100), + kafka_config.batch_timeout_ms.unwrap_or(1000), + ))); + tracing::info!("Kafka sink created for topic: {}", kafka_config.topic); + } + Err(e) => { + tracing::error!("Failed to create Kafka producer: {}", e); + } + } + } + #[cfg(feature = "webhook")] + SinkConfig::Webhook(webhook_config) => { + sinks.push(Arc::new(webhook::WebhookSink::new( + webhook_config.endpoint.clone(), + webhook_config.auth_token.clone(), + webhook_config.max_retries.unwrap_or(3), + webhook_config.retry_delay_ms.unwrap_or(100), + ))); + tracing::info!("Webhook sink created for endpoint: {}", webhook_config.endpoint); + } + + #[cfg(feature = "file")] + SinkConfig::File(file_config) => { + tracing::debug!("FileSink: Using path: {}", file_config.path); + match file::FileSink::new( + file_config.path.clone(), + file_config.buffer_size.unwrap_or(8192), + file_config.flush_interval_ms.unwrap_or(1000), + file_config.flush_threshold.unwrap_or(100), + ) + .await + { + Ok(sink) => { + sinks.push(Arc::new(sink)); + tracing::info!("File sink created for path: {}", file_config.path); + } + Err(e) => { + tracing::error!("Failed to create File sink: {}", e); + } + } + } + #[cfg(any(not(feature = "kafka"), not(target_os = "linux")))] + SinkConfig::Kafka(_) => { + tracing::warn!("Kafka sink is configured but the 'kafka' feature is not enabled"); + } + #[cfg(not(feature = "webhook"))] + SinkConfig::Webhook(_) => { + tracing::warn!("Webhook sink is configured but the 'webhook' feature is not enabled"); + } + #[cfg(not(feature = "file"))] + SinkConfig::File(_) => { + tracing::warn!("File sink is configured but the 'file' feature is not enabled"); + } + } + } + + sinks +} diff --git a/crates/obs/src/sinks/webhook.rs b/crates/obs/src/sinks/webhook.rs new file mode 100644 index 000000000..77a874d9f --- /dev/null +++ b/crates/obs/src/sinks/webhook.rs @@ -0,0 +1,70 @@ +use crate::sinks::Sink; +use crate::UnifiedLogEntry; +use async_trait::async_trait; + +/// Webhook Sink Implementation +pub struct WebhookSink { + endpoint: String, + auth_token: String, + client: reqwest::Client, + max_retries: usize, + retry_delay_ms: u64, +} + +impl WebhookSink { + pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self { + WebhookSink { + endpoint, + auth_token, + client: reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()), + max_retries, + retry_delay_ms, + } + } +} + +#[async_trait] +impl Sink for WebhookSink { + async fn write(&self, entry: &UnifiedLogEntry) { + let mut retries = 0; + let url = self.endpoint.clone(); + let entry_clone = entry.clone(); + let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap(); + while retries < self.max_retries { + match self + .client + .post(&url) + .header(reqwest::header::AUTHORIZATION, auth_value.clone()) + .json(&entry_clone) + .send() + .await + { + Ok(response) if response.status().is_success() => { + return; + } + _ => { + retries += 1; + if retries < self.max_retries { + tokio::time::sleep(tokio::time::Duration::from_millis( + self.retry_delay_ms * (1 << retries), // Exponential backoff + )) + .await; + } + } + } + } + + eprintln!("Failed to send log to webhook after {} retries", self.max_retries); + } +} + +impl Drop for WebhookSink { + fn drop(&mut self) { + // Perform any necessary cleanup here + // For example, you might want to log that the sink is being dropped + eprintln!("Dropping WebhookSink with URL: {}", self.endpoint); + } +} diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs index 612527041..cba605d3e 100644 --- a/crates/obs/src/telemetry.rs +++ b/crates/obs/src/telemetry.rs @@ -1,5 +1,3 @@ -use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT}; -use crate::utils::get_local_ip_with_default; use crate::OtelConfig; use opentelemetry::trace::TracerProvider; use opentelemetry::{global, KeyValue}; @@ -15,6 +13,8 @@ use opentelemetry_semantic_conventions::{ attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION}, SCHEMA_URL, }; +use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT}; +use rustfs_utils::get_local_ip_with_default; use smallvec::SmallVec; use std::borrow::Cow; use std::io::IsTerminal; @@ -70,7 +70,7 @@ impl Drop for OtelGuard { /// create OpenTelemetry Resource fn resource(config: &OtelConfig) -> Resource { Resource::builder() - .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(SERVICE_NAME)).to_string()) + .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(APP_NAME)).to_string()) .with_schema_url( [ KeyValue::new( @@ -101,8 +101,8 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { let endpoint = &config.endpoint; let use_stdout = config.use_stdout.unwrap_or(USE_STDOUT); let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL); - let logger_level = config.logger_level.as_deref().unwrap_or(LOGGER_LEVEL); - let service_name = config.service_name.as_deref().unwrap_or(SERVICE_NAME); + let logger_level = config.logger_level.as_deref().unwrap_or(DEFAULT_LOG_LEVEL); + let service_name = config.service_name.as_deref().unwrap_or(APP_NAME); // Pre-create resource objects to avoid repeated construction let res = resource(config); @@ -210,7 +210,8 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { .with_thread_names(true) .with_thread_ids(true) .with_file(true) - .with_line_number(true); + .with_line_number(true) + .with_filter(build_env_filter(logger_level, None)); let filter = build_env_filter(logger_level, None); let otel_filter = build_env_filter(logger_level, None); @@ -218,7 +219,7 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string()); // Configure registry to avoid repeated calls to filter methods - let _registry = tracing_subscriber::registry() + tracing_subscriber::registry() .with(filter) .with(ErrorLayer::default()) .with(if config.local_logging_enabled.unwrap_or(false) { @@ -230,10 +231,13 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { .with(otel_layer) .with(MetricsLayer::new(meter_provider.clone())) .init(); + if !endpoint.is_empty() { info!( - "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}", - endpoint, logger_level + "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {},RUST_LOG env: {}", + endpoint, + logger_level, + std::env::var("RUST_LOG").unwrap_or_else(|_| "未设置".to_string()) ); } } @@ -248,7 +252,6 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { fn build_env_filter(logger_level: &str, default_level: Option<&str>) -> EnvFilter { let level = default_level.unwrap_or(logger_level); let mut filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level)); - if !matches!(logger_level, "trace" | "debug") { let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"]; for directive in directives { diff --git a/crates/obs/src/worker.rs b/crates/obs/src/worker.rs index 2d7ee2e14..cfe2f26ce 100644 --- a/crates/obs/src/worker.rs +++ b/crates/obs/src/worker.rs @@ -1,9 +1,9 @@ -use crate::{sink::Sink, UnifiedLogEntry}; +use crate::{sinks::Sink, UnifiedLogEntry}; use std::sync::Arc; use tokio::sync::mpsc::Receiver; /// Start the log processing worker thread -pub async fn start_worker(receiver: Receiver, sinks: Vec>) { +pub(crate) async fn start_worker(receiver: Receiver, sinks: Vec>) { let mut receiver = receiver; while let Some(entry) = receiver.recv().await { for sink in &sinks { diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml new file mode 100644 index 000000000..76cac05c7 --- /dev/null +++ b/crates/utils/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "rustfs-utils" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +local-ip-address = { workspace = true, optional = true } +rustfs-config = { workspace = true } +rustls = { workspace = true, optional = true } +rustls-pemfile = { workspace = true, optional = true } +rustls-pki-types = { workspace = true, optional = true } +tracing = { workspace = true } + +[lints] +workspace = true + +[features] +default = ["ip"] # features that are enabled by default +ip = ["dep:local-ip-address"] # ip characteristics and their dependencies +tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies +net = ["ip"] # empty network features +full = ["ip", "tls", "net"] # all features \ No newline at end of file diff --git a/rustfs/src/utils/mod.rs b/crates/utils/src/certs.rs similarity index 76% rename from rustfs/src/utils/mod.rs rename to crates/utils/src/certs.rs index fad2718c8..021c5915e 100644 --- a/rustfs/src/utils/mod.rs +++ b/crates/utils/src/certs.rs @@ -1,57 +1,48 @@ -use crate::config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; +use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni}; use rustls::sign::CertifiedKey; use rustls_pemfile::{certs, private_key}; use rustls_pki_types::{CertificateDer, PrivateKeyDer}; use std::collections::HashMap; -use std::fmt::Debug; use std::io::Error; -use std::net::IpAddr; use std::path::Path; use std::sync::Arc; use std::{fs, io}; use tracing::{debug, warn}; -/// Get the local IP address. -/// This function retrieves the local IP address of the machine. -pub(crate) fn get_local_ip() -> Option { - match local_ip_address::local_ip() { - Ok(IpAddr::V4(ip)) => Some(ip), - Err(_) => None, - Ok(IpAddr::V6(_)) => todo!(), - } -} - /// Load public certificate from file. /// This function loads a public certificate from the specified file. -pub(crate) fn load_certs(filename: &str) -> io::Result>> { +pub fn load_certs(filename: &str) -> io::Result>> { // Open certificate file. - let cert_file = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; + let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {}: {}", filename, e)))?; let mut reader = io::BufReader::new(cert_file); // Load and return certificate. let certs = certs(&mut reader) .collect::, _>>() - .map_err(|_| error(format!("certificate file {} format error", filename)))?; + .map_err(|e| certs_error(format!("certificate file {} format error:{:?}", filename, e)))?; if certs.is_empty() { - return Err(error(format!("No valid certificate was found in the certificate file {}", filename))); + return Err(certs_error(format!( + "No valid certificate was found in the certificate file {}", + filename + ))); } Ok(certs) } /// Load private key from file. /// This function loads a private key from the specified file. -pub(crate) fn load_private_key(filename: &str) -> io::Result> { +pub fn load_private_key(filename: &str) -> io::Result> { // Open keyfile. - let keyfile = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?; + let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {}: {}", filename, e)))?; let mut reader = io::BufReader::new(keyfile); // Load and return a single private key. - private_key(&mut reader)?.ok_or_else(|| error(format!("no private key found in {}", filename))) + private_key(&mut reader)?.ok_or_else(|| certs_error(format!("no private key found in {}", filename))) } /// error function -pub(crate) fn error(err: String) -> Error { +pub fn certs_error(err: String) -> Error { Error::new(io::ErrorKind::Other, err) } @@ -59,14 +50,14 @@ pub(crate) fn error(err: String) -> Error { /// This function loads all certificate and private key pairs from the specified directory. /// It looks for files named `rustfs_cert.pem` and `rustfs_key.pem` in each subdirectory. /// The root directory can also contain a default certificate/private key pair. -pub(crate) fn load_all_certs_from_directory( +pub fn load_all_certs_from_directory( dir_path: &str, ) -> io::Result>, PrivateKeyDer<'static>)>> { let mut cert_key_pairs = HashMap::new(); let dir = Path::new(dir_path); if !dir.exists() || !dir.is_dir() { - return Err(error(format!( + return Err(certs_error(format!( "The certificate directory does not exist or is not a directory: {}", dir_path ))); @@ -80,10 +71,10 @@ pub(crate) fn load_all_certs_from_directory( debug!("find the root directory certificate: {:?}", root_cert_path); let root_cert_str = root_cert_path .to_str() - .ok_or_else(|| error(format!("Invalid UTF-8 in root certificate path: {:?}", root_cert_path)))?; + .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root certificate path: {:?}", root_cert_path)))?; let root_key_str = root_key_path .to_str() - .ok_or_else(|| error(format!("Invalid UTF-8 in root key path: {:?}", root_key_path)))?; + .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root key path: {:?}", root_key_path)))?; match load_cert_key_pair(root_cert_str, root_key_str) { Ok((certs, key)) => { // The root directory certificate is used as the default certificate and is stored using special keys. @@ -104,7 +95,7 @@ pub(crate) fn load_all_certs_from_directory( let domain_name = path .file_name() .and_then(|name| name.to_str()) - .ok_or_else(|| error(format!("invalid domain name directory:{:?}", path)))?; + .ok_or_else(|| certs_error(format!("invalid domain name directory:{:?}", path)))?; // find certificate and private key files let cert_path = path.join(RUSTFS_TLS_CERT); // e.g., rustfs_cert.pem @@ -125,7 +116,10 @@ pub(crate) fn load_all_certs_from_directory( } if cert_key_pairs.is_empty() { - return Err(error(format!("No valid certificate/private key pair found in directory {}", dir_path))); + return Err(certs_error(format!( + "No valid certificate/private key pair found in directory {}", + dir_path + ))); } Ok(cert_key_pairs) @@ -171,7 +165,7 @@ pub fn create_multi_cert_resolver( for (domain, (certs, key)) in cert_key_pairs { // create a signature let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key) - .map_err(|_| error(format!("unsupported private key types:{}", domain)))?; + .map_err(|e| certs_error(format!("unsupported private key types:{}, err:{:?}", domain, e)))?; // create a CertifiedKey let certified_key = CertifiedKey::new(certs, signing_key); @@ -181,7 +175,7 @@ pub fn create_multi_cert_resolver( // add certificate to resolver resolver .add(&domain, certified_key) - .map_err(|e| error(format!("failed to add a domain name certificate:{},err: {:?}", domain, e)))?; + .map_err(|e| certs_error(format!("failed to add a domain name certificate:{},err: {:?}", domain, e)))?; } } diff --git a/crates/obs/src/utils.rs b/crates/utils/src/ip.rs similarity index 91% rename from crates/obs/src/utils.rs rename to crates/utils/src/ip.rs index 774594577..3b63b12fc 100644 --- a/crates/obs/src/utils.rs +++ b/crates/utils/src/ip.rs @@ -1,4 +1,3 @@ -use local_ip_address::{local_ip, local_ipv6}; use std::net::{IpAddr, Ipv4Addr}; /// Get the IP address of the machine @@ -11,7 +10,9 @@ use std::net::{IpAddr, Ipv4Addr}; /// * `Some(IpAddr)` - Native IP address (IPv4 or IPv6) /// * `None` - Unable to obtain any native IP address pub fn get_local_ip() -> Option { - local_ip().ok().or_else(|| local_ipv6().ok()) + local_ip_address::local_ip() + .ok() + .or_else(|| local_ip_address::local_ipv6().ok()) } /// Get the IP address of the machine as a string diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs new file mode 100644 index 000000000..cda53d088 --- /dev/null +++ b/crates/utils/src/lib.rs @@ -0,0 +1,8 @@ +mod certs; +mod ip; +mod net; + +#[cfg(feature = "ip")] +pub use certs::*; +#[cfg(feature = "ip")] +pub use ip::*; diff --git a/crates/utils/src/net.rs b/crates/utils/src/net.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/crates/utils/src/net.rs @@ -0,0 +1 @@ + diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml new file mode 100644 index 000000000..a73f083b0 --- /dev/null +++ b/crates/zip/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "zip" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + + +[dependencies] +async-compression = { version = "0.4.0", features = [ + "tokio", + "bzip2", + "gzip", + "zlib", + "zstd", + "xz", +] } +# async_zip = { version = "0.0.17", features = ["tokio"] } +# rc-zip-tokio = "4.2.6" +tokio = { version = "1.45.0", features = ["full"] } +tokio-stream = "0.1.17" +tokio-tar = { workspace = true } +xz2 = { version = "0.1", optional = true, features = ["static"] } + + +[lints] +workspace = true diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs new file mode 100644 index 000000000..0da854c55 --- /dev/null +++ b/crates/zip/src/lib.rs @@ -0,0 +1,124 @@ +use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder}; +use tokio::io::{self, AsyncRead, BufReader}; +use tokio_stream::StreamExt; +use tokio_tar::Archive; + +#[derive(Debug, PartialEq)] +pub enum CompressionFormat { + Gzip, //.gz + Bzip2, //.bz2 + // Lz4, //.lz4 + Zip, + Xz, //.xz + Zlib, //.z + Zstd, //.zst + Unknown, +} + +impl CompressionFormat { + pub fn from_extension(ext: &str) -> Self { + match ext { + "gz" => CompressionFormat::Gzip, + "bz2" => CompressionFormat::Bzip2, + // "lz4" => CompressionFormat::Lz4, + "zip" => CompressionFormat::Zip, + "xz" => CompressionFormat::Xz, + "zlib" => CompressionFormat::Zlib, + "zst" => CompressionFormat::Zstd, + _ => CompressionFormat::Unknown, + } + } + + pub fn get_decoder(&self, input: R) -> io::Result> + where + R: AsyncRead + Send + Unpin + 'static, + { + let reader = BufReader::new(input); + + let decoder: Box = match self { + CompressionFormat::Gzip => Box::new(GzipDecoder::new(reader)), + CompressionFormat::Bzip2 => Box::new(BzDecoder::new(reader)), + // CompressionFormat::Lz4 => Box::new(Lz4Decoder::new(reader)), + CompressionFormat::Zlib => Box::new(ZlibDecoder::new(reader)), + CompressionFormat::Xz => Box::new(XzDecoder::new(reader)), + CompressionFormat::Zstd => Box::new(ZstdDecoder::new(reader)), + _ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format")), + }; + + Ok(decoder) + } +} + +pub async fn decompress(input: R, format: CompressionFormat, mut callback: F) -> io::Result<()> +where + R: AsyncRead + Send + Unpin + 'static, + F: AsyncFnMut(tokio_tar::Entry>>) -> std::io::Result<()> + Send + 'static, +{ + // 打开输入文件 + // println!("format {:?}", format); + + let decoder = format.get_decoder(input)?; + + // let reader: BufReader = BufReader::new(input); + + // // 根据文件扩展名选择解压器 + // let decoder: Box = match format { + // CompressionFormat::Gzip => Box::new(GzipDecoder::new(reader)), + // CompressionFormat::Bzip2 => Box::new(BzDecoder::new(reader)), + // // CompressionFormat::Lz4 => Box::new(Lz4Decoder::new(reader)), + // CompressionFormat::Zlib => Box::new(ZlibDecoder::new(reader)), + // CompressionFormat::Xz => Box::new(XzDecoder::new(reader)), + // CompressionFormat::Zstd => Box::new(ZstdDecoder::new(reader)), + // // CompressionFormat::Zip => Box::new(DeflateDecoder::new(reader)), + // _ => { + // return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format")); + // } + // }; + + let mut ar = Archive::new(decoder); + let mut entries = ar.entries().unwrap(); + while let Some(entry) = entries.next().await { + let f = match entry { + Ok(f) => f, + Err(e) => { + println!("Error reading entry: {}", e); + return Err(e); + } + }; + // println!("{}", f.path().unwrap().display()); + callback(f).await?; + } + + Ok(()) +} + +// #[tokio::test] +// async fn test_decompress() -> io::Result<()> { +// use std::path::Path; +// use tokio::fs::File; + +// let input_path = "/Users/weisd/Downloads/wsd.tar.gz"; // 替换为你的压缩文件路径 + +// let f = File::open(input_path).await?; + +// let Some(ext) = Path::new(input_path).extension().and_then(|s| s.to_str()) else { +// return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format")); +// }; + +// match decompress( +// f, +// CompressionFormat::from_extension(ext), +// |entry: tokio_tar::Entry>>| async move { +// let path = entry.path().unwrap(); +// println!("Extracted: {}", path.display()); +// Ok(()) +// }, +// ) +// .await +// { +// Ok(_) => println!("解压成功！"), +// Err(e) => println!("解压失败: {}", e), +// } + +// Ok(()) +// } diff --git a/crypto/Cargo.toml b/crypto/Cargo.toml index 5b0f58969..7822ee5a6 100644 --- a/crypto/Cargo.toml +++ b/crypto/Cargo.toml @@ -17,7 +17,7 @@ chacha20poly1305 = { version = "0.10.1", optional = true } jsonwebtoken = { workspace = true } pbkdf2 = { version = "0.12.2", optional = true } rand = { workspace = true, optional = true } -sha2 = { version = "0.10.8", optional = true } +sha2 = { workspace = true, optional = true } thiserror.workspace = true serde_json.workspace = true diff --git a/deploy/config/.example.obs.env b/deploy/config/.example.obs.env index a1ea67b4f..edbf3e887 100644 --- a/deploy/config/.example.obs.env +++ b/deploy/config/.example.obs.env @@ -1,27 +1,28 @@ -OBSERVABILITY__ENDPOINT=http://localhost:4317 -OBSERVABILITY__USE_STDOUT=true -OBSERVABILITY__SAMPLE_RATIO=2.0 -OBSERVABILITY__METER_INTERVAL=30 -OBSERVABILITY__SERVICE_NAME=rustfs -OBSERVABILITY__SERVICE_VERSION=0.1.0 -OBSERVABILITY__ENVIRONMENT=develop -OBSERVABILITY__LOGGER_LEVEL=debug - -SINKS__KAFKA__ENABLED=false -SINKS__KAFKA__BOOTSTRAP_SERVERS=localhost:9092 -SINKS__KAFKA__TOPIC=logs -SINKS__KAFKA__BATCH_SIZE=100 -SINKS__KAFKA__BATCH_TIMEOUT_MS=1000 - -SINKS__WEBHOOK__ENABLED=false -SINKS__WEBHOOK__ENDPOINT=http://localhost:8080/webhook -SINKS__WEBHOOK__AUTH_TOKEN= -SINKS__WEBHOOK__BATCH_SIZE=100 -SINKS__WEBHOOK__BATCH_TIMEOUT_MS=1000 - -SINKS__FILE__ENABLED=true -SINKS__FILE__PATH=./deploy/logs/rustfs.log -SINKS__FILE__BATCH_SIZE=10 -SINKS__FILE__BATCH_TIMEOUT_MS=1000 - -LOGGER__QUEUE_CAPACITY=10 \ No newline at end of file +#RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 +#RUSTFS__OBSERVABILITY__USE_STDOUT=true +#RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 +#RUSTFS__OBSERVABILITY__METER_INTERVAL=30 +#RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs +#RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 +#RUSTFS__OBSERVABILITY__ENVIRONMENT=develop +#RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug +# +#RUSTFS__SINKS_0__type=Kakfa +#RUSTFS__SINKS_0__brokers=localhost:9092 +#RUSTFS__SINKS_0__topic=logs +#RUSTFS__SINKS_0__batch_size=100 +#RUSTFS__SINKS_0__batch_timeout_ms=1000 +# +#RUSTFS__SINKS_1__type=Webhook +#RUSTFS__SINKS_1__endpoint=http://localhost:8080/webhook +#RUSTFS__SINKS_1__auth_token=you-auth-token +#RUSTFS__SINKS_1__batch_size=100 +#RUSTFS__SINKS_1__batch_timeout_ms=1000 +# +#RUSTFS__SINKS_2__type=File +#RUSTFS__SINKS_2__path=./deploy/logs/rustfs.log +#RUSTFS__SINKS_2__buffer_size=10 +#RUSTFS__SINKS_2__flush_interval_ms=1000 +#RUSTFS__SINKS_2__flush_threshold=100 +# +#RUSTFS__LOGGER__QUEUE_CAPACITY=10 \ No newline at end of file diff --git a/deploy/config/obs-zh.example.toml b/deploy/config/obs-zh.example.toml index 712ae6cef..0474391bd 100644 --- a/deploy/config/obs-zh.example.toml +++ b/deploy/config/obs-zh.example.toml @@ -9,26 +9,26 @@ environments = "develop" # 运行环境，如开发环境 (develop) logger_level = "debug" # 日志级别，可选 debug/info/warn/error 等 local_logging_enabled = true # 是否启用本地 stdout 日志记录，true 表示启用，false 表示禁用 -[sinks] -[sinks.kafka] # Kafka 接收器配置 -enabled = false # 是否启用 Kafka 接收器，默认禁用 -bootstrap_servers = "localhost:9092" # Kafka 服务器地址 -topic = "logs" # Kafka 主题名称 -batch_size = 100 # 批处理大小，每次发送的消息数量 -batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 +#[[sinks]] # Kafka 接收器配置 +#type = "Kafka" # 是否启用 Kafka 接收器，默认禁用 +#brokers = "localhost:9092" # Kafka 服务器地址 +#topic = "logs" # Kafka 主题名称 +#batch_size = 100 # 批处理大小，每次发送的消息数量 +#batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 -[sinks.webhook] # Webhook 接收器配置 -enabled = false # 是否启用 Webhook 接收器 -endpoint = "http://localhost:8080/webhook" # Webhook 接收地址 -auth_token = "" # 认证令牌 -batch_size = 100 # 批处理大小 -batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 +#[[sinks]] # Webhook 接收器配置 +#type = "Webhook" # 是否启用 Webhook 接收器 +#endpoint = "http://localhost:8080/webhook" # Webhook 接收地址 +#auth_token = "" # 认证令牌 +#batch_size = 100 # 批处理大小 +#batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 -[sinks.file] # 文件接收器配置 -enabled = true # 是否启用文件接收器 +[[sinks]] # 文件接收器配置 +type = "File" # 是否启用文件接收器 path = "./deploy/logs/rustfs.log" # 日志文件路径 -batch_size = 10 # 批处理大小 -batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒 +buffer_size = 10 # 缓冲区大小，表示每次写入的字节数 +flush_interval_ms = 100 # 批处理超时时间，单位为毫秒 +flush_threshold = 100 # 刷新阈值，表示在达到该数量时刷新日志 [logger] # 日志器配置 queue_capacity = 10 # 日志队列容量，表示可以缓存的日志条数 \ No newline at end of file diff --git a/deploy/config/obs.example.toml b/deploy/config/obs.example.toml index e38e06f81..e6c898339 100644 --- a/deploy/config/obs.example.toml +++ b/deploy/config/obs.example.toml @@ -4,31 +4,31 @@ use_stdout = false # Output with stdout, true output, false no output sample_ratio = 2.0 meter_interval = 30 service_name = "rustfs" -service_version = "0.1.0" +service_version = "0.0.1" environment = "develop" -logger_level = "error" +logger_level = "info" local_logging_enabled = true -[sinks] -[sinks.kafka] # Kafka sink is disabled by default -enabled = false -bootstrap_servers = "localhost:9092" -topic = "logs" -batch_size = 100 # Default is 100 if not specified -batch_timeout_ms = 1000 # Default is 1000ms if not specified +#[[sinks]] +#type = "Kafka" +#brokers = "localhost:9092" +#topic = "logs" +#batch_size = 100 # Default is 100 if not specified +#batch_timeout_ms = 100 # Default is 1000ms if not specified +# +#[[sinks]] +#type = "Webhook" +#endpoint = "http://localhost:8080/webhook" +#auth_token = "" +#batch_size = 100 # Default is 3 if not specified +#batch_timeout_ms = 100 # Default is 100ms if not specified -[sinks.webhook] -enabled = false -endpoint = "http://localhost:8080/webhook" -auth_token = "" -batch_size = 100 # Default is 3 if not specified -batch_timeout_ms = 1000 # Default is 100ms if not specified - -[sinks.file] -enabled = true -path = "./deploy/logs/rustfs.log" -batch_size = 100 -batch_timeout_ms = 1000 # Default is 8192 bytes if not specified +[[sinks]] +type = "File" +path = "deploy/logs/rustfs.log" +buffer_size = 101 # Default is 8192 bytes if not specified +flush_interval_ms = 1000 +flush_threshold = 100 [logger] queue_capacity = 10000 diff --git a/deploy/config/rustfs-zh.env b/deploy/config/rustfs-zh.env index b9700adb8..fd2a5178b 100644 --- a/deploy/config/rustfs-zh.env +++ b/deploy/config/rustfs-zh.env @@ -23,4 +23,6 @@ RUSTFS_LICENSE="license content" # 可观测性配置文件路径：deploy/config/obs.example.toml RUSTFS_OBS_CONFIG=/etc/default/obs.toml # TLS 证书目录路径：deploy/certs -RUSTFS_TLS_PATH=/etc/default/tls \ No newline at end of file +RUSTFS_TLS_PATH=/etc/default/tls +# 事件通知配置文件路径：deploy/config/event.example.toml +RUSTFS_EVENT_CONFIG=/etc/default/event.toml \ No newline at end of file diff --git a/deploy/config/rustfs.env b/deploy/config/rustfs.env index ec7d31b48..3f50033f4 100644 --- a/deploy/config/rustfs.env +++ b/deploy/config/rustfs.env @@ -23,4 +23,6 @@ RUSTFS_LICENSE="license content" # Observability configuration file path: deploy/config/obs.example.toml RUSTFS_OBS_CONFIG=/etc/default/obs.toml # TLS certificates directory path: deploy/certs -RUSTFS_TLS_PATH=/etc/default/tls \ No newline at end of file +RUSTFS_TLS_PATH=/etc/default/tls +# event notification configuration file path: deploy/config/event.example.toml +RUSTFS_EVENT_CONFIG=/etc/default/event.toml \ No newline at end of file diff --git a/docker-compose-obs.yaml b/docker-compose-obs.yaml index 505f287b5..a709587b8 100644 --- a/docker-compose-obs.yaml +++ b/docker-compose-obs.yaml @@ -1,6 +1,6 @@ services: otel-collector: - image: otel/opentelemetry-collector-contrib:0.120.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.124.0 environment: - TZ=Asia/Shanghai volumes: @@ -16,7 +16,7 @@ services: networks: - rustfs-network jaeger: - image: jaegertracing/jaeger:2.4.0 + image: jaegertracing/jaeger:2.5.0 environment: - TZ=Asia/Shanghai ports: @@ -26,7 +26,7 @@ services: networks: - rustfs-network prometheus: - image: prom/prometheus:v3.2.1 + image: prom/prometheus:v3.3.0 environment: - TZ=Asia/Shanghai volumes: @@ -36,7 +36,7 @@ services: networks: - rustfs-network loki: - image: grafana/loki:3.4.2 + image: grafana/loki:3.5.0 environment: - TZ=Asia/Shanghai volumes: @@ -47,7 +47,7 @@ services: networks: - rustfs-network grafana: - image: grafana/grafana:11.6.0 + image: grafana/grafana:11.6.1 ports: - "3000:3000" # Web UI environment: @@ -63,10 +63,10 @@ services: container_name: node1 environment: - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4} - - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002 - - RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml + - RUSTFS_CONSOLE_ADDRESS=:9002 + - RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml platform: linux/amd64 ports: - "9001:9000" # 映射宿主机的 9001 端口到容器的 9000 端口 @@ -83,11 +83,11 @@ services: dockerfile: Dockerfile.obs container_name: node2 environment: - - RUSTFS_VOLUMES=/root/data/target/volume/test{1...4} - - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4} + - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002 - - RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml + - RUSTFS_CONSOLE_ADDRESS=:9002 + - RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml platform: linux/amd64 ports: - "9002:9000" # 映射宿主机的 9002 端口到容器的 9000 端口 @@ -105,10 +105,10 @@ services: container_name: node3 environment: - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4} - - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002 - - RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml + - RUSTFS_CONSOLE_ADDRESS=:9002 + - RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml platform: linux/amd64 ports: - "9003:9000" # 映射宿主机的 9003 端口到容器的 9000 端口 @@ -126,10 +126,10 @@ services: container_name: node4 environment: - RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4} - - RUSTFS_ADDRESS=0.0.0.0:9000 + - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002 - - RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml + - RUSTFS_CONSOLE_ADDRESS=:9002 + - RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml platform: linux/amd64 ports: - "9004:9000" # 映射宿主机的 9004 端口到容器的 9000 端口 diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 721c902fe..0f9431fce 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -11,14 +11,15 @@ rust-version.workspace = true workspace = true [dependencies] +rustfs-config = { workspace = true } async-trait.workspace = true backon.workspace = true -blake2 = "0.10.6" +blake2 = { workspace = true } bytes.workspace = true common.workspace = true policy.workspace = true chrono.workspace = true -glob = "0.3.2" +glob = { workspace = true } thiserror.workspace = true flatbuffers.workspace = true futures.workspace = true @@ -30,16 +31,16 @@ serde_json.workspace = true tracing-error.workspace = true s3s.workspace = true http.workspace = true -highway = "1.3.0" +highway = { workspace = true } url.workspace = true uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] } -reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] } +reed-solomon-erasure = { workspace = true } transform-stream = "0.3.1" lazy_static.workspace = true lock.workspace = true -regex = "1.11.1" -netif = "0.1.6" -nix = { version = "0.29.0", features = ["fs"] } +regex = { workspace = true } +netif = { workspace = true } +nix = { workspace = true } path-absolutize = "3.1.1" protos.workspace = true rmp.workspace = true @@ -53,13 +54,13 @@ hex-simd = "0.8.0" path-clean = "1.0.1" tempfile.workspace = true tokio = { workspace = true, features = ["io-util", "sync", "signal"] } -tokio-stream = "0.1.17" +tokio-stream = { workspace = true } tonic.workspace = true tower.workspace = true byteorder = "1.5.0" xxhash-rust = { version = "0.8.15", features = ["xxh64"] } num = "0.4.3" -num_cpus = "1.16" +num_cpus = { workspace = true } s3s-policy.workspace = true rand.workspace = true pin-project-lite.workspace = true @@ -68,20 +69,18 @@ madmin.workspace = true workers.workspace = true reqwest = { workspace = true } urlencoding = "2.1.3" -smallvec = "1.15.0" +smallvec = { workspace = true } shadow-rs.workspace = true [target.'cfg(not(windows))'.dependencies] - -nix = { version = "0.29.0", features = ["fs"] } +nix = { workspace = true } [target.'cfg(windows)'.dependencies] - -winapi = "0.3.9" +winapi = { workspace = true } [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } [build-dependencies] -shadow-rs.workspace = true +shadow-rs = { workspace = true, features = ["build", "metadata"] } diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index b64c84b3b..05e55a430 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -534,6 +534,7 @@ impl Writer for BitrotFileWriter { self } + #[tracing::instrument(level = "info", skip_all)] async fn write(&mut self, buf: Bytes) -> Result<()> { if buf.is_empty() { return Ok(()); diff --git a/ecstore/src/cache_value/metacache_set.rs b/ecstore/src/cache_value/metacache_set.rs index e716a3051..1e32b4391 100644 --- a/ecstore/src/cache_value/metacache_set.rs +++ b/ecstore/src/cache_value/metacache_set.rs @@ -7,7 +7,7 @@ use common::error::{Error, Result}; use futures::future::join_all; use std::{future::Future, pin::Pin, sync::Arc}; use tokio::{spawn, sync::broadcast::Receiver as B_Receiver}; -use tracing::{error, info}; +use tracing::error; pub type AgreedFn = Box Pin + Send>> + Send + 'static>; pub type PartialFn = Box]) -> Pin + Send>> + Send + 'static>; @@ -54,7 +54,6 @@ impl Clone for ListPathRawOptions { pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) -> Result<()> { // println!("list_path_raw {},{}", &opts.bucket, &opts.path); if opts.disks.is_empty() { - info!("list_path_raw 0 drives provided"); return Err(Error::from_string("list_path_raw: 0 drives provided")); } @@ -205,7 +204,7 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - continue; } // If exact match, we agree. - if let Ok((_, true)) = current.matches(&entry, true) { + if let (_, true) = current.matches(Some(&entry), true) { top_entries[i] = Some(entry); agree += 1; @@ -214,17 +213,18 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - // If only the name matches we didn't agree, but add it for resolution. if entry.name == current.name { top_entries[i] = Some(entry); - continue; } // We got different entries if entry.name > current.name { continue; } - // We got a new, better current. - // Clear existing entries. - top_entries = vec![None; top_entries.len()]; - agree += 1; + + for item in top_entries.iter_mut().take(i) { + *item = None; + } + + agree = 1; top_entries[i] = Some(entry.clone()); current = entry; } @@ -272,6 +272,7 @@ pub async fn list_path_raw(mut rx: B_Receiver, opts: ListPathRawOptions) - for r in readers.iter_mut() { let _ = r.skip(1).await; } + if let Some(agreed_fn) = opts.agreed.as_ref() { agreed_fn(current).await; } diff --git a/ecstore/src/config/com.rs b/ecstore/src/config/com.rs index 383359ea1..43e48c6d2 100644 --- a/ecstore/src/config/com.rs +++ b/ecstore/src/config/com.rs @@ -204,7 +204,7 @@ async fn apply_dynamic_config_for_sub_sys(cfg: &mut Config, api: } } Err(err) => { - error!("init storageclass err:{:?}", &err); + error!("init storage class err:{:?}", &err); break; } } diff --git a/ecstore/src/config/mod.rs b/ecstore/src/config/mod.rs index ffe00477c..10a5d50fd 100644 --- a/ecstore/src/config/mod.rs +++ b/ecstore/src/config/mod.rs @@ -141,7 +141,7 @@ impl Config { } pub fn merge(&self) -> Config { - // TODO: merge defauls + // TODO: merge default self.clone() } } @@ -158,6 +158,6 @@ pub fn register_default_kvs(kvs: HashMap) { pub fn init() { let mut kvs = HashMap::new(); kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DefaultKVS.clone()); - // TODO: other defauls + // TODO: other default register_default_kvs(kvs) } diff --git a/ecstore/src/config/storageclass.rs b/ecstore/src/config/storageclass.rs index edb2095f9..ef5199fc6 100644 --- a/ecstore/src/config/storageclass.rs +++ b/ecstore/src/config/storageclass.rs @@ -8,8 +8,8 @@ use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; use tracing::warn; -// default_partiy_count 默认配置，根据磁盘总数分配校验磁盘数量 -pub fn default_partiy_count(drive: usize) -> usize { +// default_parity_count 默认配置，根据磁盘总数分配校验磁盘数量 +pub fn default_parity_count(drive: usize) -> usize { match drive { 1 => 0, 2 | 3 => 1, @@ -158,7 +158,7 @@ pub fn lookup_config(kvs: &KVS, set_drive_count: usize) -> Result { parse_storage_class(&ssc_str)? } else { StorageClass { - parity: default_partiy_count(set_drive_count), + parity: default_parity_count(set_drive_count), } } }; diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 45abdbf5f..a814baca3 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -38,7 +38,9 @@ use crate::set_disk::{ CHECK_PART_VOLUME_NOT_FOUND, }; use crate::store_api::{BitrotAlgorithm, StorageAPI}; -use crate::utils::fs::{access, lstat, remove, remove_all, rename, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; +use crate::utils::fs::{ + access, lstat, lstat_std, remove, remove_all, remove_all_std, remove_std, rename, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY, +}; use crate::utils::os::get_info; use crate::utils::path::{ self, clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, GLOBAL_DIR_SUFFIX, @@ -259,7 +261,7 @@ impl LocalDisk { #[tracing::instrument(level = "debug", skip(self))] async fn check_format_json(&self) -> Result { - let md = fs::metadata(&self.format_path).await.map_err(|e| match e.kind() { + let md = std::fs::metadata(&self.format_path).map_err(|e| match e.kind() { ErrorKind::NotFound => DiskError::DiskNotFound, ErrorKind::PermissionDenied => DiskError::FileAccessDenied, _ => { @@ -315,9 +317,9 @@ impl LocalDisk { #[allow(unused_variables)] pub async fn move_to_trash(&self, delete_path: &PathBuf, recursive: bool, immediate_purge: bool) -> Result<()> { if recursive { - remove_all(delete_path).await?; + remove_all_std(delete_path)?; } else { - remove(delete_path).await?; + remove_std(delete_path)?; } return Ok(()); @@ -365,7 +367,7 @@ impl LocalDisk { Ok(()) } - // #[tracing::instrument(skip(self))] + #[tracing::instrument(level = "debug", skip(self))] pub async fn delete_file( &self, base_path: &PathBuf, @@ -688,6 +690,7 @@ impl LocalDisk { } // write_all_private with check_path_length + #[tracing::instrument(level = "debug", skip_all)] pub async fn write_all_private( &self, volume: &str, @@ -1213,7 +1216,7 @@ impl DiskAPI for LocalDisk { Ok(data) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(level = "debug", skip_all)] async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { self.write_all_public(volume, path, data).await } @@ -1334,10 +1337,10 @@ impl DiskAPI for LocalDisk { let src_volume_dir = self.get_bucket_path(src_volume)?; let dst_volume_dir = self.get_bucket_path(dst_volume)?; if !skip_access_checks(src_volume) { - utils::fs::access(&src_volume_dir).await.map_err(map_err_not_exists)? + utils::fs::access_std(&src_volume_dir).map_err(map_err_not_exists)? } if !skip_access_checks(dst_volume) { - utils::fs::access(&dst_volume_dir).await.map_err(map_err_not_exists)? + utils::fs::access_std(&dst_volume_dir).map_err(map_err_not_exists)? } let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); @@ -1360,7 +1363,7 @@ impl DiskAPI for LocalDisk { check_path_length(dst_file_path.to_string_lossy().as_ref())?; if src_is_dir { - let meta_op = match lstat(&src_file_path).await { + let meta_op = match lstat_std(&src_file_path) { Ok(meta) => Some(meta), Err(e) => { if is_sys_err_io(&e) { @@ -1381,7 +1384,7 @@ impl DiskAPI for LocalDisk { } } - if let Err(e) = utils::fs::remove(&dst_file_path).await { + if let Err(e) = utils::fs::remove_std(&dst_file_path) { if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) { warn!("rename_part remove dst failed {:?} err {:?}", &dst_file_path, e); return Err(Error::new(DiskError::FileAccessDenied)); @@ -1721,7 +1724,7 @@ impl DiskAPI for LocalDisk { Ok(()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(level = "debug", skip(self))] async fn rename_data( &self, src_volume: &str, @@ -1732,7 +1735,7 @@ impl DiskAPI for LocalDisk { ) -> Result { let src_volume_dir = self.get_bucket_path(src_volume)?; if !skip_access_checks(src_volume) { - if let Err(e) = utils::fs::access(&src_volume_dir).await { + if let Err(e) = utils::fs::access_std(&src_volume_dir) { info!("access checks failed, src_volume_dir: {:?}, err: {}", src_volume_dir, e.to_string()); return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); } @@ -1740,7 +1743,7 @@ impl DiskAPI for LocalDisk { let dst_volume_dir = self.get_bucket_path(dst_volume)?; if !skip_access_checks(dst_volume) { - if let Err(e) = utils::fs::access(&dst_volume_dir).await { + if let Err(e) = utils::fs::access_std(&dst_volume_dir) { info!("access checks failed, dst_volume_dir: {:?}, err: {}", dst_volume_dir, e.to_string()); return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); } @@ -1913,7 +1916,7 @@ impl DiskAPI for LocalDisk { if let Some(src_file_path_parent) = src_file_path.parent() { if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { - let _ = utils::fs::remove(src_file_path_parent).await; + let _ = utils::fs::remove_std(src_file_path_parent); } else { let _ = self .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index ddefd1bed..cda289c2b 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -786,46 +786,62 @@ impl MetaCacheEntry { fm.into_file_info_versions(bucket, self.name.as_str(), false) } - pub fn matches(&self, other: &MetaCacheEntry, strict: bool) -> Result<(Option, bool)> { + pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option, bool) { + if other.is_none() { + return (None, false); + } + + let other = other.unwrap(); + let mut prefer = None; if self.name != other.name { if self.name < other.name { - return Ok((Some(self.clone()), false)); + return (Some(self.clone()), false); } - return Ok((Some(other.clone()), false)); + return (Some(other.clone()), false); } if other.is_dir() || self.is_dir() { if self.is_dir() { - return Ok((Some(self.clone()), other.is_dir())); + return (Some(self.clone()), other.is_dir() == self.is_dir()); } - return Ok((Some(other.clone()), other.is_dir() == self.is_dir())); + return (Some(other.clone()), other.is_dir() == self.is_dir()); } let self_vers = match &self.cached { Some(file_meta) => file_meta.clone(), - None => FileMeta::load(&self.metadata)?, + None => match FileMeta::load(&self.metadata) { + Ok(meta) => meta, + Err(_) => { + return (None, false); + } + }, }; let other_vers = match &other.cached { Some(file_meta) => file_meta.clone(), - None => FileMeta::load(&other.metadata)?, + None => match FileMeta::load(&other.metadata) { + Ok(meta) => meta, + Err(_) => { + return (None, false); + } + }, }; if self_vers.versions.len() != other_vers.versions.len() { match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) { Ordering::Greater => { - return Ok((Some(self.clone()), false)); + return (Some(self.clone()), false); } Ordering::Less => { - return Ok((Some(self.clone()), false)); + return (Some(other.clone()), false); } _ => {} } if self_vers.versions.len() > other_vers.versions.len() { - return Ok((Some(self.clone()), false)); + return (Some(self.clone()), false); } - return Ok((Some(self.clone()), false)); + return (Some(other.clone()), false); } for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) { @@ -853,14 +869,14 @@ impl MetaCacheEntry { } if prefer.is_some() { - return Ok((prefer, false)); + return (prefer, false); } if s_version.header.sorts_before(&o_version.header) { - return Ok((Some(self.clone()), false)); + return (Some(self.clone()), false); } - return Ok((Some(other.clone()), false)); + return (Some(other.clone()), false); } } @@ -868,7 +884,7 @@ impl MetaCacheEntry { prefer = Some(self.clone()); } - Ok((prefer, true)) + (prefer, true) } pub fn xl_meta(&mut self) -> Result { @@ -900,9 +916,10 @@ impl MetaCacheEntries { pub fn as_ref(&self) -> &[Option] { &self.0 } - pub fn resolve(&self, mut params: MetadataResolutionParams) -> Result> { + pub fn resolve(&self, mut params: MetadataResolutionParams) -> Option { if self.0.is_empty() { - return Ok(None); + warn!("decommission_pool: entries resolve empty"); + return None; } let mut dir_exists = 0; @@ -913,76 +930,104 @@ impl MetaCacheEntries { let mut objs_valid = 0; for entry in self.0.iter().flatten() { + let mut entry = entry.clone(); + + warn!("decommission_pool: entries resolve entry {:?}", entry.name); if entry.name.is_empty() { continue; } if entry.is_dir() { dir_exists += 1; selected = Some(entry.clone()); + warn!("decommission_pool: entries resolve entry dir {:?}", entry.name); continue; } + let xl = match entry.xl_meta() { + Ok(xl) => xl, + Err(e) => { + warn!("decommission_pool: entries resolve entry xl_meta {:?}", e); + continue; + } + }; + objs_valid += 1; - match &entry.cached { - Some(file_meta) => { - params.candidates.push(file_meta.versions.clone()); - } - None => { - params.candidates.push(FileMeta::load(&entry.metadata)?.versions); - } - } + params.candidates.push(xl.versions.clone()); if selected.is_none() { selected = Some(entry.clone()); objs_agree = 1; + warn!("decommission_pool: entries resolve entry selected {:?}", entry.name); continue; } - if let (Some(prefer), true) = entry.matches(selected.as_ref().unwrap(), params.strict)? { - selected = Some(prefer); + if let (prefer, true) = entry.matches(selected.as_ref(), params.strict) { + selected = prefer; objs_agree += 1; + warn!("decommission_pool: entries resolve entry prefer {:?}", entry.name); continue; } } - // Return dir entries, if enough... - if selected.is_some() && selected.as_ref().unwrap().is_dir() && dir_exists >= params.dir_quorum { - return Ok(selected); + let Some(selected) = selected else { + warn!("decommission_pool: entries resolve entry no selected"); + return None; + }; + + if selected.is_dir() && dir_exists >= params.dir_quorum { + warn!("decommission_pool: entries resolve entry dir selected {:?}", selected.name); + return Some(selected); } + // If we would never be able to reach read quorum. if objs_valid < params.obj_quorum { - return Ok(None); + warn!( + "decommission_pool: entries resolve entry not enough objects {} < {}", + objs_valid, params.obj_quorum + ); + return None; } - // If all objects agree. - if selected.is_some() && objs_agree == objs_valid { - return Ok(selected); + + if objs_agree == objs_valid { + warn!("decommission_pool: entries resolve entry all agree {} == {}", objs_agree, objs_valid); + return Some(selected); } - // If cached is nil we shall skip the entry. - if selected.is_none() || (selected.is_some() && selected.as_ref().unwrap().cached.is_none()) { - return Ok(None); + + let Some(cached) = selected.cached else { + warn!("decommission_pool: entries resolve entry no cached"); + return None; + }; + + let versions = merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, ¶ms.candidates); + if versions.is_empty() { + warn!("decommission_pool: entries resolve entry no versions"); + return None; } + + let metadata = match cached.marshal_msg() { + Ok(meta) => meta, + Err(e) => { + warn!("decommission_pool: entries resolve entry marshal_msg {:?}", e); + return None; + } + }; + // Merge if we have disagreement. // Create a new merged result. - selected = Some(MetaCacheEntry { - name: selected.as_ref().unwrap().name.clone(), + let new_selected = MetaCacheEntry { + name: selected.name.clone(), cached: Some(FileMeta { - meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver, + meta_ver: cached.meta_ver, + versions, ..Default::default() }), reusable: true, - ..Default::default() - }); + metadata, + }; - selected.as_mut().unwrap().cached.as_mut().unwrap().versions = - merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, ¶ms.candidates); - if selected.as_ref().unwrap().cached.as_ref().unwrap().versions.is_empty() { - return Ok(None); - } - - selected.as_mut().unwrap().metadata = selected.as_ref().unwrap().cached.as_ref().unwrap().marshal_msg()?; - - Ok(selected) + warn!("decommission_pool: entries resolve entry selected {:?}", new_selected.name); + Some(new_selected) } pub fn first_found(&self) -> (Option, usize) { diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index ae88611a8..41056a45a 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -9,7 +9,6 @@ use crate::{ }; use common::error::{Error, Result}; use tokio::fs; -use tracing::info; use super::error::{os_err_to_file_err, os_is_exist, DiskError}; @@ -108,6 +107,7 @@ pub async fn read_dir(path: impl AsRef, count: i32) -> Result> Ok(volumes) } +#[tracing::instrument(level = "debug", skip_all)] pub async fn rename_all( src_file_path: impl AsRef, dst_file_path: impl AsRef, @@ -136,15 +136,15 @@ pub async fn reliable_rename( base_dir: impl AsRef, ) -> io::Result<()> { if let Some(parent) = dst_file_path.as_ref().parent() { - if !file_exists(parent).await { - info!("reliable_rename reliable_mkdir_all parent: {:?}", parent); + if !file_exists(parent) { + // info!("reliable_rename reliable_mkdir_all parent: {:?}", parent); reliable_mkdir_all(parent, base_dir.as_ref()).await?; } } let mut i = 0; loop { - if let Err(e) = utils::fs::rename(src_file_path.as_ref(), dst_file_path.as_ref()).await { + if let Err(e) = utils::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) { if os_is_not_exist(&e) && i == 0 { i += 1; continue; @@ -221,6 +221,6 @@ pub async fn os_mkdir_all(dir_path: impl AsRef, base_dir: impl AsRef Ok(()) } -pub async fn file_exists(path: impl AsRef) -> bool { - fs::metadata(path.as_ref()).await.map(|_| true).unwrap_or(false) +pub fn file_exists(path: impl AsRef) -> bool { + std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false) } diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 590889e78..702d393ee 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -1,5 +1,6 @@ use crate::bitrot::{BitrotReader, BitrotWriter}; use crate::error::clone_err; +use crate::io::Etag; use crate::quorum::{object_op_ignored_errs, reduce_write_quorum_errs}; use bytes::{Bytes, BytesMut}; use common::error::{Error, Result}; @@ -8,8 +9,10 @@ use reed_solomon_erasure::galois_8::ReedSolomon; use smallvec::SmallVec; use std::any::Any; use std::io::ErrorKind; +use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::mpsc; use tracing::warn; use tracing::{error, info}; // use tracing::debug; @@ -26,7 +29,7 @@ pub struct Erasure { encoder: Option, pub block_size: usize, _id: Uuid, - buf: Vec, + _buf: Vec, } impl Erasure { @@ -46,61 +49,65 @@ impl Erasure { block_size, encoder, _id: Uuid::new_v4(), - buf: vec![0u8; block_size], + _buf: vec![0u8; block_size], } } - #[tracing::instrument(level = "debug", skip(self, reader, writers))] + #[tracing::instrument(level = "info", skip(self, reader, writers))] pub async fn encode( - &mut self, - reader: &mut S, + self: Arc, + mut reader: S, writers: &mut [Option], // block_size: usize, total_size: usize, write_quorum: usize, - ) -> Result + ) -> Result<(usize, String)> where - S: AsyncRead + Unpin + Send + 'static, + S: AsyncRead + Etag + Unpin + Send + 'static, { - // pin_mut!(body); - // let mut reader = tokio_util::io::StreamReader::new( - // body.map(|f| f.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))), - // ); + let (tx, mut rx) = mpsc::channel(5); + let task = tokio::spawn(async move { + let mut buf = vec![0u8; self.block_size]; + let mut total: usize = 0; + loop { + if total_size > 0 { + let new_len = { + let remain = total_size - total; + if remain > self.block_size { + self.block_size + } else { + remain + } + }; - let mut total: usize = 0; - let mut blocks = >::new(); - - loop { - if total_size > 0 { - let new_len = { - let remain = total_size - total; - if remain > self.block_size { - self.block_size - } else { - remain + if new_len == 0 && total > 0 { + break; } - }; - if new_len == 0 && total > 0 { + buf.resize(new_len, 0u8); + match reader.read_exact(&mut buf).await { + Ok(res) => res, + Err(e) => { + if let ErrorKind::UnexpectedEof = e.kind() { + break; + } else { + return Err(Error::new(e)); + } + } + }; + total += buf.len(); + } + let blocks = Arc::new(Box::pin(self.clone().encode_data(&buf)?)); + let _ = tx.send(blocks).await; + if total_size == 0 { break; } - - self.buf.resize(new_len, 0u8); - match reader.read_exact(&mut self.buf).await { - Ok(res) => res, - Err(e) => { - if let ErrorKind::UnexpectedEof = e.kind() { - break; - } else { - return Err(Error::new(e)); - } - } - }; - total += self.buf.len(); } + let etag = reader.etag().await; + Ok((total, etag)) + }); - self.encode_data(&self.buf, &mut blocks)?; - + while let Some(blocks) = rx.recv().await { let write_futures = writers.iter_mut().enumerate().map(|(i, w_op)| { let i_inner = i; let blocks_inner = blocks.clone(); @@ -125,84 +132,8 @@ impl Erasure { warn!("Erasure encode errs {:?}", &errs); return Err(err); } - - if total_size == 0 { - break; - } } - - Ok(total) - - // // let stream = ChunkedStream::new(body, self.block_size); - // let stream = ChunkedStream::new(body, total_size, self.block_size, false); - // let mut total: usize = 0; - // // let mut idx = 0; - // pin_mut!(stream); - - // // warn!("encode start..."); - - // loop { - // match stream.next().await { - // Some(result) => match result { - // Ok(data) => { - // total += data.len(); - - // // EOF - // if data.is_empty() { - // break; - // } - - // // idx += 1; - // // warn!("encode {} get data {:?}", data.len(), data.to_vec()); - - // let blocks = self.encode_data(data.as_ref())?; - - // // warn!( - // // "encode shard size: {}/{} from block_size {}, total_size {} ", - // // blocks[0].len(), - // // blocks.len(), - // // data.len(), - // // total_size - // // ); - - // let mut errs = Vec::new(); - - // for (i, w_op) in writers.iter_mut().enumerate() { - // if let Some(w) = w_op { - // match w.write(blocks[i].as_ref()).await { - // Ok(_) => errs.push(None), - // Err(e) => errs.push(Some(e)), - // } - // } else { - // errs.push(Some(Error::new(DiskError::DiskNotFound))); - // } - // } - - // let none_count = errs.iter().filter(|&x| x.is_none()).count(); - // if none_count >= write_quorum { - // continue; - // } - - // if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) { - // warn!("Erasure encode errs {:?}", &errs); - // return Err(err); - // } - // } - // Err(e) => { - // warn!("poll result err {:?}", &e); - // return Err(Error::msg(e.to_string())); - // } - // }, - // None => { - // // warn!("poll empty result"); - // break; - // } - // } - // } - - // let _ = close_bitrot_writers(writers).await?; - - // Ok(total) + task.await? } pub async fn decode( @@ -356,8 +287,8 @@ impl Erasure { self.data_shards + self.parity_shards } - #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))] - pub fn encode_data(&self, data: &[u8], shards: &mut SmallVec<[Bytes; 16]>) -> Result<()> { + #[tracing::instrument(level = "info", skip_all, fields(data_len=data.len()))] + pub fn encode_data(self: Arc, data: &[u8]) -> Result> { let (shard_size, total_size) = self.need_size(data.len()); // 生成一个新的 所需的所有分片数据长度 @@ -379,14 +310,13 @@ impl Erasure { // 零拷贝分片，所有 shard 引用 data_buffer let mut data_buffer = data_buffer.freeze(); - shards.clear(); - shards.reserve(self.total_shard_count()); + let mut shards = Vec::with_capacity(self.total_shard_count()); for _ in 0..self.total_shard_count() { let shard = data_buffer.split_to(shard_size); shards.push(shard); } - Ok(()) + Ok(shards) } pub fn decode_data(&self, shards: &mut [Option>]) -> Result<()> { @@ -617,7 +547,6 @@ impl ShardReader { #[cfg(test)] mod test { - use super::*; #[test] @@ -626,8 +555,7 @@ mod test { let parity_shards = 2; let data: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; let ec = Erasure::new(data_shards, parity_shards, 1); - let mut shards = SmallVec::new(); - ec.encode_data(data, &mut shards).unwrap(); + let shards = Arc::new(ec).encode_data(data).unwrap(); println!("shards:{:?}", shards); let mut s: Vec<_> = shards @@ -643,6 +571,7 @@ mod test { println!("sss:{:?}", &s); + let ec = Erasure::new(data_shards, parity_shards, 1); ec.decode_data(&mut s).unwrap(); // ec.encoder.reconstruct(&mut s).unwrap(); diff --git a/ecstore/src/file_meta.rs b/ecstore/src/file_meta.rs index e4da3882a..6bb644bab 100644 --- a/ecstore/src/file_meta.rs +++ b/ecstore/src/file_meta.rs @@ -58,10 +58,12 @@ impl FileMeta { } // isXL2V1Format + #[tracing::instrument(level = "debug", skip_all)] pub fn is_xl2_v1_format(buf: &[u8]) -> bool { !matches!(Self::check_xl2_v1(buf), Err(_e)) } + #[tracing::instrument(level = "debug", skip_all)] pub fn load(buf: &[u8]) -> Result { let mut xl = FileMeta::default(); xl.unmarshal_msg(buf)?; @@ -245,7 +247,7 @@ impl FileMeta { } } - #[tracing::instrument] + #[tracing::instrument(level = "debug", skip_all)] pub fn marshal_msg(&self) -> Result> { let mut wr = Vec::new(); @@ -363,6 +365,7 @@ impl FileMeta { } // shard_data_dir_count 查询 vid下data_dir的数量 + #[tracing::instrument(level = "debug", skip_all)] pub fn shard_data_dir_count(&self, vid: &Option, data_dir: &Option) -> usize { self.versions .iter() @@ -434,6 +437,7 @@ impl FileMeta { } // 添加版本 + #[tracing::instrument(level = "debug", skip_all)] pub fn add_version(&mut self, fi: FileInfo) -> Result<()> { let vid = fi.version_id; @@ -916,11 +920,20 @@ impl FileMetaVersionHeader { } pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool { + let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o); if self.version_id.is_none() { - return self.version_id == o.version_id && self.version_type == o.version_type && self.mod_time == o.mod_time; + ok = ok && self.mod_time == o.mod_time; } - self.version_id == o.version_id && self.version_type == o.version_type + ok + } + + pub fn matches_ec(&self, o: &FileMetaVersionHeader) -> bool { + if self.has_ec() && o.has_ec() { + return self.ec_n == o.ec_n && self.ec_m == o.ec_m; + } + + true } pub fn free_version(&self) -> bool { diff --git a/ecstore/src/global.rs b/ecstore/src/global.rs index 162aa9d3e..c60c6c594 100644 --- a/ecstore/src/global.rs +++ b/ecstore/src/global.rs @@ -20,8 +20,6 @@ pub const DISK_MIN_INODES: u64 = 1000; pub const DISK_FILL_FRACTION: f64 = 0.99; pub const DISK_RESERVE_FRACTION: f64 = 0.15; -pub const DEFAULT_PORT: u16 = 9000; - lazy_static! { static ref GLOBAL_RUSTFS_PORT: OnceLock = OnceLock::new(); pub static ref GLOBAL_OBJECT_API: OnceLock> = OnceLock::new(); @@ -41,31 +39,37 @@ lazy_static! { pub static ref GLOBAL_BOOT_TIME: OnceCell = OnceCell::new(); } +/// Get the global rustfs port pub fn global_rustfs_port() -> u16 { if let Some(p) = GLOBAL_RUSTFS_PORT.get() { *p } else { - DEFAULT_PORT + rustfs_config::DEFAULT_PORT } } +/// Set the global rustfs port pub fn set_global_rustfs_port(value: u16) { GLOBAL_RUSTFS_PORT.set(value).expect("set_global_rustfs_port fail"); } +/// Get the global rustfs port pub fn set_global_deployment_id(id: Uuid) { globalDeploymentIDPtr.set(id).unwrap(); } + +/// Get the global deployment id pub fn get_global_deployment_id() -> Option { globalDeploymentIDPtr.get().map(|v| v.to_string()) } - +/// Get the global deployment id pub fn set_global_endpoints(eps: Vec) { GLOBAL_Endpoints .set(EndpointServerPools::from(eps)) .expect("GLOBAL_Endpoints set failed") } +/// Get the global endpoints pub fn get_global_endpoints() -> EndpointServerPools { if let Some(eps) = GLOBAL_Endpoints.get() { eps.clone() diff --git a/ecstore/src/heal/data_scanner.rs b/ecstore/src/heal/data_scanner.rs index b1b110a54..c806267f2 100644 --- a/ecstore/src/heal/data_scanner.rs +++ b/ecstore/src/heal/data_scanner.rs @@ -867,7 +867,7 @@ impl FolderScanner { // return; // } let entry = match entries.resolve(resolver_partial) { - Ok(Some(entry)) => entry, + Some(entry) => entry, _ => match entries.first_found() { (Some(entry), _) => entry, _ => return, diff --git a/ecstore/src/heal/heal_ops.rs b/ecstore/src/heal/heal_ops.rs index 6f4614578..f2e065fc7 100644 --- a/ecstore/src/heal/heal_ops.rs +++ b/ecstore/src/heal/heal_ops.rs @@ -17,7 +17,7 @@ use crate::{ global::GLOBAL_IsDistErasure, heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN}, new_object_layer_fn, - utils::path::has_profix, + utils::path::has_prefix, }; use crate::{ heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT}, @@ -786,7 +786,7 @@ impl AllHealState { let _ = self.mu.write().await; for (k, v) in self.heal_seq_map.read().await.iter() { - if (has_profix(k, path_s) || has_profix(path_s, k)) && !v.has_ended().await { + if (has_prefix(k, path_s) || has_prefix(path_s, k)) && !v.has_ended().await { return Err(Error::from_string(format!( "The provided heal sequence path overlaps with an existing heal path: {}", k diff --git a/ecstore/src/io.rs b/ecstore/src/io.rs index f2affe8cb..2bd02c117 100644 --- a/ecstore/src/io.rs +++ b/ecstore/src/io.rs @@ -1,8 +1,12 @@ +use async_trait::async_trait; use bytes::Bytes; use futures::TryStreamExt; use md5::Digest; use md5::Md5; +use pin_project_lite::pin_project; +use std::io; use std::pin::Pin; +use std::task::ready; use std::task::Context; use std::task::Poll; use tokio::io::AsyncRead; @@ -125,10 +129,17 @@ impl AsyncRead for HttpFileReader { } } -pub struct EtagReader { - inner: R, - bytes_tx: mpsc::Sender, - md5_rx: oneshot::Receiver, +#[async_trait] +pub trait Etag { + async fn etag(self) -> String; +} + +pin_project! { + pub struct EtagReader { + inner: R, + bytes_tx: mpsc::Sender, + md5_rx: oneshot::Receiver, + } } impl EtagReader { @@ -148,8 +159,11 @@ impl EtagReader { EtagReader { inner, bytes_tx, md5_rx } } +} - pub async fn etag(self) -> String { +#[async_trait] +impl Etag for EtagReader { + async fn etag(self) -> String { drop(self.inner); drop(self.bytes_tx); self.md5_rx.await.unwrap() @@ -157,21 +171,28 @@ impl EtagReader { } impl AsyncRead for EtagReader { - #[tracing::instrument(level = "debug", skip_all)] - fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - let poll = Pin::new(&mut self.inner).poll_read(cx, buf); - if let Poll::Ready(Ok(())) = &poll { - if buf.remaining() == 0 { + #[tracing::instrument(level = "info", skip_all)] + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let me = self.project(); + + loop { + let rem = buf.remaining(); + if rem != 0 { + ready!(Pin::new(&mut *me.inner).poll_read(cx, buf))?; + if buf.remaining() == rem { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "early eof")).into(); + } + } else { let bytes = buf.filled(); let bytes = Bytes::copy_from_slice(bytes); - let tx = self.bytes_tx.clone(); + let tx = me.bytes_tx.clone(); tokio::spawn(async move { if let Err(e) = tx.send(bytes).await { warn!("EtagReader send error: {:?}", e); } }); + return Poll::Ready(Ok(())); } } - poll } } diff --git a/ecstore/src/pools.rs b/ecstore/src/pools.rs index f6f996941..8a0761006 100644 --- a/ecstore/src/pools.rs +++ b/ecstore/src/pools.rs @@ -31,6 +31,7 @@ use std::io::{Cursor, Write}; use std::path::PathBuf; use std::sync::Arc; use time::{Duration, OffsetDateTime}; +use tokio::io::AsyncReadExt; use tokio::sync::broadcast::Receiver as B_Receiver; use tracing::{error, info, warn}; @@ -910,7 +911,11 @@ impl ECStore { let wk = wk.clone(); let set = set.clone(); let rcfg = rcfg.clone(); - Box::pin(async move { this.decommission_entry(idx, entry, bucket, set, wk, rcfg).await }) + + Box::pin(async move { + wk.take().await; + this.decommission_entry(idx, entry, bucket, set, wk, rcfg).await + }) } }); @@ -918,6 +923,7 @@ impl ECStore { let mut rx = rx.resubscribe(); let bi = bi.clone(); let set_id = set_idx; + let wk_clone = wk.clone(); tokio::spawn(async move { loop { if rx.try_recv().is_ok() { @@ -945,6 +951,8 @@ impl ECStore { } } } + + wk_clone.give().await; }); } @@ -1166,7 +1174,7 @@ impl ECStore { #[tracing::instrument(skip(self, rd))] async fn decommission_object(self: Arc, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> { - warn!("decommission_object: {} {}", &bucket, &rd.object_info.name); + warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name); let object_info = rd.object_info.clone(); // TODO: check : use size or actual_size ? @@ -1194,8 +1202,6 @@ impl ECStore { } }; - // TODO: defer abort_multipart_upload - defer!(|| async { if let Err(err) = self .abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default()) @@ -1210,9 +1216,13 @@ impl ECStore { let mut reader = rd.stream; for (i, part) in object_info.parts.iter().enumerate() { - // 每次从reader中读取一个part上传 + let mut chunk = vec![0u8; part.size]; - let mut data = PutObjReader::new(reader, part.size); + reader.read_exact(&mut chunk).await?; + + // 每次从reader中读取一个part上传 + let rd = Box::new(Cursor::new(chunk)); + let mut data = PutObjReader::new(rd, part.size); let pi = match self .put_object_part( @@ -1230,18 +1240,17 @@ impl ECStore { { Ok(pi) => pi, Err(err) => { - error!("decommission_object: put_object_part err {:?}", &err); + error!("decommission_object: put_object_part {} err {:?}", i, &err); return Err(err); } }; + warn!("decommission_object: put_object_part {} done {} {}", i, &bucket, &object_info.name); + parts[i] = CompletePart { part_num: pi.part_num, e_tag: pi.etag, }; - - // 把reader所有权拿回来? - reader = data.stream; } if let Err(err) = self @@ -1262,6 +1271,7 @@ impl ECStore { return Err(err); } + warn!("decommission_object: complete_multipart_upload done {} {}", &bucket, &object_info.name); return Ok(()); } @@ -1289,6 +1299,7 @@ impl ECStore { return Err(err); } + warn!("decommission_object: put_object done {} {}", &bucket, &object_info.name); Ok(()) } } @@ -1319,6 +1330,8 @@ impl SetDisks { ..Default::default() }; + let cb1 = cb_func.clone(); + list_path_raw( rx, ListPathRawOptions { @@ -1327,25 +1340,21 @@ impl SetDisks { path: bucket_info.prefix.clone(), recursice: true, min_disks: listing_quorum, + agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))), partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { let resolver = resolver.clone(); let cb_func = cb_func.clone(); - match entries.resolve(resolver) { - Ok(Some(entry)) => { + Some(entry) => { warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name); Box::pin(async move { cb_func(entry).await; }) } - Ok(None) => { + None => { warn!("decommission_pool: list_objects_to_decommission get none"); Box::pin(async {}) } - Err(err) => { - error!("decommission_pool: list_objects_to_decommission get err {:?}", &err); - Box::pin(async {}) - } } })), ..Default::default() diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index d89fa523b..d38177ca5 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -147,6 +147,7 @@ pub fn reduce_read_quorum_errs( // 根据写quorum验证错误数量 // 返回最大错误数量的下标，或QuorumError +#[tracing::instrument(level = "info", skip_all)] pub fn reduce_write_quorum_errs( errs: &[Option], ignored_errs: &[Box], diff --git a/ecstore/src/rebalance.rs b/ecstore/src/rebalance.rs index 62c5d696f..007cf7a4b 100644 --- a/ecstore/src/rebalance.rs +++ b/ecstore/src/rebalance.rs @@ -18,6 +18,7 @@ use common::defer; use common::error::{Error, Result}; use http::HeaderMap; use serde::{Deserialize, Serialize}; +use tokio::io::AsyncReadExt; use tokio::sync::broadcast::{self, Receiver as B_Receiver}; use tokio::time::{Duration, Instant}; use tracing::{error, info, warn}; @@ -370,7 +371,7 @@ impl ECStore { let rebalance_meta = self.rebalance_meta.read().await; if let Some(meta) = rebalance_meta.as_ref() { if let Some(pool_stat) = meta.pool_stats.get(pool_index) { - if pool_stat.info.status != RebalStatus::Completed || !pool_stat.participating { + if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating { return Ok(None); } @@ -389,10 +390,15 @@ impl ECStore { let mut rebalance_meta = self.rebalance_meta.write().await; if let Some(meta) = rebalance_meta.as_mut() { if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { - if let Some(idx) = pool_stat.buckets.iter().position(|b| *b == bucket) { + warn!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets); + if let Some(idx) = pool_stat.buckets.iter().position(|b| b.as_str() == bucket.as_str()) { + warn!("bucket_rebalance_done: bucket {} rebalanced", &bucket); pool_stat.buckets.remove(idx); pool_stat.rebalanced_buckets.push(bucket); + return Ok(()); + } else { + warn!("bucket_rebalance_done: bucket {} not found", bucket); } } } @@ -581,25 +587,26 @@ impl ECStore { } }); - tracing::warn!("Pool {} rebalancing is started", pool_index + 1); + warn!("Pool {} rebalancing is started", pool_index + 1); while let Some(bucket) = self.next_rebal_bucket(pool_index).await? { - tracing::info!("Rebalancing bucket: {}", bucket); + warn!("Rebalancing bucket: start {}", bucket); if let Err(err) = self.rebalance_bucket(rx.resubscribe(), bucket.clone(), pool_index).await { if err.to_string().contains("not initialized") { warn!("rebalance_bucket: rebalance not initialized, continue"); continue; } - tracing::error!("Error rebalancing bucket {}: {:?}", bucket, err); + error!("Error rebalancing bucket {}: {:?}", bucket, err); done_tx.send(Err(err)).await.ok(); break; } + warn!("Rebalance bucket: done {} ", bucket); self.bucket_rebalance_done(pool_index, bucket).await?; } - tracing::warn!("Pool {} rebalancing is done", pool_index + 1); + warn!("Pool {} rebalancing is done", pool_index + 1); done_tx.send(Ok(())).await.ok(); save_task.await.ok(); @@ -632,7 +639,6 @@ impl ECStore { false } - #[allow(unused_assignments)] #[tracing::instrument(skip(self, wk, set))] async fn rebalance_entry( &self, @@ -838,8 +844,6 @@ impl ECStore { } }; - // TODO: defer abort_multipart_upload - defer!(|| async { if let Err(err) = self .abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default()) @@ -856,7 +860,13 @@ impl ECStore { for (i, part) in object_info.parts.iter().enumerate() { // 每次从reader中读取一个part上传 - let mut data = PutObjReader::new(reader, part.size); + let mut chunk = vec![0u8; part.size]; + + reader.read_exact(&mut chunk).await?; + + // 每次从reader中读取一个part上传 + let rd = Box::new(Cursor::new(chunk)); + let mut data = PutObjReader::new(rd, part.size); let pi = match self .put_object_part( @@ -883,9 +893,6 @@ impl ECStore { part_num: pi.part_num, e_tag: pi.etag, }; - - // 把reader所有权拿回来? - reader = data.stream; } if let Err(err) = self @@ -939,7 +946,7 @@ impl ECStore { #[tracing::instrument(skip(self, rx))] async fn rebalance_bucket(self: &Arc, rx: B_Receiver, bucket: String, pool_index: usize) -> Result<()> { // Placeholder for actual bucket rebalance logic - tracing::info!("Rebalancing bucket {} in pool {}", bucket, pool_index); + warn!("Rebalancing bucket {} in pool {}", bucket, pool_index); // TODO: other config // if bucket != RUSTFS_META_BUCKET{ @@ -977,14 +984,12 @@ impl ECStore { let bucket = bucket.clone(); let wk = wk.clone(); tokio::spawn(async move { - defer!(|| async { - wk.clone().give().await; - }); if let Err(err) = set.list_objects_to_rebalance(rx, bucket, rebalance_entry).await { error!("Rebalance worker {} error: {}", set_idx, err); } else { info!("Rebalance worker {} done", set_idx); } + wk.clone().give().await; }); } @@ -1068,25 +1073,21 @@ impl SetDisks { bucket: bucket.clone(), recursice: true, min_disks: listing_quorum, - agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry.clone())))), + agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))), partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option]| { // let cb = cb.clone(); let resolver = resolver.clone(); let cb = cb.clone(); match entries.resolve(resolver) { - Ok(Some(entry)) => { + Some(entry) => { warn!("rebalance: list_objects_to_decommission get {}", &entry.name); Box::pin(async move { cb(entry).await }) } - Ok(None) => { + None => { warn!("rebalance: list_objects_to_decommission get none"); Box::pin(async {}) } - Err(err) => { - error!("rebalance: list_objects_to_decommission get err {:?}", &err); - Box::pin(async {}) - } } })), ..Default::default() diff --git a/ecstore/src/set_disk.rs b/ecstore/src/set_disk.rs index bd55ab39f..fe992df18 100644 --- a/ecstore/src/set_disk.rs +++ b/ecstore/src/set_disk.rs @@ -284,10 +284,20 @@ impl SetDisks { // let mut ress = Vec::with_capacity(disks.len()); let mut errs = Vec::with_capacity(disks.len()); - for (i, disk) in disks.iter().enumerate() { - let mut file_info = file_infos[i].clone(); + let src_bucket = Arc::new(src_bucket.to_string()); + let src_object = Arc::new(src_object.to_string()); + let dst_bucket = Arc::new(dst_bucket.to_string()); + let dst_object = Arc::new(dst_object.to_string()); - futures.push(async move { + for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() { + let mut file_info = file_info.clone(); + let disk = disk.clone(); + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + let dst_object = dst_object.clone(); + let dst_bucket = dst_bucket.clone(); + + futures.push(tokio::spawn(async move { if file_info.erasure.index == 0 { file_info.erasure.index = i + 1; } @@ -297,12 +307,12 @@ impl SetDisks { } if let Some(disk) = disk { - disk.rename_data(src_bucket, src_object, file_info, dst_bucket, dst_object) + disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) .await } else { Err(Error::new(DiskError::DiskNotFound)) } - }) + })); } let mut disk_versions = vec![None; disks.len()]; @@ -311,15 +321,13 @@ impl SetDisks { let results = join_all(futures).await; for (idx, result) in results.iter().enumerate() { - match result { + match result.as_ref().map_err(|_| Error::new(DiskError::Unexpected))? { Ok(res) => { data_dirs[idx] = res.old_data_dir; disk_versions[idx].clone_from(&res.sign); - // ress.push(Some(res)); errs.push(None); } Err(e) => { - // ress.push(None); errs.push(Some(clone_err(e))); } } @@ -336,11 +344,14 @@ impl SetDisks { if let Some(disk) = disks[i].as_ref() { let fi = file_infos[i].clone(); let old_data_dir = data_dirs[i]; - futures.push(async move { + let disk = disk.clone(); + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + futures.push(tokio::spawn(async move { let _ = disk .delete_version( - src_bucket, - src_object, + &src_bucket, + &src_object, fi, false, DeleteOptions { @@ -354,7 +365,7 @@ impl SetDisks { debug!("rename_data delete_version err {:?}", e); e }); - }); + })); } } @@ -416,41 +427,38 @@ impl SetDisks { data_dir: &str, write_quorum: usize, ) -> Result<()> { - let file_path = format!("{}/{}", object, data_dir); - - let mut futures = Vec::with_capacity(disks.len()); - let mut errs = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { + let file_path = Arc::new(format!("{}/{}", object, data_dir)); + let bucket = Arc::new(bucket.to_string()); + let futures = disks.iter().map(|disk| { let file_path = file_path.clone(); - futures.push(async move { + let bucket = bucket.clone(); + let disk = disk.clone(); + tokio::spawn(async move { if let Some(disk) = disk { - disk.delete( - bucket, - &file_path, - DeleteOptions { - recursive: true, - ..Default::default() - }, - ) - .await + (disk + .delete( + &bucket, + &file_path, + DeleteOptions { + recursive: true, + ..Default::default() + }, + ) + .await) + .err() } else { - Err(Error::new(DiskError::DiskNotFound)) + Some(Error::new(DiskError::DiskNotFound)) } - }); - } - - let results = join_all(futures).await; - for result in results { - match result { - Ok(_) => { - errs.push(None); - } - Err(e) => { - errs.push(Some(e)); - } - } - } + }) + }); + let errs: Vec> = join_all(futures) + .await + .into_iter() + .map(|e| match e { + Ok(e) => e, + Err(_) => Some(Error::new(DiskError::Unexpected)), + }) + .collect(); if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) { return Err(err); @@ -502,24 +510,33 @@ impl SetDisks { meta: Vec, write_quorum: usize, ) -> Result>> { - let mut futures = Vec::with_capacity(disks.len()); + let src_bucket = Arc::new(src_bucket.to_string()); + let src_object = Arc::new(src_object.to_string()); + let dst_bucket = Arc::new(dst_bucket.to_string()); + let dst_object = Arc::new(dst_object.to_string()); let mut errs = Vec::with_capacity(disks.len()); - for disk in disks.iter() { + let futures = disks.iter().map(|disk| { + let disk = disk.clone(); let meta = meta.clone(); - futures.push(async move { + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + let dst_bucket = dst_bucket.clone(); + let dst_object = dst_object.clone(); + tokio::spawn(async move { if let Some(disk) = disk { - disk.rename_part(src_bucket, src_object, dst_bucket, dst_object, meta).await + disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta) + .await } else { Err(Error::new(DiskError::DiskNotFound)) } }) - } + }); let results = join_all(futures).await; for result in results { - match result { + match result? { Ok(_) => { errs.push(None); } @@ -531,7 +548,7 @@ impl SetDisks { if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) { warn!("rename_part errs {:?}", &errs); - Self::cleanup_multipart_path(disks, &[dst_object.to_owned(), format!("{}.meta", dst_object)]).await; + Self::cleanup_multipart_path(disks, &[dst_object.to_string(), format!("{}.meta", dst_object)]).await; return Err(err); } @@ -933,7 +950,7 @@ impl SetDisks { let disks = disks.clone(); let (parts_metadata, errs) = - Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await; + Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await?; let map_err_notfound = |err: Error| { if is_err_object_not_found(&err) { @@ -1103,39 +1120,47 @@ impl SetDisks { version_id: &str, read_data: bool, healing: bool, - ) -> (Vec, Vec>) { - let mut futures = Vec::with_capacity(disks.len()); + ) -> Result<(Vec, Vec>)> { let mut ress = Vec::with_capacity(disks.len()); let mut errors = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { - let opts = ReadOptions { - read_data, - healing, - ..Default::default() - }; - futures.push(async move { + let opts = Arc::new(ReadOptions { + read_data, + healing, + ..Default::default() + }); + let org_bucket = Arc::new(org_bucket.to_string()); + let bucket = Arc::new(bucket.to_string()); + let object = Arc::new(object.to_string()); + let version_id = Arc::new(version_id.to_string()); + let futures = disks.iter().map(|disk| { + let disk = disk.clone(); + let opts = opts.clone(); + let org_bucket = org_bucket.clone(); + let bucket = bucket.clone(); + let object = object.clone(); + let version_id = version_id.clone(); + tokio::spawn(async move { if let Some(disk) = disk { if version_id.is_empty() { - match disk.read_xl(bucket, object, read_data).await { + match disk.read_xl(&bucket, &object, read_data).await { Ok(info) => { - let fi = file_info_from_raw(info, bucket, object, read_data).await?; + let fi = file_info_from_raw(info, &bucket, &object, read_data).await?; Ok(fi) } Err(err) => Err(err), } } else { - disk.read_version(org_bucket, bucket, object, version_id, &opts).await + disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await } } else { Err(Error::new(DiskError::DiskNotFound)) } }) - } + }); let results = join_all(futures).await; for result in results { - match result { + match result? { Ok(res) => { ress.push(res); errors.push(None); @@ -1146,7 +1171,7 @@ impl SetDisks { } } } - (ress, errors) + Ok((ress, errors)) } async fn read_all_xl( @@ -1759,7 +1784,7 @@ impl SetDisks { let vid = opts.version_id.clone().unwrap_or_default(); // TODO: 优化并发 可用数量中断 - let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await; + let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await?; // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata); // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs); @@ -2052,7 +2077,7 @@ impl SetDisks { let bucket_partial = bucket_partial.clone(); async move { let entry = match entries.resolve(resolver_partial) { - Ok(Some(entry)) => entry, + Some(entry) => entry, _ => match entries.first_found() { (Some(entry), _) => entry, _ => return, @@ -2166,7 +2191,7 @@ impl SetDisks { let disks = { self.disks.read().await.clone() }; - let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await; + let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await?; if is_all_not_found(&errs) { warn!( "heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}", @@ -3505,7 +3530,7 @@ impl SetDisks { let heal_entry = heal_entry.clone(); let resolver = resolver.clone(); async move { - let entry = if let Ok(Some(entry)) = entries.resolve(resolver) { + let entry = if let Some(entry) = entries.resolve(resolver) { entry } else if let (Some(entry), _) = entries.first_found() { entry @@ -3759,7 +3784,7 @@ impl ObjectIO for SetDisks { let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap()); - let mut erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); let is_inline_buffer = { if let Some(sc) = GLOBAL_StorageClass.get() { @@ -3798,19 +3823,18 @@ impl ObjectIO for SetDisks { } let stream = replace(&mut data.stream, Box::new(empty())); - let mut etag_stream = EtagReader::new(stream); + let etag_stream = EtagReader::new(stream); // TODO: etag from header - let w_size = erasure - .encode(&mut etag_stream, &mut writers, data.content_length, write_quorum) + let (w_size, etag) = Arc::new(erasure) + .encode(etag_stream, &mut writers, data.content_length, write_quorum) .await?; // TODO: 出错，删除临时目录 if let Err(err) = close_bitrot_writers(&mut writers).await { error!("close_bitrot_writers err {:?}", err); } - let etag = etag_stream.etag().await; //TODO: userDefined user_defined.insert("etag".to_owned(), etag.clone()); @@ -3948,7 +3972,7 @@ impl StorageAPI for SetDisks { let (mut metas, errs) = { if let Some(vid) = &src_opts.version_id { - Self::read_all_fileinfo(&disks, "", src_bucket, src_object, vid, true, false).await + Self::read_all_fileinfo(&disks, "", src_bucket, src_object, vid, true, false).await? } else { Self::read_all_xl(&disks, src_bucket, src_object, true, false).await } @@ -4242,7 +4266,7 @@ impl StorageAPI for SetDisks { false, false, ) - .await + .await? } else { Self::read_all_xl(&disks, bucket, object, false, false).await } @@ -4382,47 +4406,57 @@ impl StorageAPI for SetDisks { let part_suffix = format!("part.{}", part_id); let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp()); - let tmp_part_path = format!("{}/{}", tmp_part, part_suffix); + let tmp_part_path = Arc::new(format!("{}/{}", tmp_part, part_suffix)); let mut writers = Vec::with_capacity(disks.len()); let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let shared_size = erasure.shard_size(erasure.block_size); - for disk in disks.iter() { - if let Some(disk) = disk { - // let writer = disk.append_file(RUSTFS_META_TMP_BUCKET, &tmp_part_path).await?; - // let filewriter = disk - // .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, data.content_length) - // .await?; - let writer = new_bitrot_filewriter( - disk.clone(), - RUSTFS_META_TMP_BUCKET, - &tmp_part_path, - false, - DEFAULT_BITROT_ALGO, - erasure.shard_size(erasure.block_size), - ) - .await?; - writers.push(Some(writer)); - } else { - writers.push(None); - } + let futures = disks.iter().map(|disk| { + let disk = disk.clone(); + let tmp_part_path = tmp_part_path.clone(); + tokio::spawn(async move { + if let Some(disk) = disk { + // let writer = disk.append_file(RUSTFS_META_TMP_BUCKET, &tmp_part_path).await?; + // let filewriter = disk + // .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, data.content_length) + // .await?; + match new_bitrot_filewriter( + disk.clone(), + RUSTFS_META_TMP_BUCKET, + &tmp_part_path, + false, + DEFAULT_BITROT_ALGO, + shared_size, + ) + .await + { + Ok(writer) => Ok(Some(writer)), + Err(e) => Err(e), + } + } else { + Ok(None) + } + }) + }); + for x in join_all(futures).await { + let x = x??; + writers.push(x); } - let mut erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); let stream = replace(&mut data.stream, Box::new(empty())); - let mut etag_stream = EtagReader::new(stream); + let etag_stream = EtagReader::new(stream); - let w_size = erasure - .encode(&mut etag_stream, &mut writers, data.content_length, write_quorum) + let (w_size, mut etag) = Arc::new(erasure) + .encode(etag_stream, &mut writers, data.content_length, write_quorum) .await?; if let Err(err) = close_bitrot_writers(&mut writers).await { error!("close_bitrot_writers err {:?}", err); } - let mut etag = etag_stream.etag().await; - if let Some(ref tag) = opts.preserve_etag { etag = tag.clone(); } @@ -5036,7 +5070,7 @@ impl StorageAPI for SetDisks { let disks = self.disks.read().await; let disks = disks.clone(); - let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false).await; + let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false).await?; if is_all_not_found(&errs) { warn!( "heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}", diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 5a2075775..eb28236e7 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -524,7 +524,9 @@ impl ECStore { // TODO: 并发 for (idx, pool) in self.pools.iter().enumerate() { - // TODO: IsSuspended + if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await { + continue; + } n_sets[idx] = pool.set_count; @@ -714,16 +716,14 @@ impl ECStore { let mut def_pool = PoolObjInfo::default(); let mut has_def_pool = false; - let pool_meta = self.pool_meta.read().await; for pinfo in ress.iter() { - if opts.skip_decommissioned && pool_meta.is_suspended(pinfo.index) { + if opts.skip_decommissioned && self.is_suspended(pinfo.index).await { continue; } - // TODO:SkipRebalancing - // if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) { - // continue - // } + if opts.skip_rebalancing && self.is_pool_rebalancing(pinfo.index).await { + continue; + } if pinfo.err.is_none() { return Ok((pinfo.clone(), self.pools_with_object(&ress, opts).await)); @@ -756,15 +756,15 @@ impl ECStore { async fn pools_with_object(&self, pools: &[PoolObjInfo], opts: &ObjectOptions) -> Vec { let mut errs = Vec::new(); - let pool_meta = self.pool_meta.read().await; + for pool in pools.iter() { - if opts.skip_decommissioned && pool_meta.is_suspended(pool.index) { + if opts.skip_decommissioned && self.is_suspended(pool.index).await { + continue; + } + + if opts.skip_rebalancing && self.is_pool_rebalancing(pool.index).await { continue; } - // TODO:SkipRebalancing - // if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) { - // continue - // } if let Some(err) = &pool.err { if is_err_read_quorum(err) { @@ -1865,7 +1865,9 @@ impl StorageAPI for ECStore { } for pool in self.pools.iter() { - // TODO: IsSuspended + if self.is_suspended(pool.pool_idx).await { + continue; + } let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await { Ok(res) => return Ok(res), Err(err) => { @@ -1915,6 +1917,9 @@ impl StorageAPI for ECStore { let mut uploads = Vec::new(); for pool in self.pools.iter() { + if self.is_suspended(pool.pool_idx).await { + continue; + } let res = pool .list_multipart_uploads( bucket, @@ -1948,7 +1953,9 @@ impl StorageAPI for ECStore { } for (idx, pool) in self.pools.iter().enumerate() { - // // TODO: IsSuspended + if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await { + continue; + } let res = pool .list_multipart_uploads(bucket, object, None, None, None, MAX_UPLOADS_LIST) .await?; @@ -1983,8 +1990,8 @@ impl StorageAPI for ECStore { return self.pools[0].get_multipart_info(bucket, object, upload_id, opts).await; } - for (idx, pool) in self.pools.iter().enumerate() { - if self.is_suspended(idx).await { + for pool in self.pools.iter() { + if self.is_suspended(pool.pool_idx).await { continue; } @@ -2017,7 +2024,9 @@ impl StorageAPI for ECStore { } for pool in self.pools.iter() { - // TODO: IsSuspended + if self.is_suspended(pool.pool_idx).await { + continue; + } let err = match pool.abort_multipart_upload(bucket, object, upload_id, opts).await { Ok(_) => return Ok(()), @@ -2060,7 +2069,9 @@ impl StorageAPI for ECStore { } for pool in self.pools.iter() { - // TODO: IsSuspended + if self.is_suspended(pool.pool_idx).await { + continue; + } let err = match pool .complete_multipart_upload(bucket, object, upload_id, uploaded_parts.clone(), opts) @@ -2549,6 +2560,7 @@ fn check_abort_multipart_args(bucket: &str, object: &str, upload_id: &str) -> Re check_multipart_object_args(bucket, object, upload_id) } +#[tracing::instrument(level = "debug")] fn check_put_object_args(bucket: &str, object: &str) -> Result<()> { if !is_meta_bucketname(bucket) && check_valid_bucket_name_strict(bucket).is_err() { return Err(Error::new(StorageError::BucketNameInvalid(bucket.to_string()))); diff --git a/ecstore/src/store_list_objects.rs b/ecstore/src/store_list_objects.rs index 8b9118de3..445c56f0c 100644 --- a/ecstore/src/store_list_objects.rs +++ b/ecstore/src/store_list_objects.rs @@ -778,7 +778,7 @@ impl ECStore { let value = tx2.clone(); let resolver = resolver.clone(); async move { - if let Ok(Some(entry)) = entries.resolve(resolver) { + if let Some(entry) = entries.resolve(resolver) { if let Err(err) = value.send(entry).await { error!("list_path send fail {:?}", err); } @@ -1296,7 +1296,7 @@ impl SetDisks { let value = tx2.clone(); let resolver = resolver.clone(); async move { - if let Ok(Some(entry)) = entries.resolve(resolver) { + if let Some(entry) = entries.resolve(resolver) { if let Err(err) = value.send(entry).await { error!("list_path send fail {:?}", err); } diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index 463989c0f..d8110ca63 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -106,14 +106,24 @@ pub async fn access(path: impl AsRef) -> io::Result<()> { Ok(()) } +pub fn access_std(path: impl AsRef) -> io::Result<()> { + std::fs::metadata(path)?; + Ok(()) +} + pub async fn lstat(path: impl AsRef) -> io::Result { fs::metadata(path).await } +pub fn lstat_std(path: impl AsRef) -> io::Result { + std::fs::metadata(path) +} + pub async fn make_dir_all(path: impl AsRef) -> io::Result<()> { fs::create_dir_all(path.as_ref()).await } +#[tracing::instrument(level = "debug", skip_all)] pub async fn remove(path: impl AsRef) -> io::Result<()> { let meta = fs::metadata(path.as_ref()).await?; if meta.is_dir() { @@ -132,6 +142,25 @@ pub async fn remove_all(path: impl AsRef) -> io::Result<()> { } } +#[tracing::instrument(level = "debug", skip_all)] +pub fn remove_std(path: impl AsRef) -> io::Result<()> { + let meta = std::fs::metadata(path.as_ref())?; + if meta.is_dir() { + std::fs::remove_dir(path.as_ref()) + } else { + std::fs::remove_file(path.as_ref()) + } +} + +pub fn remove_all_std(path: impl AsRef) -> io::Result<()> { + let meta = std::fs::metadata(path.as_ref())?; + if meta.is_dir() { + std::fs::remove_dir_all(path.as_ref()) + } else { + std::fs::remove_file(path.as_ref()) + } +} + pub async fn mkdir(path: impl AsRef) -> io::Result<()> { fs::create_dir(path.as_ref()).await } @@ -140,6 +169,11 @@ pub async fn rename(from: impl AsRef, to: impl AsRef) -> io::Result< fs::rename(from, to).await } +pub fn rename_std(from: impl AsRef, to: impl AsRef) -> io::Result<()> { + std::fs::rename(from, to) +} + +#[tracing::instrument(level = "debug", skip_all)] pub async fn read_file(path: impl AsRef) -> io::Result> { fs::read(path.as_ref()).await } diff --git a/ecstore/src/utils/path.rs b/ecstore/src/utils/path.rs index 0e38eac2f..0c63b960a 100644 --- a/ecstore/src/utils/path.rs +++ b/ecstore/src/utils/path.rs @@ -52,7 +52,7 @@ pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool { s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix)) } -pub fn has_profix(s: &str, prefix: &str) -> bool { +pub fn has_prefix(s: &str, prefix: &str) -> bool { if cfg!(target_os = "windows") { return strings_has_prefix_fold(s, prefix); } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 3019c060e..a238665d9 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -15,6 +15,8 @@ path = "src/main.rs" workspace = true [dependencies] +zip = { workspace = true } +tokio-tar = { workspace = true } madmin = { workspace = true } api = { workspace = true } appauth = { workspace = true } @@ -30,7 +32,7 @@ clap.workspace = true crypto = { workspace = true } datafusion = { workspace = true } common.workspace = true -const-str = { version = "0.6.1", features = ["std", "proc"] } +const-str = { workspace = true } ecstore.workspace = true policy.workspace = true flatbuffers.workspace = true @@ -42,7 +44,6 @@ http.workspace = true http-body.workspace = true iam = { workspace = true } lock.workspace = true -local-ip-address = { workspace = true } matchit = { workspace = true } mime.workspace = true mime_guess = { workspace = true } @@ -51,17 +52,18 @@ pin-project-lite.workspace = true protos.workspace = true query = { workspace = true } rmp-serde.workspace = true +rustfs-config = { workspace = true } rustfs-event-notifier = { workspace = true } rustfs-obs = { workspace = true } +rustfs-utils = { workspace = true, features = ["full"] } rustls.workspace = true -rustls-pemfile.workspace = true -rustls-pki-types.workspace = true rust-embed = { workspace = true, features = ["interpolate-folder-path"] } s3s.workspace = true serde.workspace = true serde_json.workspace = true serde_urlencoded = { workspace = true } shadow-rs = { workspace = true, features = ["build", "metadata"] } +socket2 = { workspace = true } tracing.workspace = true time = { workspace = true, features = ["parsing", "formatting", "serde"] } tokio-util.workspace = true @@ -77,7 +79,12 @@ tokio-stream.workspace = true tonic = { workspace = true } tower.workspace = true transform-stream.workspace = true -tower-http = { workspace = true, features = ["trace", "compression-deflate", "compression-gzip", "cors"] } +tower-http = { workspace = true, features = [ + "trace", + "compression-deflate", + "compression-gzip", + "cors", +] } uuid = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] diff --git a/rustfs/README.md b/rustfs/README.md index 86b2ab767..147e7ed7e 100644 --- a/rustfs/README.md +++ b/rustfs/README.md @@ -1,30 +1,36 @@ -rustfs/ -├── Cargo.toml -├── src/ -│ ├── main.rs # 主入口 -│ ├── admin/ -│ │ └── mod.rs # 管理接口 -│ ├── auth/ -│ │ └── mod.rs # 认证模块 -│ ├── config/ -│ │ ├── mod.rs # 配置模块 -│ │ └── options.rs # 命令行参数 -│ ├── console/ -│ │ ├── mod.rs # 控制台模块 -│ │ └── server.rs # 控制台服务器 -│ ├── grpc/ -│ │ └── mod.rs # gRPC 服务 -│ ├── license/ -│ │ └── mod.rs # 许可证管理 -│ ├── logging/ -│ │ └── mod.rs # 日志管理 -│ ├── server/ -│ │ ├── mod.rs # 服务器实现 -│ │ ├── connection.rs # 连接处理 -│ │ ├── service.rs # 服务实现 -│ │ └── state.rs # 状态管理 -│ ├── storage/ -│ │ ├── mod.rs # 存储模块 -│ │ └── fs.rs # 文件系统实现 -│ └── utils/ -│ └── mod.rs # 工具函数 \ No newline at end of file +# RustFS + +RustFS is a simple file system written in Rust. It is designed to be a learning project for those who want to understand +how file systems work and how to implement them in Rust. + +## Features + +- Simple file system structure +- Basic file operations (create, read, write, delete) +- Directory support +- File metadata (size, creation time, etc.) +- Basic error handling +- Unit tests for core functionality +- Documentation for public API +- Example usage +- License information +- Contributing guidelines +- Changelog +- Code of conduct +- Acknowledgements +- Contact information +- Links to additional resources + +## Getting Started + +To get started with RustFS, clone the repository and build the project: + +```bash +git clone git@github.com:rustfs/s3-rustfs.git +cd rustfs +cargo build +``` + +## Usage + +To use RustFS, you can create a new file system instance and perform basic file operations. Here is an example: diff --git a/rustfs/src/admin/handlers.rs b/rustfs/src/admin/handlers.rs index dfeeeb876..45ee477a8 100644 --- a/rustfs/src/admin/handlers.rs +++ b/rustfs/src/admin/handlers.rs @@ -48,6 +48,7 @@ use tokio::{select, spawn}; use tokio_stream::wrappers::ReceiverStream; use tracing::{error, info, warn}; +pub mod event; pub mod group; pub mod policys; pub mod pools; diff --git a/rustfs/src/admin/handlers/event.rs b/rustfs/src/admin/handlers/event.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/rustfs/src/admin/handlers/event.rs @@ -0,0 +1 @@ + diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index 57293d4ea..f0229f201 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -25,7 +25,7 @@ pub fn make_admin_route() -> Result { r.insert(Method::POST, "/", AdminOperation(&sts::AssumeRoleHandle {}))?; regist_rpc_route(&mut r)?; - regist_user_route(&mut r)?; + register_user_route(&mut r)?; r.insert( Method::POST, @@ -124,7 +124,7 @@ pub fn make_admin_route() -> Result { Ok(r) } -fn regist_user_route(r: &mut S3Router) -> Result<()> { +fn register_user_route(r: &mut S3Router) -> Result<()> { // 1 r.insert( Method::GET, diff --git a/rustfs/src/config/mod.rs b/rustfs/src/config/mod.rs index 7a7bdc4c4..a93b6ddaf 100644 --- a/rustfs/src/config/mod.rs +++ b/rustfs/src/config/mod.rs @@ -1,40 +1,8 @@ use clap::Parser; use const_str::concat; -use ecstore::global::DEFAULT_PORT; use std::string::ToString; shadow_rs::shadow!(build); -/// Default Access Key -/// Default value: rustfsadmin -/// Environment variable: RUSTFS_ACCESS_KEY -/// Command line argument: --access-key -/// Example: RUSTFS_ACCESS_KEY=rustfsadmin -/// Example: --access-key rustfsadmin -pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin"; -/// Default Secret Key -/// Default value: rustfsadmin -/// Environment variable: RUSTFS_SECRET_KEY -/// Command line argument: --secret-key -/// Example: RUSTFS_SECRET_KEY=rustfsadmin -/// Example: --secret-key rustfsadmin -pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; -/// Default configuration file for observability -/// Default value: config/obs.toml -/// Environment variable: RUSTFS_OBS_CONFIG -/// Command line argument: --obs-config -/// Example: RUSTFS_OBS_CONFIG=config/obs.toml -/// Example: --obs-config config/obs.toml -/// Example: --obs-config /etc/rustfs/obs.toml -pub const DEFAULT_OBS_CONFIG: &str = "config/obs.toml"; - -/// Default TLS key for rustfs -/// This is the default key for TLS. -pub(crate) const RUSTFS_TLS_KEY: &str = "rustfs_key.pem"; - -/// Default TLS cert for rustfs -/// This is the default cert for TLS. -pub(crate) const RUSTFS_TLS_CERT: &str = "rustfs_cert.pem"; - #[allow(clippy::const_is_empty)] const SHORT_VERSION: &str = { if !build::TAG.is_empty() { @@ -67,7 +35,7 @@ pub struct Opt { pub volumes: Vec, /// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname - #[arg(long, default_value_t = format!("0.0.0.0:{}", DEFAULT_PORT), env = "RUSTFS_ADDRESS")] + #[arg(long, default_value_t = rustfs_config::DEFAULT_ADDRESS.to_string(), env = "RUSTFS_ADDRESS")] pub address: String, /// Domain name used for virtual-hosted-style requests. @@ -75,17 +43,19 @@ pub struct Opt { pub server_domains: Vec, /// Access key used for authentication. - #[arg(long, default_value_t = DEFAULT_ACCESS_KEY.to_string(), env = "RUSTFS_ACCESS_KEY")] + #[arg(long, default_value_t = rustfs_config::DEFAULT_ACCESS_KEY.to_string(), env = "RUSTFS_ACCESS_KEY")] pub access_key: String, /// Secret key used for authentication. - #[arg(long, default_value_t = DEFAULT_SECRET_KEY.to_string(), env = "RUSTFS_SECRET_KEY")] + #[arg(long, default_value_t = rustfs_config::DEFAULT_SECRET_KEY.to_string(), env = "RUSTFS_SECRET_KEY")] pub secret_key: String, + /// Enable console server #[arg(long, default_value_t = false, env = "RUSTFS_CONSOLE_ENABLE")] pub console_enable: bool, - #[arg(long, default_value_t = format!("127.0.0.1:{}", 9002), env = "RUSTFS_CONSOLE_ADDRESS")] + /// Console server bind address + #[arg(long, default_value_t = rustfs_config::DEFAULT_CONSOLE_ADDRESS.to_string(), env = "RUSTFS_CONSOLE_ADDRESS")] pub console_address: String, /// rustfs endpoint for console @@ -94,7 +64,7 @@ pub struct Opt { /// Observability configuration file /// Default value: config/obs.toml - #[arg(long, default_value_t = DEFAULT_OBS_CONFIG.to_string(), env = "RUSTFS_OBS_CONFIG")] + #[arg(long, default_value_t = rustfs_config::DEFAULT_OBS_CONFIG.to_string(), env = "RUSTFS_OBS_CONFIG")] pub obs_config: String, /// tls path for rustfs api and console. diff --git a/rustfs/src/console.rs b/rustfs/src/console.rs index c3edbaaa1..0523991d6 100644 --- a/rustfs/src/console.rs +++ b/rustfs/src/console.rs @@ -1,4 +1,3 @@ -use crate::config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use crate::license::get_license; use axum::{ body::Body, @@ -8,6 +7,7 @@ use axum::{ Router, }; use axum_extra::extract::Host; +use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use std::io; use axum::response::Redirect; @@ -17,7 +17,7 @@ use mime_guess::from_path; use rust_embed::RustEmbed; use serde::Serialize; use shadow_rs::shadow; -use std::net::{Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, SocketAddr}; use std::sync::OnceLock; use std::time::Duration; use tokio::signal; @@ -33,7 +33,8 @@ const RUSTFS_ADMIN_PREFIX: &str = "/rustfs/admin/v3"; #[folder = "$CARGO_MANIFEST_DIR/static"] struct StaticFiles; -async fn static_handler(uri: axum::http::Uri) -> impl IntoResponse { +/// Static file handler +async fn static_handler(uri: Uri) -> impl IntoResponse { let mut path = uri.path().trim_start_matches('/'); if path.is_empty() { path = "index.html" @@ -72,7 +73,7 @@ pub(crate) struct Config { } impl Config { - fn new(local_ip: Ipv4Addr, port: u16, version: &str, date: &str) -> Self { + fn new(local_ip: IpAddr, port: u16, version: &str, date: &str) -> Self { Config { port, api: Api { @@ -143,7 +144,7 @@ struct License { pub(crate) static CONSOLE_CONFIG: OnceLock = OnceLock::new(); #[allow(clippy::const_is_empty)] -pub(crate) fn init_console_cfg(local_ip: Ipv4Addr, port: u16) { +pub(crate) fn init_console_cfg(local_ip: IpAddr, port: u16) { CONSOLE_CONFIG.get_or_init(|| { let ver = { if !build::TAG.is_empty() { @@ -193,7 +194,7 @@ fn _is_private_ip(ip: std::net::IpAddr) -> bool { async fn config_handler(uri: Uri, Host(host): Host) -> impl IntoResponse { let scheme = uri.scheme().map(|s| s.as_str()).unwrap_or("http"); - // 从 uri 中获取 host，如果没有则使用 Host extractor 的值 + // Get the host from the uri and use the value of the host extractor if it doesn't have one let host = uri.host().unwrap_or(host.as_str()); let host = if host.contains(':') { @@ -203,7 +204,7 @@ async fn config_handler(uri: Uri, Host(host): Host) -> impl IntoResponse { host }; - // 将当前配置复制一份 + // Make a copy of the current configuration let mut cfg = CONSOLE_CONFIG.get().unwrap().clone(); let url = format!("{}://{}:{}", scheme, host, cfg.port); @@ -219,14 +220,14 @@ async fn config_handler(uri: Uri, Host(host): Host) -> impl IntoResponse { pub async fn start_static_file_server( addrs: &str, - local_ip: Ipv4Addr, + local_ip: IpAddr, access_key: &str, secret_key: &str, tls_path: Option, ) { - // 配置 CORS + // Configure CORS let cors = CorsLayer::new() - .allow_origin(Any) // 生产环境建议指定具体域名 + .allow_origin(Any) // In the production environment, we recommend that you specify a specific domain name .allow_methods([http::Method::GET, http::Method::POST]) .allow_headers([header::CONTENT_TYPE]); // Create a route @@ -298,7 +299,7 @@ async fn start_server(server_addr: SocketAddr, tls_path: Option, app: Ro } #[allow(dead_code)] -/// HTTP 到 HTTPS 的 308 重定向 +/// 308 redirect for HTTP to HTTPS fn redirect_to_https(https_port: u16) -> Router { Router::new().route( "/*path", diff --git a/rustfs/src/event.rs b/rustfs/src/event.rs new file mode 100644 index 000000000..99e2a75c3 --- /dev/null +++ b/rustfs/src/event.rs @@ -0,0 +1,21 @@ +use rustfs_event_notifier::NotifierConfig; +use tracing::{error, info, instrument}; + +#[instrument] +pub(crate) async fn init_event_notifier(notifier_config: Option) { + // Initialize event notifier + if notifier_config.is_some() { + info!("event_config is not empty"); + tokio::spawn(async move { + let config = NotifierConfig::event_load_config(notifier_config); + let result = rustfs_event_notifier::initialize(config).await; + if let Err(e) = result { + error!("Failed to initialize event notifier: {}", e); + } else { + info!("Event notifier initialized successfully"); + } + }); + } else { + info!("event_config is empty"); + } +} diff --git a/rustfs/src/license.rs b/rustfs/src/license.rs index 25d09c82c..2206c40e2 100644 --- a/rustfs/src/license.rs +++ b/rustfs/src/license.rs @@ -10,6 +10,7 @@ lazy_static::lazy_static! { static ref LICENSE: OnceLock = OnceLock::new(); } +/// Initialize the license pub fn init_license(license: Option) { if license.is_none() { error!("License is None"); @@ -23,10 +24,13 @@ pub fn init_license(license: Option) { }); } +/// Get the license pub fn get_license() -> Option { LICENSE.get().cloned() } +/// Check the license +/// This function checks if the license is valid. #[allow(unreachable_code)] pub fn license_check() -> Result<()> { return Ok(()); diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 7871fba58..bf94a6f6b 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -2,18 +2,18 @@ mod admin; mod auth; mod config; mod console; +mod event; mod grpc; pub mod license; mod logging; mod server; mod service; mod storage; -mod utils; + use crate::auth::IAMAuth; use crate::console::{init_console_cfg, CONSOLE_CONFIG}; // Ensure the correct path for parse_license is imported use crate::server::{wait_for_shutdown, ServiceState, ServiceStateManager, ShutdownSignal, SHUTDOWN_TIMEOUT}; -use crate::utils::error; use bytes::Bytes; use chrono::Datelike; use clap::Parser; @@ -21,7 +21,6 @@ use common::{ error::{Error, Result}, globals::set_global_addr, }; -use config::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use ecstore::bucket::metadata_sys::init_bucket_metadata_sys; use ecstore::config as ecconfig; use ecstore::config::GLOBAL_ConfigSys; @@ -48,11 +47,12 @@ use hyper_util::{ use iam::init_iam_sys; use license::init_license; use protos::proto_gen::node_service::node_service_server::NodeServiceServer; -use rustfs_event_notifier::NotifierConfig; +use rustfs_config::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; use rustfs_obs::{init_obs, init_process_observer, load_config, set_global_guard}; use rustls::ServerConfig; use s3s::{host::MultiDomain, service::S3ServiceBuilder}; use service::hybrid; +use socket2::SockRef; use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; @@ -62,8 +62,10 @@ use tokio_rustls::TlsAcceptor; use tonic::{metadata::MetadataValue, Request, Status}; use tower_http::cors::CorsLayer; use tower_http::trace::TraceLayer; -use tracing::Span; -use tracing::{debug, error, info, info_span, warn}; +use tracing::{debug, error, info, warn}; +use tracing::{instrument, Span}; + +const MI_B: usize = 1024 * 1024; #[cfg(all(target_os = "linux", target_env = "gnu"))] #[global_allocator] @@ -77,12 +79,12 @@ fn check_auth(req: Request<()>) -> Result, Status> { _ => Err(Status::unauthenticated("No valid auth token")), } } - +#[instrument] fn print_server_info() { let cfg = CONSOLE_CONFIG.get().unwrap(); let current_year = chrono::Utc::now().year(); - // 使用自定义宏打印服务器信息 + // Use custom macros to print server information info!("RustFS Object Storage Server"); info!("Copyright: 2024-{} RustFS, Inc", current_year); info!("License: {}", cfg.license()); @@ -95,8 +97,7 @@ async fn main() -> Result<()> { // Parse the obtained parameters let opt = config::Opt::parse(); - // config::init_config(opt.clone()); - + // Initialize the configuration init_license(opt.license.clone()); // Load the configuration file @@ -108,50 +109,34 @@ async fn main() -> Result<()> { // Store in global storage set_global_guard(guard)?; - // Initialize event notifier - let notifier_config = opt.clone().event_config; - if notifier_config.is_some() { - info!("event_config is not empty"); - tokio::spawn(async move { - let config = NotifierConfig::event_load_config(notifier_config); - let result = rustfs_event_notifier::initialize(config).await; - if let Err(e) = result { - error!("Failed to initialize event notifier: {}", e); - } else { - info!("Event notifier initialized successfully"); - } - }); - } else { - info!("event_config is empty"); - } - // Run parameters run(opt).await } -// #[tokio::main] +#[instrument(skip(opt))] async fn run(opt: config::Opt) -> Result<()> { - let span = info_span!("trace-main-run"); - let _enter = span.enter(); - debug!("opt: {:?}", &opt); + // Initialize event notifier + event::init_event_notifier(opt.event_config).await; + let server_addr = net::parse_and_resolve_address(opt.address.as_str())?; let server_port = server_addr.port(); let server_address = server_addr.to_string(); debug!("server_address {}", &server_address); - //设置 AK 和 SK + // Set up AK and SK iam::init_global_action_cred(Some(opt.access_key.clone()), Some(opt.secret_key.clone()))?; set_global_rustfs_port(server_port); - //监听地址，端口从参数中获取 + // The listening address and port are obtained from the parameters let listener = TcpListener::bind(server_address.clone()).await?; - //获取监听地址 + // Obtain the listener address let local_addr: SocketAddr = listener.local_addr()?; - let local_ip = utils::get_local_ip().ok_or(local_addr.ip()).unwrap(); + // let local_ip = utils::get_local_ip().ok_or(local_addr.ip()).unwrap(); + let local_ip = rustfs_utils::get_local_ip().ok_or(local_addr.ip()).unwrap(); // 用于 rpc let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address.clone().as_str(), opt.volumes.clone()) @@ -175,7 +160,7 @@ async fn run(opt: config::Opt) -> Result<()> { // Detailed endpoint information (showing all API endpoints) let api_endpoints = format!("http://{}:{}", local_ip, server_port); let localhost_endpoint = format!("http://127.0.0.1:{}", server_port); - info!("API: {} {}", api_endpoints, localhost_endpoint); + info!(" API: {} {}", api_endpoints, localhost_endpoint); info!(" RootUser: {}", opt.access_key.clone()); info!(" RootPass: {}", opt.secret_key.clone()); if DEFAULT_ACCESS_KEY.eq(&opt.access_key) && DEFAULT_SECRET_KEY.eq(&opt.secret_key) { @@ -198,13 +183,13 @@ async fn run(opt: config::Opt) -> Result<()> { set_global_endpoints(endpoint_pools.as_ref().clone()); update_erasure_type(setup_type).await; - // 初始化本地磁盘 + // Initialize the local disk init_local_disks(endpoint_pools.clone()) .await .map_err(|err| Error::from_string(err.to_string()))?; // Setup S3 service - // 本项目使用 s3s 库来实现 s3 服务 + // This project uses the S3S library to implement S3 services let s3_service = { let store = storage::ecfs::FS::new(); // let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(server_address.clone(), endpoint_pools).await?); @@ -212,7 +197,7 @@ async fn run(opt: config::Opt) -> Result<()> { let access_key = opt.access_key.clone(); let secret_key = opt.secret_key.clone(); - //显示 info 信息 + // Displays info information debug!("authentication is enabled {}, {}", &access_key, &secret_key); b.set_auth(IAMAuth::new(access_key, secret_key)); @@ -263,7 +248,7 @@ async fn run(opt: config::Opt) -> Result<()> { debug!("Found TLS directory, checking for certificates"); // 1. Try to load all certificates directly (including root and subdirectories) - match utils::load_all_certs_from_directory(&tls_path) { + match rustfs_utils::load_all_certs_from_directory(&tls_path) { Ok(cert_key_pairs) if !cert_key_pairs.is_empty() => { debug!("Found {} certificates, starting with HTTPS", cert_key_pairs.len()); let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -271,7 +256,7 @@ async fn run(opt: config::Opt) -> Result<()> { // create a multi certificate configuration let mut server_config = ServerConfig::builder() .with_no_client_auth() - .with_cert_resolver(Arc::new(utils::create_multi_cert_resolver(cert_key_pairs)?)); + .with_cert_resolver(Arc::new(rustfs_utils::create_multi_cert_resolver(cert_key_pairs)?)); server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()]; Some(TlsAcceptor::from(Arc::new(server_config))) @@ -286,12 +271,14 @@ async fn run(opt: config::Opt) -> Result<()> { if has_single_cert { debug!("Found legacy single TLS certificate, starting with HTTPS"); let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - let certs = utils::load_certs(cert_path.as_str()).map_err(|e| error(e.to_string()))?; - let key = utils::load_private_key(key_path.as_str()).map_err(|e| error(e.to_string()))?; + let certs = + rustfs_utils::load_certs(cert_path.as_str()).map_err(|e| rustfs_utils::certs_error(e.to_string()))?; + let key = rustfs_utils::load_private_key(key_path.as_str()) + .map_err(|e| rustfs_utils::certs_error(e.to_string()))?; let mut server_config = ServerConfig::builder() .with_no_client_auth() .with_single_cert(certs, key) - .map_err(|e| error(e.to_string()))?; + .map_err(|e| rustfs_utils::certs_error(e.to_string()))?; server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()]; Some(TlsAcceptor::from(Arc::new(server_config))) } else { @@ -339,7 +326,7 @@ async fn run(opt: config::Opt) -> Result<()> { .layer( TraceLayer::new_for_http() .make_span_with(|request: &HttpRequest<_>| { - let span = tracing::debug_span!("http-request", + let span = tracing::info_span!("http-request", status_code = tracing::field::Empty, method = %request.method(), uri = %request.uri(), @@ -368,13 +355,14 @@ async fn run(opt: config::Opt) -> Result<()> { debug!("http response generated in {:?}", latency) }) .on_body_chunk(|chunk: &Bytes, latency: Duration, _span: &Span| { - info!(histogram.request.body.len = chunk.len(), "histogram request body lenght",); + info!(histogram.request.body.len = chunk.len(), "histogram request body length",); debug!("http body sending {} bytes in {:?}", chunk.len(), latency) }) .on_eos(|_trailers: Option<&HeaderMap>, stream_duration: Duration, _span: &Span| { debug!("http stream closed after {:?}", stream_duration) }) .on_failure(|_error, latency: Duration, _span: &Span| { + info!(counter.rustfs_api_requests_failure_total = 1_u64, "handle request api failure total"); debug!("http request failure error: {:?} in {:?}", _error, latency) }), ) @@ -425,15 +413,22 @@ async fn run(opt: config::Opt) -> Result<()> { } }; - if let Err(err) = socket.set_nodelay(true) { + let socket_ref = SockRef::from(&socket); + if let Err(err) = socket_ref.set_nodelay(true) { warn!(?err, "Failed to set TCP_NODELAY"); } + if let Err(err) = socket_ref.set_recv_buffer_size(4 * MI_B) { + warn!(?err, "Failed to set set_recv_buffer_size"); + } + if let Err(err) = socket_ref.set_send_buffer_size(4 * MI_B) { + warn!(?err, "Failed to set set_send_buffer_size"); + } if has_tls_certs { debug!("TLS certificates found, starting with SIGINT"); let tls_socket = match tls_acceptor .as_ref() - .ok_or_else(|| error("TLS not configured".to_string())) + .ok_or_else(|| rustfs_utils::certs_error("TLS not configured".to_string())) .unwrap() .accept(socket) .await diff --git a/rustfs/src/server/service_state.rs b/rustfs/src/server/service_state.rs index ad137f66d..5390fb1e4 100644 --- a/rustfs/src/server/service_state.rs +++ b/rustfs/src/server/service_state.rs @@ -129,7 +129,7 @@ impl Default for ServiceStateManager { } } -// 使用示例 +// Example of use #[cfg(test)] mod tests { use super::*; @@ -138,18 +138,18 @@ mod tests { fn test_service_state_manager() { let manager = ServiceStateManager::new(); - // 初始状态应该是 Starting + // The initial state should be Starting assert_eq!(manager.current_state(), ServiceState::Starting); - // 更新状态到 Ready + // Update the status to Ready manager.update(ServiceState::Ready); assert_eq!(manager.current_state(), ServiceState::Ready); - // 更新状态到 Stopping + // Update the status to Stopping manager.update(ServiceState::Stopping); assert_eq!(manager.current_state(), ServiceState::Stopping); - // 更新状态到 Stopped + // Update the status to Stopped manager.update(ServiceState::Stopped); assert_eq!(manager.current_state(), ServiceState::Stopped); } diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index e2cad0c07..3589923f8 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -20,6 +20,7 @@ pub(crate) struct ReqInfo { pub version_id: Option, } +/// Authorizes the request based on the action and credentials. pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3Result<()> { let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 252cd316f..418ecf077 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -4,6 +4,10 @@ use super::options::extract_metadata; use super::options::put_opts; use crate::auth::get_condition_values; use crate::storage::access::ReqInfo; +use crate::storage::error::to_s3_error; +use crate::storage::options::copy_dst_opts; +use crate::storage::options::copy_src_opts; +use crate::storage::options::{extract_metadata_from_mime, get_opts}; use api::query::Context; use api::query::Query; use api::server::dbms::DatabaseManagerSystem; @@ -61,9 +65,12 @@ use s3s::S3Result; use s3s::S3; use s3s::{S3Request, S3Response}; use std::fmt::Debug; +use std::path::Path; use std::str::FromStr; +use std::sync::Arc; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; +use tokio_tar::Archive; use tokio_util::io::ReaderStream; use tokio_util::io::StreamReader; use tracing::debug; @@ -72,11 +79,7 @@ use tracing::info; use tracing::warn; use transform_stream::AsyncTryStream; use uuid::Uuid; - -use crate::storage::error::to_s3_error; -use crate::storage::options::copy_dst_opts; -use crate::storage::options::copy_src_opts; -use crate::storage::options::{extract_metadata_from_mime, get_opts}; +use zip::CompressionFormat; macro_rules! try_ { ($result:expr) => { @@ -106,6 +109,108 @@ impl FS { // let store: ECStore = ECStore::new(address, endpoint_pools).await?; Self {} } + + async fn put_object_extract(&self, req: S3Request) -> S3Result> { + let PutObjectInput { body, bucket, key, .. } = req.input; + + let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; + + let body = StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))); + + // let etag_stream = EtagReader::new(body); + + let Some(ext) = Path::new(&key).extension().and_then(|s| s.to_str()) else { + return Err(s3_error!(InvalidArgument, "key extension not found")); + }; + + let ext = ext.to_owned(); + + // TODO: spport zip + let decoder = CompressionFormat::from_extension(&ext).get_decoder(body).map_err(|e| { + error!("get_decoder err {:?}", e); + s3_error!(InvalidArgument, "get_decoder err") + })?; + + let mut ar = Archive::new(decoder); + let mut entries = ar.entries().map_err(|e| { + error!("get entries err {:?}", e); + s3_error!(InvalidArgument, "get entries err") + })?; + + let Some(store) = new_object_layer_fn() else { + return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); + }; + + let prefix = req + .headers + .get("X-Amz-Meta-RustFs-Snowball-Prefix") + .map(|v| v.to_str().unwrap_or_default()) + .unwrap_or_default(); + + while let Some(entry) = entries.next().await { + let f = match entry { + Ok(f) => f, + Err(e) => { + println!("Error reading entry: {}", e); + return Err(s3_error!(InvalidArgument, "Error reading entry {:?}", e)); + } + }; + + if f.header().entry_type().is_dir() { + continue; + } + + if let Ok(fpath) = f.path() { + let mut fpath = fpath.to_string_lossy().to_string(); + + if !prefix.is_empty() { + fpath = format!("{}/{}", prefix, fpath); + } + + let size = f.header().size().unwrap_or_default() as usize; + + println!("Extracted: {}, size {}", fpath, size); + + let mut reader = PutObjReader::new(Box::new(f), size); + + let _obj_info = store + .put_object(&bucket, &fpath, &mut reader, &ObjectOptions::default()) + .await + .map_err(to_s3_error)?; + + // let e_tag = obj_info.etag; + + // // store.put_object(bucket, object, data, opts); + + // let output = PutObjectOutput { + // e_tag, + // ..Default::default() + // }; + } + } + + // match decompress( + // body, + // CompressionFormat::from_extension(&ext), + // |entry: tokio_tar::Entry>>| async move { + // let path = entry.path().unwrap(); + // println!("Extracted: {}", path.display()); + // Ok(()) + // }, + // ) + // .await + // { + // Ok(_) => println!("解压成功！"), + // Err(e) => println!("解压失败: {}", e), + // } + + // TODO: etag + let output = PutObjectOutput { + // e_tag: Some(etag_stream.etag().await), + ..Default::default() + }; + Ok(S3Response::new(output)) + } } #[async_trait::async_trait] impl S3 for FS { @@ -141,6 +246,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Copy an object from one location to another #[tracing::instrument(level = "debug", skip(self, req))] async fn copy_object(&self, req: S3Request) -> S3Result> { let CopyObjectInput { @@ -227,6 +333,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Delete a bucket #[tracing::instrument(level = "debug", skip(self, req))] async fn delete_bucket(&self, req: S3Request) -> S3Result> { let input = req.input; @@ -249,6 +356,7 @@ impl S3 for FS { Ok(S3Response::new(DeleteBucketOutput {})) } + /// Delete an object #[tracing::instrument(level = "debug", skip(self, req))] async fn delete_object(&self, req: S3Request) -> S3Result> { let DeleteObjectInput { @@ -308,6 +416,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Delete multiple objects #[tracing::instrument(level = "debug", skip(self, req))] async fn delete_objects(&self, req: S3Request) -> S3Result> { // info!("delete_objects args {:?}", req.input); @@ -367,6 +476,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Get bucket location #[tracing::instrument(level = "debug", skip(self, req))] async fn get_bucket_location(&self, req: S3Request) -> S3Result> { // mc get 1 @@ -385,6 +495,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + /// Get bucket notification #[tracing::instrument( level = "debug", skip(self, req), @@ -402,6 +513,8 @@ impl S3 for FS { .. } = req.input; + // TODO: getObjectInArchiveFileHandler object = xxx.zip/xxx/xxx.xxx + // let range = HTTPRangeSpec::nil(); let h = HeaderMap::new(); @@ -797,8 +910,16 @@ impl S3 for FS { Ok(S3Response::new(output)) } - #[tracing::instrument(level = "debug", skip(self, req))] + // #[tracing::instrument(level = "debug", skip(self, req))] async fn put_object(&self, req: S3Request) -> S3Result> { + if req + .headers + .get("X-Amz-Meta-Snowball-Auto-Extract") + .is_some_and(|v| v.to_str().unwrap_or_default() == "true") + { + return self.put_object_extract(req).await; + } + let input = req.input; if let Some(ref storage_class) = input.storage_class { @@ -1890,14 +2011,14 @@ impl S3 for FS { ) -> S3Result> { info!("handle select_object_content"); - let input = req.input; + let input = Arc::new(req.input); info!("{:?}", input); let db = make_rustfsms(input.clone(), false).await.map_err(|e| { error!("make db failed, {}", e.to_string()); s3_error!(InternalError, "{}", e.to_string()) })?; - let query = Query::new(Context { input: input.clone() }, input.request.expression); + let query = Query::new(Context { input: input.clone() }, input.request.expression.clone()); let result = db .execute(&query) .await diff --git a/rustfs/src/storage/event_notifier.rs b/rustfs/src/storage/event_notifier.rs new file mode 100644 index 000000000..292b45e31 --- /dev/null +++ b/rustfs/src/storage/event_notifier.rs @@ -0,0 +1,17 @@ +use rustfs_event_notifier::{Event, Metadata}; + +/// Create a new metadata object +#[allow(dead_code)] +pub(crate) fn create_metadata() -> Metadata { + // Create a new metadata object + let mut metadata = Metadata::new(); + metadata.set_configuration_id("test-config".to_string()); + // Return the created metadata object + metadata +} + +/// Create a new event object +#[allow(dead_code)] +pub(crate) async fn send_event(event: Event) -> Result<(), Box> { + rustfs_event_notifier::send_event(event).await.map_err(|e| e.into()) +} diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index d7d9d87ea..2f8ec8b88 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -1,4 +1,5 @@ pub mod access; pub mod ecfs; pub mod error; +mod event_notifier; pub mod options; diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 18a13974a..ae0df5391 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -8,6 +8,7 @@ use lazy_static::lazy_static; use std::collections::HashMap; use uuid::Uuid; +/// Creates options for deleting an object in a bucket. pub async fn del_opts( bucket: &str, object: &str, @@ -56,6 +57,7 @@ pub async fn del_opts( Ok(opts) } +/// Creates options for getting an object from a bucket. pub async fn get_opts( bucket: &str, object: &str, @@ -105,6 +107,7 @@ pub async fn get_opts( Ok(opts) } +/// Creates options for putting an object in a bucket. pub async fn put_opts( bucket: &str, object: &str, @@ -151,6 +154,7 @@ pub async fn put_opts( Ok(opts) } +/// Creates options for copying an object in a bucket. pub async fn copy_dst_opts( bucket: &str, object: &str, @@ -172,6 +176,7 @@ pub fn put_opts_from_headers( get_default_opts(headers, metadata, false) } +/// Creates default options for getting an object from a bucket. pub fn get_default_opts( _headers: &HeaderMap, metadata: Option>, @@ -183,6 +188,7 @@ pub fn get_default_opts( }) } +/// Extracts metadata from headers and returns it as a HashMap. pub fn extract_metadata(headers: &HeaderMap) -> HashMap { let mut metadata = HashMap::new(); @@ -191,6 +197,7 @@ pub fn extract_metadata(headers: &HeaderMap) -> HashMap, metadata: &mut HashMap) { for (k, v) in headers.iter() { if let Some(key) = k.as_str().strip_prefix("x-amz-meta-") { @@ -219,7 +226,9 @@ pub fn extract_metadata_from_mime(headers: &HeaderMap, metadata: &m metadata.insert("content-type".to_owned(), "binary/octet-stream".to_owned()); } } + lazy_static! { + /// List of supported headers. static ref SUPPORTED_HEADERS: Vec<&'static str> = vec![ "content-type", "cache-control", diff --git a/s3select/api/Cargo.toml b/s3select/api/Cargo.toml index 27e260c41..125849cdc 100644 --- a/s3select/api/Cargo.toml +++ b/s3select/api/Cargo.toml @@ -7,12 +7,14 @@ edition.workspace = true async-trait.workspace = true bytes.workspace = true chrono.workspace = true +common.workspace = true datafusion = { workspace = true } ecstore.workspace = true futures = { workspace = true } futures-core = { workspace = true } http.workspace = true object_store = { workspace = true } +pin-project-lite.workspace = true s3s.workspace = true snafu = { workspace = true, features = ["backtrace"] } tokio.workspace = true diff --git a/s3select/api/src/object_store.rs b/s3select/api/src/object_store.rs index 7772936ce..52132bc1a 100644 --- a/s3select/api/src/object_store.rs +++ b/s3select/api/src/object_store.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use bytes::Bytes; use chrono::Utc; +use common::DEFAULT_DELIMITER; use ecstore::io::READ_BUFFER_SIZE; use ecstore::new_object_layer_fn; use ecstore::store::ECStore; @@ -24,29 +25,54 @@ use object_store::PutOptions; use object_store::PutPayload; use object_store::PutResult; use object_store::{Error as o_Error, Result}; +use pin_project_lite::pin_project; use s3s::dto::SelectObjectContentInput; use s3s::s3_error; use s3s::S3Result; use std::ops::Range; +use std::pin::Pin; use std::sync::Arc; +use std::task::ready; +use std::task::Poll; +use tokio::io::AsyncRead; use tokio_util::io::ReaderStream; use tracing::info; use transform_stream::AsyncTryStream; #[derive(Debug)] pub struct EcObjectStore { - input: SelectObjectContentInput, + input: Arc, + need_convert: bool, + delimiter: String, store: Arc, } - impl EcObjectStore { - pub fn new(input: SelectObjectContentInput) -> S3Result { + pub fn new(input: Arc) -> S3Result { let Some(store) = new_object_layer_fn() else { return Err(s3_error!(InternalError, "ec store not inited")); }; - Ok(Self { input, store }) + let (need_convert, delimiter) = if let Some(csv) = input.request.input_serialization.csv.as_ref() { + if let Some(delimiter) = csv.field_delimiter.as_ref() { + if delimiter.len() > 1 { + (true, delimiter.to_owned()) + } else { + (false, String::new()) + } + } else { + (false, String::new()) + } + } else { + (false, String::new()) + }; + + Ok(Self { + input, + need_convert, + delimiter, + store, + }) } } @@ -79,16 +105,6 @@ impl ObjectStore for EcObjectStore { source: "can not get object info".into(), })?; - // let stream = stream::unfold(reader.stream, |mut blob| async move { - // match blob.next().await { - // Some(Ok(chunk)) => { - // let bytes = chunk; - // Some((Ok(bytes), blob)) - // } - // _ => None, - // } - // }) - // .boxed(); let meta = ObjectMeta { location: location.clone(), last_modified: Utc::now(), @@ -98,10 +114,21 @@ impl ObjectStore for EcObjectStore { }; let attributes = Attributes::default(); - Ok(GetResult { - payload: object_store::GetResultPayload::Stream( + let payload = if self.need_convert { + object_store::GetResultPayload::Stream( + bytes_stream( + ReaderStream::with_capacity(ConvertStream::new(reader.stream, self.delimiter.clone()), READ_BUFFER_SIZE), + reader.object_info.size, + ) + .boxed(), + ) + } else { + object_store::GetResultPayload::Stream( bytes_stream(ReaderStream::with_capacity(reader.stream, READ_BUFFER_SIZE), reader.object_info.size).boxed(), - ), + ) + }; + Ok(GetResult { + payload, meta, range: 0..reader.object_info.size, attributes, @@ -154,6 +181,54 @@ impl ObjectStore for EcObjectStore { } } +pin_project! { + struct ConvertStream { + inner: R, + delimiter: Vec, + } +} + +impl ConvertStream { + fn new(inner: R, delimiter: String) -> Self { + ConvertStream { + inner, + delimiter: delimiter.as_bytes().to_vec(), + } + } +} + +impl AsyncRead for ConvertStream { + #[tracing::instrument(level = "debug", skip_all)] + fn poll_read( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let me = self.project(); + ready!(Pin::new(&mut *me.inner).poll_read(cx, buf))?; + let bytes = buf.filled(); + let replaced = replace_symbol(me.delimiter, bytes); + buf.clear(); + buf.put_slice(&replaced); + Poll::Ready(Ok(())) + } +} + +fn replace_symbol(delimiter: &[u8], slice: &[u8]) -> Vec { + let mut result = Vec::with_capacity(slice.len()); + let mut i = 0; + while i < slice.len() { + if slice[i..].starts_with(delimiter) { + result.push(DEFAULT_DELIMITER); + i += delimiter.len(); + } else { + result.push(slice[i]); + i += 1; + } + } + result +} + pub fn bytes_stream(stream: S, content_length: usize) -> impl Stream> + Send + 'static where S: Stream> + Send + 'static, @@ -175,3 +250,21 @@ where Ok(()) }) } + +#[cfg(test)] +mod test { + use super::replace_symbol; + + #[test] + fn test_replace() { + let ss = String::from("dandan&&is&&best"); + let slice = ss.as_bytes(); + let delimiter = b"&&"; + println!("len: {}", "╦".len()); + let result = replace_symbol(delimiter, slice); + match String::from_utf8(result) { + Ok(s) => println!("slice: {}", s), + Err(e) => eprintln!("Error converting to string: {}", e), + } + } +} diff --git a/s3select/api/src/query/mod.rs b/s3select/api/src/query/mod.rs index 6ddd2dc86..93736be2b 100644 --- a/s3select/api/src/query/mod.rs +++ b/s3select/api/src/query/mod.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use s3s::dto::SelectObjectContentInput; pub mod analyzer; @@ -16,7 +18,7 @@ pub mod session; #[derive(Clone)] pub struct Context { // maybe we need transfer some info? - pub input: SelectObjectContentInput, + pub input: Arc, } #[derive(Clone)] diff --git a/s3select/api/src/query/session.rs b/s3select/api/src/query/session.rs index 286ee9f8e..581cdf39b 100644 --- a/s3select/api/src/query/session.rs +++ b/s3select/api/src/query/session.rs @@ -66,7 +66,7 @@ impl SessionCtxFactory { 9,Ivy,24,Marketing,4800 10,Jack,38,Finance,7500"; let data_bytes = data.as_bytes(); - // let data = r#""year"╦"gender"╦"ethnicity"╦"firstname"╦"count"╦"rank" + // let data = r#""year"╦"gender"╦"ethnicity"╦"firstname"╦"count"╦"rank" // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"SOPHIA"╦"119"╦"1" // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"CHLOE"╦"106"╦"2" // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMILY"╦"93"╦"3" diff --git a/s3select/query/src/dispatcher/manager.rs b/s3select/query/src/dispatcher/manager.rs index 80543ed69..ee5386e20 100644 --- a/s3select/query/src/dispatcher/manager.rs +++ b/s3select/query/src/dispatcher/manager.rs @@ -49,7 +49,7 @@ lazy_static! { #[derive(Clone)] pub struct SimpleQueryDispatcher { - input: SelectObjectContentInput, + input: Arc, // client for default tenant _default_table_provider: TableHandleProviderRef, session_factory: Arc, @@ -164,7 +164,9 @@ impl SimpleQueryDispatcher { .map(|e| e.as_bytes().first().copied().unwrap_or_default()), ); if let Some(delimiter) = csv.field_delimiter.as_ref() { - file_format = file_format.with_delimiter(delimiter.as_bytes().first().copied().unwrap_or_default()); + if delimiter.len() == 1 { + file_format = file_format.with_delimiter(delimiter.as_bytes()[0]); + } } // TODO waiting for processing @junxiang Mu // if csv.file_header_info.is_some() {} @@ -272,7 +274,7 @@ impl Stream for TrackedRecordBatchStream { #[derive(Default, Clone)] pub struct SimpleQueryDispatcherBuilder { - input: Option, + input: Option>, default_table_provider: Option, session_factory: Option>, parser: Option>, @@ -283,7 +285,7 @@ pub struct SimpleQueryDispatcherBuilder { } impl SimpleQueryDispatcherBuilder { - pub fn with_input(mut self, input: SelectObjectContentInput) -> Self { + pub fn with_input(mut self, input: Arc) -> Self { self.input = Some(input); self } diff --git a/s3select/query/src/instance.rs b/s3select/query/src/instance.rs index 44952a4ac..344920631 100644 --- a/s3select/query/src/instance.rs +++ b/s3select/query/src/instance.rs @@ -63,7 +63,7 @@ where } } -pub async fn make_rustfsms(input: SelectObjectContentInput, is_test: bool) -> QueryResult { +pub async fn make_rustfsms(input: Arc, is_test: bool) -> QueryResult { // init Function Manager, we can define some UDF if need let func_manager = SimpleFunctionMetadataManager::default(); // TODO session config need load global system config @@ -95,6 +95,8 @@ pub async fn make_rustfsms(input: SelectObjectContentInput, is_test: bool) -> Qu #[cfg(test)] mod tests { + use std::sync::Arc; + use api::{ query::{Context, Query}, server::dbms::DatabaseManagerSystem, @@ -111,7 +113,7 @@ mod tests { #[ignore] async fn test_simple_sql() { let sql = "select * from S3Object"; - let input = SelectObjectContentInput { + let input = Arc::new(SelectObjectContentInput { bucket: "dandan".to_string(), expected_bucket_owner: None, key: "test.csv".to_string(), @@ -135,7 +137,7 @@ mod tests { request_progress: None, scan_range: None, }, - }; + }); let db = make_rustfsms(input.clone(), true).await.unwrap(); let query = Query::new(Context { input }, sql.to_string()); @@ -167,8 +169,8 @@ mod tests { #[tokio::test] #[ignore] async fn test_func_sql() { - let sql = "SELECT s._1 FROM S3Object s"; - let input = SelectObjectContentInput { + let sql = "SELECT * FROM S3Object s"; + let input = Arc::new(SelectObjectContentInput { bucket: "dandan".to_string(), expected_bucket_owner: None, key: "test.csv".to_string(), @@ -194,7 +196,7 @@ mod tests { request_progress: None, scan_range: None, }, - }; + }); let db = make_rustfsms(input.clone(), true).await.unwrap(); let query = Query::new(Context { input }, sql.to_string()); diff --git a/scripts/build.py b/scripts/build.py index 106a25fed..f1beb0789 100755 --- a/scripts/build.py +++ b/scripts/build.py @@ -34,7 +34,7 @@ def main(args: CliArgs): use_zigbuild = True use_old_glibc = True - if args.target and args.target == "x86_64-unknown-linux-musl": + if args.target and args.target != "x86_64-unknown-linux-gnu": shell("rustup target add " + args.target) cmd = ["cargo", "build"] diff --git a/scripts/run.sh b/scripts/run.sh index 690ac8511..58d0f2642 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -33,30 +33,41 @@ export RUSTFS_CONSOLE_ENABLE=true export RUSTFS_CONSOLE_ADDRESS=":9002" # export RUSTFS_SERVER_DOMAINS="localhost:9000" # HTTPS 证书目录 - export RUSTFS_TLS_PATH="./deploy/certs" +# export RUSTFS_TLS_PATH="./deploy/certs" # 具体路径修改为配置文件真实路径，obs.example.toml 仅供参考 其中`RUSTFS_OBS_CONFIG` 和下面变量二选一 export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml" # 如下变量需要必须参数都有值才可以，以及会覆盖配置文件中的值 -export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 -export RUSTFS__OBSERVABILITY__USE_STDOUT=false -export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 -export RUSTFS__OBSERVABILITY__METER_INTERVAL=30 -export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs -export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 -export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop -export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug -export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true -export RUSTFS__SINKS__FILE__ENABLED=true -export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log" -export RUSTFS__SINKS__WEBHOOK__ENABLED=false -export RUSTFS__SINKS__WEBHOOK__ENDPOINT="" -export RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN="" -export RUSTFS__SINKS__KAFKA__ENABLED=false -export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS="" -export RUSTFS__SINKS__KAFKA__TOPIC="" -export RUSTFS__LOGGER__QUEUE_CAPACITY=10 +#export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 +#export RUSTFS__OBSERVABILITY__USE_STDOUT=false +#export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 +#export RUSTFS__OBSERVABILITY__METER_INTERVAL=31 +#export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs +#export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 +#export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop +#export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug +#export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true +# +#export RUSTFS__SINKS_0__type=File +#export RUSTFS__SINKS_0__path=./deploy/logs/rustfs.log +#export RUSTFS__SINKS_0__buffer_size=12 +#export RUSTFS__SINKS_0__flush_interval_ms=1000 +#export RUSTFS__SINKS_0__flush_threshold=100 +# +#export RUSTFS__SINKS_1__type=Kakfa +#export RUSTFS__SINKS_1__brokers=localhost:9092 +#export RUSTFS__SINKS_1__topic=logs +#export RUSTFS__SINKS_1__batch_size=100 +#export RUSTFS__SINKS_1__batch_timeout_ms=1000 +# +#export RUSTFS__SINKS_2__type=Webhook +#export RUSTFS__SINKS_2__endpoint=http://localhost:8080/webhook +#export RUSTFS__SINKS_2__auth_token=you-auth-token +#export RUSTFS__SINKS_2__batch_size=100 +#export RUSTFS__SINKS_2__batch_timeout_ms=1000 +# +#export RUSTFS__LOGGER__QUEUE_CAPACITY=10 export OTEL_INSTRUMENTATION_NAME="rustfs" export OTEL_INSTRUMENTATION_VERSION="0.1.1" @@ -64,13 +75,13 @@ export OTEL_INSTRUMENTATION_SCHEMA_URL="https://opentelemetry.io/schemas/1.31.0" export OTEL_INSTRUMENTATION_ATTRIBUTES="env=production" # 事件消息配置 -export RUSTFS_EVENT_CONFIG="./deploy/config/event.example.toml" +#export RUSTFS_EVENT_CONFIG="./deploy/config/event.example.toml" if [ -n "$1" ]; then export RUSTFS_VOLUMES="$1" fi # 启动 webhook 服务器 -cargo run --example webhook -p rustfs-event-notifier & +#cargo run --example webhook -p rustfs-event-notifier & # 启动主服务 cargo run --bin rustfs \ No newline at end of file