gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 06:13:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/observability-metrics

Browse Source 
# Conflicts:
#	Cargo.toml
#	crates/obs/examples/config.toml
#	crates/obs/src/telemetry.rs
This commit is contained in:
houseme
2025-05-12 15:21:30 +08:00
parent 19cdf9660b 571cedf4ce
commit 76e38526c6
123 changed files with 3434 additions and 1688 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cargo/config.toml
-7
View File
@@ -1,7 +0,0 @@
[target.x86_64-unknown-linux-gnu]
rustflags = [
"-C", "link-arg=-fuse-ld=bfd"
]
[target.x86_64-unknown-linux-musl]
linker = "x86_64-linux-musl-gcc"
.docker/observability/README_ZH.md
+2 -2
View File
@@ -9,7 +9,7 @@ Jaeger、Prometheus 等）而需要运行和维护多个代理/收集器的必
2. 执行以下命令启动服务：
```bash
docker compose up -d -f docker-compose.yml
docker compose -f docker-compose.yml up -d
```
### 访问监控面板
@@ -34,7 +34,7 @@ docker compose up -d -f docker-compose.yml
| service_name | 服务名称 | rustfs |
| service_version | 服务版本 | 1.0.0 |
| environment | 运行环境 | production |
| meter_interval | 指标导出间隔 (秒) | 30 |
| meter_interval | 指标导出间隔 (秒) | 30 |
| sample_ratio | 采样率 | 1.0 |
| use_stdout | 是否输出到控制台 | true/false |
| logger_level | 日志级别 | info |
.docker/observability/config/obs-multi.toml
+34
View File
@@ -0,0 +1,34 @@
[observability]
endpoint = "http://otel-collector:4317" # Default is "http://localhost:4317" if not specified
use_stdout = false # Output with stdout, true output, false no output
sample_ratio = 2.0
meter_interval = 30
service_name = "rustfs"
service_version = "0.1.0"
environments = "production"
logger_level = "debug"
local_logging_enabled = true
#[[sinks]]
#type = "Kafka"
#brokers = "localhost:9092"
#topic = "logs"
#batch_size = 100 # Default is 100 if not specified
#batch_timeout_ms = 1000 # Default is 1000ms if not specified
#
#[[sinks]]
#type = "Webhook"
#endpoint = "http://localhost:8080/webhook"
#auth_token = ""
#batch_size = 100 # Default is 3 if not specified
#batch_timeout_ms = 1000 # Default is 100ms if not specified
[[sinks]]
type = "File"
path = "/root/data/logs/rustfs.log"
buffer_size = 100 # Default is 8192 bytes if not specified
flush_interval_ms = 1000
flush_threshold = 100
[logger]
queue_capacity = 10
.docker/observability/config/obs.toml
+20 -20
View File
@@ -1,5 +1,5 @@
[observability]
endpoint = "http://otel-collector:4317" # Default is "http://localhost:4317" if not specified
endpoint = "http://localhost:4317" # Default is "http://localhost:4317" if not specified
use_stdout = false # Output with stdout, true output, false no output
sample_ratio = 2.0
meter_interval = 30
@@ -9,26 +9,26 @@ environments = "production"
logger_level = "debug"
local_logging_enabled = true
[sinks]
[sinks.kafka] # Kafka sink is disabled by default
enabled = false
bootstrap_servers = "localhost:9092"
topic = "logs"
batch_size = 100 # Default is 100 if not specified
batch_timeout_ms = 1000 # Default is 1000ms if not specified
#[[sinks]]
#type = "Kafka"
#brokers = "localhost:9092"
#topic = "logs"
#batch_size = 100 # Default is 100 if not specified
#batch_timeout_ms = 1000 # Default is 1000ms if not specified
#
#[[sinks]]
#type = "Webhook"
#endpoint = "http://localhost:8080/webhook"
#auth_token = ""
#batch_size = 100 # Default is 3 if not specified
#batch_timeout_ms = 1000 # Default is 100ms if not specified
[sinks.webhook]
enabled = false
endpoint = "http://localhost:8080/webhook"
auth_token = ""
batch_size = 100 # Default is 3 if not specified
batch_timeout_ms = 1000 # Default is 100ms if not specified
[sinks.file]
enabled = true
path = "/root/data/logs/app.log"
batch_size = 10
batch_timeout_ms = 1000 # Default is 8192 bytes if not specified
[[sinks]]
type = "File"
path = "/root/data/logs/rustfs.log"
buffer_size = 100 # Default is 8192 bytes if not specified
flush_interval_ms = 1000
flush_threshold = 100
[logger]
queue_capacity = 10
.docker/observability/docker-compose.yml
+5 -5
View File
@@ -1,6 +1,6 @@
services:
otel-collector:
image: otel/opentelemetry-collector-contrib:0.120.0
image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.124.0
environment:
- TZ=Asia/Shanghai
volumes:
@@ -16,7 +16,7 @@ services:
networks:
- otel-network
jaeger:
image: jaegertracing/jaeger:2.4.0
image: jaegertracing/jaeger:2.5.0
environment:
- TZ=Asia/Shanghai
ports:
@@ -26,7 +26,7 @@ services:
networks:
- otel-network
prometheus:
image: prom/prometheus:v3.2.1
image: prom/prometheus:v3.3.0
environment:
- TZ=Asia/Shanghai
volumes:
@@ -36,7 +36,7 @@ services:
networks:
- otel-network
loki:
image: grafana/loki:3.4.2
image: grafana/loki:3.5.0
environment:
- TZ=Asia/Shanghai
volumes:
@@ -47,7 +47,7 @@ services:
networks:
- otel-network
grafana:
image: grafana/grafana:11.6.0
image: grafana/grafana:11.6.1
ports:
- "3000:3000" # Web UI
environment:
.github/actions/setup/action.yml
+9 -1
View File
@@ -6,17 +6,25 @@ inputs:
rust-version:
required: true
default: "stable"
description: "Rust version to use"
cache-shared-key:
required: true
default: ""
description: "Cache key for shared cache"
cache-save-if:
required: true
default: true
description: "Cache save condition"
run-os:
required: true
default: "ubuntu-latest"
description: "Running system"
runs:
using: "composite"
steps:
- name: Install system dependencies
if: inputs.run-os == 'ubuntu-latest'
shell: bash
run: |
sudo apt update
@@ -42,5 +50,5 @@ runs:
shared-key: ${{ inputs.cache-shared-key }}
save-if: ${{ inputs.cache-save-if }}
- uses: mlugg/setup-zig@v1
- uses: mlugg/setup-zig@v2
- uses: taiki-e/install-action@cargo-zigbuild
.github/workflows/build.yml
+329 -107
View File
@@ -1,4 +1,4 @@
name: Build
name: Build RustFS And GUI
on:
workflow_dispatch:
@@ -11,66 +11,262 @@ on:
jobs:
build-rustfs:
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-latest, macos-latest, windows-latest ]
variant:
# - { profile: dev, target: x86_64-unknown-linux-musl, glibc: "default" }
- { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" }
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
# - { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" }
- { profile: release, target: aarch64-apple-darwin, glibc: "default" }
#- { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" }
#- { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" }
#- { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" }
exclude:
# Linux targets on non-Linux systems
- os: macos-latest
variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
- os: macos-latest
variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" }
- os: macos-latest
variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" }
- os: macos-latest
variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" }
- os: windows-latest
variant: { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
- os: windows-latest
variant: { profile: release, target: x86_64-unknown-linux-musl, glibc: "default" }
- os: windows-latest
variant: { profile: release, target: aarch64-unknown-linux-gnu, glibc: "default" }
- os: windows-latest
variant: { profile: release, target: aarch64-unknown-linux-musl, glibc: "default" }
# Apple targets on non-macOS systems
- os: ubuntu-latest
variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" }
- os: windows-latest
variant: { profile: release, target: aarch64-apple-darwin, glibc: "default" }
# Windows targets on non-Windows systems
- os: ubuntu-latest
variant: { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" }
- os: macos-latest
variant: { profile: release, target: x86_64-pc-windows-msvc, glibc: "default" }
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup
- name: Checkout repository
uses: actions/checkout@v4.2.2
with:
cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }}
fetch-depth: 0
# Installation system dependencies
- name: Install system dependencies (Ubuntu)
if: runner.os == 'Linux'
run: |
sudo apt update
sudo apt install -y musl-tools build-essential lld libdbus-1-dev libwayland-dev libwebkit2gtk-4.1-dev libxdo-dev
shell: bash
#Install Rust using dtolnay/rust-toolchain
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
targets: ${{ matrix.variant.target }}
components: rustfmt, clippy
# Install system dependencies
- name: Cache Protoc
id: cache-protoc
uses: actions/cache@v4.2.3
with:
path: /Users/runner/hostedtoolcache/protoc
key: protoc-${{ runner.os }}-30.2
restore-keys: |
protoc-${{ runner.os }}-
- name: Install Protoc
if: steps.cache-protoc.outputs.cache-hit != 'true'
uses: arduino/setup-protoc@v3
with:
version: '30.2'
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Flatc
uses: Nugine/setup-flatc@v1
with:
version: "25.2.10"
# Cache Cargo dependencies
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
cache-all-crates: true
shared-key: rustfs-${{ matrix.os }}-${{ matrix.variant.profile }}-${{ matrix.variant.target }}-${{ matrix.variant.glibc }}-${{ hashFiles('**/Cargo.lock') }}
save-if: ${{ github.event_name != 'pull_request' }}
# Set up Zig for cross-compilation
- uses: mlugg/setup-zig@v2
if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'linux')
- uses: taiki-e/install-action@cargo-zigbuild
if: matrix.variant.glibc != 'default' || contains(matrix.variant.target, 'linux')
# Download static resources
- name: Download and Extract Static Assets
run: |
url="https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip"
mkdir -p static
curl -L -o static_assets.zip "$url"
unzip -o static_assets.zip -d ./rustfs/static
rm static_assets.zip
ls -la ./rustfs/static
- name: Build
# Create a static resource directory
mkdir -p ./rustfs/static
# Download static resources
echo "::group::Downloading static assets"
curl -L -o static_assets.zip "$url" --retry 3
# Unzip static resources
echo "::group::Extracting static assets"
if [ "${{ runner.os }}" = "Windows" ]; then
7z x static_assets.zip -o./rustfs/static
del static_assets.zip
else
unzip -o static_assets.zip -d ./rustfs/static
rm static_assets.zip
fi
echo "::group::Static assets content"
ls -la ./rustfs/static
shell: bash
# Build rustfs
- name: Build rustfs
id: build
shell: bash
run: |
./scripts/build.py \
--profile ${{ matrix.variant.profile }} \
--target ${{ matrix.variant.target }} \
--glibc ${{ matrix.variant.glibc }}
echo "::group::Setting up build parameters"
PROFILE="${{ matrix.variant.profile }}"
TARGET="${{ matrix.variant.target }}"
GLIBC="${{ matrix.variant.glibc }}"
# Determine whether to use zigbuild
USE_ZIGBUILD=false
if [[ "$GLIBC" != "default" || "$TARGET" == *"linux"* ]]; then
USE_ZIGBUILD=true
echo "Using zigbuild for cross-compilation"
fi
# Determine the target parameters
TARGET_ARG="$TARGET"
if [[ "$GLIBC" != "default" ]]; then
TARGET_ARG="${TARGET}.${GLIBC}"
echo "Using custom glibc target: $TARGET_ARG"
fi
# Confirm the profile directory name
if [[ "$PROFILE" == "dev" ]]; then
PROFILE_DIR="debug"
else
PROFILE_DIR="$PROFILE"
fi
# Determine the binary suffix
BIN_SUFFIX=""
if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then
BIN_SUFFIX=".exe"
fi
# Determine the binary name - Use the appropriate extension for Windows
BIN_NAME="rustfs.${PROFILE}.${TARGET}"
if [[ "$GLIBC" != "default" ]]; then
BIN_NAME="${BIN_NAME}.glibc${GLIBC}"
fi
# Windows systems use exe suffix, and other systems do not have suffix
if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then
BIN_NAME="${BIN_NAME}.exe"
else
BIN_NAME="${BIN_NAME}.bin"
fi
echo "Binary name will be: $BIN_NAME"
echo "::group::Building rustfs"
# Refresh build information
touch rustfs/build.rs
# Identify the build command and execute it
if [[ "$USE_ZIGBUILD" == "true" ]]; then
echo "Build command: cargo zigbuild --profile $PROFILE --target $TARGET_ARG -p rustfs --bins"
cargo zigbuild --profile $PROFILE --target $TARGET_ARG -p rustfs --bins
else
echo "Build command: cargo build --profile $PROFILE --target $TARGET_ARG -p rustfs --bins"
cargo build --profile $PROFILE --target $TARGET_ARG -p rustfs --bins
fi
# Determine the binary path and output path
BIN_PATH="target/${TARGET_ARG}/${PROFILE_DIR}/rustfs${BIN_SUFFIX}"
OUT_PATH="target/artifacts/${BIN_NAME}"
# Create a target directory
mkdir -p target/artifacts
echo "Copying binary from ${BIN_PATH} to ${OUT_PATH}"
cp "${BIN_PATH}" "${OUT_PATH}"
# Record the output path for use in the next steps
echo "bin_path=${OUT_PATH}" >> $GITHUB_OUTPUT
echo "bin_name=${BIN_NAME}" >> $GITHUB_OUTPUT
- name: Package Binary and Static Assets
id: package
run: |
# Create artifact filename
# Create component file name
ARTIFACT_NAME="rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}"
if [ "${{ matrix.variant.glibc }}" != "default" ]; then
ARTIFACT_NAME="${ARTIFACT_NAME}-glibc${{ matrix.variant.glibc }}"
fi
echo "artifact_name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT
# Determine binary path
bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.bin"
if [ -f "target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin" ]; then
bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin"
# Get the binary path
BIN_PATH="${{ steps.build.outputs.bin_path }}"
# Create a packaged directory structure - only contains bin and docs directories
mkdir -p ${ARTIFACT_NAME}/{bin,docs}
# Copy binary files (note the difference between Windows and other systems)
if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then
cp "${BIN_PATH}" ${ARTIFACT_NAME}/bin/rustfs.exe
else
cp "${BIN_PATH}" ${ARTIFACT_NAME}/bin/rustfs
fi
# Create package
mkdir -p ${ARTIFACT_NAME}
cp "$bin_path" ${ARTIFACT_NAME}/rustfs
zip -r ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME}
ls -la
# copy documents and licenses
if [ -f "LICENSE" ]; then
cp LICENSE ${ARTIFACT_NAME}/docs/
fi
if [ -f "README.md" ]; then
cp README.md ${ARTIFACT_NAME}/docs/
fi
# Packaged as zip
if [ "${{ runner.os }}" = "Windows" ]; then
7z a ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME}
else
zip -r ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME}
fi
echo "Created artifact: ${ARTIFACT_NAME}.zip"
ls -la ${ARTIFACT_NAME}.zip
shell: bash
- uses: actions/upload-artifact@v4
with:
name: ${{ steps.package.outputs.artifact_name }}
path: ${{ steps.package.outputs.artifact_name }}.zip
retention-days: 7
- name: Upload to Aliyun OSS
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main'
uses: JohnGuan/oss-upload-action@main
with:
key-id: ${{ secrets.ALICLOUDOSS_KEY_ID }}
@@ -79,100 +275,124 @@ jobs:
bucket: rustfs-artifacts
assets: |
${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.zip
${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.latest.zip
${{ steps.package.outputs.artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.artifact_name }}.latest.zip
build-rustfs-gui:
runs-on: ubuntu-latest
needs: build-rustfs
strategy:
matrix:
variant:
- { profile: release, target: x86_64-unknown-linux-gnu }
# - { profile: release, target: x86_64-apple-darwin }
if: startsWith(github.ref, 'refs/tags/')
steps:
- uses: actions/checkout@v4
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: "rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}"
- name: Display structure of downloaded files
# Determine whether to perform GUI construction based on conditions
- name: Prepare for GUI build
if: startsWith(github.ref, 'refs/tags/')
id: prepare_gui
run: |
ls -R
unzip -o -j "rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}.zip" -d ./cli/rustfs-gui/embedded-rustfs/
ls -la cli/rustfs-gui/embedded-rustfs
# - name: Cache dioxus-cli
# uses: actions/cache@v4
# with:
# path: ~/.cargo/bin/dx
# key: ${{ runner.os }}-dioxus-cli-${{ hashFiles('**/Cargo.lock') }}
# restore-keys: |
# ${{ runner.os }}-dioxus-cli-
#
# - name: Install dioxus-cli
# run: |
# if [ ! -f ~/.cargo/bin/dx ]; then
# cargo install dioxus-cli
# fi
# Create a target directory
mkdir -p ./cli/rustfs-gui/embedded-rustfs/
# Copy the currently built binary to the embedded-rustfs directory
if [[ "${{ matrix.variant.target }}" == *"windows"* ]]; then
cp "${{ steps.build.outputs.bin_path }}" ./cli/rustfs-gui/embedded-rustfs/rustfs.exe
else
cp "${{ steps.build.outputs.bin_path }}" ./cli/rustfs-gui/embedded-rustfs/rustfs
fi
echo "Copied binary to embedded-rustfs directory"
ls -la ./cli/rustfs-gui/embedded-rustfs/
shell: bash
#Install the dioxus-cli tool
- uses: taiki-e/cache-cargo-install-action@v2
if: startsWith(github.ref, 'refs/tags/')
with:
tool: dioxus-cli
# Build and package GUI applications
- name: Build and Bundle rustfs-gui
if: startsWith(github.ref, 'refs/tags/')
id: build_gui
shell: bash
run: |
ls -la
echo "::group::Setting up build parameters for GUI"
PROFILE="${{ matrix.variant.profile }}"
TARGET="${{ matrix.variant.target }}"
GLIBC="${{ matrix.variant.glibc }}"
RELEASE_PATH="target/artifacts/$TARGET"
release_path="target/${{ matrix.variant.target }}"
mkdir -p ${release_path}
cd cli/rustfs-gui
ls -la embedded-rustfs
# Make sure the output directory exists
mkdir -p ${RELEASE_PATH}
# Configure the linker based on the target
case "${{ matrix.target }}" in
# Configure the target platform linker
echo "::group::Configuring linker for $TARGET"
case "$TARGET" in
"x86_64-unknown-linux-gnu")
# Default gcc
export CC_x86_64_unknown_linux_gnu=gcc
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc
;;
export CC_x86_64_unknown_linux_gnu=gcc
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc
;;
"x86_64-unknown-linux-musl")
export CC_x86_64_unknown_linux_musl=musl-gcc
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=musl-gcc
;;
"aarch64-unknown-linux-gnu")
# AArch64 Cross-compiler
export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
;;
"x86_64-apple-darwin")
# macOS default clang
export CC_x86_64_apple_darwin=clang
export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=clang
;;
export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
;;
"aarch64-unknown-linux-musl")
export CC_aarch64_unknown_linux_musl=aarch64-linux-musl-gcc
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-musl-gcc
;;
"aarch64-apple-darwin")
# macOS ARM64 used clang
export CC_aarch64_apple_darwin=clang
export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=clang
;;
export CC_aarch64_apple_darwin=clang
export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=clang
;;
"x86_64-pc-windows-msvc")
export CC_x86_64_pc_windows_msvc=cl
export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER=link
;;
esac
# Validating Environment Variables (for Debugging)
echo "CC for ${{ matrix.target }}: $CC_${{ matrix.target }}"
echo "Linker for ${{ matrix.target }}: $CARGO_TARGET_${{ matrix.target }}_LINKER"
if [[ "${{ matrix.variant.target }}" == *"apple-darwin"* ]]; then
dx bundle --platform macos --package-types "macos" --package-types "dmg" --package-types "ios" --release --profile release --out-dir ../../${release_path}
elif [[ "${{ matrix.variant.target }}" == *"windows-msvc"* ]]; then
dx bundle --platform windows --package-types "msi" --release --profile release --out-dir ../../${release_path}
elif [[ "${{ matrix.variant.target }}" == *"unknown-linux-gnu"* ]]; then
dx bundle --platform linux --package-types "deb" --package-types "rpm" --package-types "appimage" --release --profile release --out-dir ../../${release_path}
echo "::group::Building GUI application"
cd cli/rustfs-gui
# Building according to the target platform
if [[ "$TARGET" == *"apple-darwin"* ]]; then
echo "Building for macOS"
dx bundle --platform macos --package-types "macos" --package-types "dmg" --release --profile ${PROFILE} --out-dir ../../${RELEASE_PATH}
elif [[ "$TARGET" == *"windows-msvc"* ]]; then
echo "Building for Windows"
dx bundle --platform windows --package-types "msi" --release --profile ${PROFILE} --out-dir ../../${RELEASE_PATH}
elif [[ "$TARGET" == *"linux"* ]]; then
echo "Building for Linux"
dx bundle --platform linux --package-types "deb" --package-types "rpm" --package-types "appimage" --release --profile ${PROFILE} --out-dir ../../${RELEASE_PATH}
fi
cd ../..
GUI_ARTIFACT_NAME="rustfs-gui-${{ matrix.variant.profile }}-${{ matrix.variant.target }}"
zip -r ${GUI_ARTIFACT_NAME}.zip ${release_path}/*
echo "gui_artifact_name=${GUI_ARTIFACT_NAME}" >> $GITHUB_OUTPUT
ls -la ${release_path}
cd ../..
# Create component name
GUI_ARTIFACT_NAME="rustfs-gui-${PROFILE}-${TARGET}"
if [ "$GLIBC" != "default" ]; then
GUI_ARTIFACT_NAME="${GUI_ARTIFACT_NAME}-glibc${GLIBC}"
fi
echo "::group::Packaging GUI application"
# Select packaging method according to the operating system
if [ "${{ runner.os }}" = "Windows" ]; then
7z a ${GUI_ARTIFACT_NAME}.zip ${RELEASE_PATH}/*
else
zip -r ${GUI_ARTIFACT_NAME}.zip ${RELEASE_PATH}/*
fi
echo "gui_artifact_name=${GUI_ARTIFACT_NAME}" >> $GITHUB_OUTPUT
echo "Created GUI artifact: ${GUI_ARTIFACT_NAME}.zip"
ls -la ${GUI_ARTIFACT_NAME}.zip
# Upload GUI components
- uses: actions/upload-artifact@v4
if: startsWith(github.ref, 'refs/tags/')
with:
name: ${{ steps.package.outputs.gui_artifact_name }}
path: ${{ steps.package.outputs.gui_artifact_name }}.zip
name: ${{ steps.build_gui.outputs.gui_artifact_name }}
path: ${{ steps.build_gui.outputs.gui_artifact_name }}.zip
retention-days: 7
- name: Upload to Aliyun OSS
# Upload GUI to Alibaba Cloud OSS
- name: Upload GUI to Aliyun OSS
if: startsWith(github.ref, 'refs/tags/')
uses: JohnGuan/oss-upload-action@main
with:
key-id: ${{ secrets.ALICLOUDOSS_KEY_ID }}
@@ -180,15 +400,17 @@ jobs:
region: oss-cn-beijing
bucket: rustfs-artifacts
assets: |
${{ steps.package.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.gui_artifact_name }}.zip
${{ steps.package.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.package.outputs.gui_artifact_name }}.latest.zip
${{ steps.build_gui.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.zip
${{ steps.build_gui.outputs.gui_artifact_name }}.zip:/artifacts/rustfs/${{ steps.build_gui.outputs.gui_artifact_name }}.latest.zip
merge:
runs-on: ubuntu-latest
needs: [ build-rustfs, build-rustfs-gui ]
needs: [ build-rustfs ]
if: startsWith(github.ref, 'refs/tags/')
steps:
- uses: actions/upload-artifact/merge@v4
with:
name: rustfs-packages
pattern: "rustfs-*"
delete-merged: true
delete-merged: true
.gitignore
+3 -1
View File
@@ -12,4 +12,6 @@ cli/rustfs-gui/embedded-rustfs/rustfs
deploy/config/obs.toml
*.log
deploy/certs/*
*jsonl
*jsonl
.env
.rustfs.sys
Cargo.lock
Generated
+160 -54
View File
@@ -185,12 +185,14 @@ dependencies = [
"async-trait",
"bytes",
"chrono",
"common",
"datafusion",
"ecstore",
"futures",
"futures-core",
"http",
"object_store",
"pin-project-lite",
"s3s",
"snafu",
"tokio",
@@ -711,9 +713,9 @@ dependencies = [
[[package]]
name = "axum"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de45108900e1f9b9242f7f2e254aa3e2c029c921c258fe9e6b4217eeebd54288"
checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5"
dependencies = [
"axum-core",
"bytes",
@@ -799,7 +801,7 @@ dependencies = [
"hyper",
"hyper-util",
"pin-project-lite",
"rustls 0.23.26",
"rustls 0.23.27",
"rustls-pemfile",
"rustls-pki-types",
"tokio",
@@ -1204,9 +1206,9 @@ dependencies = [
[[package]]
name = "chrono"
version = "0.4.40"
version = "0.4.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d"
dependencies = [
"android-tzdata",
"iana-time-zone",
@@ -1430,7 +1432,7 @@ dependencies = [
"lazy_static",
"scopeguard",
"tokio",
"tonic 0.13.0",
"tonic 0.13.1",
"tracing-error",
]
@@ -1744,7 +1746,7 @@ dependencies = [
"pbkdf2",
"rand 0.8.5",
"serde_json",
"sha2 0.10.8",
"sha2 0.10.9",
"test-case",
"thiserror 2.0.12",
"time",
@@ -2139,7 +2141,7 @@ dependencies = [
"md-5",
"rand 0.8.5",
"regex",
"sha2 0.10.8",
"sha2 0.10.9",
"unicode-segmentation",
"uuid",
]
@@ -3018,6 +3020,12 @@ dependencies = [
"const-random",
]
[[package]]
name = "dotenvy"
version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
[[package]]
name = "dpi"
version = "0.1.1"
@@ -3061,7 +3069,7 @@ dependencies = [
"serde",
"serde_json",
"tokio",
"tonic 0.13.0",
"tonic 0.13.1",
"tower 0.5.2",
"url",
]
@@ -3091,7 +3099,7 @@ dependencies = [
"madmin",
"md-5",
"netif",
"nix",
"nix 0.30.1",
"num",
"num_cpus",
"path-absolutize",
@@ -3105,6 +3113,7 @@ dependencies = [
"reqwest",
"rmp",
"rmp-serde",
"rustfs-config",
"s3s",
"s3s-policy",
"serde",
@@ -3119,7 +3128,7 @@ dependencies = [
"tokio",
"tokio-stream",
"tokio-util",
"tonic 0.13.0",
"tonic 0.13.1",
"tower 0.5.2",
"tracing",
"tracing-error",
@@ -3267,6 +3276,18 @@ dependencies = [
"rustc_version",
]
[[package]]
name = "filetime"
version = "0.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
dependencies = [
"cfg-if",
"libc",
"libredox",
"windows-sys 0.59.0",
]
[[package]]
name = "fixedbitset"
version = "0.5.7"
@@ -4129,7 +4150,7 @@ dependencies = [
"http",
"hyper",
"hyper-util",
"rustls 0.23.26",
"rustls 0.23.27",
"rustls-pki-types",
"tokio",
"tokio-rustls 0.26.2",
@@ -4776,22 +4797,23 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.9.0",
"libc",
"redox_syscall 0.5.11",
]
[[package]]
name = "libsystemd"
version = "0.7.1"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b85fe9dc49de659d05829fdf72b5770c0a5952d1055c34a39f6d4e932bce175d"
checksum = "19c97a761fc86953c5b885422b22c891dbf5bcb9dcc99d0110d6ce4c052759f0"
dependencies = [
"hmac 0.12.1",
"libc",
"log",
"nix",
"nix 0.29.0",
"nom 8.0.0",
"once_cell",
"serde",
"sha2 0.10.8",
"sha2 0.10.9",
"thiserror 2.0.12",
"uuid",
]
@@ -4847,13 +4869,13 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
[[package]]
name = "local-ip-address"
version = "0.6.3"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3669cf5561f8d27e8fc84cc15e58350e70f557d4d65f70e3154e54cd2f8e1782"
checksum = "656b3b27f8893f7bbf9485148ff9a65f019e3f33bd5cdc87c83cab16b3fd9ec8"
dependencies = [
"libc",
"neli",
"thiserror 1.0.69",
"thiserror 2.0.12",
"windows-sys 0.59.0",
]
@@ -4870,7 +4892,7 @@ dependencies = [
"serde",
"serde_json",
"tokio",
"tonic 0.13.0",
"tonic 0.13.1",
"tracing",
"tracing-error",
"url",
@@ -5225,6 +5247,18 @@ dependencies = [
"memoffset",
]
[[package]]
name = "nix"
version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
dependencies = [
"bitflags 2.9.0",
"cfg-if",
"cfg_aliases",
"libc",
]
[[package]]
name = "nodrop"
version = "0.1.14"
@@ -6092,7 +6126,7 @@ checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0"
dependencies = [
"once_cell",
"pest",
"sha2 0.10.8",
"sha2 0.10.9",
]
[[package]]
@@ -6587,7 +6621,7 @@ dependencies = [
"prost-build",
"protobuf",
"tokio",
"tonic 0.13.0",
"tonic 0.13.1",
"tonic-build",
"tower 0.5.2",
]
@@ -6641,7 +6675,7 @@ dependencies = [
"quinn-proto",
"quinn-udp",
"rustc-hash 2.1.1",
"rustls 0.23.26",
"rustls 0.23.27",
"socket2",
"thiserror 2.0.12",
"tokio",
@@ -6660,7 +6694,7 @@ dependencies = [
"rand 0.9.1",
"ring",
"rustc-hash 2.1.1",
"rustls 0.23.26",
"rustls 0.23.27",
"rustls-pki-types",
"slab",
"thiserror 2.0.12",
@@ -6879,6 +6913,15 @@ dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.5.11"
@@ -6986,7 +7029,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls 0.23.26",
"rustls 0.23.27",
"rustls-pemfile",
"rustls-pki-types",
"serde",
@@ -7143,9 +7186,9 @@ dependencies = [
[[package]]
name = "rust-embed"
version = "8.7.0"
version = "8.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5fbc0ee50fcb99af7cebb442e5df7b5b45e9460ffa3f8f549cd26b862bec49d"
checksum = "60e425e204264b144d4c929d126d0de524b40a961686414bab5040f7465c71be"
dependencies = [
"rust-embed-impl",
"rust-embed-utils",
@@ -7172,7 +7215,7 @@ version = "8.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d55b95147fe01265d06b3955db798bdaed52e60e2211c41137701b3aba8e21"
dependencies = [
"sha2 0.10.8",
"sha2 0.10.9",
"walkdir",
]
@@ -7244,7 +7287,6 @@ dependencies = [
"iam",
"lazy_static",
"libsystemd",
"local-ip-address",
"lock",
"madmin",
"matchit",
@@ -7259,29 +7301,42 @@ dependencies = [
"query",
"rmp-serde",
"rust-embed",
"rustfs-config",
"rustfs-event-notifier",
"rustfs-obs",
"rustls 0.23.26",
"rustls-pemfile",
"rustls-pki-types",
"rustfs-utils",
"rustls 0.23.27",
"s3s",
"serde",
"serde_json",
"serde_urlencoded",
"shadow-rs",
"socket2",
"tikv-jemallocator",
"time",
"tokio",
"tokio-rustls 0.26.2",
"tokio-stream",
"tokio-tar",
"tokio-util",
"tonic 0.13.0",
"tonic 0.13.1",
"tonic-build",
"tower 0.5.2",
"tower-http",
"tracing",
"transform-stream",
"uuid",
"zip",
]
[[package]]
name = "rustfs-config"
version = "0.0.1"
dependencies = [
"config",
"const-str",
"serde",
"serde_json",
]
[[package]]
@@ -7291,6 +7346,7 @@ dependencies = [
"async-trait",
"axum",
"config",
"dotenvy",
"http",
"rdkafka",
"reqwest",
@@ -7322,7 +7378,7 @@ dependencies = [
"rust-embed",
"serde",
"serde_json",
"sha2 0.10.8",
"sha2 0.10.9",
"tokio",
"tracing-appender",
"tracing-subscriber",
@@ -7345,6 +7401,8 @@ dependencies = [
"opentelemetry_sdk",
"rdkafka",
"reqwest",
"rustfs-config",
"rustfs-utils",
"serde",
"serde_json",
"smallvec",
@@ -7358,6 +7416,18 @@ dependencies = [
"tracing-subscriber",
]
[[package]]
name = "rustfs-utils"
version = "0.0.1"
dependencies = [
"local-ip-address",
"rustfs-config",
"rustls 0.23.27",
"rustls-pemfile",
"rustls-pki-types",
"tracing",
]
[[package]]
name = "rustix"
version = "0.38.44"
@@ -7400,16 +7470,16 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.26"
version = "0.23.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0"
checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321"
dependencies = [
"aws-lc-rs",
"log",
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki 0.103.1",
"rustls-webpki 0.103.2",
"subtle",
"zeroize",
]
@@ -7458,9 +7528,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.1"
version = "0.103.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03"
checksum = "7149975849f1abb3832b246010ef62ccc80d3a76169517ada7188252b9cfb437"
dependencies = [
"aws-lc-rs",
"ring",
@@ -7858,9 +7928,9 @@ dependencies = [
[[package]]
name = "sha2"
version = "0.10.8"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
@@ -8596,9 +8666,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.44.2"
version = "1.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48"
checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165"
dependencies = [
"backtrace",
"bytes",
@@ -8641,7 +8711,7 @@ version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b"
dependencies = [
"rustls 0.23.26",
"rustls 0.23.27",
"tokio",
]
@@ -8657,10 +8727,25 @@ dependencies = [
]
[[package]]
name = "tokio-util"
version = "0.7.14"
name = "tokio-tar"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034"
checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75"
dependencies = [
"filetime",
"futures-core",
"libc",
"redox_syscall 0.3.5",
"tokio",
"tokio-stream",
"xattr",
]
[[package]]
name = "tokio-util"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df"
dependencies = [
"bytes",
"futures-core",
@@ -8755,9 +8840,9 @@ dependencies = [
[[package]]
name = "tonic"
version = "0.13.0"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85839f0b32fd242bb3209262371d07feda6d780d16ee9d2bc88581b89da1549b"
checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9"
dependencies = [
"async-trait",
"axum",
@@ -8785,9 +8870,9 @@ dependencies = [
[[package]]
name = "tonic-build"
version = "0.13.0"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d85f0383fadd15609306383a90e85eaed44169f931a5d2be1b42c76ceff1825e"
checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847"
dependencies = [
"prettyplease",
"proc-macro2",
@@ -10142,7 +10227,7 @@ dependencies = [
"once_cell",
"percent-encoding",
"raw-window-handle 0.6.2",
"sha2 0.10.8",
"sha2 0.10.9",
"soup3",
"tao-macros",
"thiserror 1.0.69",
@@ -10176,6 +10261,16 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "xattr"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e"
dependencies = [
"libc",
"rustix 1.0.5",
]
[[package]]
name = "xdg-home"
version = "1.3.0"
@@ -10252,7 +10347,7 @@ dependencies = [
"futures-sink",
"futures-util",
"hex",
"nix",
"nix 0.29.0",
"ordered-stream",
"rand 0.8.5",
"serde",
@@ -10283,7 +10378,7 @@ dependencies = [
"futures-core",
"futures-lite",
"hex",
"nix",
"nix 0.29.0",
"ordered-stream",
"serde",
"serde_repr",
@@ -10439,6 +10534,17 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "zip"
version = "0.0.1"
dependencies = [
"async-compression",
"tokio",
"tokio-stream",
"tokio-tar",
"xz2",
]
[[package]]
name = "zstd"
version = "0.13.3"
Cargo.toml
+45 -25
View File
@@ -1,21 +1,24 @@
[workspace]
members = [
"madmin", # Management dashboard and admin API interface
"rustfs", # Core file system implementation
"ecstore", # Erasure coding storage implementation
"e2e_test", # End-to-end test suite
"appauth", # Application authentication and authorization
"cli/rustfs-gui", # Graphical user interface client
"common/common", # Shared utilities and data structures
"common/lock", # Distributed locking implementation
"common/protos", # Protocol buffer definitions
"common/workers", # Worker thread pools and task scheduling
"iam", # Identity and Access Management
"crypto", # Cryptography and security features
"cli/rustfs-gui", # Graphical user interface client
"crates/obs", # Observability utilities
"crates/config", # Configuration management
"crates/event-notifier", # Event notification system
"crates/obs", # Observability utilities
"crates/utils", # Utility functions and helpers
"crypto", # Cryptography and security features
"ecstore", # Erasure coding storage implementation
"e2e_test", # End-to-end test suite
"iam", # Identity and Access Management
"madmin", # Management dashboard and admin API interface
"rustfs", # Core file system implementation
"s3select/api", # S3 Select API interface
"s3select/query", # S3 Select query engine
"appauth", # Application authentication and authorization
"crates/zip",
]
resolver = "2"
@@ -45,22 +48,28 @@ policy = { path = "./policy", version = "0.0.1" }
protos = { path = "./common/protos", version = "0.0.1" }
query = { path = "./s3select/query", version = "0.0.1" }
rustfs = { path = "./rustfs", version = "0.0.1" }
zip = { path = "./crates/zip", version = "0.0.1" }
rustfs-config = { path = "./crates/config", version = "0.0.1" }
rustfs-obs = { path = "crates/obs", version = "0.0.1" }
rustfs-event-notifier = { path = "crates/event-notifier", version = "0.0.1" }
rustfs-utils = { path = "crates/utils", version = "0.0.1" }
workers = { path = "./common/workers", version = "0.0.1" }
tokio-tar = "0.3.1"
atoi = "2.0.0"
async-recursion = "1.1.1"
async-trait = "0.1.88"
atomic_enum = "0.3.0"
axum = "0.8.3"
axum = "0.8.4"
axum-extra = "0.10.1"
axum-server = { version = "0.7.2", features = ["tls-rustls"] }
backon = "1.5.0"
blake2 = "0.10.6"
bytes = "1.10.1"
bytesize = "2.0.1"
chrono = { version = "0.4.40", features = ["serde"] }
chrono = { version = "0.4.41", features = ["serde"] }
clap = { version = "4.5.37", features = ["derive", "env"] }
config = "0.15.11"
const-str = { version = "0.6.2", features = ["std", "proc"] }
datafusion = "46.0.1"
derive_builder = "0.20.2"
dioxus = { version = "0.6.3", features = ["router"] }
@@ -69,7 +78,9 @@ flatbuffers = "25.2.10"
futures = "0.3.31"
futures-core = "0.3.31"
futures-util = "0.3.31"
glob = "0.3.2"
hex = "0.4.3"
highway = { version = "1.3.0" }
hyper = "1.6.0"
hyper-util = { version = "0.1.11", features = [
"tokio",
@@ -86,13 +97,15 @@ keyring = { version = "3.6.2", features = [
"sync-secret-service",
] }
lazy_static = "1.5.0"
libsystemd = { version = "0.7.1" }
local-ip-address = "0.6.3"
libsystemd = { version = "0.7.2" }
local-ip-address = "0.6.5"
matchit = "0.8.4"
md-5 = "0.10.6"
mime = "0.3.17"
mime_guess = "2.0.5"
netif = "0.1.6"
nix = { version = "0.30.1", features = ["fs"] }
num_cpus = { version = "1.16.0" }
nvml-wrapper = "0.10.0"
object_store = "0.11.2"
opentelemetry = { version = "0.29.1" }
@@ -103,7 +116,9 @@ opentelemetry-appender-tracing = { version = "0.29.1", features = [
opentelemetry_sdk = { version = "0.29.0" }
opentelemetry-stdout = { version = "0.29.0" }
opentelemetry-otlp = { version = "0.29.0" }
opentelemetry-semantic-conventions = { version = "0.29.0", features = ["semconv_experimental"] }
opentelemetry-semantic-conventions = { version = "0.29.0", features = [
"semconv_experimental",
] }
parking_lot = "0.12.3"
pin-project-lite = "0.2.16"
# pin-utils = "0.1.0"
@@ -113,6 +128,8 @@ prost-types = "0.13.5"
protobuf = "3.7"
rand = "0.8.5"
rdkafka = { version = "0.37.0", features = ["tokio"] }
reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] }
regex = { version = "1.11.1" }
reqwest = { version = "0.12.15", default-features = false, features = [
"rustls-tls",
"charset",
@@ -129,8 +146,8 @@ rfd = { version = "0.15.3", default-features = false, features = [
rmp = "0.8.14"
rmp-serde = "1.3.0"
rumqttc = { version = "0.24" }
rust-embed = "8.7.0"
rustls = { version = "0.23.26" }
rust-embed = { version = "8.7.1" }
rustls = { version = "0.23.27" }
rustls-pki-types = "1.11.0"
rustls-pemfile = "2.2.0"
s3s = { git = "https://github.com/Nugine/s3s.git", rev = "4733cdfb27b2713e832967232cbff413bb768c10" }
@@ -140,9 +157,10 @@ serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.140"
serde_urlencoded = "0.7.1"
serde_with = "3.12.0"
sha2 = "0.10.8"
sha2 = "0.10.9"
smallvec = { version = "1.15.0", features = ["serde"] }
snafu = "0.8.5"
socket2 = "0.5.9"
strum = { version = "0.27.1", features = ["derive"] }
sysinfo = "0.34.2"
tempfile = "3.19.1"
@@ -155,12 +173,13 @@ time = { version = "0.3.41", features = [
"macros",
"serde",
] }
tokio = { version = "1.44.2", features = ["fs", "rt-multi-thread"] }
tokio = { version = "1.45.0", features = ["fs", "rt-multi-thread"] }
tokio-rustls = { version = "0.26.2", default-features = false }
tokio-stream = "0.1.17"
tokio-util = { version = "0.7.14", features = ["io", "compat"] }
tonic = { version = "0.13.0", features = ["gzip"] }
tonic-build = "0.13.0"
tokio-stream = { version = "0.1.17" }
tokio-util = { version = "0.7.15", features = ["io", "compat"] }
tonic = { version = "0.13.1", features = ["gzip"] }
tonic-build = { version = "0.13.1" }
tower = { version = "0.5.2", features = ["timeout"] }
tower-http = { version = "0.6.2", features = ["cors"] }
tracing = "0.1.41"
@@ -176,6 +195,7 @@ uuid = { version = "1.16.0", features = [
"fast-rng",
"macro-diagnostics",
] }
winapi = { version = "0.3.9" }
[profile.wasm-dev]
@@ -190,10 +210,10 @@ inherits = "dev"
[profile.release]
opt-level = 3
lto = "fat"
lto = "thin"
codegen-units = 1
panic = "abort" # Optional, remove the panic expansion code
strip = true # strip symbol information to reduce binary size
panic = "abort" # Optional, remove the panic expansion code
strip = true # strip symbol information to reduce binary size
[profile.production]
inherits = "release"
LICENSE
View File
README.md
+5 -23
View File
@@ -59,30 +59,12 @@ export RUSTFS_ADDRESS="0.0.0.0:9000"
export RUSTFS_CONSOLE_ENABLE=true
export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001"
# Observability config (option 1: config file)
# Observability config
export RUSTFS_OBS_CONFIG="./deploy/config/obs.toml"
# Observability config (option 2: environment variables)
export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317
export RUSTFS__OBSERVABILITY__USE_STDOUT=true
export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0
export RUSTFS__OBSERVABILITY__METER_INTERVAL=30
export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs
export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0
export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop
export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info
export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true
# Event message configuration
#export RUSTFS_EVENT_CONFIG="./deploy/config/event.toml"
# Logging sinks
export RUSTFS__SINKS__FILE__ENABLED=true
export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log"
export RUSTFS__SINKS__WEBHOOK__ENABLED=false
export RUSTFS__SINKS__WEBHOOK__ENDPOINT=""
export RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN=""
export RUSTFS__SINKS__KAFKA__ENABLED=false
export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS=""
export RUSTFS__SINKS__KAFKA__TOPIC=""
export RUSTFS__LOGGER__QUEUE_CAPACITY=10
```
#### Start the service
@@ -102,7 +84,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10
2. Start the observability stack:
```bash
docker compose up -d -f docker-compose.yml
docker compose -f docker-compose.yml up -d
```
#### Access Monitoring Dashboards
@@ -116,7 +98,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10
1. Copy the example configuration:
```bash
cd deploy/config
cp obs.toml.example obs.toml
cp obs.example.toml obs.toml
```
2. Edit `obs.toml` with the following parameters:
README_ZH.md
+5 -24
View File
@@ -59,30 +59,11 @@ export RUSTFS_ADDRESS="0.0.0.0:9000"
export RUSTFS_CONSOLE_ENABLE=true
export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001"
# 可观测性配置（方式一：配置文件）
# 可观测性配置
export RUSTFS_OBS_CONFIG="./deploy/config/obs.toml"
# 可观测性配置（方式二：环境变量）
export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317
export RUSTFS__OBSERVABILITY__USE_STDOUT=true
export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0
export RUSTFS__OBSERVABILITY__METER_INTERVAL=30
export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs
export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0
export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop
export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info
export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true
# 日志接收器
export RUSTFS__SINKS__FILE__ENABLED=true
export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log"
export RUSTFS__SINKS__WEBHOOK__ENABLED=false
export RUSTFS__SINKS__WEBHOOK__ENDPOINT=""
export RUSTFS__SINKS__WEBHOOK__AUTH_TOKEN=""
export RUSTFS__SINKS__KAFKA__ENABLED=false
export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS=""
export RUSTFS__SINKS__KAFKA__TOPIC=""
export RUSTFS__LOGGER__QUEUE_CAPACITY=10
# 事件消息配置
#export RUSTFS_EVENT_CONFIG="./deploy/config/event.toml"
```
#### 启动服务
@@ -102,7 +83,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10
2. 启动可观测性系统：
```bash
docker compose up -d -f docker-compose.yml
docker compose -f docker-compose.yml up -d
```
#### 访问监控面板
@@ -116,7 +97,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10
1. 复制示例配置：
```bash
cd deploy/config
cp obs.toml.example obs.toml
cp obs.example.toml obs.toml
```
2. 编辑 `obs.toml` 配置文件，参数如下：
common/common/src/lib.rs
+3
View File
@@ -2,6 +2,9 @@ pub mod error;
pub mod globals;
pub mod last_minute;
// is ','
pub static DEFAULT_DELIMITER: u8 = 44;
/// Defers evaluation of a block of code until the end of the scope.
#[macro_export]
macro_rules! defer {
common/workers/src/workers.rs
+10 -10
View File
@@ -3,13 +3,13 @@ use tokio::sync::{Mutex, Notify};
use tracing::info;
pub struct Workers {
available: Mutex<usize>, // 可用的工作槽
notify: Notify, // 用于通知等待的任务
limit: usize, // 最大并发工作数
available: Mutex<usize>, // Available working slots
notify: Notify, // Used to notify waiting tasks
limit: usize, // Maximum number of concurrent jobs
}
impl Workers {
// 创建 Workers 对象，允许最多 n 个作业并发执行
// Create a Workers object that allows up to n jobs to execute concurrently
pub fn new(n: usize) -> Result<Arc<Workers>, &'static str> {
if n == 0 {
return Err("n must be > 0");
@@ -22,7 +22,7 @@ impl Workers {
}))
}
// 让一个作业获得执行的机会
// Give a job a chance to be executed
pub async fn take(&self) {
loop {
let mut available = self.available.lock().await;
@@ -37,15 +37,15 @@ impl Workers {
}
}
// 让一个作业释放其机会
// Release a job's slot
pub async fn give(&self) {
let mut available = self.available.lock().await;
info!("worker give, {}", *available);
*available += 1; // 增加可用槽
self.notify.notify_one(); // 通知一个等待的任务
*available += 1; // Increase available slots
self.notify.notify_one(); // Notify a waiting task
}
// 等待所有并发作业完成
// Wait for all concurrent jobs to complete
pub async fn wait(&self) {
loop {
{
@@ -54,7 +54,7 @@ impl Workers {
break;
}
}
// 等待直到所有槽都被释放
// Wait until all slots are freed
self.notify.notified().await;
}
info!("worker wait end");
crates/config/Cargo.toml
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "rustfs-config"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[dependencies]
config = { workspace = true }
const-str = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
[lints]
workspace = true
crates/config/src/config.rs
+23
View File
@@ -0,0 +1,23 @@
use crate::event::config::NotifierConfig;
use crate::ObservabilityConfig;
/// RustFs configuration
pub struct RustFsConfig {
pub observability: ObservabilityConfig,
pub event: NotifierConfig,
}
impl RustFsConfig {
pub fn new() -> Self {
Self {
observability: ObservabilityConfig::new(),
event: NotifierConfig::new(),
}
}
}
impl Default for RustFsConfig {
fn default() -> Self {
Self::new()
}
}
crates/config/src/constants/app.rs
+91
View File
@@ -0,0 +1,91 @@
use const_str::concat;
/// Application name
/// Default value: RustFs
/// Environment variable: RUSTFS_APP_NAME
pub const APP_NAME: &str = "RustFs";
/// Application version
/// Default value: 1.0.0
/// Environment variable: RUSTFS_VERSION
pub const VERSION: &str = "0.0.1";
/// Default configuration logger level
/// Default value: info
/// Environment variable: RUSTFS_LOG_LEVEL
pub const DEFAULT_LOG_LEVEL: &str = "info";
/// Default configuration use stdout
/// Default value: true
pub const USE_STDOUT: bool = true;
/// Default configuration sample ratio
/// Default value: 1.0
pub const SAMPLE_RATIO: f64 = 1.0;
/// Default configuration meter interval
/// Default value: 30
pub const METER_INTERVAL: u64 = 30;
/// Default configuration service version
/// Default value: 0.0.1
pub const SERVICE_VERSION: &str = "0.0.1";
/// Default configuration environment
/// Default value: production
pub const ENVIRONMENT: &str = "production";
/// maximum number of connections
/// This is the maximum number of connections that the server will accept.
/// This is used to limit the number of connections to the server.
pub const MAX_CONNECTIONS: usize = 100;
/// timeout for connections
/// This is the timeout for connections to the server.
/// This is used to limit the time that a connection can be open.
pub const DEFAULT_TIMEOUT_MS: u64 = 3000;
/// Default Access Key
/// Default value: rustfsadmin
/// Environment variable: RUSTFS_ACCESS_KEY
/// Command line argument: --access-key
/// Example: RUSTFS_ACCESS_KEY=rustfsadmin
/// Example: --access-key rustfsadmin
pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
/// Default Secret Key
/// Default value: rustfsadmin
/// Environment variable: RUSTFS_SECRET_KEY
/// Command line argument: --secret-key
/// Example: RUSTFS_SECRET_KEY=rustfsadmin
/// Example: --secret-key rustfsadmin
pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
/// Default configuration file for observability
/// Default value: config/obs.toml
/// Environment variable: RUSTFS_OBS_CONFIG
/// Command line argument: --obs-config
/// Example: RUSTFS_OBS_CONFIG=config/obs.toml
/// Example: --obs-config config/obs.toml
/// Example: --obs-config /etc/rustfs/obs.toml
pub const DEFAULT_OBS_CONFIG: &str = "./deploy/config/obs.toml";
/// Default TLS key for rustfs
/// This is the default key for TLS.
pub const RUSTFS_TLS_KEY: &str = "rustfs_key.pem";
/// Default TLS cert for rustfs
/// This is the default cert for TLS.
pub const RUSTFS_TLS_CERT: &str = "rustfs_cert.pem";
/// Default port for rustfs
/// This is the default port for rustfs.
/// This is used to bind the server to a specific port.
pub const DEFAULT_PORT: u16 = 9000;
/// Default address for rustfs
/// This is the default address for rustfs.
pub const DEFAULT_ADDRESS: &str = concat!(":", DEFAULT_PORT);
/// Default port for rustfs console
/// This is the default port for rustfs console.
pub const DEFAULT_CONSOLE_PORT: u16 = 9002;
/// Default address for rustfs console
/// This is the default address for rustfs console.
pub const DEFAULT_CONSOLE_ADDRESS: &str = concat!(":", DEFAULT_CONSOLE_PORT);
crates/config/src/constants/mod.rs
+1
View File
@@ -0,0 +1 @@
pub(crate) mod app;
crates/config/src/event/adapters.rs
+27
View File
@@ -0,0 +1,27 @@
use crate::event::kafka::KafkaAdapter;
use crate::event::mqtt::MqttAdapter;
use crate::event::webhook::WebhookAdapter;
use serde::{Deserialize, Serialize};
/// Configuration for the notification system.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum AdapterConfig {
Webhook(WebhookAdapter),
Kafka(KafkaAdapter),
Mqtt(MqttAdapter),
}
impl AdapterConfig {
/// create a new configuration with default values
pub fn new() -> Self {
Self::Webhook(WebhookAdapter::new())
}
}
impl Default for AdapterConfig {
/// create a new configuration with default values
fn default() -> Self {
Self::new()
}
}
crates/config/src/event/config.rs
+43
View File
@@ -0,0 +1,43 @@
use crate::event::adapters::AdapterConfig;
use serde::{Deserialize, Serialize};
use std::env;
#[allow(dead_code)]
const DEFAULT_CONFIG_FILE: &str = "event";
/// Configuration for the notification system.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NotifierConfig {
#[serde(default = "default_store_path")]
pub store_path: String,
#[serde(default = "default_channel_capacity")]
pub channel_capacity: usize,
pub adapters: Vec<AdapterConfig>,
}
impl Default for NotifierConfig {
fn default() -> Self {
Self::new()
}
}
impl NotifierConfig {
/// create a new configuration with default values
pub fn new() -> Self {
Self {
store_path: default_store_path(),
channel_capacity: default_channel_capacity(),
adapters: vec![AdapterConfig::new()],
}
}
}
/// Provide temporary directories as default storage paths
fn default_store_path() -> String {
env::temp_dir().join("event-notification").to_string_lossy().to_string()
}
/// Provides the recommended default channel capacity for high concurrency systems
fn default_channel_capacity() -> usize {
10000 // Reasonable default values for high concurrency systems
}
crates/config/src/event/kafka.rs
+29
View File
@@ -0,0 +1,29 @@
use serde::{Deserialize, Serialize};
/// Configuration for the Kafka adapter.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KafkaAdapter {
pub brokers: String,
pub topic: String,
pub max_retries: u32,
pub timeout: u64,
}
impl KafkaAdapter {
/// create a new configuration with default values
pub fn new() -> Self {
Self {
brokers: "localhost:9092".to_string(),
topic: "kafka_topic".to_string(),
max_retries: 3,
timeout: 1000,
}
}
}
impl Default for KafkaAdapter {
/// create a new configuration with default values
fn default() -> Self {
Self::new()
}
}
crates/config/src/event/mod.rs
+5
View File
@@ -0,0 +1,5 @@
pub(crate) mod adapters;
pub(crate) mod config;
pub(crate) mod kafka;
pub(crate) mod mqtt;
pub(crate) mod webhook;
crates/config/src/event/mqtt.rs
+31
View File
@@ -0,0 +1,31 @@
use serde::{Deserialize, Serialize};
/// Configuration for the MQTT adapter.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MqttAdapter {
pub broker: String,
pub port: u16,
pub client_id: String,
pub topic: String,
pub max_retries: u32,
}
impl MqttAdapter {
/// create a new configuration with default values
pub fn new() -> Self {
Self {
broker: "localhost".to_string(),
port: 1883,
client_id: "mqtt_client".to_string(),
topic: "mqtt_topic".to_string(),
max_retries: 3,
}
}
}
impl Default for MqttAdapter {
/// create a new configuration with default values
fn default() -> Self {
Self::new()
}
}
crates/config/src/event/webhook.rs
+51
View File
@@ -0,0 +1,51 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
/// Configuration for the notification system.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebhookAdapter {
pub endpoint: String,
pub auth_token: Option<String>,
pub custom_headers: Option<HashMap<String, String>>,
pub max_retries: u32,
pub timeout: u64,
}
impl WebhookAdapter {
/// verify that the configuration is valid
pub fn validate(&self) -> Result<(), String> {
// verify that endpoint cannot be empty
if self.endpoint.trim().is_empty() {
return Err("Webhook endpoint cannot be empty".to_string());
}
// verification timeout must be reasonable
if self.timeout == 0 {
return Err("Webhook timeout must be greater than 0".to_string());
}
// Verify that the maximum number of retry is reasonable
if self.max_retries > 10 {
return Err("Maximum retry count cannot exceed 10".to_string());
}
Ok(())
}
/// Get the default configuration
pub fn new() -> Self {
Self {
endpoint: "".to_string(),
auth_token: None,
custom_headers: Some(HashMap::new()),
max_retries: 3,
timeout: 1000,
}
}
}
impl Default for WebhookAdapter {
fn default() -> Self {
Self::new()
}
}
crates/config/src/lib.rs
+11
View File
@@ -0,0 +1,11 @@
use crate::observability::config::ObservabilityConfig;
mod config;
mod constants;
mod event;
mod observability;
pub use config::RustFsConfig;
pub use constants::app::*;
pub use event::config::NotifierConfig;
crates/config/src/observability/config.rs
+28
View File
@@ -0,0 +1,28 @@
use crate::observability::logger::LoggerConfig;
use crate::observability::otel::OtelConfig;
use crate::observability::sink::SinkConfig;
use serde::{Deserialize, Serialize};
/// Observability configuration
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct ObservabilityConfig {
pub otel: OtelConfig,
pub sinks: Vec<SinkConfig>,
pub logger: Option<LoggerConfig>,
}
impl ObservabilityConfig {
pub fn new() -> Self {
Self {
otel: OtelConfig::new(),
sinks: vec![SinkConfig::new()],
logger: Some(LoggerConfig::new()),
}
}
}
impl Default for ObservabilityConfig {
fn default() -> Self {
Self::new()
}
}
crates/config/src/observability/file.rs
+59
View File
@@ -0,0 +1,59 @@
use serde::{Deserialize, Serialize};
use std::env;
/// File sink configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileSink {
pub path: String,
#[serde(default = "default_buffer_size")]
pub buffer_size: Option<usize>,
#[serde(default = "default_flush_interval_ms")]
pub flush_interval_ms: Option<u64>,
#[serde(default = "default_flush_threshold")]
pub flush_threshold: Option<usize>,
}
impl FileSink {
pub fn new() -> Self {
Self {
path: env::var("RUSTFS_SINKS_FILE_PATH")
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(default_path),
buffer_size: default_buffer_size(),
flush_interval_ms: default_flush_interval_ms(),
flush_threshold: default_flush_threshold(),
}
}
}
impl Default for FileSink {
fn default() -> Self {
Self::new()
}
}
fn default_buffer_size() -> Option<usize> {
Some(8192)
}
fn default_flush_interval_ms() -> Option<u64> {
Some(1000)
}
fn default_flush_threshold() -> Option<usize> {
Some(100)
}
fn default_path() -> String {
let temp_dir = env::temp_dir().join("rustfs");
if let Err(e) = std::fs::create_dir_all(&temp_dir) {
eprintln!("Failed to create log directory: {}", e);
return "rustfs/rustfs.log".to_string();
}
temp_dir
.join("rustfs.log")
.to_str()
.unwrap_or("rustfs/rustfs.log")
.to_string()
}
crates/config/src/observability/kafka.rs
+36
View File
@@ -0,0 +1,36 @@
use serde::{Deserialize, Serialize};
/// Kafka sink configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KafkaSink {
pub brokers: String,
pub topic: String,
#[serde(default = "default_batch_size")]
pub batch_size: Option<usize>,
#[serde(default = "default_batch_timeout_ms")]
pub batch_timeout_ms: Option<u64>,
}
impl KafkaSink {
pub fn new() -> Self {
Self {
brokers: "localhost:9092".to_string(),
topic: "rustfs".to_string(),
batch_size: default_batch_size(),
batch_timeout_ms: default_batch_timeout_ms(),
}
}
}
impl Default for KafkaSink {
fn default() -> Self {
Self::new()
}
}
fn default_batch_size() -> Option<usize> {
Some(100)
}
fn default_batch_timeout_ms() -> Option<u64> {
Some(1000)
}
crates/config/src/observability/logger.rs
+21
View File
@@ -0,0 +1,21 @@
use serde::{Deserialize, Serialize};
/// Logger configuration
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct LoggerConfig {
pub queue_capacity: Option<usize>,
}
impl LoggerConfig {
pub fn new() -> Self {
Self {
queue_capacity: Some(10000),
}
}
}
impl Default for LoggerConfig {
fn default() -> Self {
Self::new()
}
}
crates/config/src/observability/mod.rs
+7
View File
@@ -0,0 +1,7 @@
pub(crate) mod config;
pub(crate) mod file;
pub(crate) mod kafka;
pub(crate) mod logger;
pub(crate) mod otel;
pub(crate) mod sink;
pub(crate) mod webhook;
crates/config/src/observability/otel.rs
+69
View File
@@ -0,0 +1,69 @@
use crate::constants::app::{ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT};
use crate::{APP_NAME, DEFAULT_LOG_LEVEL};
use serde::{Deserialize, Serialize};
use std::env;
/// OpenTelemetry configuration
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct OtelConfig {
pub endpoint: String, // Endpoint for metric collection
pub use_stdout: Option<bool>, // Output to stdout
pub sample_ratio: Option<f64>, // Trace sampling ratio
pub meter_interval: Option<u64>, // Metric collection interval
pub service_name: Option<String>, // Service name
pub service_version: Option<String>, // Service version
pub environment: Option<String>, // Environment
pub logger_level: Option<String>, // Logger level
pub local_logging_enabled: Option<bool>, // Local logging enabled
}
impl OtelConfig {
pub fn new() -> Self {
extract_otel_config_from_env()
}
}
impl Default for OtelConfig {
fn default() -> Self {
Self::new()
}
}
// Helper function: Extract observable configuration from environment variables
fn extract_otel_config_from_env() -> OtelConfig {
OtelConfig {
endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()),
use_stdout: env::var("RUSTFS_OBSERVABILITY_USE_STDOUT")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(USE_STDOUT)),
sample_ratio: env::var("RUSTFS_OBSERVABILITY_SAMPLE_RATIO")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(SAMPLE_RATIO)),
meter_interval: env::var("RUSTFS_OBSERVABILITY_METER_INTERVAL")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(METER_INTERVAL)),
service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(APP_NAME.to_string())),
service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(SERVICE_VERSION.to_string())),
environment: env::var("RUSTFS_OBSERVABILITY_ENVIRONMENT")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(ENVIRONMENT.to_string())),
logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(DEFAULT_LOG_LEVEL.to_string())),
local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(false)),
}
}
crates/config/src/observability/sink.rs
+25
View File
@@ -0,0 +1,25 @@
use crate::observability::file::FileSink;
use crate::observability::kafka::KafkaSink;
use crate::observability::webhook::WebhookSink;
use serde::{Deserialize, Serialize};
/// Sink configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum SinkConfig {
Kafka(KafkaSink),
Webhook(WebhookSink),
File(FileSink),
}
impl SinkConfig {
pub fn new() -> Self {
Self::File(FileSink::new())
}
}
impl Default for SinkConfig {
fn default() -> Self {
Self::new()
}
}
crates/config/src/observability/webhook.rs
+39
View File
@@ -0,0 +1,39 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
/// Webhook sink configuration
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct WebhookSink {
pub endpoint: String,
pub auth_token: String,
pub headers: Option<HashMap<String, String>>,
#[serde(default = "default_max_retries")]
pub max_retries: Option<usize>,
#[serde(default = "default_retry_delay_ms")]
pub retry_delay_ms: Option<u64>,
}
impl WebhookSink {
pub fn new() -> Self {
Self {
endpoint: "".to_string(),
auth_token: "".to_string(),
headers: Some(HashMap::new()),
max_retries: default_max_retries(),
retry_delay_ms: default_retry_delay_ms(),
}
}
}
impl Default for WebhookSink {
fn default() -> Self {
Self::new()
}
}
fn default_max_retries() -> Option<usize> {
Some(3)
}
fn default_retry_delay_ms() -> Option<u64> {
Some(100)
}
crates/event-notifier/Cargo.toml
+5 -2
View File
@@ -9,13 +9,12 @@ version.workspace = true
[features]
default = ["webhook"]
webhook = ["dep:reqwest"]
kafka = ["rdkafka"]
mqtt = ["rumqttc"]
kafka = ["dep:rdkafka"]
[dependencies]
async-trait = { workspace = true }
config = { workspace = true }
rdkafka = { workspace = true, features = ["tokio"], optional = true }
reqwest = { workspace = true, optional = true }
rumqttc = { workspace = true, optional = true }
serde = { workspace = true }
@@ -29,12 +28,16 @@ tokio = { workspace = true, features = ["sync", "net", "macros", "signal", "rt-m
tokio-util = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde"] }
# Only enable kafka features and related dependencies on Linux
[target.'cfg(target_os = "linux")'.dependencies]
rdkafka = { workspace = true, features = ["tokio"], optional = true }
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
tracing-subscriber = { workspace = true }
http = { workspace = true }
axum = { workspace = true }
dotenvy = "0.15.7"
[lints]
workspace = true
crates/event-notifier/examples/.env-zh.example
-28
View File
@@ -1,28 +0,0 @@
# ===== 全局配置 =====
NOTIFIER__STORE_PATH=/var/log/event-notification
NOTIFIER__CHANNEL_CAPACITY=5000
# ===== 适配器配置（数组格式） =====
# Webhook 适配器（索引 0）
NOTIFIER__ADAPTERS_0__type=Webhook
NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
NOTIFIER__ADAPTERS_0__max_retries=3
NOTIFIER__ADAPTERS_0__timeout=50
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value
# Kafka 适配器（索引 1）
NOTIFIER__ADAPTERS_1__type=Kafka
NOTIFIER__ADAPTERS_1__brokers=localhost:9092
NOTIFIER__ADAPTERS_1__topic=notifications
NOTIFIER__ADAPTERS_1__max_retries=3
NOTIFIER__ADAPTERS_1__timeout=60
# MQTT 适配器（索引 2）
NOTIFIER__ADAPTERS_2__type=Mqtt
NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
NOTIFIER__ADAPTERS_2__port=1883
NOTIFIER__ADAPTERS_2__client_id=event-notifier
NOTIFIER__ADAPTERS_2__topic=events
NOTIFIER__ADAPTERS_2__max_retries=3
crates/event-notifier/examples/.env.example
+28 -28
View File
@@ -1,28 +1,28 @@
# ===== global configuration =====
NOTIFIER__STORE_PATH=/var/log/event-notification
NOTIFIER__CHANNEL_CAPACITY=5000
# ===== adapter configuration array format =====
# webhook adapter index 0
NOTIFIER__ADAPTERS_0__type=Webhook
NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
NOTIFIER__ADAPTERS_0__max_retries=3
NOTIFIER__ADAPTERS_0__timeout=50
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value
# kafka adapter index 1
NOTIFIER__ADAPTERS_1__type=Kafka
NOTIFIER__ADAPTERS_1__brokers=localhost:9092
NOTIFIER__ADAPTERS_1__topic=notifications
NOTIFIER__ADAPTERS_1__max_retries=3
NOTIFIER__ADAPTERS_1__timeout=60
# mqtt adapter index 2
NOTIFIER__ADAPTERS_2__type=Mqtt
NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
NOTIFIER__ADAPTERS_2__port=1883
NOTIFIER__ADAPTERS_2__client_id=event-notifier
NOTIFIER__ADAPTERS_2__topic=events
NOTIFIER__ADAPTERS_2__max_retries=3
## ===== global configuration =====
#NOTIFIER__STORE_PATH=/var/log/event-notification
#NOTIFIER__CHANNEL_CAPACITY=5000
#
## ===== adapter configuration array format =====
## webhook adapter index 0
#NOTIFIER__ADAPTERS_0__type=Webhook
#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
#NOTIFIER__ADAPTERS_0__max_retries=3
#NOTIFIER__ADAPTERS_0__timeout=50
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value
#
## kafka adapter index 1
#NOTIFIER__ADAPTERS_1__type=Kafka
#NOTIFIER__ADAPTERS_1__brokers=localhost:9092
#NOTIFIER__ADAPTERS_1__topic=notifications
#NOTIFIER__ADAPTERS_1__max_retries=3
#NOTIFIER__ADAPTERS_1__timeout=60
#
## mqtt adapter index 2
#NOTIFIER__ADAPTERS_2__type=Mqtt
#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
#NOTIFIER__ADAPTERS_2__port=1883
#NOTIFIER__ADAPTERS_2__client_id=event-notifier
#NOTIFIER__ADAPTERS_2__topic=events
#NOTIFIER__ADAPTERS_2__max_retries=3
crates/event-notifier/examples/.env.zh.example
+28
View File
@@ -0,0 +1,28 @@
## ===== 全局配置 =====
#NOTIFIER__STORE_PATH=/var/log/event-notification
#NOTIFIER__CHANNEL_CAPACITY=5000
#
## ===== 适配器配置（数组格式） =====
## Webhook 适配器（索引 0）
#NOTIFIER__ADAPTERS_0__type=Webhook
#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
#NOTIFIER__ADAPTERS_0__max_retries=3
#NOTIFIER__ADAPTERS_0__timeout=50
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value
#
## Kafka 适配器（索引 1）
#NOTIFIER__ADAPTERS_1__type=Kafka
#NOTIFIER__ADAPTERS_1__brokers=localhost:9092
#NOTIFIER__ADAPTERS_1__topic=notifications
#NOTIFIER__ADAPTERS_1__max_retries=3
#NOTIFIER__ADAPTERS_1__timeout=60
#
## MQTT 适配器（索引 2）
#NOTIFIER__ADAPTERS_2__type=Mqtt
#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
#NOTIFIER__ADAPTERS_2__port=1883
#NOTIFIER__ADAPTERS_2__client_id=event-notifier
#NOTIFIER__ADAPTERS_2__topic=events
#NOTIFIER__ADAPTERS_2__max_retries=3
crates/event-notifier/examples/simple.rs
+3 -1
View File
@@ -33,7 +33,9 @@ async fn main() -> Result<(), Box<dyn error::Error>> {
// loading configuration from environment variables
let _config = NotifierConfig::event_load_config(Some("./crates/event-notifier/examples/event.toml".to_string()));
tracing::info!("event_load_config config: {:?} \n", _config);
dotenvy::dotenv()?;
let _config = NotifierConfig::event_load_config(None);
tracing::info!("event_load_config config: {:?} \n", _config);
let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?));
let adapters = create_adapters(&config.adapters)?;
crates/event-notifier/examples/webhook.rs
+4 -4
View File
@@ -37,7 +37,7 @@ async fn receive_webhook(Json(payload): Json<Value>) -> StatusCode {
println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second);
println!(
"received a webhook request time:{} content:\n {}",
seconds.to_string(),
seconds,
serde_json::to_string_pretty(&payload).unwrap()
);
StatusCode::OK
@@ -66,10 +66,10 @@ fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) {
// calculate month
let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
for m in 0..12 {
if total_seconds >= days_in_month[m] * seconds_per_day {
for m in &days_in_month {
if total_seconds >= m * seconds_per_day {
month += 1;
total_seconds -= days_in_month[m] * seconds_per_day;
total_seconds -= m * seconds_per_day;
} else {
break;
}
crates/event-notifier/src/adapter/mod.rs
+3 -3
View File
@@ -4,7 +4,7 @@ use crate::Event;
use async_trait::async_trait;
use std::sync::Arc;
#[cfg(feature = "kafka")]
#[cfg(all(feature = "kafka", target_os = "linux"))]
pub(crate) mod kafka;
#[cfg(feature = "mqtt")]
pub(crate) mod mqtt;
@@ -31,7 +31,7 @@ pub fn create_adapters(configs: &[AdapterConfig]) -> Result<Vec<Arc<dyn ChannelA
webhook_config.validate().map_err(Error::ConfigError)?;
adapters.push(Arc::new(webhook::WebhookAdapter::new(webhook_config.clone())));
}
#[cfg(feature = "kafka")]
#[cfg(all(feature = "kafka", target_os = "linux"))]
AdapterConfig::Kafka(kafka_config) => {
adapters.push(Arc::new(kafka::KafkaAdapter::new(kafka_config)?));
}
@@ -43,7 +43,7 @@ pub fn create_adapters(configs: &[AdapterConfig]) -> Result<Vec<Arc<dyn ChannelA
}
#[cfg(not(feature = "webhook"))]
AdapterConfig::Webhook(_) => return Err(Error::FeatureDisabled("webhook")),
#[cfg(not(feature = "kafka"))]
#[cfg(any(not(feature = "kafka"), not(target_os = "linux")))]
AdapterConfig::Kafka(_) => return Err(Error::FeatureDisabled("kafka")),
#[cfg(not(feature = "mqtt"))]
AdapterConfig::Mqtt(_) => return Err(Error::FeatureDisabled("mqtt")),
crates/event-notifier/src/bus.rs
+2
View File
@@ -7,11 +7,13 @@ use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::mpsc;
use tokio::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::instrument;
/// Handles incoming events from the producer.
///
/// This function is responsible for receiving events from the producer and sending them to the appropriate adapters.
/// It also handles the shutdown process and saves any pending logs to the event store.
#[instrument(skip_all)]
pub async fn event_bus(
mut rx: mpsc::Receiver<Event>,
adapters: Vec<Arc<dyn ChannelAdapter>>,
crates/event-notifier/src/config.rs
+2 -11
View File
@@ -1,4 +1,4 @@
use config::{Config, Environment, File, FileFormat};
use config::{Config, File, FileFormat};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::env;
@@ -138,15 +138,6 @@ impl NotifierConfig {
let app_config = Config::builder()
.add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false))
.add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false))
.add_source(
Environment::default()
.prefix("NOTIFIER")
.prefix_separator("__")
.separator("__")
.list_separator("_")
.with_list_parse_key("adapters")
.try_parsing(true),
)
.build()
.unwrap_or_default();
match app_config.try_deserialize::<NotifierConfig>() {
@@ -162,7 +153,7 @@ impl NotifierConfig {
}
}
const DEFAULT_CONFIG_FILE: &str = "obs";
const DEFAULT_CONFIG_FILE: &str = "event";
/// Provide temporary directories as default storage paths
fn default_store_path() -> String {
crates/event-notifier/src/error.rs
+1 -1
View File
@@ -15,7 +15,7 @@ pub enum Error {
Serde(#[from] serde_json::Error),
#[error("HTTP error: {0}")]
Http(#[from] reqwest::Error),
#[cfg(feature = "kafka")]
#[cfg(all(feature = "kafka", target_os = "linux"))]
#[error("Kafka error: {0}")]
Kafka(#[from] rdkafka::error::KafkaError),
#[cfg(feature = "mqtt")]
crates/event-notifier/src/event.rs
+169
View File
@@ -15,6 +15,18 @@ pub struct Identity {
pub principal_id: String,
}
impl Identity {
/// Create a new Identity instance
pub fn new(principal_id: String) -> Self {
Self { principal_id }
}
/// Set the principal ID
pub fn set_principal_id(&mut self, principal_id: String) {
self.principal_id = principal_id;
}
}
/// A struct representing the bucket information
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Bucket {
@@ -24,6 +36,32 @@ pub struct Bucket {
pub arn: String,
}
impl Bucket {
/// Create a new Bucket instance
pub fn new(name: String, owner_identity: Identity, arn: String) -> Self {
Self {
name,
owner_identity,
arn,
}
}
/// Set the name of the bucket
pub fn set_name(&mut self, name: String) {
self.name = name;
}
/// Set the ARN of the bucket
pub fn set_arn(&mut self, arn: String) {
self.arn = arn;
}
/// Set the owner identity of the bucket
pub fn set_owner_identity(&mut self, owner_identity: Identity) {
self.owner_identity = owner_identity;
}
}
/// A struct representing the object information
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Object {
@@ -41,6 +79,64 @@ pub struct Object {
pub sequencer: String,
}
impl Object {
/// Create a new Object instance
pub fn new(
key: String,
size: Option<i64>,
etag: Option<String>,
content_type: Option<String>,
user_metadata: Option<HashMap<String, String>>,
version_id: Option<String>,
sequencer: String,
) -> Self {
Self {
key,
size,
etag,
content_type,
user_metadata,
version_id,
sequencer,
}
}
/// Set the key
pub fn set_key(&mut self, key: String) {
self.key = key;
}
/// Set the size
pub fn set_size(&mut self, size: Option<i64>) {
self.size = size;
}
/// Set the etag
pub fn set_etag(&mut self, etag: Option<String>) {
self.etag = etag;
}
/// Set the content type
pub fn set_content_type(&mut self, content_type: Option<String>) {
self.content_type = content_type;
}
/// Set the user metadata
pub fn set_user_metadata(&mut self, user_metadata: Option<HashMap<String, String>>) {
self.user_metadata = user_metadata;
}
/// Set the version ID
pub fn set_version_id(&mut self, version_id: Option<String>) {
self.version_id = version_id;
}
/// Set the sequencer
pub fn set_sequencer(&mut self, sequencer: String) {
self.sequencer = sequencer;
}
}
/// A struct representing the metadata of the event
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Metadata {
@@ -52,6 +148,57 @@ pub struct Metadata {
pub object: Object,
}
impl Default for Metadata {
fn default() -> Self {
Self::new()
}
}
impl Metadata {
/// Create a new Metadata instance with default values
pub fn new() -> Self {
Self {
schema_version: "1.0".to_string(),
configuration_id: "default".to_string(),
bucket: Bucket::new(
"default".to_string(),
Identity::new("default".to_string()),
"arn:aws:s3:::default".to_string(),
),
object: Object::new("default".to_string(), None, None, None, None, None, "default".to_string()),
}
}
/// Create a new Metadata instance
pub fn create(schema_version: String, configuration_id: String, bucket: Bucket, object: Object) -> Self {
Self {
schema_version,
configuration_id,
bucket,
object,
}
}
/// Set the schema version
pub fn set_schema_version(&mut self, schema_version: String) {
self.schema_version = schema_version;
}
/// Set the configuration ID
pub fn set_configuration_id(&mut self, configuration_id: String) {
self.configuration_id = configuration_id;
}
/// Set the bucket
pub fn set_bucket(&mut self, bucket: Bucket) {
self.bucket = bucket;
}
/// Set the object
pub fn set_object(&mut self, object: Object) {
self.object = object;
}
}
/// A struct representing the source of the event
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Source {
@@ -61,6 +208,28 @@ pub struct Source {
pub user_agent: String,
}
impl Source {
/// Create a new Source instance
pub fn new(host: String, port: String, user_agent: String) -> Self {
Self { host, port, user_agent }
}
/// Set the host
pub fn set_host(&mut self, host: String) {
self.host = host;
}
/// Set the port
pub fn set_port(&mut self, port: String) {
self.port = port;
}
/// Set the user agent
pub fn set_user_agent(&mut self, user_agent: String) {
self.user_agent = user_agent;
}
}
/// Builder for creating an Event.
///
/// This struct is used to build an Event object with various parameters.
crates/event-notifier/src/global.rs
+5 -2
View File
@@ -1,6 +1,7 @@
use crate::{create_adapters, Error, Event, NotifierConfig, NotifierSystem};
use std::sync::{atomic, Arc};
use tokio::sync::{Mutex, OnceCell};
use tracing::instrument;
static GLOBAL_SYSTEM: OnceCell<Arc<Mutex<NotifierSystem>>> = OnceCell::const_new();
static INITIALIZED: atomic::AtomicBool = atomic::AtomicBool::new(false);
@@ -113,6 +114,7 @@ pub fn is_ready() -> bool {
/// - The system is not initialized.
/// - The system is not ready.
/// - Sending the event fails.
#[instrument(fields(event))]
pub async fn send_event(event: Event) -> Result<(), Error> {
if !READY.load(atomic::Ordering::SeqCst) {
return Err(Error::custom("Notification system not ready, please wait for initialization to complete"));
@@ -124,6 +126,7 @@ pub async fn send_event(event: Event) -> Result<(), Error> {
}
/// Shuts down the notification system.
#[instrument]
pub async fn shutdown() -> Result<(), Error> {
if let Some(system) = GLOBAL_SYSTEM.get() {
tracing::info!("Shutting down notification system start");
@@ -189,7 +192,7 @@ mod tests {
let config = NotifierConfig::default();
let _ = initialize(config.clone()).await; // first initialization
let result = initialize(config).await; // second initialization
assert!(!result.is_ok(), "Initialization should succeed");
assert!(result.is_err(), "Initialization should succeed");
assert!(result.is_err(), "Re-initialization should fail");
}
@@ -211,7 +214,7 @@ mod tests {
..Default::default()
};
let result = initialize(config).await;
assert!(!result.is_err(), "Initialization with invalid config should fail");
assert!(result.is_ok(), "Initialization with invalid config should fail");
assert!(is_initialized(), "System should not be marked as initialized after failure");
assert!(is_ready(), "System should not be marked as ready after failure");
}
crates/event-notifier/src/lib.rs
+2 -2
View File
@@ -8,7 +8,7 @@ mod notifier;
mod store;
pub use adapter::create_adapters;
#[cfg(feature = "kafka")]
#[cfg(all(feature = "kafka", target_os = "linux"))]
pub use adapter::kafka::KafkaAdapter;
#[cfg(feature = "mqtt")]
pub use adapter::mqtt::MqttAdapter;
@@ -16,7 +16,7 @@ pub use adapter::mqtt::MqttAdapter;
pub use adapter::webhook::WebhookAdapter;
pub use adapter::ChannelAdapter;
pub use bus::event_bus;
#[cfg(feature = "kafka")]
#[cfg(all(feature = "kafka", target_os = "linux"))]
pub use config::KafkaConfig;
#[cfg(feature = "mqtt")]
pub use config::MqttConfig;
crates/event-notifier/src/notifier.rs
+9 -1
View File
@@ -2,6 +2,7 @@ use crate::{event_bus, ChannelAdapter, Error, Event, EventStore, NotifierConfig}
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::instrument;
/// The `NotificationSystem` struct represents the notification system.
/// It manages the event bus and the adapters.
@@ -18,6 +19,7 @@ pub struct NotifierSystem {
impl NotifierSystem {
/// Creates a new `NotificationSystem` instance.
#[instrument(skip(config))]
pub async fn new(config: NotifierConfig) -> Result<Self, Error> {
let (tx, rx) = mpsc::channel::<Event>(config.channel_capacity);
let store = Arc::new(EventStore::new(&config.store_path).await?);
@@ -44,6 +46,7 @@ impl NotifierSystem {
/// Starts the notification system.
/// It initializes the event bus and the producer.
#[instrument(skip_all)]
pub async fn start(&mut self, adapters: Vec<Arc<dyn ChannelAdapter>>) -> Result<(), Error> {
if self.shutdown.is_cancelled() {
let error = Error::custom("System is shutting down");
@@ -67,6 +70,7 @@ impl NotifierSystem {
/// Sends an event to the notification system.
/// This method is used to send events to the event bus.
#[instrument(skip(self))]
pub async fn send_event(&self, event: Event) -> Result<(), Error> {
self.log(tracing::Level::DEBUG, "send_event", &format!("Sending event: {:?}", event));
if self.shutdown.is_cancelled() {
@@ -85,6 +89,7 @@ impl NotifierSystem {
/// Shuts down the notification system.
/// This method is used to cancel the event bus and producer tasks.
#[instrument(skip(self))]
pub async fn shutdown(&mut self) -> Result<(), Error> {
tracing::info!("Shutting down the notification system");
self.shutdown.cancel();
@@ -112,10 +117,13 @@ impl NotifierSystem {
self.shutdown.is_cancelled()
}
fn handle_error(&self, context: &str, error: &Error) {
#[instrument(skip(self))]
pub fn handle_error(&self, context: &str, error: &Error) {
self.log(tracing::Level::ERROR, context, &format!("{:?}", error));
// TODO Can be extended to record to files or send to monitoring systems
}
#[instrument(skip(self))]
fn log(&self, level: tracing::Level, context: &str, message: &str) {
match level {
tracing::Level::ERROR => tracing::error!("[{}] {}", context, message),
crates/event-notifier/src/store.rs
+2
View File
@@ -5,6 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use tokio::fs::{create_dir_all, File, OpenOptions};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter};
use tokio::sync::RwLock;
use tracing::instrument;
/// `EventStore` is a struct that manages the storage of event logs.
pub struct EventStore {
@@ -21,6 +22,7 @@ impl EventStore {
})
}
#[instrument(skip(self))]
pub async fn save_logs(&self, logs: &[Log]) -> Result<(), Error> {
let _guard = self.lock.write().await;
let file_path = format!(
crates/obs/Cargo.toml
+6 -3
View File
@@ -13,11 +13,11 @@ workspace = true
default = ["file"]
file = []
gpu = ["dep:nvml-wrapper"]
kafka = ["dep:rdkafka"]
webhook = ["dep:reqwest"]
full = ["file", "gpu", "kafka", "webhook"]
kafka = ["dep:rdkafka"]
[dependencies]
rustfs-config = { workspace = true }
async-trait = { workspace = true }
chrono = { workspace = true }
config = { workspace = true }
@@ -29,6 +29,7 @@ opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
opentelemetry-stdout = { workspace = true }
opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "gzip-tonic"] }
opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
rustfs-utils = { workspace = true, features = ["ip"] }
serde = { workspace = true }
smallvec = { workspace = true, features = ["serde"] }
tracing = { workspace = true, features = ["std", "attributes"] }
@@ -37,12 +38,14 @@ tracing-error = { workspace = true }
tracing-opentelemetry = { workspace = true }
tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
rdkafka = { workspace = true, features = ["tokio"], optional = true }
reqwest = { workspace = true, optional = true, default-features = false }
serde_json = { workspace = true }
sysinfo = { workspace = true }
thiserror = { workspace = true }
# Only enable kafka features and related dependencies on Linux
[target.'cfg(target_os = "linux")'.dependencies]
rdkafka = { workspace = true, features = ["tokio"], optional = true }
[dev-dependencies]
crates/obs/examples/config.toml
+19 -18
View File
@@ -9,26 +9,27 @@ environments = "develop"
logger_level = "debug"
local_logging_enabled = true # Default is false if not specified
[sinks]
[sinks.kafka]
enabled = false
bootstrap_servers = "localhost:9092"
topic = "logs"
batch_size = 100 # Default is 100 if not specified
batch_timeout_ms = 1000 # Default is 1000ms if not specified
[sinks.webhook]
enabled = false
endpoint = "http://localhost:8080/webhook"
auth_token = ""
batch_size = 100 # Default is 3 if not specified
batch_timeout_ms = 1000 # Default is 100ms if not specified
#[[sinks]]
#type = "Kafka"
#bootstrap_servers = "localhost:9092"
#topic = "logs"
#batch_size = 100 # Default is 100 if not specified
#batch_timeout_ms = 100 # Default is 1000ms if not specified
#
#[[sinks]]
#type = "Webhook"
#endpoint = "http://localhost:8080/webhook"
#auth_token = ""
#batch_size = 100 # Default is 3 if not specified
#batch_timeout_ms = 100 # Default is 100ms if not specified
[sinks.file]
enabled = true
path = "deploy/logs/app.log"
batch_size = 100
batch_timeout_ms = 1000 # Default is 8192 bytes if not specified
[[sinks]]
type = "File"
path = "deploy/logs/rustfs.log"
buffer_size = 102 # Default is 8192 bytes if not specified
flush_interval_ms = 1000
flush_threshold = 100
[logger]
queue_capacity = 10000
crates/obs/src/config.rs
+103 -47
View File
@@ -1,6 +1,6 @@
use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
use config::{Config, Environment, File, FileFormat};
use serde::Deserialize;
use config::{Config, File, FileFormat};
use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT};
use serde::{Deserialize, Serialize};
use std::env;
/// OpenTelemetry Configuration
@@ -11,7 +11,7 @@ use std::env;
/// Add use_stdout for output to stdout
/// Add logger level for log level
/// Add local_logging_enabled for local logging enabled
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct OtelConfig {
pub endpoint: String, // Endpoint for metric collection
pub use_stdout: Option<bool>, // Output to stdout
@@ -24,7 +24,7 @@ pub struct OtelConfig {
pub local_logging_enabled: Option<bool>, // Local logging enabled
}
// Helper function: Extract observable configuration from environment variables
/// Helper function: Extract observable configuration from environment variables
fn extract_otel_config_from_env() -> OtelConfig {
OtelConfig {
endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()),
@@ -43,7 +43,7 @@ fn extract_otel_config_from_env() -> OtelConfig {
service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(SERVICE_NAME.to_string())),
.or(Some(APP_NAME.to_string())),
service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION")
.ok()
.and_then(|v| v.parse().ok())
@@ -55,7 +55,7 @@ fn extract_otel_config_from_env() -> OtelConfig {
logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL")
.ok()
.and_then(|v| v.parse().ok())
.or(Some(LOGGER_LEVEL.to_string())),
.or(Some(DEFAULT_LOG_LEVEL.to_string())),
local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED")
.ok()
.and_then(|v| v.parse().ok())
@@ -63,36 +63,89 @@ fn extract_otel_config_from_env() -> OtelConfig {
}
}
impl Default for OtelConfig {
fn default() -> Self {
impl OtelConfig {
/// Create a new instance of OtelConfig with default values
///
/// # Returns
/// A new instance of OtelConfig
pub fn new() -> Self {
extract_otel_config_from_env()
}
}
impl Default for OtelConfig {
fn default() -> Self {
Self::new()
}
}
/// Kafka Sink Configuration - Add batch parameters
#[derive(Debug, Deserialize, Clone, Default)]
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct KafkaSinkConfig {
pub enabled: bool,
pub bootstrap_servers: String,
pub brokers: String,
pub topic: String,
pub batch_size: Option<usize>, // Batch size, default 100
pub batch_timeout_ms: Option<u64>, // Batch timeout time, default 1000ms
}
impl KafkaSinkConfig {
pub fn new() -> Self {
Self {
brokers: env::var("RUSTFS_SINKS_KAFKA_BROKERS")
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| "localhost:9092".to_string()),
topic: env::var("RUSTFS_SINKS_KAFKA_TOPIC")
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| "default_topic".to_string()),
batch_size: Some(100),
batch_timeout_ms: Some(1000),
}
}
}
impl Default for KafkaSinkConfig {
fn default() -> Self {
Self::new()
}
}
/// Webhook Sink Configuration - Add Retry Parameters
#[derive(Debug, Deserialize, Clone, Default)]
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct WebhookSinkConfig {
pub enabled: bool,
pub endpoint: String,
pub auth_token: String,
pub max_retries: Option<usize>, // Maximum number of retry times, default 3
pub retry_delay_ms: Option<u64>, // Retry the delay cardinality, default 100ms
}
impl WebhookSinkConfig {
pub fn new() -> Self {
Self {
endpoint: env::var("RUSTFS_SINKS_WEBHOOK_ENDPOINT")
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| "http://localhost:8080".to_string()),
auth_token: env::var("RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN")
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| "default_token".to_string()),
max_retries: Some(3),
retry_delay_ms: Some(100),
}
}
}
impl Default for WebhookSinkConfig {
fn default() -> Self {
Self::new()
}
}
/// File Sink Configuration - Add buffering parameters
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct FileSinkConfig {
pub enabled: bool,
pub path: String,
pub buffer_size: Option<usize>, // Write buffer size, default 8192
pub flush_interval_ms: Option<u64>, // Refresh interval time, default 1000ms
@@ -107,19 +160,15 @@ impl FileSinkConfig {
eprintln!("Failed to create log directory: {}", e);
return "rustfs/rustfs.log".to_string();
}
println!("Using log directory: {:?}", temp_dir);
temp_dir
.join("rustfs.log")
.to_str()
.unwrap_or("rustfs/rustfs.log")
.to_string()
}
}
impl Default for FileSinkConfig {
fn default() -> Self {
FileSinkConfig {
enabled: true,
pub fn new() -> Self {
Self {
path: env::var("RUSTFS_SINKS_FILE_PATH")
.ok()
.filter(|s| !s.trim().is_empty())
@@ -131,38 +180,53 @@ impl Default for FileSinkConfig {
}
}
impl Default for FileSinkConfig {
fn default() -> Self {
Self::new()
}
}
/// Sink configuration collection
#[derive(Debug, Deserialize, Clone)]
pub struct SinkConfig {
pub kafka: Option<KafkaSinkConfig>,
pub webhook: Option<WebhookSinkConfig>,
pub file: Option<FileSinkConfig>,
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum SinkConfig {
File(FileSinkConfig),
Kafka(KafkaSinkConfig),
Webhook(WebhookSinkConfig),
}
impl SinkConfig {
pub fn new() -> Self {
Self::File(FileSinkConfig::new())
}
}
impl Default for SinkConfig {
fn default() -> Self {
SinkConfig {
kafka: None,
webhook: None,
file: Some(FileSinkConfig::default()),
}
Self::new()
}
}
///Logger Configuration
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct LoggerConfig {
pub queue_capacity: Option<usize>,
}
impl Default for LoggerConfig {
fn default() -> Self {
LoggerConfig {
impl LoggerConfig {
pub fn new() -> Self {
Self {
queue_capacity: Some(10000),
}
}
}
impl Default for LoggerConfig {
fn default() -> Self {
Self::new()
}
}
/// Overall application configuration
/// Add observability, sinks, and logger configuration
///
@@ -180,7 +244,7 @@ impl Default for LoggerConfig {
#[derive(Debug, Deserialize, Clone)]
pub struct AppConfig {
pub observability: OtelConfig,
pub sinks: SinkConfig,
pub sinks: Vec<SinkConfig>,
pub logger: Option<LoggerConfig>,
}
@@ -192,7 +256,7 @@ impl AppConfig {
pub fn new() -> Self {
Self {
observability: OtelConfig::default(),
sinks: SinkConfig::default(),
sinks: vec![SinkConfig::default()],
logger: Some(LoggerConfig::default()),
}
}
@@ -258,14 +322,6 @@ pub fn load_config(config_dir: Option<String>) -> AppConfig {
let app_config = Config::builder()
.add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false))
.add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false))
.add_source(
Environment::default()
.prefix("RUSTFS")
.prefix_separator("__")
.separator("__")
.with_list_parse_key("volumes")
.try_parsing(true),
)
.build()
.unwrap_or_default();
crates/obs/src/global.rs
-8
View File
@@ -3,14 +3,6 @@ use std::sync::{Arc, Mutex};
use tokio::sync::{OnceCell, SetError};
use tracing::{error, info};
pub(crate) const USE_STDOUT: bool = true;
pub(crate) const SERVICE_NAME: &str = "RustFS";
pub(crate) const SAMPLE_RATIO: f64 = 1.0;
pub(crate) const METER_INTERVAL: u64 = 60;
pub(crate) const SERVICE_VERSION: &str = "0.1.0";
pub(crate) const ENVIRONMENT: &str = "production";
pub(crate) const LOGGER_LEVEL: &str = "info";
/// Global guard for OpenTelemetry tracing
static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
crates/obs/src/lib.rs
+2 -9
View File
@@ -32,20 +32,13 @@ mod config;
mod entry;
mod global;
mod logger;
mod sink;
mod sinks;
mod system;
mod telemetry;
mod utils;
mod worker;
use crate::logger::InitLogStatus;
pub use config::load_config;
#[cfg(feature = "file")]
pub use config::FileSinkConfig;
#[cfg(feature = "kafka")]
pub use config::KafkaSinkConfig;
#[cfg(feature = "webhook")]
pub use config::WebhookSinkConfig;
pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig};
pub use entry::args::Args;
pub use entry::audit::{ApiDetails, AuditLogEntry};
@@ -79,7 +72,7 @@ use tracing::{error, info};
/// ```
pub async fn init_obs(config: AppConfig) -> (Arc<Mutex<Logger>>, telemetry::OtelGuard) {
let guard = init_telemetry(&config.observability);
let sinks = sink::create_sinks(&config).await;
let sinks = sinks::create_sinks(&config).await;
let logger = init_global_logger(&config, sinks).await;
let obs_config = config.observability.clone();
tokio::spawn(async move {
crates/obs/src/logger.rs
+4 -4
View File
@@ -1,6 +1,6 @@
use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION};
use crate::sink::Sink;
use crate::sinks::Sink;
use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry};
use rustfs_config::{APP_NAME, ENVIRONMENT, SERVICE_VERSION};
use std::sync::Arc;
use std::time::SystemTime;
use tokio::sync::mpsc::{self, Receiver, Sender};
@@ -428,7 +428,7 @@ impl Default for InitLogStatus {
fn default() -> Self {
Self {
timestamp: SystemTime::now(),
service_name: String::from(SERVICE_NAME),
service_name: String::from(APP_NAME),
version: SERVICE_VERSION.to_string(),
environment: ENVIRONMENT.to_string(),
}
@@ -442,7 +442,7 @@ impl InitLogStatus {
let version = config.service_version.unwrap_or(SERVICE_VERSION.to_string());
Self {
timestamp: SystemTime::now(),
service_name: String::from(SERVICE_NAME),
service_name: String::from(APP_NAME),
version,
environment,
}
crates/obs/src/sink.rs
-497
View File
@@ -1,497 +0,0 @@
use crate::{AppConfig, LogRecord, UnifiedLogEntry};
use async_trait::async_trait;
use std::sync::Arc;
use tokio::fs::OpenOptions;
use tokio::io;
use tokio::io::AsyncWriteExt;
/// Sink Trait definition, asynchronously write logs
#[async_trait]
pub trait Sink: Send + Sync {
async fn write(&self, entry: &UnifiedLogEntry);
}
#[cfg(feature = "kafka")]
/// Kafka Sink Implementation
pub struct KafkaSink {
producer: rdkafka::producer::FutureProducer,
topic: String,
batch_size: usize,
batch_timeout_ms: u64,
entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
last_flush: Arc<std::sync::atomic::AtomicU64>,
}
#[cfg(feature = "kafka")]
impl KafkaSink {
/// Create a new KafkaSink instance
pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self {
// Create Arc-wrapped values first
let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size)));
let last_flush = Arc::new(std::sync::atomic::AtomicU64::new(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64,
));
let sink = KafkaSink {
producer: producer.clone(),
topic: topic.clone(),
batch_size,
batch_timeout_ms,
entries: entries.clone(),
last_flush: last_flush.clone(),
};
// Start background flusher
tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms));
sink
}
/// Add a getter method to read the batch_timeout_ms field
#[allow(dead_code)]
pub fn batch_timeout(&self) -> u64 {
self.batch_timeout_ms
}
/// Add a method to dynamically adjust the timeout if needed
#[allow(dead_code)]
pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) {
self.batch_timeout_ms = new_timeout_ms;
}
async fn periodic_flush(
producer: rdkafka::producer::FutureProducer,
topic: String,
entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
last_flush: Arc<std::sync::atomic::AtomicU64>,
timeout_ms: u64,
) {
loop {
tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await;
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
let last = last_flush.load(std::sync::atomic::Ordering::Relaxed);
if now - last >= timeout_ms {
let mut batch = entries.lock().await;
if !batch.is_empty() {
Self::send_batch(&producer, &topic, batch.drain(..).collect()).await;
last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
}
}
}
}
async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec<UnifiedLogEntry>) {
for entry in entries {
let payload = match serde_json::to_string(&entry) {
Ok(p) => p,
Err(e) => {
eprintln!("Failed to serialize log entry: {}", e);
continue;
}
};
let span_id = entry.get_timestamp().to_rfc3339();
let _ = producer
.send(
rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id),
std::time::Duration::from_secs(5),
)
.await;
}
}
}
#[cfg(feature = "kafka")]
#[async_trait]
impl Sink for KafkaSink {
async fn write(&self, entry: &UnifiedLogEntry) {
let mut batch = self.entries.lock().await;
batch.push(entry.clone());
let should_flush_by_size = batch.len() >= self.batch_size;
let should_flush_by_time = {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
now - last >= self.batch_timeout_ms
};
if should_flush_by_size || should_flush_by_time {
// Existing flush logic
let entries_to_send: Vec<UnifiedLogEntry> = batch.drain(..).collect();
let producer = self.producer.clone();
let topic = self.topic.clone();
self.last_flush.store(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64,
std::sync::atomic::Ordering::Relaxed,
);
tokio::spawn(async move {
KafkaSink::send_batch(&producer, &topic, entries_to_send).await;
});
}
}
}
#[cfg(feature = "kafka")]
impl Drop for KafkaSink {
fn drop(&mut self) {
// Perform any necessary cleanup here
// For example, you might want to flush any remaining entries
let producer = self.producer.clone();
let topic = self.topic.clone();
let entries = self.entries.clone();
let last_flush = self.last_flush.clone();
tokio::spawn(async move {
let mut batch = entries.lock().await;
if !batch.is_empty() {
KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await;
last_flush.store(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64,
std::sync::atomic::Ordering::Relaxed,
);
}
});
eprintln!("Dropping KafkaSink with topic: {}", self.topic);
}
}
#[cfg(feature = "webhook")]
/// Webhook Sink Implementation
pub struct WebhookSink {
endpoint: String,
auth_token: String,
client: reqwest::Client,
max_retries: usize,
retry_delay_ms: u64,
}
#[cfg(feature = "webhook")]
impl WebhookSink {
pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self {
WebhookSink {
endpoint,
auth_token,
client: reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(10))
.build()
.unwrap_or_else(|_| reqwest::Client::new()),
max_retries,
retry_delay_ms,
}
}
}
#[cfg(feature = "webhook")]
#[async_trait]
impl Sink for WebhookSink {
async fn write(&self, entry: &UnifiedLogEntry) {
let mut retries = 0;
let url = self.endpoint.clone();
let entry_clone = entry.clone();
let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap();
while retries < self.max_retries {
match self
.client
.post(&url)
.header(reqwest::header::AUTHORIZATION, auth_value.clone())
.json(&entry_clone)
.send()
.await
{
Ok(response) if response.status().is_success() => {
return;
}
_ => {
retries += 1;
if retries < self.max_retries {
tokio::time::sleep(tokio::time::Duration::from_millis(
self.retry_delay_ms * (1 << retries), // Exponential backoff
))
.await;
}
}
}
}
eprintln!("Failed to send log to webhook after {} retries", self.max_retries);
}
}
#[cfg(feature = "webhook")]
impl Drop for WebhookSink {
fn drop(&mut self) {
// Perform any necessary cleanup here
// For example, you might want to log that the sink is being dropped
eprintln!("Dropping WebhookSink with URL: {}", self.endpoint);
}
}
#[cfg(feature = "file")]
/// File Sink Implementation
pub struct FileSink {
path: String,
buffer_size: usize,
writer: Arc<tokio::sync::Mutex<io::BufWriter<tokio::fs::File>>>,
entry_count: std::sync::atomic::AtomicUsize,
last_flush: std::sync::atomic::AtomicU64,
flush_interval_ms: u64, // Time between flushes
flush_threshold: usize, // Number of entries before flush
}
#[cfg(feature = "file")]
impl FileSink {
/// Create a new FileSink instance
pub async fn new(
path: String,
buffer_size: usize,
flush_interval_ms: u64,
flush_threshold: usize,
) -> Result<Self, io::Error> {
// check if the file exists
let file_exists = tokio::fs::metadata(&path).await.is_ok();
// if the file not exists, create it
if !file_exists {
tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?;
tracing::debug!("File does not exist, creating it. Path: {:?}", path)
}
let file = if file_exists {
// If the file exists, open it in append mode
tracing::debug!("FileSink: File exists, opening in append mode.");
OpenOptions::new().append(true).create(true).open(&path).await?
} else {
// If the file does not exist, create it
tracing::debug!("FileSink: File does not exist, creating a new file.");
// Create the file and write a header or initial content if needed
OpenOptions::new().create(true).truncate(true).write(true).open(&path).await?
};
let writer = io::BufWriter::with_capacity(buffer_size, file);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
Ok(FileSink {
path,
buffer_size,
writer: Arc::new(tokio::sync::Mutex::new(writer)),
entry_count: std::sync::atomic::AtomicUsize::new(0),
last_flush: std::sync::atomic::AtomicU64::new(now),
flush_interval_ms,
flush_threshold,
})
}
#[allow(dead_code)]
async fn initialize_writer(&mut self) -> io::Result<()> {
let file = tokio::fs::File::create(&self.path).await?;
// Use buffer_size to create a buffer writer with a specified capacity
let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file);
// Replace the original writer with the new Mutex
self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer));
Ok(())
}
// Get the current buffer size
#[allow(dead_code)]
pub fn buffer_size(&self) -> usize {
self.buffer_size
}
// How to dynamically adjust the buffer size
#[allow(dead_code)]
pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> {
if self.buffer_size != new_size {
self.buffer_size = new_size;
// Reinitialize the writer directly, without checking is_some()
self.initialize_writer().await?;
}
Ok(())
}
// Check if flushing is needed based on count or time
fn should_flush(&self) -> bool {
// Check entry count threshold
if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold {
return true;
}
// Check time threshold
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
now - last >= self.flush_interval_ms
}
}
#[cfg(feature = "file")]
#[async_trait]
impl Sink for FileSink {
async fn write(&self, entry: &UnifiedLogEntry) {
let line = format!("{:?}\n", entry);
let mut writer = self.writer.lock().await;
if let Err(e) = writer.write_all(line.as_bytes()).await {
eprintln!(
"Failed to write log to file {}: {},entry timestamp:{:?}",
self.path,
e,
entry.get_timestamp()
);
return;
}
// Only flush periodically to improve performance
// Logic to determine when to flush could be added here
// Increment the entry count
self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
// Check if we should flush
if self.should_flush() {
if let Err(e) = writer.flush().await {
eprintln!("Failed to flush log file {}: {}", self.path, e);
return;
}
// Reset counters
self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
}
}
}
#[cfg(feature = "file")]
impl Drop for FileSink {
fn drop(&mut self) {
let writer = self.writer.clone();
let path = self.path.clone();
tokio::task::spawn_blocking(move || {
let rt = tokio::runtime::Runtime::new().unwrap();
rt.block_on(async {
let mut writer = writer.lock().await;
if let Err(e) = writer.flush().await {
eprintln!("Failed to flush log file {}: {}", path, e);
}
});
});
}
}
/// Create a list of Sink instances
pub async fn create_sinks(config: &AppConfig) -> Vec<Arc<dyn Sink>> {
let mut sinks: Vec<Arc<dyn Sink>> = Vec::new();
#[cfg(feature = "kafka")]
{
match &config.sinks.kafka {
Some(sink_kafka) => {
if sink_kafka.enabled {
match rdkafka::config::ClientConfig::new()
.set("bootstrap.servers", &sink_kafka.bootstrap_servers)
.set("message.timeout.ms", "5000")
.create()
{
Ok(producer) => {
sinks.push(Arc::new(KafkaSink::new(
producer,
sink_kafka.topic.clone(),
sink_kafka.batch_size.unwrap_or(100),
sink_kafka.batch_timeout_ms.unwrap_or(1000),
)));
}
Err(e) => {
tracing::error!("Failed to create Kafka producer: {}", e);
}
}
} else {
tracing::info!("Kafka sink is disabled in the configuration");
}
}
_ => {
tracing::info!("Kafka sink is not configured or disabled");
}
}
}
#[cfg(feature = "webhook")]
{
match &config.sinks.webhook {
Some(sink_webhook) => {
if sink_webhook.enabled {
sinks.push(Arc::new(WebhookSink::new(
sink_webhook.endpoint.clone(),
sink_webhook.auth_token.clone(),
sink_webhook.max_retries.unwrap_or(3),
sink_webhook.retry_delay_ms.unwrap_or(100),
)));
} else {
tracing::info!("Webhook sink is disabled in the configuration");
}
}
_ => {
tracing::info!("Webhook sink is not configured or disabled");
}
}
}
#[cfg(feature = "file")]
{
// let config = config.clone();
match &config.sinks.file {
Some(sink_file) => {
tracing::info!("File sink is enabled in the configuration");
let path = if sink_file.enabled {
sink_file.path.clone()
} else {
"rustfs.log".to_string()
};
tracing::debug!("FileSink: Using path: {}", path);
sinks.push(Arc::new(
FileSink::new(
path.clone(),
sink_file.buffer_size.unwrap_or(8192),
sink_file.flush_interval_ms.unwrap_or(1000),
sink_file.flush_threshold.unwrap_or(100),
)
.await
.unwrap(),
));
}
_ => {
tracing::info!("File sink is not configured or disabled");
}
}
}
sinks
}
crates/obs/src/sinks/file.rs
+164
View File
@@ -0,0 +1,164 @@
use crate::sinks::Sink;
use crate::{LogRecord, UnifiedLogEntry};
use async_trait::async_trait;
use std::sync::Arc;
use tokio::fs::OpenOptions;
use tokio::io;
use tokio::io::AsyncWriteExt;
/// File Sink Implementation
pub struct FileSink {
path: String,
buffer_size: usize,
writer: Arc<tokio::sync::Mutex<io::BufWriter<tokio::fs::File>>>,
entry_count: std::sync::atomic::AtomicUsize,
last_flush: std::sync::atomic::AtomicU64,
flush_interval_ms: u64, // Time between flushes
flush_threshold: usize, // Number of entries before flush
}
impl FileSink {
/// Create a new FileSink instance
pub async fn new(
path: String,
buffer_size: usize,
flush_interval_ms: u64,
flush_threshold: usize,
) -> Result<Self, io::Error> {
// check if the file exists
let file_exists = tokio::fs::metadata(&path).await.is_ok();
// if the file not exists, create it
if !file_exists {
tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?;
tracing::debug!("File does not exist, creating it. Path: {:?}", path)
}
let file = if file_exists {
// If the file exists, open it in append mode
tracing::debug!("FileSink: File exists, opening in append mode.");
OpenOptions::new().append(true).create(true).open(&path).await?
} else {
// If the file does not exist, create it
tracing::debug!("FileSink: File does not exist, creating a new file.");
// Create the file and write a header or initial content if needed
OpenOptions::new().create(true).truncate(true).write(true).open(&path).await?
};
let writer = io::BufWriter::with_capacity(buffer_size, file);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
Ok(FileSink {
path,
buffer_size,
writer: Arc::new(tokio::sync::Mutex::new(writer)),
entry_count: std::sync::atomic::AtomicUsize::new(0),
last_flush: std::sync::atomic::AtomicU64::new(now),
flush_interval_ms,
flush_threshold,
})
}
#[allow(dead_code)]
async fn initialize_writer(&mut self) -> io::Result<()> {
let file = tokio::fs::File::create(&self.path).await?;
// Use buffer_size to create a buffer writer with a specified capacity
let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file);
// Replace the original writer with the new Mutex
self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer));
Ok(())
}
// Get the current buffer size
#[allow(dead_code)]
pub fn buffer_size(&self) -> usize {
self.buffer_size
}
// How to dynamically adjust the buffer size
#[allow(dead_code)]
pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> {
if self.buffer_size != new_size {
self.buffer_size = new_size;
// Reinitialize the writer directly, without checking is_some()
self.initialize_writer().await?;
}
Ok(())
}
// Check if flushing is needed based on count or time
fn should_flush(&self) -> bool {
// Check entry count threshold
if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold {
return true;
}
// Check time threshold
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
now - last >= self.flush_interval_ms
}
}
#[async_trait]
impl Sink for FileSink {
async fn write(&self, entry: &UnifiedLogEntry) {
let line = format!("{:?}\n", entry);
let mut writer = self.writer.lock().await;
if let Err(e) = writer.write_all(line.as_bytes()).await {
eprintln!(
"Failed to write log to file {}: {},entry timestamp:{:?}",
self.path,
e,
entry.get_timestamp()
);
return;
}
// Only flush periodically to improve performance
// Logic to determine when to flush could be added here
// Increment the entry count
self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
// Check if we should flush
if self.should_flush() {
if let Err(e) = writer.flush().await {
eprintln!("Failed to flush log file {}: {}", self.path, e);
return;
}
// Reset counters
self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
}
}
}
impl Drop for FileSink {
fn drop(&mut self) {
let writer = self.writer.clone();
let path = self.path.clone();
tokio::task::spawn_blocking(move || {
let rt = tokio::runtime::Runtime::new().unwrap();
rt.block_on(async {
let mut writer = writer.lock().await;
if let Err(e) = writer.flush().await {
eprintln!("Failed to flush log file {}: {}", path, e);
}
});
});
}
}
crates/obs/src/sinks/kafka.rs
+165
View File
@@ -0,0 +1,165 @@
use crate::sinks::Sink;
use crate::{LogRecord, UnifiedLogEntry};
use async_trait::async_trait;
use std::sync::Arc;
/// Kafka Sink Implementation
pub struct KafkaSink {
producer: rdkafka::producer::FutureProducer,
topic: String,
batch_size: usize,
batch_timeout_ms: u64,
entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
last_flush: Arc<std::sync::atomic::AtomicU64>,
}
impl KafkaSink {
/// Create a new KafkaSink instance
pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self {
// Create Arc-wrapped values first
let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size)));
let last_flush = Arc::new(std::sync::atomic::AtomicU64::new(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64,
));
let sink = KafkaSink {
producer: producer.clone(),
topic: topic.clone(),
batch_size,
batch_timeout_ms,
entries: entries.clone(),
last_flush: last_flush.clone(),
};
// Start background flusher
tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms));
sink
}
/// Add a getter method to read the batch_timeout_ms field
#[allow(dead_code)]
pub fn batch_timeout(&self) -> u64 {
self.batch_timeout_ms
}
/// Add a method to dynamically adjust the timeout if needed
#[allow(dead_code)]
pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) {
self.batch_timeout_ms = new_timeout_ms;
}
async fn periodic_flush(
producer: rdkafka::producer::FutureProducer,
topic: String,
entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
last_flush: Arc<std::sync::atomic::AtomicU64>,
timeout_ms: u64,
) {
loop {
tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await;
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
let last = last_flush.load(std::sync::atomic::Ordering::Relaxed);
if now - last >= timeout_ms {
let mut batch = entries.lock().await;
if !batch.is_empty() {
Self::send_batch(&producer, &topic, batch.drain(..).collect()).await;
last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
}
}
}
}
async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec<UnifiedLogEntry>) {
for entry in entries {
let payload = match serde_json::to_string(&entry) {
Ok(p) => p,
Err(e) => {
eprintln!("Failed to serialize log entry: {}", e);
continue;
}
};
let span_id = entry.get_timestamp().to_rfc3339();
let _ = producer
.send(
rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id),
std::time::Duration::from_secs(5),
)
.await;
}
}
}
#[async_trait]
impl Sink for KafkaSink {
async fn write(&self, entry: &UnifiedLogEntry) {
let mut batch = self.entries.lock().await;
batch.push(entry.clone());
let should_flush_by_size = batch.len() >= self.batch_size;
let should_flush_by_time = {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64;
let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
now - last >= self.batch_timeout_ms
};
if should_flush_by_size || should_flush_by_time {
// Existing flush logic
let entries_to_send: Vec<UnifiedLogEntry> = batch.drain(..).collect();
let producer = self.producer.clone();
let topic = self.topic.clone();
self.last_flush.store(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64,
std::sync::atomic::Ordering::Relaxed,
);
tokio::spawn(async move {
KafkaSink::send_batch(&producer, &topic, entries_to_send).await;
});
}
}
}
impl Drop for KafkaSink {
fn drop(&mut self) {
// Perform any necessary cleanup here
// For example, you might want to flush any remaining entries
let producer = self.producer.clone();
let topic = self.topic.clone();
let entries = self.entries.clone();
let last_flush = self.last_flush.clone();
tokio::spawn(async move {
let mut batch = entries.lock().await;
if !batch.is_empty() {
KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await;
last_flush.store(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as u64,
std::sync::atomic::Ordering::Relaxed,
);
}
});
eprintln!("Dropping KafkaSink with topic: {}", self.topic);
}
}
crates/obs/src/sinks/mod.rs
+92
View File
@@ -0,0 +1,92 @@
use crate::{AppConfig, SinkConfig, UnifiedLogEntry};
use async_trait::async_trait;
use std::sync::Arc;
#[cfg(feature = "file")]
mod file;
#[cfg(all(feature = "kafka", target_os = "linux"))]
mod kafka;
#[cfg(feature = "webhook")]
mod webhook;
/// Sink Trait definition, asynchronously write logs
#[async_trait]
pub trait Sink: Send + Sync {
async fn write(&self, entry: &UnifiedLogEntry);
}
/// Create a list of Sink instances
pub async fn create_sinks(config: &AppConfig) -> Vec<Arc<dyn Sink>> {
let mut sinks: Vec<Arc<dyn Sink>> = Vec::new();
for sink_config in &config.sinks {
match sink_config {
#[cfg(all(feature = "kafka", target_os = "linux"))]
SinkConfig::Kafka(kafka_config) => {
match rdkafka::config::ClientConfig::new()
.set("bootstrap.servers", &kafka_config.brokers)
.set("message.timeout.ms", "5000")
.create()
{
Ok(producer) => {
sinks.push(Arc::new(kafka::KafkaSink::new(
producer,
kafka_config.topic.clone(),
kafka_config.batch_size.unwrap_or(100),
kafka_config.batch_timeout_ms.unwrap_or(1000),
)));
tracing::info!("Kafka sink created for topic: {}", kafka_config.topic);
}
Err(e) => {
tracing::error!("Failed to create Kafka producer: {}", e);
}
}
}
#[cfg(feature = "webhook")]
SinkConfig::Webhook(webhook_config) => {
sinks.push(Arc::new(webhook::WebhookSink::new(
webhook_config.endpoint.clone(),
webhook_config.auth_token.clone(),
webhook_config.max_retries.unwrap_or(3),
webhook_config.retry_delay_ms.unwrap_or(100),
)));
tracing::info!("Webhook sink created for endpoint: {}", webhook_config.endpoint);
}
#[cfg(feature = "file")]
SinkConfig::File(file_config) => {
tracing::debug!("FileSink: Using path: {}", file_config.path);
match file::FileSink::new(
file_config.path.clone(),
file_config.buffer_size.unwrap_or(8192),
file_config.flush_interval_ms.unwrap_or(1000),
file_config.flush_threshold.unwrap_or(100),
)
.await
{
Ok(sink) => {
sinks.push(Arc::new(sink));
tracing::info!("File sink created for path: {}", file_config.path);
}
Err(e) => {
tracing::error!("Failed to create File sink: {}", e);
}
}
}
#[cfg(any(not(feature = "kafka"), not(target_os = "linux")))]
SinkConfig::Kafka(_) => {
tracing::warn!("Kafka sink is configured but the 'kafka' feature is not enabled");
}
#[cfg(not(feature = "webhook"))]
SinkConfig::Webhook(_) => {
tracing::warn!("Webhook sink is configured but the 'webhook' feature is not enabled");
}
#[cfg(not(feature = "file"))]
SinkConfig::File(_) => {
tracing::warn!("File sink is configured but the 'file' feature is not enabled");
}
}
}
sinks
}
crates/obs/src/sinks/webhook.rs
+70
View File
@@ -0,0 +1,70 @@
use crate::sinks::Sink;
use crate::UnifiedLogEntry;
use async_trait::async_trait;
/// Webhook Sink Implementation
pub struct WebhookSink {
endpoint: String,
auth_token: String,
client: reqwest::Client,
max_retries: usize,
retry_delay_ms: u64,
}
impl WebhookSink {
pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self {
WebhookSink {
endpoint,
auth_token,
client: reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(10))
.build()
.unwrap_or_else(|_| reqwest::Client::new()),
max_retries,
retry_delay_ms,
}
}
}
#[async_trait]
impl Sink for WebhookSink {
async fn write(&self, entry: &UnifiedLogEntry) {
let mut retries = 0;
let url = self.endpoint.clone();
let entry_clone = entry.clone();
let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap();
while retries < self.max_retries {
match self
.client
.post(&url)
.header(reqwest::header::AUTHORIZATION, auth_value.clone())
.json(&entry_clone)
.send()
.await
{
Ok(response) if response.status().is_success() => {
return;
}
_ => {
retries += 1;
if retries < self.max_retries {
tokio::time::sleep(tokio::time::Duration::from_millis(
self.retry_delay_ms * (1 << retries), // Exponential backoff
))
.await;
}
}
}
}
eprintln!("Failed to send log to webhook after {} retries", self.max_retries);
}
}
impl Drop for WebhookSink {
fn drop(&mut self) {
// Perform any necessary cleanup here
// For example, you might want to log that the sink is being dropped
eprintln!("Dropping WebhookSink with URL: {}", self.endpoint);
}
}
crates/obs/src/telemetry.rs
+13 -10
View File
@@ -1,5 +1,3 @@
use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
use crate::utils::get_local_ip_with_default;
use crate::OtelConfig;
use opentelemetry::trace::TracerProvider;
use opentelemetry::{global, KeyValue};
@@ -15,6 +13,8 @@ use opentelemetry_semantic_conventions::{
attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
SCHEMA_URL,
};
use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT};
use rustfs_utils::get_local_ip_with_default;
use smallvec::SmallVec;
use std::borrow::Cow;
use std::io::IsTerminal;
@@ -70,7 +70,7 @@ impl Drop for OtelGuard {
/// create OpenTelemetry Resource
fn resource(config: &OtelConfig) -> Resource {
Resource::builder()
.with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(SERVICE_NAME)).to_string())
.with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(APP_NAME)).to_string())
.with_schema_url(
[
KeyValue::new(
@@ -101,8 +101,8 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
let endpoint = &config.endpoint;
let use_stdout = config.use_stdout.unwrap_or(USE_STDOUT);
let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL);
let logger_level = config.logger_level.as_deref().unwrap_or(LOGGER_LEVEL);
let service_name = config.service_name.as_deref().unwrap_or(SERVICE_NAME);
let logger_level = config.logger_level.as_deref().unwrap_or(DEFAULT_LOG_LEVEL);
let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
// Pre-create resource objects to avoid repeated construction
let res = resource(config);
@@ -210,7 +210,8 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
.with_thread_names(true)
.with_thread_ids(true)
.with_file(true)
.with_line_number(true);
.with_line_number(true)
.with_filter(build_env_filter(logger_level, None));
let filter = build_env_filter(logger_level, None);
let otel_filter = build_env_filter(logger_level, None);
@@ -218,7 +219,7 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string());
// Configure registry to avoid repeated calls to filter methods
let _registry = tracing_subscriber::registry()
tracing_subscriber::registry()
.with(filter)
.with(ErrorLayer::default())
.with(if config.local_logging_enabled.unwrap_or(false) {
@@ -230,10 +231,13 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
.with(otel_layer)
.with(MetricsLayer::new(meter_provider.clone()))
.init();
if !endpoint.is_empty() {
info!(
"OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
endpoint, logger_level
"OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {},RUST_LOG env: {}",
endpoint,
logger_level,
std::env::var("RUST_LOG").unwrap_or_else(|_| "未设置".to_string())
);
}
}
@@ -248,7 +252,6 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
fn build_env_filter(logger_level: &str, default_level: Option<&str>) -> EnvFilter {
let level = default_level.unwrap_or(logger_level);
let mut filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level));
if !matches!(logger_level, "trace" | "debug") {
let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
for directive in directives {
crates/obs/src/worker.rs
+2 -2
View File
@@ -1,9 +1,9 @@
use crate::{sink::Sink, UnifiedLogEntry};
use crate::{sinks::Sink, UnifiedLogEntry};
use std::sync::Arc;
use tokio::sync::mpsc::Receiver;
/// Start the log processing worker thread
pub async fn start_worker(receiver: Receiver<UnifiedLogEntry>, sinks: Vec<Arc<dyn Sink>>) {
pub(crate) async fn start_worker(receiver: Receiver<UnifiedLogEntry>, sinks: Vec<Arc<dyn Sink>>) {
let mut receiver = receiver;
while let Some(entry) = receiver.recv().await {
for sink in &sinks {
crates/utils/Cargo.toml
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "rustfs-utils"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[dependencies]
local-ip-address = { workspace = true, optional = true }
rustfs-config = { workspace = true }
rustls = { workspace = true, optional = true }
rustls-pemfile = { workspace = true, optional = true }
rustls-pki-types = { workspace = true, optional = true }
tracing = { workspace = true }
[lints]
workspace = true
[features]
default = ["ip"] # features that are enabled by default
ip = ["dep:local-ip-address"] # ip characteristics and their dependencies
tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies
net = ["ip"] # empty network features
full = ["ip", "tls", "net"] # all features
rustfs/src/utils/mod.rs → crates/utils/src/certs.rs
+23 -29
View File
@@ -1,57 +1,48 @@
use crate::config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY};
use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY};
use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni};
use rustls::sign::CertifiedKey;
use rustls_pemfile::{certs, private_key};
use rustls_pki_types::{CertificateDer, PrivateKeyDer};
use std::collections::HashMap;
use std::fmt::Debug;
use std::io::Error;
use std::net::IpAddr;
use std::path::Path;
use std::sync::Arc;
use std::{fs, io};
use tracing::{debug, warn};
/// Get the local IP address.
/// This function retrieves the local IP address of the machine.
pub(crate) fn get_local_ip() -> Option<std::net::Ipv4Addr> {
match local_ip_address::local_ip() {
Ok(IpAddr::V4(ip)) => Some(ip),
Err(_) => None,
Ok(IpAddr::V6(_)) => todo!(),
}
}
/// Load public certificate from file.
/// This function loads a public certificate from the specified file.
pub(crate) fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
// Open certificate file.
let cert_file = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?;
let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {}: {}", filename, e)))?;
let mut reader = io::BufReader::new(cert_file);
// Load and return certificate.
let certs = certs(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|_| error(format!("certificate file {} format error", filename)))?;
.map_err(|e| certs_error(format!("certificate file {} format error:{:?}", filename, e)))?;
if certs.is_empty() {
return Err(error(format!("No valid certificate was found in the certificate file {}", filename)));
return Err(certs_error(format!(
"No valid certificate was found in the certificate file {}",
filename
)));
}
Ok(certs)
}
/// Load private key from file.
/// This function loads a private key from the specified file.
pub(crate) fn load_private_key(filename: &str) -> io::Result<PrivateKeyDer<'static>> {
pub fn load_private_key(filename: &str) -> io::Result<PrivateKeyDer<'static>> {
// Open keyfile.
let keyfile = fs::File::open(filename).map_err(|e| error(format!("failed to open {}: {}", filename, e)))?;
let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {}: {}", filename, e)))?;
let mut reader = io::BufReader::new(keyfile);
// Load and return a single private key.
private_key(&mut reader)?.ok_or_else(|| error(format!("no private key found in {}", filename)))
private_key(&mut reader)?.ok_or_else(|| certs_error(format!("no private key found in {}", filename)))
}
/// error function
pub(crate) fn error(err: String) -> Error {
pub fn certs_error(err: String) -> Error {
Error::new(io::ErrorKind::Other, err)
}
@@ -59,14 +50,14 @@ pub(crate) fn error(err: String) -> Error {
/// This function loads all certificate and private key pairs from the specified directory.
/// It looks for files named `rustfs_cert.pem` and `rustfs_key.pem` in each subdirectory.
/// The root directory can also contain a default certificate/private key pair.
pub(crate) fn load_all_certs_from_directory(
pub fn load_all_certs_from_directory(
dir_path: &str,
) -> io::Result<HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>> {
let mut cert_key_pairs = HashMap::new();
let dir = Path::new(dir_path);
if !dir.exists() || !dir.is_dir() {
return Err(error(format!(
return Err(certs_error(format!(
"The certificate directory does not exist or is not a directory: {}",
dir_path
)));
@@ -80,10 +71,10 @@ pub(crate) fn load_all_certs_from_directory(
debug!("find the root directory certificate: {:?}", root_cert_path);
let root_cert_str = root_cert_path
.to_str()
.ok_or_else(|| error(format!("Invalid UTF-8 in root certificate path: {:?}", root_cert_path)))?;
.ok_or_else(|| certs_error(format!("Invalid UTF-8 in root certificate path: {:?}", root_cert_path)))?;
let root_key_str = root_key_path
.to_str()
.ok_or_else(|| error(format!("Invalid UTF-8 in root key path: {:?}", root_key_path)))?;
.ok_or_else(|| certs_error(format!("Invalid UTF-8 in root key path: {:?}", root_key_path)))?;
match load_cert_key_pair(root_cert_str, root_key_str) {
Ok((certs, key)) => {
// The root directory certificate is used as the default certificate and is stored using special keys.
@@ -104,7 +95,7 @@ pub(crate) fn load_all_certs_from_directory(
let domain_name = path
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| error(format!("invalid domain name directory:{:?}", path)))?;
.ok_or_else(|| certs_error(format!("invalid domain name directory:{:?}", path)))?;
// find certificate and private key files
let cert_path = path.join(RUSTFS_TLS_CERT); // e.g., rustfs_cert.pem
@@ -125,7 +116,10 @@ pub(crate) fn load_all_certs_from_directory(
}
if cert_key_pairs.is_empty() {
return Err(error(format!("No valid certificate/private key pair found in directory {}", dir_path)));
return Err(certs_error(format!(
"No valid certificate/private key pair found in directory {}",
dir_path
)));
}
Ok(cert_key_pairs)
@@ -171,7 +165,7 @@ pub fn create_multi_cert_resolver(
for (domain, (certs, key)) in cert_key_pairs {
// create a signature
let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
.map_err(|_| error(format!("unsupported private key types:{}", domain)))?;
.map_err(|e| certs_error(format!("unsupported private key types:{}, err:{:?}", domain, e)))?;
// create a CertifiedKey
let certified_key = CertifiedKey::new(certs, signing_key);
@@ -181,7 +175,7 @@ pub fn create_multi_cert_resolver(
// add certificate to resolver
resolver
.add(&domain, certified_key)
.map_err(|e| error(format!("failed to add a domain name certificate:{},err: {:?}", domain, e)))?;
.map_err(|e| certs_error(format!("failed to add a domain name certificate:{},err: {:?}", domain, e)))?;
}
}
crates/obs/src/utils.rs → crates/utils/src/ip.rs
+3 -2
View File
@@ -1,4 +1,3 @@
use local_ip_address::{local_ip, local_ipv6};
use std::net::{IpAddr, Ipv4Addr};
/// Get the IP address of the machine
@@ -11,7 +10,9 @@ use std::net::{IpAddr, Ipv4Addr};
/// * `Some(IpAddr)` - Native IP address (IPv4 or IPv6)
/// * `None` - Unable to obtain any native IP address
pub fn get_local_ip() -> Option<IpAddr> {
local_ip().ok().or_else(|| local_ipv6().ok())
local_ip_address::local_ip()
.ok()
.or_else(|| local_ip_address::local_ipv6().ok())
}
/// Get the IP address of the machine as a string
crates/utils/src/lib.rs
+8
View File
@@ -0,0 +1,8 @@
mod certs;
mod ip;
mod net;
#[cfg(feature = "ip")]
pub use certs::*;
#[cfg(feature = "ip")]
pub use ip::*;
crates/utils/src/net.rs
+1
View File
@@ -0,0 +1 @@
crates/zip/Cargo.toml
+28
View File
@@ -0,0 +1,28 @@
[package]
name = "zip"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
[dependencies]
async-compression = { version = "0.4.0", features = [
"tokio",
"bzip2",
"gzip",
"zlib",
"zstd",
"xz",
] }
# async_zip = { version = "0.0.17", features = ["tokio"] }
# rc-zip-tokio = "4.2.6"
tokio = { version = "1.45.0", features = ["full"] }
tokio-stream = "0.1.17"
tokio-tar = { workspace = true }
xz2 = { version = "0.1", optional = true, features = ["static"] }
[lints]
workspace = true
crates/zip/src/lib.rs
+124
View File
@@ -0,0 +1,124 @@
use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder};
use tokio::io::{self, AsyncRead, BufReader};
use tokio_stream::StreamExt;
use tokio_tar::Archive;
#[derive(Debug, PartialEq)]
pub enum CompressionFormat {
Gzip, //.gz
Bzip2, //.bz2
// Lz4, //.lz4
Zip,
Xz, //.xz
Zlib, //.z
Zstd, //.zst
Unknown,
}
impl CompressionFormat {
pub fn from_extension(ext: &str) -> Self {
match ext {
"gz" => CompressionFormat::Gzip,
"bz2" => CompressionFormat::Bzip2,
// "lz4" => CompressionFormat::Lz4,
"zip" => CompressionFormat::Zip,
"xz" => CompressionFormat::Xz,
"zlib" => CompressionFormat::Zlib,
"zst" => CompressionFormat::Zstd,
_ => CompressionFormat::Unknown,
}
}
pub fn get_decoder<R>(&self, input: R) -> io::Result<Box<dyn AsyncRead + Send + Unpin>>
where
R: AsyncRead + Send + Unpin + 'static,
{
let reader = BufReader::new(input);
let decoder: Box<dyn AsyncRead + Send + Unpin + 'static> = match self {
CompressionFormat::Gzip => Box::new(GzipDecoder::new(reader)),
CompressionFormat::Bzip2 => Box::new(BzDecoder::new(reader)),
// CompressionFormat::Lz4 => Box::new(Lz4Decoder::new(reader)),
CompressionFormat::Zlib => Box::new(ZlibDecoder::new(reader)),
CompressionFormat::Xz => Box::new(XzDecoder::new(reader)),
CompressionFormat::Zstd => Box::new(ZstdDecoder::new(reader)),
_ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format")),
};
Ok(decoder)
}
}
pub async fn decompress<R, F>(input: R, format: CompressionFormat, mut callback: F) -> io::Result<()>
where
R: AsyncRead + Send + Unpin + 'static,
F: AsyncFnMut(tokio_tar::Entry<Archive<Box<dyn AsyncRead + Send + Unpin + 'static>>>) -> std::io::Result<()> + Send + 'static,
{
// 打开输入文件
// println!("format {:?}", format);
let decoder = format.get_decoder(input)?;
// let reader: BufReader<R> = BufReader::new(input);
// // 根据文件扩展名选择解压器
// let decoder: Box<dyn AsyncRead + Send + Unpin> = match format {
// CompressionFormat::Gzip => Box::new(GzipDecoder::new(reader)),
// CompressionFormat::Bzip2 => Box::new(BzDecoder::new(reader)),
// // CompressionFormat::Lz4 => Box::new(Lz4Decoder::new(reader)),
// CompressionFormat::Zlib => Box::new(ZlibDecoder::new(reader)),
// CompressionFormat::Xz => Box::new(XzDecoder::new(reader)),
// CompressionFormat::Zstd => Box::new(ZstdDecoder::new(reader)),
// // CompressionFormat::Zip => Box::new(DeflateDecoder::new(reader)),
// _ => {
// return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format"));
// }
// };
let mut ar = Archive::new(decoder);
let mut entries = ar.entries().unwrap();
while let Some(entry) = entries.next().await {
let f = match entry {
Ok(f) => f,
Err(e) => {
println!("Error reading entry: {}", e);
return Err(e);
}
};
// println!("{}", f.path().unwrap().display());
callback(f).await?;
}
Ok(())
}
// #[tokio::test]
// async fn test_decompress() -> io::Result<()> {
// use std::path::Path;
// use tokio::fs::File;
// let input_path = "/Users/weisd/Downloads/wsd.tar.gz"; // 替换为你的压缩文件路径
// let f = File::open(input_path).await?;
// let Some(ext) = Path::new(input_path).extension().and_then(|s| s.to_str()) else {
// return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format"));
// };
// match decompress(
// f,
// CompressionFormat::from_extension(ext),
// |entry: tokio_tar::Entry<Archive<Box<dyn AsyncRead + Send + Unpin>>>| async move {
// let path = entry.path().unwrap();
// println!("Extracted: {}", path.display());
// Ok(())
// },
// )
// .await
// {
// Ok(_) => println!("解压成功！"),
// Err(e) => println!("解压失败: {}", e),
// }
// Ok(())
// }
crypto/Cargo.toml
+1 -1
View File
@@ -17,7 +17,7 @@ chacha20poly1305 = { version = "0.10.1", optional = true }
jsonwebtoken = { workspace = true }
pbkdf2 = { version = "0.12.2", optional = true }
rand = { workspace = true, optional = true }
sha2 = { version = "0.10.8", optional = true }
sha2 = { workspace = true, optional = true }
thiserror.workspace = true
serde_json.workspace = true
deploy/config/.example.obs.env
+28 -27
View File
@@ -1,27 +1,28 @@
OBSERVABILITY__ENDPOINT=http://localhost:4317
OBSERVABILITY__USE_STDOUT=true
OBSERVABILITY__SAMPLE_RATIO=2.0
OBSERVABILITY__METER_INTERVAL=30
OBSERVABILITY__SERVICE_NAME=rustfs
OBSERVABILITY__SERVICE_VERSION=0.1.0
OBSERVABILITY__ENVIRONMENT=develop
OBSERVABILITY__LOGGER_LEVEL=debug
SINKS__KAFKA__ENABLED=false
SINKS__KAFKA__BOOTSTRAP_SERVERS=localhost:9092
SINKS__KAFKA__TOPIC=logs
SINKS__KAFKA__BATCH_SIZE=100
SINKS__KAFKA__BATCH_TIMEOUT_MS=1000
SINKS__WEBHOOK__ENABLED=false
SINKS__WEBHOOK__ENDPOINT=http://localhost:8080/webhook
SINKS__WEBHOOK__AUTH_TOKEN=
SINKS__WEBHOOK__BATCH_SIZE=100
SINKS__WEBHOOK__BATCH_TIMEOUT_MS=1000
SINKS__FILE__ENABLED=true
SINKS__FILE__PATH=./deploy/logs/rustfs.log
SINKS__FILE__BATCH_SIZE=10
SINKS__FILE__BATCH_TIMEOUT_MS=1000
LOGGER__QUEUE_CAPACITY=10
#RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317
#RUSTFS__OBSERVABILITY__USE_STDOUT=true
#RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0
#RUSTFS__OBSERVABILITY__METER_INTERVAL=30
#RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs
#RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0
#RUSTFS__OBSERVABILITY__ENVIRONMENT=develop
#RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug
#
#RUSTFS__SINKS_0__type=Kakfa
#RUSTFS__SINKS_0__brokers=localhost:9092
#RUSTFS__SINKS_0__topic=logs
#RUSTFS__SINKS_0__batch_size=100
#RUSTFS__SINKS_0__batch_timeout_ms=1000
#
#RUSTFS__SINKS_1__type=Webhook
#RUSTFS__SINKS_1__endpoint=http://localhost:8080/webhook
#RUSTFS__SINKS_1__auth_token=you-auth-token
#RUSTFS__SINKS_1__batch_size=100
#RUSTFS__SINKS_1__batch_timeout_ms=1000
#
#RUSTFS__SINKS_2__type=File
#RUSTFS__SINKS_2__path=./deploy/logs/rustfs.log
#RUSTFS__SINKS_2__buffer_size=10
#RUSTFS__SINKS_2__flush_interval_ms=1000
#RUSTFS__SINKS_2__flush_threshold=100
#
#RUSTFS__LOGGER__QUEUE_CAPACITY=10
deploy/config/obs-zh.example.toml
+17 -17
View File
@@ -9,26 +9,26 @@ environments = "develop" # 运行环境，如开发环境 (develop)
logger_level = "debug" # 日志级别，可选 debug/info/warn/error 等
local_logging_enabled = true # 是否启用本地 stdout 日志记录，true 表示启用，false 表示禁用
[sinks]
[sinks.kafka] # Kafka 接收器配置
enabled = false # 是否启用 Kafka 接收器，默认禁用
bootstrap_servers = "localhost:9092" # Kafka 服务器地址
topic = "logs" # Kafka 主题名称
batch_size = 100 # 批处理大小，每次发送的消息数量
batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒
#[[sinks]] # Kafka 接收器配置
#type = "Kafka" # 是否启用 Kafka 接收器，默认禁用
#brokers = "localhost:9092" # Kafka 服务器地址
#topic = "logs" # Kafka 主题名称
#batch_size = 100 # 批处理大小，每次发送的消息数量
#batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒
[sinks.webhook] # Webhook 接收器配置
enabled = false # 是否启用 Webhook 接收器
endpoint = "http://localhost:8080/webhook" # Webhook 接收地址
auth_token = "" # 认证令牌
batch_size = 100 # 批处理大小
batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒
#[[sinks]] # Webhook 接收器配置
#type = "Webhook" # 是否启用 Webhook 接收器
#endpoint = "http://localhost:8080/webhook" # Webhook 接收地址
#auth_token = "" # 认证令牌
#batch_size = 100 # 批处理大小
#batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒
[sinks.file] # 文件接收器配置
enabled = true # 是否启用文件接收器
[[sinks]] # 文件接收器配置
type = "File" # 是否启用文件接收器
path = "./deploy/logs/rustfs.log" # 日志文件路径
batch_size = 10 # 批处理大小
batch_timeout_ms = 1000 # 批处理超时时间，单位为毫秒
buffer_size = 10 # 缓冲区大小，表示每次写入的字节数
flush_interval_ms = 100 # 批处理超时时间，单位为毫秒
flush_threshold = 100 # 刷新阈值，表示在达到该数量时刷新日志
[logger] # 日志器配置
queue_capacity = 10 # 日志队列容量，表示可以缓存的日志条数
deploy/config/obs.example.toml
+21 -21
View File
@@ -4,31 +4,31 @@ use_stdout = false # Output with stdout, true output, false no output
sample_ratio = 2.0
meter_interval = 30
service_name = "rustfs"
service_version = "0.1.0"
service_version = "0.0.1"
environment = "develop"
logger_level = "error"
logger_level = "info"
local_logging_enabled = true
[sinks]
[sinks.kafka] # Kafka sink is disabled by default
enabled = false
bootstrap_servers = "localhost:9092"
topic = "logs"
batch_size = 100 # Default is 100 if not specified
batch_timeout_ms = 1000 # Default is 1000ms if not specified
#[[sinks]]
#type = "Kafka"
#brokers = "localhost:9092"
#topic = "logs"
#batch_size = 100 # Default is 100 if not specified
#batch_timeout_ms = 100 # Default is 1000ms if not specified
#
#[[sinks]]
#type = "Webhook"
#endpoint = "http://localhost:8080/webhook"
#auth_token = ""
#batch_size = 100 # Default is 3 if not specified
#batch_timeout_ms = 100 # Default is 100ms if not specified
[sinks.webhook]
enabled = false
endpoint = "http://localhost:8080/webhook"
auth_token = ""
batch_size = 100 # Default is 3 if not specified
batch_timeout_ms = 1000 # Default is 100ms if not specified
[sinks.file]
enabled = true
path = "./deploy/logs/rustfs.log"
batch_size = 100
batch_timeout_ms = 1000 # Default is 8192 bytes if not specified
[[sinks]]
type = "File"
path = "deploy/logs/rustfs.log"
buffer_size = 101 # Default is 8192 bytes if not specified
flush_interval_ms = 1000
flush_threshold = 100
[logger]
queue_capacity = 10000
deploy/config/rustfs-zh.env
+3 -1
View File
@@ -23,4 +23,6 @@ RUSTFS_LICENSE="license content"
# 可观测性配置文件路径：deploy/config/obs.example.toml
RUSTFS_OBS_CONFIG=/etc/default/obs.toml
# TLS 证书目录路径：deploy/certs
RUSTFS_TLS_PATH=/etc/default/tls
RUSTFS_TLS_PATH=/etc/default/tls
# 事件通知配置文件路径：deploy/config/event.example.toml
RUSTFS_EVENT_CONFIG=/etc/default/event.toml
deploy/config/rustfs.env
+3 -1
View File
@@ -23,4 +23,6 @@ RUSTFS_LICENSE="license content"
# Observability configuration file path: deploy/config/obs.example.toml
RUSTFS_OBS_CONFIG=/etc/default/obs.toml
# TLS certificates directory path: deploy/certs
RUSTFS_TLS_PATH=/etc/default/tls
RUSTFS_TLS_PATH=/etc/default/tls
# event notification configuration file path: deploy/config/event.example.toml
RUSTFS_EVENT_CONFIG=/etc/default/event.toml
docker-compose-obs.yaml
+18 -18
View File
@@ -1,6 +1,6 @@
services:
otel-collector:
image: otel/opentelemetry-collector-contrib:0.120.0
image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.124.0
environment:
- TZ=Asia/Shanghai
volumes:
@@ -16,7 +16,7 @@ services:
networks:
- rustfs-network
jaeger:
image: jaegertracing/jaeger:2.4.0
image: jaegertracing/jaeger:2.5.0
environment:
- TZ=Asia/Shanghai
ports:
@@ -26,7 +26,7 @@ services:
networks:
- rustfs-network
prometheus:
image: prom/prometheus:v3.2.1
image: prom/prometheus:v3.3.0
environment:
- TZ=Asia/Shanghai
volumes:
@@ -36,7 +36,7 @@ services:
networks:
- rustfs-network
loki:
image: grafana/loki:3.4.2
image: grafana/loki:3.5.0
environment:
- TZ=Asia/Shanghai
volumes:
@@ -47,7 +47,7 @@ services:
networks:
- rustfs-network
grafana:
image: grafana/grafana:11.6.0
image: grafana/grafana:11.6.1
ports:
- "3000:3000" # Web UI
environment:
@@ -63,10 +63,10 @@ services:
container_name: node1
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=0.0.0.0:9000
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002
- RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml
- RUSTFS_CONSOLE_ADDRESS=:9002
- RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml
platform: linux/amd64
ports:
- "9001:9000" # 映射宿主机的 9001 端口到容器的 9000 端口
@@ -83,11 +83,11 @@ services:
dockerfile: Dockerfile.obs
container_name: node2
environment:
- RUSTFS_VOLUMES=/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=0.0.0.0:9000
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002
- RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml
- RUSTFS_CONSOLE_ADDRESS=:9002
- RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml
platform: linux/amd64
ports:
- "9002:9000" # 映射宿主机的 9002 端口到容器的 9000 端口
@@ -105,10 +105,10 @@ services:
container_name: node3
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=0.0.0.0:9000
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002
- RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml
- RUSTFS_CONSOLE_ADDRESS=:9002
- RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml
platform: linux/amd64
ports:
- "9003:9000" # 映射宿主机的 9003 端口到容器的 9000 端口
@@ -126,10 +126,10 @@ services:
container_name: node4
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=0.0.0.0:9000
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002
- RUSTFS_OBS_CONFIG=/etc/observability/config/obs.toml
- RUSTFS_CONSOLE_ADDRESS=:9002
- RUSTFS_OBS_CONFIG=/etc/observability/config/obs-multi.toml
platform: linux/amd64
ports:
- "9004:9000" # 映射宿主机的 9004 端口到容器的 9000 端口
ecstore/Cargo.toml
+14 -15
View File
@@ -11,14 +11,15 @@ rust-version.workspace = true
workspace = true
[dependencies]
rustfs-config = { workspace = true }
async-trait.workspace = true
backon.workspace = true
blake2 = "0.10.6"
blake2 = { workspace = true }
bytes.workspace = true
common.workspace = true
policy.workspace = true
chrono.workspace = true
glob = "0.3.2"
glob = { workspace = true }
thiserror.workspace = true
flatbuffers.workspace = true
futures.workspace = true
@@ -30,16 +31,16 @@ serde_json.workspace = true
tracing-error.workspace = true
s3s.workspace = true
http.workspace = true
highway = "1.3.0"
highway = { workspace = true }
url.workspace = true
uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] }
reed-solomon-erasure = { workspace = true }
transform-stream = "0.3.1"
lazy_static.workspace = true
lock.workspace = true
regex = "1.11.1"
netif = "0.1.6"
nix = { version = "0.29.0", features = ["fs"] }
regex = { workspace = true }
netif = { workspace = true }
nix = { workspace = true }
path-absolutize = "3.1.1"
protos.workspace = true
rmp.workspace = true
@@ -53,13 +54,13 @@ hex-simd = "0.8.0"
path-clean = "1.0.1"
tempfile.workspace = true
tokio = { workspace = true, features = ["io-util", "sync", "signal"] }
tokio-stream = "0.1.17"
tokio-stream = { workspace = true }
tonic.workspace = true
tower.workspace = true
byteorder = "1.5.0"
xxhash-rust = { version = "0.8.15", features = ["xxh64"] }
num = "0.4.3"
num_cpus = "1.16"
num_cpus = { workspace = true }
s3s-policy.workspace = true
rand.workspace = true
pin-project-lite.workspace = true
@@ -68,20 +69,18 @@ madmin.workspace = true
workers.workspace = true
reqwest = { workspace = true }
urlencoding = "2.1.3"
smallvec = "1.15.0"
smallvec = { workspace = true }
shadow-rs.workspace = true
[target.'cfg(not(windows))'.dependencies]
nix = { version = "0.29.0", features = ["fs"] }
nix = { workspace = true }
[target.'cfg(windows)'.dependencies]
winapi = "0.3.9"
winapi = { workspace = true }
[dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
[build-dependencies]
shadow-rs.workspace = true
shadow-rs = { workspace = true, features = ["build", "metadata"] }
ecstore/src/bitrot.rs
+1
View File
@@ -534,6 +534,7 @@ impl Writer for BitrotFileWriter {
self
}
#[tracing::instrument(level = "info", skip_all)]
async fn write(&mut self, buf: Bytes) -> Result<()> {
if buf.is_empty() {
return Ok(());
ecstore/src/cache_value/metacache_set.rs
+9 -8
View File
@@ -7,7 +7,7 @@ use common::error::{Error, Result};
use futures::future::join_all;
use std::{future::Future, pin::Pin, sync::Arc};
use tokio::{spawn, sync::broadcast::Receiver as B_Receiver};
use tracing::{error, info};
use tracing::error;
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
pub type PartialFn = Box<dyn Fn(MetaCacheEntries, &[Option<Error>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
@@ -54,7 +54,6 @@ impl Clone for ListPathRawOptions {
pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -> Result<()> {
// println!("list_path_raw {},{}", &opts.bucket, &opts.path);
if opts.disks.is_empty() {
info!("list_path_raw 0 drives provided");
return Err(Error::from_string("list_path_raw: 0 drives provided"));
}
@@ -205,7 +204,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
continue;
}
// If exact match, we agree.
if let Ok((_, true)) = current.matches(&entry, true) {
if let (_, true) = current.matches(Some(&entry), true) {
top_entries[i] = Some(entry);
agree += 1;
@@ -214,17 +213,18 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
// If only the name matches we didn't agree, but add it for resolution.
if entry.name == current.name {
top_entries[i] = Some(entry);
continue;
}
// We got different entries
if entry.name > current.name {
continue;
}
// We got a new, better current.
// Clear existing entries.
top_entries = vec![None; top_entries.len()];
agree += 1;
for item in top_entries.iter_mut().take(i) {
*item = None;
}
agree = 1;
top_entries[i] = Some(entry.clone());
current = entry;
}
@@ -272,6 +272,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
for r in readers.iter_mut() {
let _ = r.skip(1).await;
}
if let Some(agreed_fn) = opts.agreed.as_ref() {
agreed_fn(current).await;
}
ecstore/src/config/com.rs
+1 -1
View File
@@ -204,7 +204,7 @@ async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api:
}
}
Err(err) => {
error!("init storageclass err:{:?}", &err);
error!("init storage class err:{:?}", &err);
break;
}
}
ecstore/src/config/mod.rs
+2 -2
View File
@@ -141,7 +141,7 @@ impl Config {
}
pub fn merge(&self) -> Config {
// TODO: merge defauls
// TODO: merge default
self.clone()
}
}
@@ -158,6 +158,6 @@ pub fn register_default_kvs(kvs: HashMap<String, KVS>) {
pub fn init() {
let mut kvs = HashMap::new();
kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DefaultKVS.clone());
// TODO: other defauls
// TODO: other default
register_default_kvs(kvs)
}
ecstore/src/config/storageclass.rs
+3 -3
View File
@@ -8,8 +8,8 @@ use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use tracing::warn;
// default_partiy_count 默认配置，根据磁盘总数分配校验磁盘数量
pub fn default_partiy_count(drive: usize) -> usize {
// default_parity_count 默认配置，根据磁盘总数分配校验磁盘数量
pub fn default_parity_count(drive: usize) -> usize {
match drive {
1 => 0,
2 | 3 => 1,
@@ -158,7 +158,7 @@ pub fn lookup_config(kvs: &KVS, set_drive_count: usize) -> Result<Config> {
parse_storage_class(&ssc_str)?
} else {
StorageClass {
parity: default_partiy_count(set_drive_count),
parity: default_parity_count(set_drive_count),
}
}
};
ecstore/src/disk/local.rs
+17 -14
View File
@@ -38,7 +38,9 @@ use crate::set_disk::{
CHECK_PART_VOLUME_NOT_FOUND,
};
use crate::store_api::{BitrotAlgorithm, StorageAPI};
use crate::utils::fs::{access, lstat, remove, remove_all, rename, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY};
use crate::utils::fs::{
access, lstat, lstat_std, remove, remove_all, remove_all_std, remove_std, rename, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY,
};
use crate::utils::os::get_info;
use crate::utils::path::{
self, clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, GLOBAL_DIR_SUFFIX,
@@ -259,7 +261,7 @@ impl LocalDisk {
#[tracing::instrument(level = "debug", skip(self))]
async fn check_format_json(&self) -> Result<Metadata> {
let md = fs::metadata(&self.format_path).await.map_err(|e| match e.kind() {
let md = std::fs::metadata(&self.format_path).map_err(|e| match e.kind() {
ErrorKind::NotFound => DiskError::DiskNotFound,
ErrorKind::PermissionDenied => DiskError::FileAccessDenied,
_ => {
@@ -315,9 +317,9 @@ impl LocalDisk {
#[allow(unused_variables)]
pub async fn move_to_trash(&self, delete_path: &PathBuf, recursive: bool, immediate_purge: bool) -> Result<()> {
if recursive {
remove_all(delete_path).await?;
remove_all_std(delete_path)?;
} else {
remove(delete_path).await?;
remove_std(delete_path)?;
}
return Ok(());
@@ -365,7 +367,7 @@ impl LocalDisk {
Ok(())
}
// #[tracing::instrument(skip(self))]
#[tracing::instrument(level = "debug", skip(self))]
pub async fn delete_file(
&self,
base_path: &PathBuf,
@@ -688,6 +690,7 @@ impl LocalDisk {
}
// write_all_private with check_path_length
#[tracing::instrument(level = "debug", skip_all)]
pub async fn write_all_private(
&self,
volume: &str,
@@ -1213,7 +1216,7 @@ impl DiskAPI for LocalDisk {
Ok(data)
}
#[tracing::instrument(skip(self))]
#[tracing::instrument(level = "debug", skip_all)]
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
self.write_all_public(volume, path, data).await
}
@@ -1334,10 +1337,10 @@ impl DiskAPI for LocalDisk {
let src_volume_dir = self.get_bucket_path(src_volume)?;
let dst_volume_dir = self.get_bucket_path(dst_volume)?;
if !skip_access_checks(src_volume) {
utils::fs::access(&src_volume_dir).await.map_err(map_err_not_exists)?
utils::fs::access_std(&src_volume_dir).map_err(map_err_not_exists)?
}
if !skip_access_checks(dst_volume) {
utils::fs::access(&dst_volume_dir).await.map_err(map_err_not_exists)?
utils::fs::access_std(&dst_volume_dir).map_err(map_err_not_exists)?
}
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR);
@@ -1360,7 +1363,7 @@ impl DiskAPI for LocalDisk {
check_path_length(dst_file_path.to_string_lossy().as_ref())?;
if src_is_dir {
let meta_op = match lstat(&src_file_path).await {
let meta_op = match lstat_std(&src_file_path) {
Ok(meta) => Some(meta),
Err(e) => {
if is_sys_err_io(&e) {
@@ -1381,7 +1384,7 @@ impl DiskAPI for LocalDisk {
}
}
if let Err(e) = utils::fs::remove(&dst_file_path).await {
if let Err(e) = utils::fs::remove_std(&dst_file_path) {
if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) {
warn!("rename_part remove dst failed {:?} err {:?}", &dst_file_path, e);
return Err(Error::new(DiskError::FileAccessDenied));
@@ -1721,7 +1724,7 @@ impl DiskAPI for LocalDisk {
Ok(())
}
#[tracing::instrument(skip(self))]
#[tracing::instrument(level = "debug", skip(self))]
async fn rename_data(
&self,
src_volume: &str,
@@ -1732,7 +1735,7 @@ impl DiskAPI for LocalDisk {
) -> Result<RenameDataResp> {
let src_volume_dir = self.get_bucket_path(src_volume)?;
if !skip_access_checks(src_volume) {
if let Err(e) = utils::fs::access(&src_volume_dir).await {
if let Err(e) = utils::fs::access_std(&src_volume_dir) {
info!("access checks failed, src_volume_dir: {:?}, err: {}", src_volume_dir, e.to_string());
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
@@ -1740,7 +1743,7 @@ impl DiskAPI for LocalDisk {
let dst_volume_dir = self.get_bucket_path(dst_volume)?;
if !skip_access_checks(dst_volume) {
if let Err(e) = utils::fs::access(&dst_volume_dir).await {
if let Err(e) = utils::fs::access_std(&dst_volume_dir) {
info!("access checks failed, dst_volume_dir: {:?}, err: {}", dst_volume_dir, e.to_string());
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
@@ -1913,7 +1916,7 @@ impl DiskAPI for LocalDisk {
if let Some(src_file_path_parent) = src_file_path.parent() {
if src_volume != super::RUSTFS_META_MULTIPART_BUCKET {
let _ = utils::fs::remove(src_file_path_parent).await;
let _ = utils::fs::remove_std(src_file_path_parent);
} else {
let _ = self
.delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false)
ecstore/src/disk/mod.rs
+96 -51
View File
@@ -786,46 +786,62 @@ impl MetaCacheEntry {
fm.into_file_info_versions(bucket, self.name.as_str(), false)
}
pub fn matches(&self, other: &MetaCacheEntry, strict: bool) -> Result<(Option<MetaCacheEntry>, bool)> {
pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option<MetaCacheEntry>, bool) {
if other.is_none() {
return (None, false);
}
let other = other.unwrap();
let mut prefer = None;
if self.name != other.name {
if self.name < other.name {
return Ok((Some(self.clone()), false));
return (Some(self.clone()), false);
}
return Ok((Some(other.clone()), false));
return (Some(other.clone()), false);
}
if other.is_dir() || self.is_dir() {
if self.is_dir() {
return Ok((Some(self.clone()), other.is_dir()));
return (Some(self.clone()), other.is_dir() == self.is_dir());
}
return Ok((Some(other.clone()), other.is_dir() == self.is_dir()));
return (Some(other.clone()), other.is_dir() == self.is_dir());
}
let self_vers = match &self.cached {
Some(file_meta) => file_meta.clone(),
None => FileMeta::load(&self.metadata)?,
None => match FileMeta::load(&self.metadata) {
Ok(meta) => meta,
Err(_) => {
return (None, false);
}
},
};
let other_vers = match &other.cached {
Some(file_meta) => file_meta.clone(),
None => FileMeta::load(&other.metadata)?,
None => match FileMeta::load(&other.metadata) {
Ok(meta) => meta,
Err(_) => {
return (None, false);
}
},
};
if self_vers.versions.len() != other_vers.versions.len() {
match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) {
Ordering::Greater => {
return Ok((Some(self.clone()), false));
return (Some(self.clone()), false);
}
Ordering::Less => {
return Ok((Some(self.clone()), false));
return (Some(other.clone()), false);
}
_ => {}
}
if self_vers.versions.len() > other_vers.versions.len() {
return Ok((Some(self.clone()), false));
return (Some(self.clone()), false);
}
return Ok((Some(self.clone()), false));
return (Some(other.clone()), false);
}
for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) {
@@ -853,14 +869,14 @@ impl MetaCacheEntry {
}
if prefer.is_some() {
return Ok((prefer, false));
return (prefer, false);
}
if s_version.header.sorts_before(&o_version.header) {
return Ok((Some(self.clone()), false));
return (Some(self.clone()), false);
}
return Ok((Some(other.clone()), false));
return (Some(other.clone()), false);
}
}
@@ -868,7 +884,7 @@ impl MetaCacheEntry {
prefer = Some(self.clone());
}
Ok((prefer, true))
(prefer, true)
}
pub fn xl_meta(&mut self) -> Result<FileMeta> {
@@ -900,9 +916,10 @@ impl MetaCacheEntries {
pub fn as_ref(&self) -> &[Option<MetaCacheEntry>] {
&self.0
}
pub fn resolve(&self, mut params: MetadataResolutionParams) -> Result<Option<MetaCacheEntry>> {
pub fn resolve(&self, mut params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
if self.0.is_empty() {
return Ok(None);
warn!("decommission_pool: entries resolve empty");
return None;
}
let mut dir_exists = 0;
@@ -913,76 +930,104 @@ impl MetaCacheEntries {
let mut objs_valid = 0;
for entry in self.0.iter().flatten() {
let mut entry = entry.clone();
warn!("decommission_pool: entries resolve entry {:?}", entry.name);
if entry.name.is_empty() {
continue;
}
if entry.is_dir() {
dir_exists += 1;
selected = Some(entry.clone());
warn!("decommission_pool: entries resolve entry dir {:?}", entry.name);
continue;
}
let xl = match entry.xl_meta() {
Ok(xl) => xl,
Err(e) => {
warn!("decommission_pool: entries resolve entry xl_meta {:?}", e);
continue;
}
};
objs_valid += 1;
match &entry.cached {
Some(file_meta) => {
params.candidates.push(file_meta.versions.clone());
}
None => {
params.candidates.push(FileMeta::load(&entry.metadata)?.versions);
}
}
params.candidates.push(xl.versions.clone());
if selected.is_none() {
selected = Some(entry.clone());
objs_agree = 1;
warn!("decommission_pool: entries resolve entry selected {:?}", entry.name);
continue;
}
if let (Some(prefer), true) = entry.matches(selected.as_ref().unwrap(), params.strict)? {
selected = Some(prefer);
if let (prefer, true) = entry.matches(selected.as_ref(), params.strict) {
selected = prefer;
objs_agree += 1;
warn!("decommission_pool: entries resolve entry prefer {:?}", entry.name);
continue;
}
}
// Return dir entries, if enough...
if selected.is_some() && selected.as_ref().unwrap().is_dir() && dir_exists >= params.dir_quorum {
return Ok(selected);
let Some(selected) = selected else {
warn!("decommission_pool: entries resolve entry no selected");
return None;
};
if selected.is_dir() && dir_exists >= params.dir_quorum {
warn!("decommission_pool: entries resolve entry dir selected {:?}", selected.name);
return Some(selected);
}
// If we would never be able to reach read quorum.
if objs_valid < params.obj_quorum {
return Ok(None);
warn!(
"decommission_pool: entries resolve entry not enough objects {} < {}",
objs_valid, params.obj_quorum
);
return None;
}
// If all objects agree.
if selected.is_some() && objs_agree == objs_valid {
return Ok(selected);
if objs_agree == objs_valid {
warn!("decommission_pool: entries resolve entry all agree {} == {}", objs_agree, objs_valid);
return Some(selected);
}
// If cached is nil we shall skip the entry.
if selected.is_none() || (selected.is_some() && selected.as_ref().unwrap().cached.is_none()) {
return Ok(None);
let Some(cached) = selected.cached else {
warn!("decommission_pool: entries resolve entry no cached");
return None;
};
let versions = merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, &params.candidates);
if versions.is_empty() {
warn!("decommission_pool: entries resolve entry no versions");
return None;
}
let metadata = match cached.marshal_msg() {
Ok(meta) => meta,
Err(e) => {
warn!("decommission_pool: entries resolve entry marshal_msg {:?}", e);
return None;
}
};
// Merge if we have disagreement.
// Create a new merged result.
selected = Some(MetaCacheEntry {
name: selected.as_ref().unwrap().name.clone(),
let new_selected = MetaCacheEntry {
name: selected.name.clone(),
cached: Some(FileMeta {
meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver,
meta_ver: cached.meta_ver,
versions,
..Default::default()
}),
reusable: true,
..Default::default()
});
metadata,
};
selected.as_mut().unwrap().cached.as_mut().unwrap().versions =
merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, &params.candidates);
if selected.as_ref().unwrap().cached.as_ref().unwrap().versions.is_empty() {
return Ok(None);
}
selected.as_mut().unwrap().metadata = selected.as_ref().unwrap().cached.as_ref().unwrap().marshal_msg()?;
Ok(selected)
warn!("decommission_pool: entries resolve entry selected {:?}", new_selected.name);
Some(new_selected)
}
pub fn first_found(&self) -> (Option<MetaCacheEntry>, usize) {
ecstore/src/disk/os.rs
+6 -6
View File
@@ -9,7 +9,6 @@ use crate::{
};
use common::error::{Error, Result};
use tokio::fs;
use tracing::info;
use super::error::{os_err_to_file_err, os_is_exist, DiskError};
@@ -108,6 +107,7 @@ pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> Result<Vec<String>>
Ok(volumes)
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn rename_all(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
@@ -136,15 +136,15 @@ pub async fn reliable_rename(
base_dir: impl AsRef<Path>,
) -> io::Result<()> {
if let Some(parent) = dst_file_path.as_ref().parent() {
if !file_exists(parent).await {
info!("reliable_rename reliable_mkdir_all parent: {:?}", parent);
if !file_exists(parent) {
// info!("reliable_rename reliable_mkdir_all parent: {:?}", parent);
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
}
}
let mut i = 0;
loop {
if let Err(e) = utils::fs::rename(src_file_path.as_ref(), dst_file_path.as_ref()).await {
if let Err(e) = utils::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
if os_is_not_exist(&e) && i == 0 {
i += 1;
continue;
@@ -221,6 +221,6 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
Ok(())
}
pub async fn file_exists(path: impl AsRef<Path>) -> bool {
fs::metadata(path.as_ref()).await.map(|_| true).unwrap_or(false)
pub fn file_exists(path: impl AsRef<Path>) -> bool {
std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
}
ecstore/src/erasure.rs
+53 -124
View File
@@ -1,5 +1,6 @@
use crate::bitrot::{BitrotReader, BitrotWriter};
use crate::error::clone_err;
use crate::io::Etag;
use crate::quorum::{object_op_ignored_errs, reduce_write_quorum_errs};
use bytes::{Bytes, BytesMut};
use common::error::{Error, Result};
@@ -8,8 +9,10 @@ use reed_solomon_erasure::galois_8::ReedSolomon;
use smallvec::SmallVec;
use std::any::Any;
use std::io::ErrorKind;
use std::sync::Arc;
use tokio::io::{AsyncRead, AsyncWrite};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::sync::mpsc;
use tracing::warn;
use tracing::{error, info};
// use tracing::debug;
@@ -26,7 +29,7 @@ pub struct Erasure {
encoder: Option<ReedSolomon>,
pub block_size: usize,
_id: Uuid,
buf: Vec<u8>,
_buf: Vec<u8>,
}
impl Erasure {
@@ -46,61 +49,65 @@ impl Erasure {
block_size,
encoder,
_id: Uuid::new_v4(),
buf: vec![0u8; block_size],
_buf: vec![0u8; block_size],
}
}
#[tracing::instrument(level = "debug", skip(self, reader, writers))]
#[tracing::instrument(level = "info", skip(self, reader, writers))]
pub async fn encode<S>(
&mut self,
reader: &mut S,
self: Arc<Self>,
mut reader: S,
writers: &mut [Option<BitrotWriter>],
// block_size: usize,
total_size: usize,
write_quorum: usize,
) -> Result<usize>
) -> Result<(usize, String)>
where
S: AsyncRead + Unpin + Send + 'static,
S: AsyncRead + Etag + Unpin + Send + 'static,
{
// pin_mut!(body);
// let mut reader = tokio_util::io::StreamReader::new(
// body.map(|f| f.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))),
// );
let (tx, mut rx) = mpsc::channel(5);
let task = tokio::spawn(async move {
let mut buf = vec![0u8; self.block_size];
let mut total: usize = 0;
loop {
if total_size > 0 {
let new_len = {
let remain = total_size - total;
if remain > self.block_size {
self.block_size
} else {
remain
}
};
let mut total: usize = 0;
let mut blocks = <SmallVec<[Bytes; 16]>>::new();
loop {
if total_size > 0 {
let new_len = {
let remain = total_size - total;
if remain > self.block_size {
self.block_size
} else {
remain
if new_len == 0 && total > 0 {
break;
}
};
if new_len == 0 && total > 0 {
buf.resize(new_len, 0u8);
match reader.read_exact(&mut buf).await {
Ok(res) => res,
Err(e) => {
if let ErrorKind::UnexpectedEof = e.kind() {
break;
} else {
return Err(Error::new(e));
}
}
};
total += buf.len();
}
let blocks = Arc::new(Box::pin(self.clone().encode_data(&buf)?));
let _ = tx.send(blocks).await;
if total_size == 0 {
break;
}
self.buf.resize(new_len, 0u8);
match reader.read_exact(&mut self.buf).await {
Ok(res) => res,
Err(e) => {
if let ErrorKind::UnexpectedEof = e.kind() {
break;
} else {
return Err(Error::new(e));
}
}
};
total += self.buf.len();
}
let etag = reader.etag().await;
Ok((total, etag))
});
self.encode_data(&self.buf, &mut blocks)?;
while let Some(blocks) = rx.recv().await {
let write_futures = writers.iter_mut().enumerate().map(|(i, w_op)| {
let i_inner = i;
let blocks_inner = blocks.clone();
@@ -125,84 +132,8 @@ impl Erasure {
warn!("Erasure encode errs {:?}", &errs);
return Err(err);
}
if total_size == 0 {
break;
}
}
Ok(total)
// // let stream = ChunkedStream::new(body, self.block_size);
// let stream = ChunkedStream::new(body, total_size, self.block_size, false);
// let mut total: usize = 0;
// // let mut idx = 0;
// pin_mut!(stream);
// // warn!("encode start...");
// loop {
// match stream.next().await {
// Some(result) => match result {
// Ok(data) => {
// total += data.len();
// // EOF
// if data.is_empty() {
// break;
// }
// // idx += 1;
// // warn!("encode {} get data {:?}", data.len(), data.to_vec());
// let blocks = self.encode_data(data.as_ref())?;
// // warn!(
// // "encode shard size: {}/{} from block_size {}, total_size {} ",
// // blocks[0].len(),
// // blocks.len(),
// // data.len(),
// // total_size
// // );
// let mut errs = Vec::new();
// for (i, w_op) in writers.iter_mut().enumerate() {
// if let Some(w) = w_op {
// match w.write(blocks[i].as_ref()).await {
// Ok(_) => errs.push(None),
// Err(e) => errs.push(Some(e)),
// }
// } else {
// errs.push(Some(Error::new(DiskError::DiskNotFound)));
// }
// }
// let none_count = errs.iter().filter(|&x| x.is_none()).count();
// if none_count >= write_quorum {
// continue;
// }
// if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) {
// warn!("Erasure encode errs {:?}", &errs);
// return Err(err);
// }
// }
// Err(e) => {
// warn!("poll result err {:?}", &e);
// return Err(Error::msg(e.to_string()));
// }
// },
// None => {
// // warn!("poll empty result");
// break;
// }
// }
// }
// let _ = close_bitrot_writers(writers).await?;
// Ok(total)
task.await?
}
pub async fn decode<W>(
@@ -356,8 +287,8 @@ impl Erasure {
self.data_shards + self.parity_shards
}
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
pub fn encode_data(&self, data: &[u8], shards: &mut SmallVec<[Bytes; 16]>) -> Result<()> {
#[tracing::instrument(level = "info", skip_all, fields(data_len=data.len()))]
pub fn encode_data(self: Arc<Self>, data: &[u8]) -> Result<Vec<Bytes>> {
let (shard_size, total_size) = self.need_size(data.len());
// 生成一个新的 所需的所有分片数据长度
@@ -379,14 +310,13 @@ impl Erasure {
// 零拷贝分片，所有 shard 引用 data_buffer
let mut data_buffer = data_buffer.freeze();
shards.clear();
shards.reserve(self.total_shard_count());
let mut shards = Vec::with_capacity(self.total_shard_count());
for _ in 0..self.total_shard_count() {
let shard = data_buffer.split_to(shard_size);
shards.push(shard);
}
Ok(())
Ok(shards)
}
pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> Result<()> {
@@ -617,7 +547,6 @@ impl ShardReader {
#[cfg(test)]
mod test {
use super::*;
#[test]
@@ -626,8 +555,7 @@ mod test {
let parity_shards = 2;
let data: &[u8] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
let ec = Erasure::new(data_shards, parity_shards, 1);
let mut shards = SmallVec::new();
ec.encode_data(data, &mut shards).unwrap();
let shards = Arc::new(ec).encode_data(data).unwrap();
println!("shards:{:?}", shards);
let mut s: Vec<_> = shards
@@ -643,6 +571,7 @@ mod test {
println!("sss:{:?}", &s);
let ec = Erasure::new(data_shards, parity_shards, 1);
ec.decode_data(&mut s).unwrap();
// ec.encoder.reconstruct(&mut s).unwrap();
ecstore/src/file_meta.rs
+16 -3
View File
@@ -58,10 +58,12 @@ impl FileMeta {
}
// isXL2V1Format
#[tracing::instrument(level = "debug", skip_all)]
pub fn is_xl2_v1_format(buf: &[u8]) -> bool {
!matches!(Self::check_xl2_v1(buf), Err(_e))
}
#[tracing::instrument(level = "debug", skip_all)]
pub fn load(buf: &[u8]) -> Result<FileMeta> {
let mut xl = FileMeta::default();
xl.unmarshal_msg(buf)?;
@@ -245,7 +247,7 @@ impl FileMeta {
}
}
#[tracing::instrument]
#[tracing::instrument(level = "debug", skip_all)]
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut wr = Vec::new();
@@ -363,6 +365,7 @@ impl FileMeta {
}
// shard_data_dir_count 查询 vid下data_dir的数量
#[tracing::instrument(level = "debug", skip_all)]
pub fn shard_data_dir_count(&self, vid: &Option<Uuid>, data_dir: &Option<Uuid>) -> usize {
self.versions
.iter()
@@ -434,6 +437,7 @@ impl FileMeta {
}
// 添加版本
#[tracing::instrument(level = "debug", skip_all)]
pub fn add_version(&mut self, fi: FileInfo) -> Result<()> {
let vid = fi.version_id;
@@ -916,11 +920,20 @@ impl FileMetaVersionHeader {
}
pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool {
let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o);
if self.version_id.is_none() {
return self.version_id == o.version_id && self.version_type == o.version_type && self.mod_time == o.mod_time;
ok = ok && self.mod_time == o.mod_time;
}
self.version_id == o.version_id && self.version_type == o.version_type
ok
}
pub fn matches_ec(&self, o: &FileMetaVersionHeader) -> bool {
if self.has_ec() && o.has_ec() {
return self.ec_n == o.ec_n && self.ec_m == o.ec_m;
}
true
}
pub fn free_version(&self) -> bool {
ecstore/src/global.rs
+8 -4
View File
@@ -20,8 +20,6 @@ pub const DISK_MIN_INODES: u64 = 1000;
pub const DISK_FILL_FRACTION: f64 = 0.99;
pub const DISK_RESERVE_FRACTION: f64 = 0.15;
pub const DEFAULT_PORT: u16 = 9000;
lazy_static! {
static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
@@ -41,31 +39,37 @@ lazy_static! {
pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
}
/// Get the global rustfs port
pub fn global_rustfs_port() -> u16 {
if let Some(p) = GLOBAL_RUSTFS_PORT.get() {
*p
} else {
DEFAULT_PORT
rustfs_config::DEFAULT_PORT
}
}
/// Set the global rustfs port
pub fn set_global_rustfs_port(value: u16) {
GLOBAL_RUSTFS_PORT.set(value).expect("set_global_rustfs_port fail");
}
/// Get the global rustfs port
pub fn set_global_deployment_id(id: Uuid) {
globalDeploymentIDPtr.set(id).unwrap();
}
/// Get the global deployment id
pub fn get_global_deployment_id() -> Option<String> {
globalDeploymentIDPtr.get().map(|v| v.to_string())
}
/// Get the global deployment id
pub fn set_global_endpoints(eps: Vec<PoolEndpoints>) {
GLOBAL_Endpoints
.set(EndpointServerPools::from(eps))
.expect("GLOBAL_Endpoints set failed")
}
/// Get the global endpoints
pub fn get_global_endpoints() -> EndpointServerPools {
if let Some(eps) = GLOBAL_Endpoints.get() {
eps.clone()
ecstore/src/heal/data_scanner.rs
+1 -1
View File
@@ -867,7 +867,7 @@ impl FolderScanner {
// return;
// }
let entry = match entries.resolve(resolver_partial) {
Ok(Some(entry)) => entry,
Some(entry) => entry,
_ => match entries.first_found() {
(Some(entry), _) => entry,
_ => return,
ecstore/src/heal/heal_ops.rs
+2 -2
View File
@@ -17,7 +17,7 @@ use crate::{
global::GLOBAL_IsDistErasure,
heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN},
new_object_layer_fn,
utils::path::has_profix,
utils::path::has_prefix,
};
use crate::{
heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT},
@@ -786,7 +786,7 @@ impl AllHealState {
let _ = self.mu.write().await;
for (k, v) in self.heal_seq_map.read().await.iter() {
if (has_profix(k, path_s) || has_profix(path_s, k)) && !v.has_ended().await {
if (has_prefix(k, path_s) || has_prefix(path_s, k)) && !v.has_ended().await {
return Err(Error::from_string(format!(
"The provided heal sequence path overlaps with an existing heal path: {}",
k
ecstore/src/io.rs
+33 -12
View File
@@ -1,8 +1,12 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::TryStreamExt;
use md5::Digest;
use md5::Md5;
use pin_project_lite::pin_project;
use std::io;
use std::pin::Pin;
use std::task::ready;
use std::task::Context;
use std::task::Poll;
use tokio::io::AsyncRead;
@@ -125,10 +129,17 @@ impl AsyncRead for HttpFileReader {
}
}
pub struct EtagReader<R> {
inner: R,
bytes_tx: mpsc::Sender<Bytes>,
md5_rx: oneshot::Receiver<String>,
#[async_trait]
pub trait Etag {
async fn etag(self) -> String;
}
pin_project! {
pub struct EtagReader<R> {
inner: R,
bytes_tx: mpsc::Sender<Bytes>,
md5_rx: oneshot::Receiver<String>,
}
}
impl<R> EtagReader<R> {
@@ -148,8 +159,11 @@ impl<R> EtagReader<R> {
EtagReader { inner, bytes_tx, md5_rx }
}
}
pub async fn etag(self) -> String {
#[async_trait]
impl<R: Send> Etag for EtagReader<R> {
async fn etag(self) -> String {
drop(self.inner);
drop(self.bytes_tx);
self.md5_rx.await.unwrap()
@@ -157,21 +171,28 @@ impl<R> EtagReader<R> {
}
impl<R: AsyncRead + Unpin> AsyncRead for EtagReader<R> {
#[tracing::instrument(level = "debug", skip_all)]
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<tokio::io::Result<()>> {
let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
if let Poll::Ready(Ok(())) = &poll {
if buf.remaining() == 0 {
#[tracing::instrument(level = "info", skip_all)]
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<tokio::io::Result<()>> {
let me = self.project();
loop {
let rem = buf.remaining();
if rem != 0 {
ready!(Pin::new(&mut *me.inner).poll_read(cx, buf))?;
if buf.remaining() == rem {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "early eof")).into();
}
} else {
let bytes = buf.filled();
let bytes = Bytes::copy_from_slice(bytes);
let tx = self.bytes_tx.clone();
let tx = me.bytes_tx.clone();
tokio::spawn(async move {
if let Err(e) = tx.send(bytes).await {
warn!("EtagReader send error: {:?}", e);
}
});
return Poll::Ready(Ok(()));
}
}
poll
}
}
ecstore/src/pools.rs
+26 -17
View File
@@ -31,6 +31,7 @@ use std::io::{Cursor, Write};
use std::path::PathBuf;
use std::sync::Arc;
use time::{Duration, OffsetDateTime};
use tokio::io::AsyncReadExt;
use tokio::sync::broadcast::Receiver as B_Receiver;
use tracing::{error, info, warn};
@@ -910,7 +911,11 @@ impl ECStore {
let wk = wk.clone();
let set = set.clone();
let rcfg = rcfg.clone();
Box::pin(async move { this.decommission_entry(idx, entry, bucket, set, wk, rcfg).await })
Box::pin(async move {
wk.take().await;
this.decommission_entry(idx, entry, bucket, set, wk, rcfg).await
})
}
});
@@ -918,6 +923,7 @@ impl ECStore {
let mut rx = rx.resubscribe();
let bi = bi.clone();
let set_id = set_idx;
let wk_clone = wk.clone();
tokio::spawn(async move {
loop {
if rx.try_recv().is_ok() {
@@ -945,6 +951,8 @@ impl ECStore {
}
}
}
wk_clone.give().await;
});
}
@@ -1166,7 +1174,7 @@ impl ECStore {
#[tracing::instrument(skip(self, rd))]
async fn decommission_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
warn!("decommission_object: {} {}", &bucket, &rd.object_info.name);
warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
let object_info = rd.object_info.clone();
// TODO: check : use size or actual_size ?
@@ -1194,8 +1202,6 @@ impl ECStore {
}
};
// TODO: defer abort_multipart_upload
defer!(|| async {
if let Err(err) = self
.abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default())
@@ -1210,9 +1216,13 @@ impl ECStore {
let mut reader = rd.stream;
for (i, part) in object_info.parts.iter().enumerate() {
// 每次从reader中读取一个part上传
let mut chunk = vec![0u8; part.size];
let mut data = PutObjReader::new(reader, part.size);
reader.read_exact(&mut chunk).await?;
// 每次从reader中读取一个part上传
let rd = Box::new(Cursor::new(chunk));
let mut data = PutObjReader::new(rd, part.size);
let pi = match self
.put_object_part(
@@ -1230,18 +1240,17 @@ impl ECStore {
{
Ok(pi) => pi,
Err(err) => {
error!("decommission_object: put_object_part err {:?}", &err);
error!("decommission_object: put_object_part {} err {:?}", i, &err);
return Err(err);
}
};
warn!("decommission_object: put_object_part {} done {} {}", i, &bucket, &object_info.name);
parts[i] = CompletePart {
part_num: pi.part_num,
e_tag: pi.etag,
};
// 把reader所有权拿回来?
reader = data.stream;
}
if let Err(err) = self
@@ -1262,6 +1271,7 @@ impl ECStore {
return Err(err);
}
warn!("decommission_object: complete_multipart_upload done {} {}", &bucket, &object_info.name);
return Ok(());
}
@@ -1289,6 +1299,7 @@ impl ECStore {
return Err(err);
}
warn!("decommission_object: put_object done {} {}", &bucket, &object_info.name);
Ok(())
}
}
@@ -1319,6 +1330,8 @@ impl SetDisks {
..Default::default()
};
let cb1 = cb_func.clone();
list_path_raw(
rx,
ListPathRawOptions {
@@ -1327,25 +1340,21 @@ impl SetDisks {
path: bucket_info.prefix.clone(),
recursice: true,
min_disks: listing_quorum,
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<Error>]| {
let resolver = resolver.clone();
let cb_func = cb_func.clone();
match entries.resolve(resolver) {
Ok(Some(entry)) => {
Some(entry) => {
warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name);
Box::pin(async move {
cb_func(entry).await;
})
}
Ok(None) => {
None => {
warn!("decommission_pool: list_objects_to_decommission get none");
Box::pin(async {})
}
Err(err) => {
error!("decommission_pool: list_objects_to_decommission get err {:?}", &err);
Box::pin(async {})
}
}
})),
..Default::default()
ecstore/src/quorum.rs
+1
View File
@@ -147,6 +147,7 @@ pub fn reduce_read_quorum_errs(
// 根据写quorum验证错误数量
// 返回最大错误数量的下标，或QuorumError
#[tracing::instrument(level = "info", skip_all)]
pub fn reduce_write_quorum_errs(
errs: &[Option<Error>],
ignored_errs: &[Box<dyn CheckErrorFn>],
ecstore/src/rebalance.rs
+25 -24
View File
@@ -18,6 +18,7 @@ use common::defer;
use common::error::{Error, Result};
use http::HeaderMap;
use serde::{Deserialize, Serialize};
use tokio::io::AsyncReadExt;
use tokio::sync::broadcast::{self, Receiver as B_Receiver};
use tokio::time::{Duration, Instant};
use tracing::{error, info, warn};
@@ -370,7 +371,7 @@ impl ECStore {
let rebalance_meta = self.rebalance_meta.read().await;
if let Some(meta) = rebalance_meta.as_ref() {
if let Some(pool_stat) = meta.pool_stats.get(pool_index) {
if pool_stat.info.status != RebalStatus::Completed || !pool_stat.participating {
if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating {
return Ok(None);
}
@@ -389,10 +390,15 @@ impl ECStore {
let mut rebalance_meta = self.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() {
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
if let Some(idx) = pool_stat.buckets.iter().position(|b| *b == bucket) {
warn!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets);
if let Some(idx) = pool_stat.buckets.iter().position(|b| b.as_str() == bucket.as_str()) {
warn!("bucket_rebalance_done: bucket {} rebalanced", &bucket);
pool_stat.buckets.remove(idx);
pool_stat.rebalanced_buckets.push(bucket);
return Ok(());
} else {
warn!("bucket_rebalance_done: bucket {} not found", bucket);
}
}
}
@@ -581,25 +587,26 @@ impl ECStore {
}
});
tracing::warn!("Pool {} rebalancing is started", pool_index + 1);
warn!("Pool {} rebalancing is started", pool_index + 1);
while let Some(bucket) = self.next_rebal_bucket(pool_index).await? {
tracing::info!("Rebalancing bucket: {}", bucket);
warn!("Rebalancing bucket: start {}", bucket);
if let Err(err) = self.rebalance_bucket(rx.resubscribe(), bucket.clone(), pool_index).await {
if err.to_string().contains("not initialized") {
warn!("rebalance_bucket: rebalance not initialized, continue");
continue;
}
tracing::error!("Error rebalancing bucket {}: {:?}", bucket, err);
error!("Error rebalancing bucket {}: {:?}", bucket, err);
done_tx.send(Err(err)).await.ok();
break;
}
warn!("Rebalance bucket: done {} ", bucket);
self.bucket_rebalance_done(pool_index, bucket).await?;
}
tracing::warn!("Pool {} rebalancing is done", pool_index + 1);
warn!("Pool {} rebalancing is done", pool_index + 1);
done_tx.send(Ok(())).await.ok();
save_task.await.ok();
@@ -632,7 +639,6 @@ impl ECStore {
false
}
#[allow(unused_assignments)]
#[tracing::instrument(skip(self, wk, set))]
async fn rebalance_entry(
&self,
@@ -838,8 +844,6 @@ impl ECStore {
}
};
// TODO: defer abort_multipart_upload
defer!(|| async {
if let Err(err) = self
.abort_multipart_upload(&bucket, &object_info.name, &res.upload_id, &ObjectOptions::default())
@@ -856,7 +860,13 @@ impl ECStore {
for (i, part) in object_info.parts.iter().enumerate() {
// 每次从reader中读取一个part上传
let mut data = PutObjReader::new(reader, part.size);
let mut chunk = vec![0u8; part.size];
reader.read_exact(&mut chunk).await?;
// 每次从reader中读取一个part上传
let rd = Box::new(Cursor::new(chunk));
let mut data = PutObjReader::new(rd, part.size);
let pi = match self
.put_object_part(
@@ -883,9 +893,6 @@ impl ECStore {
part_num: pi.part_num,
e_tag: pi.etag,
};
// 把reader所有权拿回来?
reader = data.stream;
}
if let Err(err) = self
@@ -939,7 +946,7 @@ impl ECStore {
#[tracing::instrument(skip(self, rx))]
async fn rebalance_bucket(self: &Arc<Self>, rx: B_Receiver<bool>, bucket: String, pool_index: usize) -> Result<()> {
// Placeholder for actual bucket rebalance logic
tracing::info!("Rebalancing bucket {} in pool {}", bucket, pool_index);
warn!("Rebalancing bucket {} in pool {}", bucket, pool_index);
// TODO: other config
// if bucket != RUSTFS_META_BUCKET{
@@ -977,14 +984,12 @@ impl ECStore {
let bucket = bucket.clone();
let wk = wk.clone();
tokio::spawn(async move {
defer!(|| async {
wk.clone().give().await;
});
if let Err(err) = set.list_objects_to_rebalance(rx, bucket, rebalance_entry).await {
error!("Rebalance worker {} error: {}", set_idx, err);
} else {
info!("Rebalance worker {} done", set_idx);
}
wk.clone().give().await;
});
}
@@ -1068,25 +1073,21 @@ impl SetDisks {
bucket: bucket.clone(),
recursice: true,
min_disks: listing_quorum,
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry.clone())))),
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<Error>]| {
// let cb = cb.clone();
let resolver = resolver.clone();
let cb = cb.clone();
match entries.resolve(resolver) {
Ok(Some(entry)) => {
Some(entry) => {
warn!("rebalance: list_objects_to_decommission get {}", &entry.name);
Box::pin(async move { cb(entry).await })
}
Ok(None) => {
None => {
warn!("rebalance: list_objects_to_decommission get none");
Box::pin(async {})
}
Err(err) => {
error!("rebalance: list_objects_to_decommission get err {:?}", &err);
Box::pin(async {})
}
}
})),
..Default::default()
ecstore/src/set_disk.rs
+139 -105
View File
@@ -284,10 +284,20 @@ impl SetDisks {
// let mut ress = Vec::with_capacity(disks.len());
let mut errs = Vec::with_capacity(disks.len());
for (i, disk) in disks.iter().enumerate() {
let mut file_info = file_infos[i].clone();
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
futures.push(async move {
for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() {
let mut file_info = file_info.clone();
let disk = disk.clone();
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
let dst_object = dst_object.clone();
let dst_bucket = dst_bucket.clone();
futures.push(tokio::spawn(async move {
if file_info.erasure.index == 0 {
file_info.erasure.index = i + 1;
}
@@ -297,12 +307,12 @@ impl SetDisks {
}
if let Some(disk) = disk {
disk.rename_data(src_bucket, src_object, file_info, dst_bucket, dst_object)
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.await
} else {
Err(Error::new(DiskError::DiskNotFound))
}
})
}));
}
let mut disk_versions = vec![None; disks.len()];
@@ -311,15 +321,13 @@ impl SetDisks {
let results = join_all(futures).await;
for (idx, result) in results.iter().enumerate() {
match result {
match result.as_ref().map_err(|_| Error::new(DiskError::Unexpected))? {
Ok(res) => {
data_dirs[idx] = res.old_data_dir;
disk_versions[idx].clone_from(&res.sign);
// ress.push(Some(res));
errs.push(None);
}
Err(e) => {
// ress.push(None);
errs.push(Some(clone_err(e)));
}
}
@@ -336,11 +344,14 @@ impl SetDisks {
if let Some(disk) = disks[i].as_ref() {
let fi = file_infos[i].clone();
let old_data_dir = data_dirs[i];
futures.push(async move {
let disk = disk.clone();
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
futures.push(tokio::spawn(async move {
let _ = disk
.delete_version(
src_bucket,
src_object,
&src_bucket,
&src_object,
fi,
false,
DeleteOptions {
@@ -354,7 +365,7 @@ impl SetDisks {
debug!("rename_data delete_version err {:?}", e);
e
});
});
}));
}
}
@@ -416,41 +427,38 @@ impl SetDisks {
data_dir: &str,
write_quorum: usize,
) -> Result<()> {
let file_path = format!("{}/{}", object, data_dir);
let mut futures = Vec::with_capacity(disks.len());
let mut errs = Vec::with_capacity(disks.len());
for disk in disks.iter() {
let file_path = Arc::new(format!("{}/{}", object, data_dir));
let bucket = Arc::new(bucket.to_string());
let futures = disks.iter().map(|disk| {
let file_path = file_path.clone();
futures.push(async move {
let bucket = bucket.clone();
let disk = disk.clone();
tokio::spawn(async move {
if let Some(disk) = disk {
disk.delete(
bucket,
&file_path,
DeleteOptions {
recursive: true,
..Default::default()
},
)
.await
(disk
.delete(
&bucket,
&file_path,
DeleteOptions {
recursive: true,
..Default::default()
},
)
.await)
.err()
} else {
Err(Error::new(DiskError::DiskNotFound))
Some(Error::new(DiskError::DiskNotFound))
}
});
}
let results = join_all(futures).await;
for result in results {
match result {
Ok(_) => {
errs.push(None);
}
Err(e) => {
errs.push(Some(e));
}
}
}
})
});
let errs: Vec<Option<Error>> = join_all(futures)
.await
.into_iter()
.map(|e| match e {
Ok(e) => e,
Err(_) => Some(Error::new(DiskError::Unexpected)),
})
.collect();
if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) {
return Err(err);
@@ -502,24 +510,33 @@ impl SetDisks {
meta: Vec<u8>,
write_quorum: usize,
) -> Result<Vec<Option<DiskStore>>> {
let mut futures = Vec::with_capacity(disks.len());
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
let mut errs = Vec::with_capacity(disks.len());
for disk in disks.iter() {
let futures = disks.iter().map(|disk| {
let disk = disk.clone();
let meta = meta.clone();
futures.push(async move {
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
let dst_bucket = dst_bucket.clone();
let dst_object = dst_object.clone();
tokio::spawn(async move {
if let Some(disk) = disk {
disk.rename_part(src_bucket, src_object, dst_bucket, dst_object, meta).await
disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
.await
} else {
Err(Error::new(DiskError::DiskNotFound))
}
})
}
});
let results = join_all(futures).await;
for result in results {
match result {
match result? {
Ok(_) => {
errs.push(None);
}
@@ -531,7 +548,7 @@ impl SetDisks {
if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) {
warn!("rename_part errs {:?}", &errs);
Self::cleanup_multipart_path(disks, &[dst_object.to_owned(), format!("{}.meta", dst_object)]).await;
Self::cleanup_multipart_path(disks, &[dst_object.to_string(), format!("{}.meta", dst_object)]).await;
return Err(err);
}
@@ -933,7 +950,7 @@ impl SetDisks {
let disks = disks.clone();
let (parts_metadata, errs) =
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await;
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await?;
let map_err_notfound = |err: Error| {
if is_err_object_not_found(&err) {
@@ -1103,39 +1120,47 @@ impl SetDisks {
version_id: &str,
read_data: bool,
healing: bool,
) -> (Vec<FileInfo>, Vec<Option<Error>>) {
let mut futures = Vec::with_capacity(disks.len());
) -> Result<(Vec<FileInfo>, Vec<Option<Error>>)> {
let mut ress = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
for disk in disks.iter() {
let opts = ReadOptions {
read_data,
healing,
..Default::default()
};
futures.push(async move {
let opts = Arc::new(ReadOptions {
read_data,
healing,
..Default::default()
});
let org_bucket = Arc::new(org_bucket.to_string());
let bucket = Arc::new(bucket.to_string());
let object = Arc::new(object.to_string());
let version_id = Arc::new(version_id.to_string());
let futures = disks.iter().map(|disk| {
let disk = disk.clone();
let opts = opts.clone();
let org_bucket = org_bucket.clone();
let bucket = bucket.clone();
let object = object.clone();
let version_id = version_id.clone();
tokio::spawn(async move {
if let Some(disk) = disk {
if version_id.is_empty() {
match disk.read_xl(bucket, object, read_data).await {
match disk.read_xl(&bucket, &object, read_data).await {
Ok(info) => {
let fi = file_info_from_raw(info, bucket, object, read_data).await?;
let fi = file_info_from_raw(info, &bucket, &object, read_data).await?;
Ok(fi)
}
Err(err) => Err(err),
}
} else {
disk.read_version(org_bucket, bucket, object, version_id, &opts).await
disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await
}
} else {
Err(Error::new(DiskError::DiskNotFound))
}
})
}
});
let results = join_all(futures).await;
for result in results {
match result {
match result? {
Ok(res) => {
ress.push(res);
errors.push(None);
@@ -1146,7 +1171,7 @@ impl SetDisks {
}
}
}
(ress, errors)
Ok((ress, errors))
}
async fn read_all_xl(
@@ -1759,7 +1784,7 @@ impl SetDisks {
let vid = opts.version_id.clone().unwrap_or_default();
// TODO: 优化并发 可用数量中断
let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await;
let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await?;
// warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
// warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
@@ -2052,7 +2077,7 @@ impl SetDisks {
let bucket_partial = bucket_partial.clone();
async move {
let entry = match entries.resolve(resolver_partial) {
Ok(Some(entry)) => entry,
Some(entry) => entry,
_ => match entries.first_found() {
(Some(entry), _) => entry,
_ => return,
@@ -2166,7 +2191,7 @@ impl SetDisks {
let disks = { self.disks.read().await.clone() };
let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await;
let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await?;
if is_all_not_found(&errs) {
warn!(
"heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}",
@@ -3505,7 +3530,7 @@ impl SetDisks {
let heal_entry = heal_entry.clone();
let resolver = resolver.clone();
async move {
let entry = if let Ok(Some(entry)) = entries.resolve(resolver) {
let entry = if let Some(entry) = entries.resolve(resolver) {
entry
} else if let (Some(entry), _) = entries.first_found() {
entry
@@ -3759,7 +3784,7 @@ impl ObjectIO for SetDisks {
let tmp_object = format!("{}/{}/part.1", tmp_dir, fi.data_dir.unwrap());
let mut erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let is_inline_buffer = {
if let Some(sc) = GLOBAL_StorageClass.get() {
@@ -3798,19 +3823,18 @@ impl ObjectIO for SetDisks {
}
let stream = replace(&mut data.stream, Box::new(empty()));
let mut etag_stream = EtagReader::new(stream);
let etag_stream = EtagReader::new(stream);
// TODO: etag from header
let w_size = erasure
.encode(&mut etag_stream, &mut writers, data.content_length, write_quorum)
let (w_size, etag) = Arc::new(erasure)
.encode(etag_stream, &mut writers, data.content_length, write_quorum)
.await?; // TODO: 出错，删除临时目录
if let Err(err) = close_bitrot_writers(&mut writers).await {
error!("close_bitrot_writers err {:?}", err);
}
let etag = etag_stream.etag().await;
//TODO: userDefined
user_defined.insert("etag".to_owned(), etag.clone());
@@ -3948,7 +3972,7 @@ impl StorageAPI for SetDisks {
let (mut metas, errs) = {
if let Some(vid) = &src_opts.version_id {
Self::read_all_fileinfo(&disks, "", src_bucket, src_object, vid, true, false).await
Self::read_all_fileinfo(&disks, "", src_bucket, src_object, vid, true, false).await?
} else {
Self::read_all_xl(&disks, src_bucket, src_object, true, false).await
}
@@ -4242,7 +4266,7 @@ impl StorageAPI for SetDisks {
false,
false,
)
.await
.await?
} else {
Self::read_all_xl(&disks, bucket, object, false, false).await
}
@@ -4382,47 +4406,57 @@ impl StorageAPI for SetDisks {
let part_suffix = format!("part.{}", part_id);
let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp());
let tmp_part_path = format!("{}/{}", tmp_part, part_suffix);
let tmp_part_path = Arc::new(format!("{}/{}", tmp_part, part_suffix));
let mut writers = Vec::with_capacity(disks.len());
let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let shared_size = erasure.shard_size(erasure.block_size);
for disk in disks.iter() {
if let Some(disk) = disk {
// let writer = disk.append_file(RUSTFS_META_TMP_BUCKET, &tmp_part_path).await?;
// let filewriter = disk
// .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, data.content_length)
// .await?;
let writer = new_bitrot_filewriter(
disk.clone(),
RUSTFS_META_TMP_BUCKET,
&tmp_part_path,
false,
DEFAULT_BITROT_ALGO,
erasure.shard_size(erasure.block_size),
)
.await?;
writers.push(Some(writer));
} else {
writers.push(None);
}
let futures = disks.iter().map(|disk| {
let disk = disk.clone();
let tmp_part_path = tmp_part_path.clone();
tokio::spawn(async move {
if let Some(disk) = disk {
// let writer = disk.append_file(RUSTFS_META_TMP_BUCKET, &tmp_part_path).await?;
// let filewriter = disk
// .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, data.content_length)
// .await?;
match new_bitrot_filewriter(
disk.clone(),
RUSTFS_META_TMP_BUCKET,
&tmp_part_path,
false,
DEFAULT_BITROT_ALGO,
shared_size,
)
.await
{
Ok(writer) => Ok(Some(writer)),
Err(e) => Err(e),
}
} else {
Ok(None)
}
})
});
for x in join_all(futures).await {
let x = x??;
writers.push(x);
}
let mut erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let stream = replace(&mut data.stream, Box::new(empty()));
let mut etag_stream = EtagReader::new(stream);
let etag_stream = EtagReader::new(stream);
let w_size = erasure
.encode(&mut etag_stream, &mut writers, data.content_length, write_quorum)
let (w_size, mut etag) = Arc::new(erasure)
.encode(etag_stream, &mut writers, data.content_length, write_quorum)
.await?;
if let Err(err) = close_bitrot_writers(&mut writers).await {
error!("close_bitrot_writers err {:?}", err);
}
let mut etag = etag_stream.etag().await;
if let Some(ref tag) = opts.preserve_etag {
etag = tag.clone();
}
@@ -5036,7 +5070,7 @@ impl StorageAPI for SetDisks {
let disks = self.disks.read().await;
let disks = disks.clone();
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false).await;
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false).await?;
if is_all_not_found(&errs) {
warn!(
"heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}",
ecstore/src/store.rs
+31 -19
View File
@@ -524,7 +524,9 @@ impl ECStore {
// TODO: 并发
for (idx, pool) in self.pools.iter().enumerate() {
// TODO: IsSuspended
if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
continue;
}
n_sets[idx] = pool.set_count;
@@ -714,16 +716,14 @@ impl ECStore {
let mut def_pool = PoolObjInfo::default();
let mut has_def_pool = false;
let pool_meta = self.pool_meta.read().await;
for pinfo in ress.iter() {
if opts.skip_decommissioned && pool_meta.is_suspended(pinfo.index) {
if opts.skip_decommissioned && self.is_suspended(pinfo.index).await {
continue;
}
// TODO:SkipRebalancing
// if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) {
// continue
// }
if opts.skip_rebalancing && self.is_pool_rebalancing(pinfo.index).await {
continue;
}
if pinfo.err.is_none() {
return Ok((pinfo.clone(), self.pools_with_object(&ress, opts).await));
@@ -756,15 +756,15 @@ impl ECStore {
async fn pools_with_object(&self, pools: &[PoolObjInfo], opts: &ObjectOptions) -> Vec<PoolErr> {
let mut errs = Vec::new();
let pool_meta = self.pool_meta.read().await;
for pool in pools.iter() {
if opts.skip_decommissioned && pool_meta.is_suspended(pool.index) {
if opts.skip_decommissioned && self.is_suspended(pool.index).await {
continue;
}
if opts.skip_rebalancing && self.is_pool_rebalancing(pool.index).await {
continue;
}
// TODO:SkipRebalancing
// if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) {
// continue
// }
if let Some(err) = &pool.err {
if is_err_read_quorum(err) {
@@ -1865,7 +1865,9 @@ impl StorageAPI for ECStore {
}
for pool in self.pools.iter() {
// TODO: IsSuspended
if self.is_suspended(pool.pool_idx).await {
continue;
}
let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await {
Ok(res) => return Ok(res),
Err(err) => {
@@ -1915,6 +1917,9 @@ impl StorageAPI for ECStore {
let mut uploads = Vec::new();
for pool in self.pools.iter() {
if self.is_suspended(pool.pool_idx).await {
continue;
}
let res = pool
.list_multipart_uploads(
bucket,
@@ -1948,7 +1953,9 @@ impl StorageAPI for ECStore {
}
for (idx, pool) in self.pools.iter().enumerate() {
// // TODO: IsSuspended
if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
continue;
}
let res = pool
.list_multipart_uploads(bucket, object, None, None, None, MAX_UPLOADS_LIST)
.await?;
@@ -1983,8 +1990,8 @@ impl StorageAPI for ECStore {
return self.pools[0].get_multipart_info(bucket, object, upload_id, opts).await;
}
for (idx, pool) in self.pools.iter().enumerate() {
if self.is_suspended(idx).await {
for pool in self.pools.iter() {
if self.is_suspended(pool.pool_idx).await {
continue;
}
@@ -2017,7 +2024,9 @@ impl StorageAPI for ECStore {
}
for pool in self.pools.iter() {
// TODO: IsSuspended
if self.is_suspended(pool.pool_idx).await {
continue;
}
let err = match pool.abort_multipart_upload(bucket, object, upload_id, opts).await {
Ok(_) => return Ok(()),
@@ -2060,7 +2069,9 @@ impl StorageAPI for ECStore {
}
for pool in self.pools.iter() {
// TODO: IsSuspended
if self.is_suspended(pool.pool_idx).await {
continue;
}
let err = match pool
.complete_multipart_upload(bucket, object, upload_id, uploaded_parts.clone(), opts)
@@ -2549,6 +2560,7 @@ fn check_abort_multipart_args(bucket: &str, object: &str, upload_id: &str) -> Re
check_multipart_object_args(bucket, object, upload_id)
}
#[tracing::instrument(level = "debug")]
fn check_put_object_args(bucket: &str, object: &str) -> Result<()> {
if !is_meta_bucketname(bucket) && check_valid_bucket_name_strict(bucket).is_err() {
return Err(Error::new(StorageError::BucketNameInvalid(bucket.to_string())));
ecstore/src/store_list_objects.rs
+2 -2
View File
@@ -778,7 +778,7 @@ impl ECStore {
let value = tx2.clone();
let resolver = resolver.clone();
async move {
if let Ok(Some(entry)) = entries.resolve(resolver) {
if let Some(entry) = entries.resolve(resolver) {
if let Err(err) = value.send(entry).await {
error!("list_path send fail {:?}", err);
}
@@ -1296,7 +1296,7 @@ impl SetDisks {
let value = tx2.clone();
let resolver = resolver.clone();
async move {
if let Ok(Some(entry)) = entries.resolve(resolver) {
if let Some(entry) = entries.resolve(resolver) {
if let Err(err) = value.send(entry).await {
error!("list_path send fail {:?}", err);
}
ecstore/src/utils/fs.rs
+34
View File
@@ -106,14 +106,24 @@ pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
Ok(())
}
pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
std::fs::metadata(path)?;
Ok(())
}
pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
fs::metadata(path).await
}
pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
std::fs::metadata(path)
}
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir_all(path.as_ref()).await
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
let meta = fs::metadata(path.as_ref()).await?;
if meta.is_dir() {
@@ -132,6 +142,25 @@ pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
}
}
#[tracing::instrument(level = "debug", skip_all)]
pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
let meta = std::fs::metadata(path.as_ref())?;
if meta.is_dir() {
std::fs::remove_dir(path.as_ref())
} else {
std::fs::remove_file(path.as_ref())
}
}
pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
let meta = std::fs::metadata(path.as_ref())?;
if meta.is_dir() {
std::fs::remove_dir_all(path.as_ref())
} else {
std::fs::remove_file(path.as_ref())
}
}
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir(path.as_ref()).await
}
@@ -140,6 +169,11 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
fs::rename(from, to).await
}
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
std::fs::rename(from, to)
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
fs::read(path.as_ref()).await
}
ecstore/src/utils/path.rs
+1 -1
View File
@@ -52,7 +52,7 @@ pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool {
s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix))
}
pub fn has_profix(s: &str, prefix: &str) -> bool {
pub fn has_prefix(s: &str, prefix: &str) -> bool {
if cfg!(target_os = "windows") {
return strings_has_prefix_fold(s, prefix);
}
rustfs/Cargo.toml
+12 -5
View File
@@ -15,6 +15,8 @@ path = "src/main.rs"
workspace = true
[dependencies]
zip = { workspace = true }
tokio-tar = { workspace = true }
madmin = { workspace = true }
api = { workspace = true }
appauth = { workspace = true }
@@ -30,7 +32,7 @@ clap.workspace = true
crypto = { workspace = true }
datafusion = { workspace = true }
common.workspace = true
const-str = { version = "0.6.1", features = ["std", "proc"] }
const-str = { workspace = true }
ecstore.workspace = true
policy.workspace = true
flatbuffers.workspace = true
@@ -42,7 +44,6 @@ http.workspace = true
http-body.workspace = true
iam = { workspace = true }
lock.workspace = true
local-ip-address = { workspace = true }
matchit = { workspace = true }
mime.workspace = true
mime_guess = { workspace = true }
@@ -51,17 +52,18 @@ pin-project-lite.workspace = true
protos.workspace = true
query = { workspace = true }
rmp-serde.workspace = true
rustfs-config = { workspace = true }
rustfs-event-notifier = { workspace = true }
rustfs-obs = { workspace = true }
rustfs-utils = { workspace = true, features = ["full"] }
rustls.workspace = true
rustls-pemfile.workspace = true
rustls-pki-types.workspace = true
rust-embed = { workspace = true, features = ["interpolate-folder-path"] }
s3s.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_urlencoded = { workspace = true }
shadow-rs = { workspace = true, features = ["build", "metadata"] }
socket2 = { workspace = true }
tracing.workspace = true
time = { workspace = true, features = ["parsing", "formatting", "serde"] }
tokio-util.workspace = true
@@ -77,7 +79,12 @@ tokio-stream.workspace = true
tonic = { workspace = true }
tower.workspace = true
transform-stream.workspace = true
tower-http = { workspace = true, features = ["trace", "compression-deflate", "compression-gzip", "cors"] }
tower-http = { workspace = true, features = [
"trace",
"compression-deflate",
"compression-gzip",
"cors",
] }
uuid = { workspace = true }
[target.'cfg(target_os = "linux")'.dependencies]

Some files were not shown because too many files have changed in this diff Show More