mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-01 19:12:14 +00:00
Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 houseme
|
b8c45fc9e3 | ||
|
weisd
|
a8a2aaa460 | ||
|
Henry Guo
|
2bf4b14394 | ||
|
安正超
|
59b5e4e722 | ||
|
安正超
|
9ec6465b26 | ||
|
安正超
|
80cb6b3939 | ||
|
weisd
|
30c8bead63 | ||
|
安正超
|
2125cffd0b | ||
|
安正超
|
aeef2e67f9 | ||
|
安正超
|
8acf2a51b9 | ||
|
安正超
|
33b50a5366 | ||
|
安正超
|
acdd2de21f | ||
|
安正超
|
b8bfaccbc5 | ||
|
安正超
|
28d9bbbdcb | ||
|
安正超
|
a9507f8ec7 | ||
|
安正超
|
722cab500c | ||
|
安正超
|
71e9178182 | ||
|
安正超
|
aee8a4df51 | ||
|
无心戈
|
57138fa660 | ||
|
GatewayJ
|
8db55de72c | ||
|
安正超
|
90e584af74 | ||
|
安正超
|
ff77cb5540 | ||
|
安正超
|
61c0da936b | ||
|
houseme
|
d62114f8d5 | ||
|
houseme
|
9899985d15 | ||
|
houseme
|
064e21062d | ||
|
安正超
|
d4ea14c2ba | ||
|
安正超
|
1977d19c29 |
@@ -62,7 +62,7 @@ runs:
|
||||
repo-token: ${{ inputs.github-token }}
|
||||
|
||||
- name: Install flatc
|
||||
uses: Nugine/setup-flatc@v1
|
||||
uses: Nugine/setup-flatc@v1.2.4
|
||||
with:
|
||||
version: "25.9.23"
|
||||
|
||||
|
||||
@@ -204,6 +204,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
# Always enable Tokio unstable features (required by dial9-tokio-telemetry).
|
||||
# The RUSTFLAGS env var takes precedence over .cargo/config.toml [build] rustflags,
|
||||
# so we must include --cfg tokio_unstable here explicitly; otherwise an empty
|
||||
@@ -320,19 +321,26 @@ jobs:
|
||||
;;
|
||||
esac
|
||||
|
||||
# Normalize version used for package filenames
|
||||
PACKAGE_VERSION="${VERSION}"
|
||||
if [[ "$PACKAGE_VERSION" == v* ]]; then
|
||||
PACKAGE_VERSION="${PACKAGE_VERSION#v}"
|
||||
fi
|
||||
|
||||
# Generate package name based on build type
|
||||
if [[ -n "$VARIANT" ]]; then
|
||||
ARCH_WITH_VARIANT="${ARCH}-${VARIANT}"
|
||||
else
|
||||
ARCH_WITH_VARIANT="${ARCH}"
|
||||
fi
|
||||
PACKAGE_BASENAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}"
|
||||
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development build: rustfs-${platform}-${arch}-${variant}-dev-${short_sha}.zip
|
||||
PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}-dev-${SHORT_SHA}"
|
||||
else
|
||||
# Release/Prerelease build: rustfs-${platform}-${arch}-${variant}-v${version}.zip
|
||||
PACKAGE_NAME="rustfs-${PLATFORM}-${ARCH_WITH_VARIANT}-v${VERSION}"
|
||||
PACKAGE_NAME="${PACKAGE_BASENAME}-v${PACKAGE_VERSION}"
|
||||
fi
|
||||
|
||||
# Create zip packages for all platforms
|
||||
@@ -402,7 +410,7 @@ jobs:
|
||||
if [[ "$BUILD_TYPE" == "release" ]] || [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
# Create latest version filename
|
||||
# Convert from rustfs-linux-x86_64-musl-v1.0.0 to rustfs-linux-x86_64-musl-latest
|
||||
LATEST_FILE="${PACKAGE_NAME%-v*}-latest.zip"
|
||||
LATEST_FILE="${PACKAGE_BASENAME}-latest.zip"
|
||||
|
||||
echo "🔄 Creating latest version: ${PACKAGE_NAME}.zip -> $LATEST_FILE"
|
||||
cp "${PACKAGE_NAME}.zip" "$LATEST_FILE"
|
||||
@@ -415,7 +423,7 @@ jobs:
|
||||
# Development builds (only main branch triggers development builds)
|
||||
# Create main-latest version filename
|
||||
# Convert from rustfs-linux-x86_64-dev-abc123 to rustfs-linux-x86_64-main-latest
|
||||
MAIN_LATEST_FILE="${PACKAGE_NAME%-dev-*}-main-latest.zip"
|
||||
MAIN_LATEST_FILE="${PACKAGE_BASENAME}-main-latest.zip"
|
||||
|
||||
echo "🔄 Creating main-latest version: ${PACKAGE_NAME}.zip -> $MAIN_LATEST_FILE"
|
||||
cp "${PACKAGE_NAME}.zip" "$MAIN_LATEST_FILE"
|
||||
|
||||
@@ -106,6 +106,8 @@ jobs:
|
||||
if: needs.skip-check.outputs.should_skip != 'true'
|
||||
runs-on: ubicloud-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
@@ -138,6 +140,8 @@ jobs:
|
||||
if: needs.skip-check.outputs.should_skip != 'true'
|
||||
runs-on: ubicloud-standard-4
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
|
||||
- name: Install Nix
|
||||
uses: DeterminateSystems/determinate-nix-action@v3
|
||||
|
||||
@@ -46,10 +46,10 @@ jobs:
|
||||
id: update
|
||||
uses: DeterminateSystems/update-flake-lock@main
|
||||
with:
|
||||
git-author-name: heihutu
|
||||
git-author-email: heihutu@gmail.com
|
||||
git-committer-name: heihutu
|
||||
git-committer-email: heihutu@gmail.com
|
||||
git-author-name: houseme
|
||||
git-author-email: housemecn@gmail.com
|
||||
git-committer-name: houseme
|
||||
git-committer-email: housemecn@gmail.com
|
||||
pr-title: "chore(deps): update flake.lock"
|
||||
pr-labels: |
|
||||
dependencies
|
||||
|
||||
@@ -42,6 +42,8 @@ jobs:
|
||||
name: Performance Profiling
|
||||
runs-on: ubicloud-standard-2
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
@@ -117,6 +119,8 @@ jobs:
|
||||
name: Benchmark Tests
|
||||
runs-on: ubicloud-standard-2
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
Generated
+50
-30
@@ -203,7 +203,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -214,7 +214,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -747,9 +747,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.128.0"
|
||||
version = "1.129.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99304b64672e0d81a3c100a589b93d9ef5e9c0ce12e21c848fd39e50f493c2a1"
|
||||
checksum = "6d4e8410fadbc0ee453145dd77a4958227b18b05bf67c2795d0a8b8596c9aa0f"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -3153,7 +3153,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3433,7 +3433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3475,9 +3475,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.4.0"
|
||||
version = "2.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a043dc74da1e37d6afe657061213aa6f425f855399a11d3463c6ecccc4dfda1f"
|
||||
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
||||
|
||||
[[package]]
|
||||
name = "ff"
|
||||
@@ -4688,7 +4688,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4765,7 +4765,7 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"serde_core",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5262,9 +5262,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.9.1"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3eede3bdf92f3b4f9dc04072a9ce5ab557d5ec9038773bf9ffcd5588b3cc05b"
|
||||
checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608"
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
@@ -5675,7 +5675,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6252,9 +6252,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "password-hash"
|
||||
version = "0.6.0"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccbd25f71dd5249dba9ed843d52500c8757a25511560d01a94f4abf56b52a1d5"
|
||||
checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b"
|
||||
dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"phc",
|
||||
@@ -7680,7 +7680,7 @@ dependencies = [
|
||||
"jiff",
|
||||
"libc",
|
||||
"libsystemd",
|
||||
"matchit 0.9.1",
|
||||
"matchit 0.9.2",
|
||||
"md5",
|
||||
"metrics",
|
||||
"mimalloc",
|
||||
@@ -7712,6 +7712,7 @@ dependencies = [
|
||||
"rustfs-madmin",
|
||||
"rustfs-metrics",
|
||||
"rustfs-notify",
|
||||
"rustfs-object-capacity",
|
||||
"rustfs-object-io",
|
||||
"rustfs-obs",
|
||||
"rustfs-policy",
|
||||
@@ -7757,7 +7758,6 @@ dependencies = [
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"walkdir",
|
||||
"zip",
|
||||
]
|
||||
|
||||
@@ -8229,6 +8229,25 @@ dependencies = [
|
||||
"wildmatch",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "0.0.5"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-utils",
|
||||
"serial_test",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-io"
|
||||
version = "0.0.5"
|
||||
@@ -8686,7 +8705,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8745,7 +8764,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9410,7 +9429,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9603,9 +9622,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "symbolic-common"
|
||||
version = "12.17.3"
|
||||
version = "12.17.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ca086c1eb5c7ee74b151ba83c6487d5d33f8c08ad991b86f3f58f6629e68d5"
|
||||
checksum = "803d14d7cb9c6fa5b95a6f3de8af95b356a528d391998fa45a07d320a5573e51"
|
||||
dependencies = [
|
||||
"debugid",
|
||||
"memmap2 0.9.10",
|
||||
@@ -9615,9 +9634,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "symbolic-demangle"
|
||||
version = "12.17.3"
|
||||
version = "12.17.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baa911a28a62823aaf2cc2e074212492a3ee69d0d926cc8f5b12b4a108ff5c0c"
|
||||
checksum = "39505731ae891b2dde47b0e4ae2ec40a7ced3476ab1129f1bf829e3fba62bb83"
|
||||
dependencies = [
|
||||
"cpp_demangle",
|
||||
"rustc-demangle",
|
||||
@@ -9726,6 +9745,7 @@ version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"parking_lot 0.12.5",
|
||||
]
|
||||
|
||||
@@ -9739,7 +9759,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9946,9 +9966,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.51.0"
|
||||
version = "1.51.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd"
|
||||
checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
@@ -10712,7 +10732,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11377,9 +11397,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "8.5.0"
|
||||
version = "8.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2726508a48f38dceb22b35ecbbd2430efe34ff05c62bd3285f965d7911b33464"
|
||||
checksum = "dcab981e19633ebcf0b001ddd37dd802996098bc1864f90b7c5d970ce76c1d59"
|
||||
dependencies = [
|
||||
"aes 0.8.4",
|
||||
"bzip2",
|
||||
|
||||
+6
-4
@@ -35,6 +35,7 @@ members = [
|
||||
"crates/metrics", # Metrics collection and reporting
|
||||
"crates/notify", # Notification system for events
|
||||
"crates/obs", # Observability utilities
|
||||
"crates/object-capacity", # Capacity scan and refresh core
|
||||
"crates/policy", # Policy management
|
||||
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
@@ -99,6 +100,7 @@ rustfs-notify = { path = "crates/notify", version = "0.0.5" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "0.0.5" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "0.0.5" }
|
||||
rustfs-object-io = { path = "crates/object-io", version = "0.0.5" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "0.0.5" }
|
||||
rustfs-obs = { path = "crates/obs", version = "0.0.5" }
|
||||
rustfs-policy = { path = "crates/policy", version = "0.0.5" }
|
||||
rustfs-protos = { path = "crates/protos", version = "0.0.5" }
|
||||
@@ -133,7 +135,7 @@ http-body = "1.0.1"
|
||||
http-body-util = "0.1.3"
|
||||
reqwest = { version = "0.13.2", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
|
||||
socket2 = { version = "0.6.3", features = ["all"] }
|
||||
tokio = { version = "1.51.0", features = ["fs", "rt-multi-thread"] }
|
||||
tokio = { version = "1.51.1", features = ["fs", "rt-multi-thread"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
|
||||
tokio-stream = { version = "0.1.18" }
|
||||
tokio-test = "0.4.5"
|
||||
@@ -192,7 +194,7 @@ atoi = "2.0.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.8.15" }
|
||||
aws-credential-types = { version = "1.2.14" }
|
||||
aws-sdk-s3 = { version = "1.128.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-s3 = { version = "1.129.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
|
||||
aws-smithy-types = { version = "1.4.7" }
|
||||
backtrace = "0.3.76"
|
||||
@@ -227,7 +229,7 @@ libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.11"
|
||||
memmap2 = "0.9.10"
|
||||
lz4 = "1.28.1"
|
||||
matchit = "0.9.1"
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
md5 = "0.8.0"
|
||||
mime_guess = "2.0.5"
|
||||
@@ -281,7 +283,7 @@ walkdir = "2.5.0"
|
||||
wildmatch = { version = "2.6.1", features = ["serde"] }
|
||||
windows = { version = "0.62.2" }
|
||||
xxhash-rust = { version = "0.8.15", features = ["xxh64", "xxh3"] }
|
||||
zip = "8.5.0"
|
||||
zip = "8.5.1"
|
||||
zstd = "0.13.3"
|
||||
|
||||
# Observability and Metrics
|
||||
|
||||
@@ -136,7 +136,7 @@ Similarly, you can run the command with podman
|
||||
podman compose --profile observability up -d
|
||||
```
|
||||
|
||||
**NOTE**: We recommend reviewing the `docker-compose.yaml` file before running. It defines several services including Grafana, Prometheus, and Jaeger, which are helpful for RustFS observability. If you wish to start Redis or Nginx containers, you can specify the corresponding profiles.
|
||||
**NOTE**: We recommend reviewing the `docker-compose.yml` file before running. It defines several services including Grafana, Prometheus, and Jaeger, which are helpful for RustFS observability. If you wish to start Redis or Nginx containers, you can specify the corresponding profiles.
|
||||
|
||||
### 3\. Build from Source (Option 3) - Advanced Users
|
||||
|
||||
|
||||
+3
-3
@@ -122,7 +122,7 @@ RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用
|
||||
docker compose --profile observability up -d
|
||||
```
|
||||
|
||||
**注意**: 我们建议您在运行前查看 `docker-compose.yaml` 文件。该文件定义了包括 Grafana、Prometheus 和 Jaeger 在内的多个服务，有助于 RustFS 的可观测性监控。如果您还想启动 Redis 或 Nginx 容器，可以指定相应的 profile。
|
||||
**注意**: 我们建议您在运行前查看 `docker-compose.yml` 文件。该文件定义了包括 Grafana、Prometheus 和 Jaeger 在内的多个服务，有助于 RustFS 的可观测性监控。如果您还想启动 Redis 或 Nginx 容器，可以指定相应的 profile。
|
||||
|
||||
### 3\. 源码编译 (选项 3) - 进阶用户
|
||||
|
||||
@@ -197,7 +197,7 @@ rustfs --help
|
||||
|
||||
### 访问 RustFS
|
||||
|
||||
1. **访问控制台**: 打开浏览器并访问 `http://localhost:9000` 进入 RustFS 控制台。
|
||||
1. **访问控制台**: 打开浏览器并访问 `http://localhost:9001` 进入 RustFS 控制台。
|
||||
- 默认账号/密码: `rustfsadmin` / `rustfsadmin`
|
||||
2. **创建存储桶**: 使用控制台为您的对象创建一个新的存储桶 (Bucket)。
|
||||
3. **上传对象**: 您可以直接通过控制台上传文件，或使用 S3 兼容的 API/客户端与您的 RustFS 实例进行交互。
|
||||
@@ -228,7 +228,7 @@ rustfs --help
|
||||
- **商务合作**: [hello@rustfs.com](mailto:hello@rustfs.com)
|
||||
- **工作机会**: [jobs@rustfs.com](mailto:jobs@rustfs.com)
|
||||
- **一般讨论**: [GitHub Discussions](https://github.com/rustfs/rustfs/discussions)
|
||||
- **贡献指南**: [CONTRIBUTING.md](https://www.google.com/search?q=CONTRIBUTING.md)
|
||||
- **贡献指南**: [CONTRIBUTING.md](CONTRIBUTING.md)
|
||||
|
||||
## 贡献者
|
||||
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
use uuid::Uuid;
|
||||
|
||||
const CAPACITY_SCOPE_REGISTRY_SOFT_LIMIT: usize = 2_048;
|
||||
const CAPACITY_SCOPE_REGISTRY_HARD_LIMIT: usize = 4_096;
|
||||
const CAPACITY_SCOPE_TTL: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct CapacityScopeDisk {
|
||||
pub endpoint: String,
|
||||
pub drive_path: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct CapacityScope {
|
||||
pub disks: Vec<CapacityScopeDisk>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct CapacityScopeEntry {
|
||||
scope: CapacityScope,
|
||||
recorded_at: Instant,
|
||||
}
|
||||
|
||||
fn capacity_scope_registry() -> &'static Mutex<HashMap<Uuid, CapacityScopeEntry>> {
|
||||
static REGISTRY: OnceLock<Mutex<HashMap<Uuid, CapacityScopeEntry>>> = OnceLock::new();
|
||||
REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
fn global_dirty_scope_registry() -> &'static Mutex<HashSet<CapacityScopeDisk>> {
|
||||
static REGISTRY: OnceLock<Mutex<HashSet<CapacityScopeDisk>>> = OnceLock::new();
|
||||
REGISTRY.get_or_init(|| Mutex::new(HashSet::new()))
|
||||
}
|
||||
|
||||
fn prune_expired_entries(entries: &mut HashMap<Uuid, CapacityScopeEntry>, now: Instant) {
|
||||
entries.retain(|_, entry| now.duration_since(entry.recorded_at) <= CAPACITY_SCOPE_TTL);
|
||||
}
|
||||
|
||||
fn enforce_hard_limit(entries: &mut HashMap<Uuid, CapacityScopeEntry>, max_len: usize) {
|
||||
if entries.len() < max_len {
|
||||
return;
|
||||
}
|
||||
|
||||
let evict_count = entries.len() - max_len + 1;
|
||||
let mut eviction_order: Vec<_> = entries.iter().map(|(token, entry)| (*token, entry.recorded_at)).collect();
|
||||
eviction_order.sort_unstable_by_key(|(_, recorded_at)| *recorded_at);
|
||||
|
||||
for (token, _) in eviction_order.into_iter().take(evict_count) {
|
||||
entries.remove(&token);
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_capacity_scopes(existing: &mut CapacityScope, incoming: CapacityScope) {
|
||||
let mut seen: HashSet<CapacityScopeDisk> = existing.disks.iter().cloned().collect();
|
||||
for disk in incoming.disks {
|
||||
if seen.insert(disk.clone()) {
|
||||
existing.disks.push(disk);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_capacity_scope(token: Uuid, scope: CapacityScope) {
|
||||
let now = Instant::now();
|
||||
let mut entries = capacity_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
|
||||
if !entries.contains_key(&token) && entries.len() >= CAPACITY_SCOPE_REGISTRY_SOFT_LIMIT {
|
||||
prune_expired_entries(&mut entries, now);
|
||||
enforce_hard_limit(&mut entries, CAPACITY_SCOPE_REGISTRY_HARD_LIMIT);
|
||||
}
|
||||
if let Some(entry) = entries.get_mut(&token) {
|
||||
merge_capacity_scopes(&mut entry.scope, scope);
|
||||
entry.recorded_at = now;
|
||||
} else {
|
||||
entries.insert(token, CapacityScopeEntry { scope, recorded_at: now });
|
||||
}
|
||||
}
|
||||
|
||||
pub fn take_capacity_scope(token: Uuid) -> Option<CapacityScope> {
|
||||
let now = Instant::now();
|
||||
let mut entries = capacity_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
|
||||
let entry = entries.remove(&token)?;
|
||||
if now.duration_since(entry.recorded_at) > CAPACITY_SCOPE_TTL {
|
||||
return None;
|
||||
}
|
||||
Some(entry.scope)
|
||||
}
|
||||
|
||||
pub fn record_global_dirty_scope(scope: CapacityScope) {
|
||||
if scope.disks.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut dirty_scopes = global_dirty_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
|
||||
dirty_scopes.extend(scope.disks);
|
||||
}
|
||||
|
||||
pub fn drain_global_dirty_scopes() -> Vec<CapacityScopeDisk> {
|
||||
let mut dirty_scopes = global_dirty_scope_registry().lock().unwrap_or_else(|p| p.into_inner());
|
||||
dirty_scopes.drain().collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
use std::thread;
|
||||
|
||||
fn test_lock() -> &'static Mutex<()> {
|
||||
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
LOCK.get_or_init(|| Mutex::new(()))
|
||||
}
|
||||
|
||||
fn clear_capacity_scope_registry_for_test() {
|
||||
capacity_scope_registry()
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clear();
|
||||
global_dirty_scope_registry()
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clear();
|
||||
}
|
||||
|
||||
fn poison_capacity_scope_registry_for_test() {
|
||||
let _ = thread::spawn(|| {
|
||||
let _guard = capacity_scope_registry()
|
||||
.lock()
|
||||
.expect("capacity scope registry lock should succeed");
|
||||
panic!("poison capacity scope registry");
|
||||
})
|
||||
.join();
|
||||
}
|
||||
|
||||
fn poison_global_dirty_scope_registry_for_test() {
|
||||
let _ = thread::spawn(|| {
|
||||
let _guard = global_dirty_scope_registry()
|
||||
.lock()
|
||||
.expect("global dirty scope registry lock should succeed");
|
||||
panic!("poison global dirty scope registry");
|
||||
})
|
||||
.join();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_and_take_capacity_scope_round_trips() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
let token = Uuid::new_v4();
|
||||
let scope = CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
};
|
||||
|
||||
record_capacity_scope(token, scope.clone());
|
||||
|
||||
assert_eq!(take_capacity_scope(token), Some(scope));
|
||||
assert_eq!(take_capacity_scope(token), None);
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_capacity_scope_merges_disks_for_same_token() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
let token = Uuid::new_v4();
|
||||
record_capacity_scope(
|
||||
token,
|
||||
CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
},
|
||||
);
|
||||
record_capacity_scope(
|
||||
token,
|
||||
CapacityScope {
|
||||
disks: vec![
|
||||
CapacityScopeDisk {
|
||||
endpoint: "node-b".to_string(),
|
||||
drive_path: "/tmp/disk-b".to_string(),
|
||||
},
|
||||
CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
let scope = take_capacity_scope(token).expect("scope should exist");
|
||||
assert_eq!(scope.disks.len(), 2);
|
||||
assert!(scope.disks.iter().any(|disk| disk.endpoint == "node-a"));
|
||||
assert!(scope.disks.iter().any(|disk| disk.endpoint == "node-b"));
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_capacity_scope_enforces_hard_limit() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
|
||||
for _ in 0..(CAPACITY_SCOPE_REGISTRY_HARD_LIMIT + 32) {
|
||||
record_capacity_scope(
|
||||
Uuid::new_v4(),
|
||||
CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let entries = capacity_scope_registry()
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
assert!(entries.len() <= CAPACITY_SCOPE_REGISTRY_HARD_LIMIT);
|
||||
drop(entries);
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_capacity_scope_recovers_from_poisoned_registry() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
poison_capacity_scope_registry_for_test();
|
||||
let token = Uuid::new_v4();
|
||||
let scope = CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
};
|
||||
|
||||
record_capacity_scope(token, scope.clone());
|
||||
|
||||
assert_eq!(take_capacity_scope(token), Some(scope));
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_and_drain_global_dirty_scope_round_trips() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
record_global_dirty_scope(CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
});
|
||||
record_global_dirty_scope(CapacityScope {
|
||||
disks: vec![
|
||||
CapacityScopeDisk {
|
||||
endpoint: "node-b".to_string(),
|
||||
drive_path: "/tmp/disk-b".to_string(),
|
||||
},
|
||||
CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
let drained = drain_global_dirty_scopes();
|
||||
assert_eq!(drained.len(), 2);
|
||||
assert!(drained.iter().any(|disk| disk.endpoint == "node-a"));
|
||||
assert!(drained.iter().any(|disk| disk.endpoint == "node-b"));
|
||||
assert!(drain_global_dirty_scopes().is_empty());
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_global_dirty_scope_recovers_from_poisoned_registry() {
|
||||
let _guard = test_lock().lock().expect("test lock poisoned");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
poison_global_dirty_scope_registry_for_test();
|
||||
|
||||
record_global_dirty_scope(CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
});
|
||||
|
||||
let drained = drain_global_dirty_scopes();
|
||||
assert_eq!(drained.len(), 1);
|
||||
assert_eq!(drained[0].endpoint, "node-a");
|
||||
clear_capacity_scope_registry_for_test();
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod bucket_stats;
|
||||
pub mod capacity_scope;
|
||||
// pub mod error;
|
||||
pub mod data_usage;
|
||||
pub mod globals;
|
||||
|
||||
@@ -17,6 +17,24 @@
|
||||
//! This module defines environment variables and default values for zero-copy
|
||||
//! read operations, which use memory mapping (mmap) to avoid data copying.
|
||||
|
||||
// =============================================================================
|
||||
// GET Fast Path Configuration
|
||||
// =============================================================================
|
||||
|
||||
/// Environment variable for the GetObject chunk fast path master switch.
|
||||
///
|
||||
/// When disabled, `GetObject` bypasses the chunk-streaming fast path entirely and
|
||||
/// always uses the legacy reader path. This provides an operational stopgap for
|
||||
/// regressions in the streaming data plane while keeping zero-copy internals
|
||||
/// configurable independently for future opt-in validation.
|
||||
pub const ENV_OBJECT_GET_CHUNK_FAST_PATH_ENABLE: &str = "RUSTFS_OBJECT_GET_CHUNK_FAST_PATH_ENABLE";
|
||||
|
||||
/// Default: GetObject chunk fast path is disabled.
|
||||
///
|
||||
/// The legacy reader path remains the safe default until the chunk-streaming
|
||||
/// path has sufficient regression coverage for full-body delivery semantics.
|
||||
pub const DEFAULT_OBJECT_GET_CHUNK_FAST_PATH_ENABLE: bool = false;
|
||||
|
||||
// =============================================================================
|
||||
// Zero-Copy Configuration
|
||||
// =============================================================================
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
@@ -28,6 +28,8 @@ mod tests {
|
||||
use std::io::{Cursor, Write};
|
||||
use std::process::Command;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions};
|
||||
|
||||
const ARCHIVE_TEST_BUCKET: &str = "archive-download-integrity";
|
||||
@@ -116,6 +118,75 @@ mod tests {
|
||||
.await?)
|
||||
}
|
||||
|
||||
fn find_header_terminator(buf: &[u8]) -> Option<usize> {
|
||||
buf.windows(4).position(|window| window == b"\r\n\r\n")
|
||||
}
|
||||
|
||||
async fn read_proxy_request(stream: &mut tokio::net::TcpStream) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mut buffer = Vec::new();
|
||||
let mut chunk = [0_u8; 4096];
|
||||
|
||||
loop {
|
||||
let read = stream.read(&mut chunk).await?;
|
||||
if read == 0 {
|
||||
return Err("proxy request ended before headers were fully received".into());
|
||||
}
|
||||
buffer.extend_from_slice(&chunk[..read]);
|
||||
if find_header_terminator(&buffer).is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn_reverse_proxy_to_presigned_url(
|
||||
target_url: String,
|
||||
) -> Result<(String, tokio::task::JoinHandle<Result<(), Box<dyn Error + Send + Sync>>>), Box<dyn Error + Send + Sync>> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let address = listener.local_addr()?;
|
||||
let proxy_url = format!("http://{address}/");
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let (mut downstream, _) = listener.accept().await?;
|
||||
read_proxy_request(&mut downstream).await?;
|
||||
|
||||
let upstream_response: Result<reqwest::Response, reqwest::Error> = local_http_client().get(&target_url).send().await;
|
||||
let (status, body, content_type) = match upstream_response {
|
||||
Ok(response) => {
|
||||
let status = response.status();
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_string);
|
||||
match response.bytes().await {
|
||||
Ok(body) => (status, body.to_vec(), content_type),
|
||||
Err(err) => {
|
||||
let body = format!("upstream body read failed: {err}").into_bytes();
|
||||
(StatusCode::BAD_GATEWAY, body, Some("text/plain".to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let body = format!("upstream request failed: {err}").into_bytes();
|
||||
(StatusCode::BAD_GATEWAY, body, Some("text/plain".to_string()))
|
||||
}
|
||||
};
|
||||
|
||||
let mut response_head = format!("HTTP/1.1 {}\r\ncontent-length: {}\r\nconnection: close\r\n", status, body.len());
|
||||
if let Some(content_type) = content_type {
|
||||
response_head.push_str(&format!("content-type: {content_type}\r\n"));
|
||||
}
|
||||
response_head.push_str("\r\n");
|
||||
|
||||
downstream.write_all(response_head.as_bytes()).await?;
|
||||
downstream.write_all(&body).await?;
|
||||
downstream.shutdown().await?;
|
||||
Ok(())
|
||||
});
|
||||
|
||||
Ok((proxy_url, handle))
|
||||
}
|
||||
|
||||
async fn signed_put_request_with_headers(
|
||||
url: &str,
|
||||
access_key: &str,
|
||||
@@ -326,4 +397,130 @@ mod tests {
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
start_rustfs_server_with_env(&mut env, &[("RUSTFS_OBJECT_GET_CHUNK_FAST_PATH_ENABLE", "true")]).await?;
|
||||
env.create_test_bucket(MULTIPART_ARCHIVE_TEST_BUCKET).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let payload = random_bytes(MULTIPART_PART_SIZE + 768 * 1024);
|
||||
let zip_bytes = build_zip_bytes(&[("payload.bin", payload.as_slice())])?;
|
||||
assert!(zip_bytes.len() > MULTIPART_PART_SIZE, "zip payload must exceed multipart threshold");
|
||||
|
||||
let create_output = client
|
||||
.create_multipart_upload()
|
||||
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
|
||||
.key("presigned-multipart-bundle.zip")
|
||||
.content_type("application/zip")
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = create_output.upload_id().expect("multipart upload id");
|
||||
|
||||
let first_part = zip_bytes[..MULTIPART_PART_SIZE].to_vec();
|
||||
let second_part = zip_bytes[MULTIPART_PART_SIZE..].to_vec();
|
||||
|
||||
let upload_part_1 = client
|
||||
.upload_part()
|
||||
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
|
||||
.key("presigned-multipart-bundle.zip")
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(first_part))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let upload_part_2 = client
|
||||
.upload_part()
|
||||
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
|
||||
.key("presigned-multipart-bundle.zip")
|
||||
.upload_id(upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(second_part))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let completed_upload = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(upload_part_1.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(upload_part_2.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(MULTIPART_ARCHIVE_TEST_BUCKET)
|
||||
.key("presigned-multipart-bundle.zip")
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(completed_upload)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let object_url = format!("{}/{}/{}", env.url, MULTIPART_ARCHIVE_TEST_BUCKET, "presigned-multipart-bundle.zip");
|
||||
let direct_response =
|
||||
presigned_get_request_with_accept_encoding(&object_url, &env.access_key, &env.secret_key, "identity").await?;
|
||||
assert_eq!(direct_response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
direct_response
|
||||
.headers()
|
||||
.get("content-length")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<usize>().ok()),
|
||||
Some(zip_bytes.len())
|
||||
);
|
||||
let direct_body = direct_response.bytes().await?;
|
||||
assert_eq!(direct_body.len(), zip_bytes.len());
|
||||
assert_eq!(direct_body.as_ref(), zip_bytes.as_slice());
|
||||
|
||||
let signed = pre_sign_v4(
|
||||
http::Request::builder()
|
||||
.method(http::Method::GET)
|
||||
.uri(object_url.parse::<http::Uri>()?)
|
||||
.header(
|
||||
HOST,
|
||||
object_url
|
||||
.parse::<http::Uri>()?
|
||||
.authority()
|
||||
.ok_or("request URL missing authority")?
|
||||
.to_string(),
|
||||
)
|
||||
.body(Body::empty())?,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"",
|
||||
"us-east-1",
|
||||
600,
|
||||
OffsetDateTime::now_utc(),
|
||||
);
|
||||
let (proxy_url, proxy_handle) = spawn_reverse_proxy_to_presigned_url(signed.uri().to_string()).await?;
|
||||
let proxied_response: reqwest::Response = local_http_client().get(&proxy_url).send().await?;
|
||||
assert_eq!(proxied_response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
proxied_response
|
||||
.headers()
|
||||
.get("content-length")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<usize>().ok()),
|
||||
Some(zip_bytes.len())
|
||||
);
|
||||
let proxied_body: bytes::Bytes = proxied_response.bytes().await?;
|
||||
assert_eq!(proxied_body.len(), zip_bytes.len());
|
||||
assert_eq!(proxied_body.as_ref(), zip_bytes.as_slice());
|
||||
|
||||
proxy_handle.await??;
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,4 +132,70 @@ mod tests {
|
||||
// Stop the RustFS server to ensure proper cleanup
|
||||
env.stop_server();
|
||||
}
|
||||
|
||||
/// Test ensuring that ListObjectsV2 returns unique keys when an explicit directory marker
|
||||
/// exists under the requested prefix and delimiter is not provided.
|
||||
///
|
||||
/// Bug Reference: Issue #2439
|
||||
/// When both "marker/subdir/" and "marker/subdir/file.txt" exist, listing with
|
||||
/// Prefix="marker/" must not duplicate "marker/subdir/file.txt" in Contents.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_list_objects_v2_unique_contents_with_explicit_directory_markers() {
|
||||
init_logging();
|
||||
info!("Starting test: ListObjectsV2 should return unique keys with explicit directory markers");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "test-list-unique-contents";
|
||||
|
||||
create_bucket(&client, bucket).await.expect("Failed to create bucket");
|
||||
|
||||
for (key, body) in [
|
||||
("marker/", ByteStream::from_static(b"")),
|
||||
("marker/subdir/", ByteStream::from_static(b"")),
|
||||
("marker/file.txt", ByteStream::from_static(b"content")),
|
||||
("marker/subdir/file.txt", ByteStream::from_static(b"nested")),
|
||||
] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("Failed to create test object {key}: {err}"));
|
||||
}
|
||||
|
||||
let result = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.prefix("marker/")
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to list objects");
|
||||
|
||||
let keys: Vec<String> = result
|
||||
.contents()
|
||||
.iter()
|
||||
.filter_map(|object| object.key().map(ToOwned::to_owned))
|
||||
.collect();
|
||||
|
||||
info!("Contents: {:?}", keys);
|
||||
|
||||
assert_eq!(
|
||||
keys,
|
||||
vec![
|
||||
"marker/".to_string(),
|
||||
"marker/file.txt".to_string(),
|
||||
"marker/subdir/".to_string(),
|
||||
"marker/subdir/file.txt".to_string(),
|
||||
]
|
||||
);
|
||||
assert_eq!(result.key_count(), Some(4));
|
||||
|
||||
env.stop_server();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1477,6 +1477,16 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let mut dir_stack: Vec<(String, bool)> = Vec::with_capacity(5);
|
||||
// Explicit directory markers and real directories can resolve to the same logical path.
|
||||
let schedule_dir = |dir_stack: &mut Vec<(String, bool)>, dir_name: String, skip_object: bool| {
|
||||
if let Some((last_dir_name, existing_skip_object)) = dir_stack.last_mut()
|
||||
&& *last_dir_name == dir_name
|
||||
{
|
||||
*existing_skip_object |= skip_object;
|
||||
} else {
|
||||
dir_stack.push((dir_name, skip_object));
|
||||
}
|
||||
};
|
||||
prefix = "".to_owned();
|
||||
|
||||
for entry in entries.iter() {
|
||||
@@ -1545,7 +1555,7 @@ impl LocalDisk {
|
||||
if !dir_name.ends_with(SLASH_SEPARATOR) {
|
||||
dir_name.push_str(SLASH_SEPARATOR);
|
||||
}
|
||||
dir_stack.push((dir_name, true));
|
||||
schedule_dir(&mut dir_stack, dir_name, true);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -1554,7 +1564,7 @@ impl LocalDisk {
|
||||
// If dirObject, but no metadata (which is unexpected) we skip it.
|
||||
if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
|
||||
meta.name.push_str(SLASH_SEPARATOR);
|
||||
dir_stack.push((meta.name, false));
|
||||
schedule_dir(&mut dir_stack, meta.name, false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3169,6 +3179,60 @@ mod test {
|
||||
assert!(names.contains(&"quux/thud".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_deduplicates_explicit_dir_marker_recursion() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().unwrap();
|
||||
let bucket = "test-bucket";
|
||||
let bucket_dir = dir.path().join(bucket);
|
||||
|
||||
fs::create_dir_all(bucket_dir.join("marker/file.txt")).await.unwrap();
|
||||
fs::create_dir_all(bucket_dir.join("marker/subdir/file.txt")).await.unwrap();
|
||||
fs::create_dir_all(bucket_dir.join(format!("marker/subdir{GLOBAL_DIR_SUFFIX}")))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
fs::write(bucket_dir.join("marker/file.txt/xl.meta"), b"meta").await.unwrap();
|
||||
fs::write(bucket_dir.join("marker/subdir/file.txt/xl.meta"), b"meta")
|
||||
.await
|
||||
.unwrap();
|
||||
fs::write(bucket_dir.join(format!("marker/subdir{GLOBAL_DIR_SUFFIX}/xl.meta")), b"meta")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
|
||||
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
|
||||
|
||||
let (reader, mut writer) = tokio::io::duplex(4096);
|
||||
let mut out = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: "marker/".to_string(),
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut objs_returned = 0;
|
||||
|
||||
disk.scan_dir("marker/".to_string(), "".to_string(), &opts, &mut out, &mut objs_returned, false)
|
||||
.await
|
||||
.unwrap();
|
||||
out.close().await.unwrap();
|
||||
|
||||
let mut reader = MetacacheReader::new(reader);
|
||||
let entries = reader.read_all().await.unwrap();
|
||||
let names: Vec<String> = entries
|
||||
.into_iter()
|
||||
.filter(|entry| !entry.metadata.is_empty())
|
||||
.map(|entry| entry.name)
|
||||
.collect();
|
||||
|
||||
assert_eq!(names.iter().filter(|name| *name == "marker/subdir/file.txt").count(), 1);
|
||||
assert_eq!(names.iter().filter(|name| *name == "marker/subdir/").count(), 1);
|
||||
assert_eq!(names.iter().filter(|name| *name == "marker/file.txt").count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_make_volume() {
|
||||
let p = "./testv0";
|
||||
|
||||
@@ -67,6 +67,7 @@ use http::HeaderMap;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::{Rng, seq::SliceRandom};
|
||||
use regex::Regex;
|
||||
use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope};
|
||||
use rustfs_common::heal_channel::{DriveState, HealChannelPriority, HealItemType, HealOpts, HealScanMode, send_heal_disk};
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_filemeta::{
|
||||
@@ -133,6 +134,36 @@ fn env_non_negative_usize(name: &str) -> Option<usize> {
|
||||
rustfs_utils::get_env_opt_usize(name)
|
||||
}
|
||||
|
||||
fn capacity_scope_from_disks(disks: &[Option<DiskStore>]) -> CapacityScope {
|
||||
let mut unique = HashSet::with_capacity(disks.len());
|
||||
let mut scoped_disks = Vec::with_capacity(disks.len());
|
||||
|
||||
for disk in disks.iter().flatten() {
|
||||
let scope_disk = CapacityScopeDisk {
|
||||
endpoint: disk.endpoint().to_string(),
|
||||
drive_path: disk.to_string(),
|
||||
};
|
||||
if unique.insert(scope_disk.clone()) {
|
||||
scoped_disks.push(scope_disk);
|
||||
}
|
||||
}
|
||||
|
||||
CapacityScope { disks: scoped_disks }
|
||||
}
|
||||
|
||||
fn record_capacity_scope_if_needed(scope_token: Option<Uuid>, disks: &[Option<DiskStore>]) {
|
||||
let scope = capacity_scope_from_disks(disks);
|
||||
if scope.disks.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
record_global_dirty_scope(scope.clone());
|
||||
|
||||
if let Some(token) = scope_token {
|
||||
record_capacity_scope(token, scope);
|
||||
}
|
||||
}
|
||||
|
||||
fn resolved_put_inline_buffer_enabled(object_size: i64, inline_by_topology: bool) -> bool {
|
||||
if !inline_by_topology || object_size < 0 {
|
||||
return false;
|
||||
@@ -1087,6 +1118,8 @@ impl ObjectIO for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
record_capacity_scope_if_needed(opts.capacity_scope_token, &online_disks);
|
||||
|
||||
fi.replication_state_internal = Some(opts.put_replication_state());
|
||||
|
||||
fi.is_latest = true;
|
||||
@@ -1681,6 +1714,8 @@ impl ObjectOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
|
||||
// TODO: add_partial
|
||||
|
||||
if dist_erasure {
|
||||
@@ -1829,6 +1864,10 @@ impl ObjectOperations for SetDisks {
|
||||
.await
|
||||
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
|
||||
|
||||
if let Ok(disks) = self.get_disks(0, 0).await {
|
||||
record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
}
|
||||
|
||||
let mut oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
oi.replication_decision = goi.replication_decision;
|
||||
return Ok(oi);
|
||||
@@ -1855,6 +1894,10 @@ impl ObjectOperations for SetDisks {
|
||||
.await
|
||||
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
|
||||
|
||||
if let Ok(disks) = self.get_disks(0, 0).await {
|
||||
record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
}
|
||||
|
||||
let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
obj_info.size = goi.size;
|
||||
Ok(obj_info)
|
||||
@@ -2157,6 +2200,8 @@ impl ObjectOperations for SetDisks {
|
||||
error = ?err,
|
||||
"transition completed on remote tier but source cleanup failed; skipping external lifecycle transition notification"
|
||||
);
|
||||
} else {
|
||||
record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
}
|
||||
|
||||
for disk in disks.iter() {
|
||||
@@ -3515,6 +3560,8 @@ impl MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
record_capacity_scope_if_needed(opts.capacity_scope_token, &online_disks);
|
||||
|
||||
fi.is_latest = true;
|
||||
|
||||
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
|
||||
|
||||
@@ -558,6 +558,8 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
record_capacity_scope_if_needed(None, &out_dated_disks);
|
||||
|
||||
Ok((result, None))
|
||||
}
|
||||
Err(err) => Ok((result, Some(err))),
|
||||
|
||||
@@ -73,6 +73,7 @@ pub struct ObjectOptions {
|
||||
pub resolved_checksum: Option<Bytes>,
|
||||
pub want_checksum: Option<Checksum>,
|
||||
pub skip_verify_bitrot: bool,
|
||||
pub capacity_scope_token: Option<Uuid>,
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
|
||||
@@ -1133,6 +1133,32 @@ mod test {
|
||||
assert!(fi.is_latest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_issue_2434_legacy_meta_v2_pool_compatibility() {
|
||||
let data = create_issue_2434_legacy_meta_v2_pool_xlmeta().expect("Failed to load issue #2434 pool fixture");
|
||||
let (major, minor, header_ver, meta_ver) = FileMeta::read_format_versions(&data).unwrap();
|
||||
assert_eq!((major, minor, header_ver, meta_ver), (1, 3, 3, 2));
|
||||
|
||||
let fm = FileMeta::load(&data).expect("Failed to parse legacy issue #2434 pool xl.meta");
|
||||
assert_eq!(fm.meta_ver, 2);
|
||||
assert_eq!(fm.versions.len(), 1);
|
||||
assert_eq!(fm.versions[0].header.version_type, VersionType::Object);
|
||||
|
||||
let fi = fm
|
||||
.into_fileinfo(".rustfs.sys", "pool.bin", "", true, false, true)
|
||||
.expect("Failed to extract file info from legacy issue #2434 pool xl.meta");
|
||||
assert_eq!(fi.size, 48);
|
||||
assert_eq!(fi.num_versions, 1);
|
||||
assert_eq!(fi.version_id, None);
|
||||
assert_eq!(fi.metadata.get("etag").map(String::as_str), Some("8d270d7a184cfa30cc0bf09ea74fd964"));
|
||||
assert_eq!(
|
||||
fi.data_dir.map(|id| id.to_string()).as_deref(),
|
||||
Some("2bcefaca-44dd-4f01-a79e-63eeb0dda396")
|
||||
);
|
||||
assert!(fi.uses_legacy_checksum);
|
||||
assert!(fi.is_latest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_legacy_v1_object_xlmeta_compatibility() {
|
||||
let data = create_legacy_v1_object_xlmeta().expect("Failed to create legacy v1 object xl.meta");
|
||||
|
||||
@@ -54,6 +54,62 @@ fn read_msgp_bin<R: std::io::Read>(rd: &mut R) -> Result<Vec<u8>> {
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
fn deserialize_legacy_uuid_bytes<'de, D>(deserializer: D) -> std::result::Result<Vec<u8>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
struct LegacyUuidBytesVisitor;
|
||||
|
||||
impl<'de> serde::de::Visitor<'de> for LegacyUuidBytesVisitor {
|
||||
type Value = Vec<u8>;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("nil or binary UUID bytes")
|
||||
}
|
||||
|
||||
fn visit_none<E>(self) -> std::result::Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
fn visit_unit<E>(self) -> std::result::Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
fn visit_bytes<E>(self, value: &[u8]) -> std::result::Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
Ok(value.to_vec())
|
||||
}
|
||||
|
||||
fn visit_byte_buf<E>(self, value: Vec<u8>) -> std::result::Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>
|
||||
where
|
||||
A: serde::de::SeqAccess<'de>,
|
||||
{
|
||||
let mut value = Vec::new();
|
||||
while let Some(byte) = seq.next_element()? {
|
||||
value.push(byte);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(LegacyUuidBytesVisitor)
|
||||
}
|
||||
|
||||
fn decode_msgp_time_payload(ext_type: i8, payload: &[u8]) -> Result<OffsetDateTime> {
|
||||
let (secs, nanos) = match (ext_type, payload.len()) {
|
||||
(MSGPACK_TIME_EXT_LEGACY, 12) => {
|
||||
@@ -179,7 +235,9 @@ struct LegacyMetaV2Version {
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LegacyMetaV2Object {
|
||||
#[serde(default, deserialize_with = "deserialize_legacy_uuid_bytes")]
|
||||
version_id: Vec<u8>,
|
||||
#[serde(default, deserialize_with = "deserialize_legacy_uuid_bytes")]
|
||||
data_dir: Vec<u8>,
|
||||
erasure_algorithm: String,
|
||||
erasure_m: usize,
|
||||
@@ -201,6 +259,7 @@ struct LegacyMetaV2Object {
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LegacyMetaV2DeleteMarker {
|
||||
#[serde(default, deserialize_with = "deserialize_legacy_uuid_bytes")]
|
||||
version_id: Vec<u8>,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
meta_sys: HashMap<String, Vec<u8>>,
|
||||
@@ -2738,6 +2797,57 @@ mod tests {
|
||||
write_version: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct LegacyDeleteMarkerNilFixture {
|
||||
version_id: Option<Vec<u8>>,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
meta_sys: HashMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct LegacyDeleteVersionNilFixture {
|
||||
version_type: LegacyDeleteVersionTypeFixture,
|
||||
object: Option<()>,
|
||||
delete_marker: Option<LegacyDeleteMarkerNilFixture>,
|
||||
write_version: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
enum LegacyObjectVersionTypeFixture {
|
||||
#[serde(rename = "Object")]
|
||||
Object,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct LegacyObjectFixture {
|
||||
version_id: Option<Vec<u8>>,
|
||||
data_dir: Option<Vec<u8>>,
|
||||
erasure_algorithm: String,
|
||||
erasure_m: usize,
|
||||
erasure_n: usize,
|
||||
erasure_block_size: usize,
|
||||
erasure_index: usize,
|
||||
erasure_dist: Vec<u8>,
|
||||
bitrot_checksum_algo: String,
|
||||
part_numbers: Vec<usize>,
|
||||
part_etags: Vec<String>,
|
||||
part_sizes: Vec<usize>,
|
||||
part_actual_sizes: Vec<i64>,
|
||||
part_indices: Vec<Vec<u8>>,
|
||||
size: i64,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
meta_sys: HashMap<String, Vec<u8>>,
|
||||
meta_user: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct LegacyObjectVersionFixture {
|
||||
version_type: LegacyObjectVersionTypeFixture,
|
||||
object: Option<LegacyObjectFixture>,
|
||||
delete_marker: Option<()>,
|
||||
write_version: u64,
|
||||
}
|
||||
|
||||
fn sample_version_id() -> Uuid {
|
||||
Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef").unwrap()
|
||||
}
|
||||
@@ -3027,4 +3137,76 @@ mod tests {
|
||||
|
||||
assert!(err.to_string().contains("legacy version_id must be 16 bytes"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_meta_v2_object_accepts_nil_uuid_fields() {
|
||||
let payload = LegacyObjectVersionFixture {
|
||||
version_type: LegacyObjectVersionTypeFixture::Object,
|
||||
object: Some(LegacyObjectFixture {
|
||||
version_id: None,
|
||||
data_dir: None,
|
||||
erasure_algorithm: "ReedSolomon".to_string(),
|
||||
erasure_m: 2,
|
||||
erasure_n: 4,
|
||||
erasure_block_size: 1_048_576,
|
||||
erasure_index: 1,
|
||||
erasure_dist: vec![1, 2, 3, 4, 5, 6],
|
||||
bitrot_checksum_algo: "HighwayHash".to_string(),
|
||||
part_numbers: vec![1],
|
||||
part_etags: vec!["etag-1".to_string()],
|
||||
part_sizes: vec![11],
|
||||
part_actual_sizes: vec![11],
|
||||
part_indices: vec![Vec::new()],
|
||||
size: 11,
|
||||
mod_time: Some(sample_mod_time()),
|
||||
meta_sys: HashMap::new(),
|
||||
meta_user: HashMap::from([("content-type".to_string(), "text/plain".to_string())]),
|
||||
}),
|
||||
delete_marker: None,
|
||||
write_version: 3,
|
||||
};
|
||||
let encoded = rmp_serde::to_vec_named(&payload).unwrap();
|
||||
|
||||
let decoded = FileMetaVersion::try_from(encoded.as_slice()).unwrap();
|
||||
let object = decoded.object.as_ref().expect("object should be decoded");
|
||||
|
||||
assert_eq!(decoded.version_type, VersionType::Object);
|
||||
assert!(decoded.uses_legacy_checksum);
|
||||
assert_eq!(object.version_id, None);
|
||||
assert_eq!(object.data_dir, None);
|
||||
|
||||
let fi = decoded.into_fileinfo("bucket", "legacy-nil.txt", true);
|
||||
assert_eq!(fi.version_id, None);
|
||||
assert_eq!(fi.data_dir, None);
|
||||
assert_eq!(fi.metadata.get("content-type").map(String::as_str), Some("text/plain"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_meta_v2_delete_marker_accepts_nil_version_id() {
|
||||
let payload = LegacyDeleteVersionNilFixture {
|
||||
version_type: LegacyDeleteVersionTypeFixture::DeleteMarker,
|
||||
object: None,
|
||||
delete_marker: Some(LegacyDeleteMarkerNilFixture {
|
||||
version_id: None,
|
||||
mod_time: Some(sample_mod_time()),
|
||||
meta_sys: HashMap::from([("x-rustfs-test".to_string(), b"gone".to_vec())]),
|
||||
}),
|
||||
write_version: 11,
|
||||
};
|
||||
let encoded = rmp_serde::to_vec_named(&payload).unwrap();
|
||||
|
||||
let decoded = FileMetaVersion::try_from(encoded.as_slice()).unwrap();
|
||||
let delete_marker = decoded.delete_marker.as_ref().expect("delete marker should be decoded");
|
||||
|
||||
assert_eq!(decoded.version_type, VersionType::Delete);
|
||||
assert!(decoded.uses_legacy_checksum);
|
||||
assert_eq!(delete_marker.version_id, None);
|
||||
assert_eq!(delete_marker.mod_time, Some(sample_mod_time()));
|
||||
|
||||
let fi = decoded.into_fileinfo("bucket", "deleted.txt", true);
|
||||
assert!(fi.deleted);
|
||||
assert_eq!(fi.version_id, None);
|
||||
assert_eq!(fi.mod_time, Some(sample_mod_time()));
|
||||
assert_eq!(fi.metadata.get("x-rustfs-test").map(String::as_str), Some("gone"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,6 +145,11 @@ pub fn create_issue_2265_legacy_meta_v2_config_xlmeta() -> Result<Vec<u8>> {
|
||||
decode_hex_fixture(include_str!("../tests/fixtures/issue_2265_legacy_meta_v2_config.hex"))
|
||||
}
|
||||
|
||||
/// Legacy pool xl.meta captured in issue #2434. Header/meta versions are 3/2.
|
||||
pub fn create_issue_2434_legacy_meta_v2_pool_xlmeta() -> Result<Vec<u8>> {
|
||||
decode_hex_fixture(include_str!("../tests/fixtures/issue_2434_legacy_meta_v2_pool.hex"))
|
||||
}
|
||||
|
||||
fn write_legacy_time(wr: &mut Vec<u8>, ts: OffsetDateTime) {
|
||||
wr.push(MSGPACK_EXT8);
|
||||
wr.push(12);
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
584c322001000300c6000000d6030201c42697c41000000000000000000000000000000000d3184f6c224d6172f1c4040000000001020001c4a994a64f626a656374dc0012c0c4102bcefaca44dd4f01a79e63eeb0dda396ab52656564536f6c6f6d6f6e0100ce00100000019101ab4869676877617948617368910191d92038643237306437613138346366613330636330626630396561373466643936349130913091c4003099cd07e9ccba112f36ce26f10ef10000008081a465746167d9203864323730643761313834636661333063633062663039656137346664393634c000ce66d2a556
|
||||
+30
-4
@@ -37,7 +37,6 @@ use rustfs_policy::policy::Args;
|
||||
use rustfs_policy::policy::opa;
|
||||
use rustfs_policy::policy::{Policy, PolicyDoc, iam_policy_claim_name_sa, policy_needs_existing_object_tag_for_args};
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
@@ -244,12 +243,11 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
pub async fn info_policy(&self, name: &str) -> Result<rustfs_madmin::PolicyInfo> {
|
||||
let d = self.store.get_policy_doc(name).await?;
|
||||
|
||||
let pdata = serde_json::to_string(&d.policy)?;
|
||||
let pdata = serde_json::to_value(&d.policy)?;
|
||||
|
||||
Ok(rustfs_madmin::PolicyInfo {
|
||||
policy_name: name.to_string(),
|
||||
policy: json!(pdata),
|
||||
policy: pdata,
|
||||
create_date: d.create_date,
|
||||
update_date: d.update_date,
|
||||
})
|
||||
@@ -2045,4 +2043,32 @@ mod tests {
|
||||
policies
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_info_policy_returns_policy_as_json_object() {
|
||||
let store = StsTestMockStore { empty_policies: false };
|
||||
let cache_manager = IamCache::new(store).await;
|
||||
let iam_sys = IamSys::new(cache_manager);
|
||||
|
||||
let policy_info = iam_sys
|
||||
.info_policy("readonly")
|
||||
.await
|
||||
.expect("info_policy should return existing default policy");
|
||||
|
||||
assert!(
|
||||
policy_info.policy.is_object(),
|
||||
"policy field should be a JSON object for MinIO-compatible policy readback; got: {}",
|
||||
policy_info.policy
|
||||
);
|
||||
assert!(
|
||||
policy_info.policy.get("Version").is_some(),
|
||||
"policy object should contain Version field; got: {}",
|
||||
policy_info.policy
|
||||
);
|
||||
assert!(
|
||||
policy_info.policy.get("Statement").is_some(),
|
||||
"policy object should contain Statement field; got: {}",
|
||||
policy_info.policy
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,12 @@ pub fn record_capacity_cache_miss() {
|
||||
counter!("rustfs.capacity.cache.misses").increment(1);
|
||||
}
|
||||
|
||||
/// Record how capacity cache was served to the caller.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_cache_served(state: &'static str) {
|
||||
counter!("rustfs.capacity.cache.served.total", "state" => state).increment(1);
|
||||
}
|
||||
|
||||
/// Record current capacity gauge.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_current_bytes(used_bytes: u64) {
|
||||
@@ -55,6 +61,44 @@ pub fn record_capacity_update_failed(source: &'static str) {
|
||||
counter!("rustfs.capacity.update.failures", "source" => source).increment(1);
|
||||
}
|
||||
|
||||
/// Record a capacity refresh request.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_refresh_request(mode: &'static str, source: &'static str) {
|
||||
counter!("rustfs.capacity.refresh.requests.total", "mode" => mode, "source" => source).increment(1);
|
||||
}
|
||||
|
||||
/// Record a refresh joiner waiting for an inflight refresh.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_refresh_joiner(source: &'static str) {
|
||||
counter!("rustfs.capacity.refresh.joiners.total", "source" => source).increment(1);
|
||||
}
|
||||
|
||||
/// Record the number of inflight capacity refreshes.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_refresh_inflight(count: usize) {
|
||||
gauge!("rustfs.capacity.refresh.inflight").set(count as f64);
|
||||
}
|
||||
|
||||
/// Record the final result of a capacity refresh.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_refresh_result(source: &'static str, result: &'static str, duration: Duration) {
|
||||
counter!("rustfs.capacity.refresh.result.total", "source" => source, "result" => result).increment(1);
|
||||
histogram!("rustfs.capacity.refresh.duration.seconds", "source" => source, "result" => result).record(duration.as_secs_f64());
|
||||
}
|
||||
|
||||
/// Record the refresh scope selected for a capacity refresh.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_refresh_scope(scope: &'static str, disk_count: usize) {
|
||||
counter!("rustfs.capacity.refresh.scope.total", "scope" => scope).increment(1);
|
||||
histogram!("rustfs.capacity.refresh.scope.disks", "scope" => scope).record(disk_count as f64);
|
||||
}
|
||||
|
||||
/// Record the current number of dirty disks tracked by capacity management.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_dirty_disk_count(count: usize) {
|
||||
gauge!("rustfs.capacity.dirty.disks").set(count as f64);
|
||||
}
|
||||
|
||||
/// Record capacity write activity.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_write_operation(write_frequency: usize) {
|
||||
@@ -98,3 +142,33 @@ pub fn record_capacity_scan_sampling(sampled_count: usize, estimated: bool) {
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Record the scan mode used for a capacity result.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_scan_mode(mode: &'static str) {
|
||||
counter!("rustfs.capacity.scan.mode.total", "mode" => mode).increment(1);
|
||||
}
|
||||
|
||||
/// Record per-disk capacity scan statistics.
|
||||
#[inline(always)]
|
||||
pub fn record_capacity_scan_disk(
|
||||
disk: &str,
|
||||
duration: Duration,
|
||||
file_count: usize,
|
||||
sampled_count: usize,
|
||||
estimated: bool,
|
||||
partial_errors: bool,
|
||||
) {
|
||||
histogram!("rustfs.capacity.scan.disk.duration.seconds", "disk" => disk.to_owned()).record(duration.as_secs_f64());
|
||||
histogram!("rustfs.capacity.scan.disk.files", "disk" => disk.to_owned()).record(file_count as f64);
|
||||
histogram!("rustfs.capacity.scan.disk.sampled", "disk" => disk.to_owned()).record(sampled_count as f64);
|
||||
counter!(
|
||||
"rustfs.capacity.scan.disk.estimated.total",
|
||||
"disk" => disk.to_owned(),
|
||||
"estimated" => if estimated { "true" } else { "false" }
|
||||
)
|
||||
.increment(1);
|
||||
if partial_errors {
|
||||
counter!("rustfs.capacity.scan.disk.partial_errors.total", "disk" => disk.to_owned()).increment(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,9 +74,12 @@ pub use adaptive_ttl::{
|
||||
|
||||
// Capacity metrics exports
|
||||
pub use capacity_metrics::{
|
||||
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_current_bytes, record_capacity_dynamic_timeout,
|
||||
record_capacity_scan_sampling, record_capacity_stall_detected, record_capacity_symlink, record_capacity_timeout_fallback,
|
||||
record_capacity_update_completed, record_capacity_update_failed, record_capacity_write_operation,
|
||||
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_current_bytes,
|
||||
record_capacity_dirty_disk_count, record_capacity_dynamic_timeout, record_capacity_refresh_inflight,
|
||||
record_capacity_refresh_joiner, record_capacity_refresh_request, record_capacity_refresh_result,
|
||||
record_capacity_refresh_scope, record_capacity_scan_disk, record_capacity_scan_mode, record_capacity_scan_sampling,
|
||||
record_capacity_stall_detected, record_capacity_symlink, record_capacity_timeout_fallback, record_capacity_update_completed,
|
||||
record_capacity_update_failed, record_capacity_write_operation,
|
||||
};
|
||||
|
||||
// I/O metrics exports
|
||||
@@ -190,6 +193,8 @@ impl IoStage {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FallbackReason {
|
||||
Unknown,
|
||||
FeatureDisabled,
|
||||
ProbeFailed,
|
||||
MmapDisabled,
|
||||
MmapUnavailable,
|
||||
SmallObject,
|
||||
@@ -210,6 +215,8 @@ impl FallbackReason {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unknown => "unknown",
|
||||
Self::FeatureDisabled => "feature_disabled",
|
||||
Self::ProbeFailed => "probe_failed",
|
||||
Self::MmapDisabled => "mmap_disabled",
|
||||
Self::MmapUnavailable => "mmap_unavailable",
|
||||
Self::SmallObject => "small_object",
|
||||
@@ -344,6 +351,59 @@ pub fn record_io_fallback(stage: IoStage, reason: FallbackReason) {
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Record a selected GET chunk fast path.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_fast_path_selected(path: &'static str, copy_mode: CopyMode, promised_bytes: i64) {
|
||||
counter!(
|
||||
metric_names::data_plane::GET_FAST_PATH_SELECTED_TOTAL,
|
||||
"path" => path.to_string(),
|
||||
"copy_mode" => copy_mode.as_str().to_string()
|
||||
)
|
||||
.increment(1);
|
||||
|
||||
if promised_bytes >= 0 {
|
||||
histogram!(metric_names::data_plane::GET_FAST_PATH_PROMISED_BYTES).record(promised_bytes as f64);
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a failed GET chunk fast path probe before the response is committed.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_fast_path_probe_failed(path: &'static str, copy_mode: CopyMode, promised_bytes: i64) {
|
||||
counter!(
|
||||
metric_names::data_plane::GET_FAST_PATH_PROBE_FAILED_TOTAL,
|
||||
"path" => path.to_string(),
|
||||
"copy_mode" => copy_mode.as_str().to_string()
|
||||
)
|
||||
.increment(1);
|
||||
|
||||
if promised_bytes >= 0 {
|
||||
histogram!(metric_names::data_plane::GET_FAST_PATH_PROMISED_BYTES).record(promised_bytes as f64);
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a GET chunk fast path mid-stream error after headers have already been committed.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_fast_path_midstream_error(
|
||||
path: &'static str,
|
||||
copy_mode: CopyMode,
|
||||
error_kind: &'static str,
|
||||
sent_bytes: usize,
|
||||
promised_bytes: i64,
|
||||
) {
|
||||
counter!(
|
||||
metric_names::data_plane::GET_FAST_PATH_MIDSTREAM_ERROR_TOTAL,
|
||||
"path" => path.to_string(),
|
||||
"copy_mode" => copy_mode.as_str().to_string(),
|
||||
"error_kind" => error_kind.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
|
||||
histogram!(metric_names::data_plane::GET_FAST_PATH_MIDSTREAM_SENT_BYTES).record(sent_bytes as f64);
|
||||
if promised_bytes >= 0 {
|
||||
histogram!(metric_names::data_plane::GET_FAST_PATH_PROMISED_BYTES).record(promised_bytes as f64);
|
||||
}
|
||||
}
|
||||
|
||||
/// Record the currently active mmap bytes held by LocalDisk chunk streams.
|
||||
#[inline(always)]
|
||||
pub fn record_local_disk_active_mmap_bytes(active_bytes: usize) {
|
||||
@@ -872,6 +932,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_fallback_reason_as_str_values_stable() {
|
||||
assert_eq!(FallbackReason::Unknown.as_str(), "unknown");
|
||||
assert_eq!(FallbackReason::FeatureDisabled.as_str(), "feature_disabled");
|
||||
assert_eq!(FallbackReason::ProbeFailed.as_str(), "probe_failed");
|
||||
assert_eq!(FallbackReason::MmapDisabled.as_str(), "mmap_disabled");
|
||||
assert_eq!(FallbackReason::MmapUnavailable.as_str(), "mmap_unavailable");
|
||||
assert_eq!(FallbackReason::SmallObject.as_str(), "small_object");
|
||||
@@ -925,6 +987,13 @@ mod tests {
|
||||
record_local_disk_compat_collect(3, 16384);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_get_object_fast_path_metrics() {
|
||||
record_get_object_fast_path_selected("direct", CopyMode::TrueZeroCopy, 8192);
|
||||
record_get_object_fast_path_probe_failed("bridge", CopyMode::SingleCopy, 4096);
|
||||
record_get_object_fast_path_midstream_error("direct", CopyMode::Reconstructed, "unexpected_eof", 2048, 8192);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_put_object_attempted_fast_path() {
|
||||
record_put_object_attempted_fast_path(1024 * 1024);
|
||||
|
||||
@@ -54,4 +54,19 @@ pub mod data_plane {
|
||||
|
||||
/// Size distribution for transformed PUT selections.
|
||||
pub const PUT_TRANSFORM_SIZE_BYTES: &str = "rustfs.io.put.transform.size.bytes";
|
||||
|
||||
/// Total number of selected GET chunk fast paths.
|
||||
pub const GET_FAST_PATH_SELECTED_TOTAL: &str = "rustfs.io.get.fast_path.selected_total";
|
||||
|
||||
/// Total number of GET chunk fast path probe failures before response commit.
|
||||
pub const GET_FAST_PATH_PROBE_FAILED_TOTAL: &str = "rustfs.io.get.fast_path.probe_failed_total";
|
||||
|
||||
/// Total number of GET chunk fast path mid-stream errors after response commit.
|
||||
pub const GET_FAST_PATH_MIDSTREAM_ERROR_TOTAL: &str = "rustfs.io.get.fast_path.midstream_error_total";
|
||||
|
||||
/// Byte distribution promised by GET chunk fast path selections or failures.
|
||||
pub const GET_FAST_PATH_PROMISED_BYTES: &str = "rustfs.io.get.fast_path.promised.bytes";
|
||||
|
||||
/// Byte distribution already sent when a GET chunk fast path fails mid-stream.
|
||||
pub const GET_FAST_PATH_MIDSTREAM_SENT_BYTES: &str = "rustfs.io.get.fast_path.midstream_sent.bytes";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-object-capacity"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "Capacity scan and refresh core for RustFS."
|
||||
keywords = ["capacity", "storage", "rustfs", "scan", "metrics"]
|
||||
categories = ["filesystem", "development-tools"]
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
[[bench]]
|
||||
name = "capacity_scan"
|
||||
harness = false
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "time"] }
|
||||
tracing = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
walkdir = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
@@ -0,0 +1,136 @@
|
||||
use criterion::{Criterion, criterion_group, criterion_main};
|
||||
use rustfs_object_capacity::{CapacityDiskRef, scan_used_capacity_disks};
|
||||
use std::fs;
|
||||
use std::hint::black_box;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
const EXACT_FILE_SIZE: usize = 4 * 1024;
|
||||
const SAMPLED_FILE_SIZE: usize = 1;
|
||||
const DEFAULT_SAMPLE_TRIGGER_FILE_COUNT: usize = 202_048;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct DiskSpec {
|
||||
file_count: usize,
|
||||
file_size: usize,
|
||||
}
|
||||
|
||||
struct CapacityScanFixture {
|
||||
_dirs: Vec<TempDir>,
|
||||
disks: Vec<CapacityDiskRef>,
|
||||
}
|
||||
|
||||
impl CapacityScanFixture {
|
||||
fn new(specs: &[DiskSpec]) -> Self {
|
||||
let mut dirs = Vec::with_capacity(specs.len());
|
||||
let mut disks = Vec::with_capacity(specs.len());
|
||||
|
||||
for (idx, spec) in specs.iter().enumerate() {
|
||||
let dir = TempDir::new().expect("create temp dir");
|
||||
populate_files(dir.path(), spec.file_count, spec.file_size).expect("populate files");
|
||||
disks.push(CapacityDiskRef {
|
||||
endpoint: format!("bench-disk-{idx}"),
|
||||
drive_path: dir.path().to_string_lossy().into_owned(),
|
||||
});
|
||||
dirs.push(dir);
|
||||
}
|
||||
|
||||
Self { _dirs: dirs, disks }
|
||||
}
|
||||
}
|
||||
|
||||
fn populate_files(root: &Path, file_count: usize, file_size: usize) -> std::io::Result<()> {
|
||||
let payload = vec![b'x'; file_size];
|
||||
let shard_count = (file_count / 512).clamp(1, 256);
|
||||
|
||||
for shard_idx in 0..shard_count {
|
||||
fs::create_dir_all(root.join(format!("bucket-{shard_idx:03}")))?;
|
||||
}
|
||||
|
||||
for file_idx in 0..file_count {
|
||||
let subdir = root.join(format!("bucket-{:03}", file_idx % shard_count));
|
||||
let file_path: PathBuf = subdir.join(format!("object-{file_idx:08}.bin"));
|
||||
fs::write(file_path, &payload)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bench_capacity_scan(c: &mut Criterion) {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("create runtime");
|
||||
|
||||
let exact_fixture = CapacityScanFixture::new(&[DiskSpec {
|
||||
file_count: 10_000,
|
||||
file_size: EXACT_FILE_SIZE,
|
||||
}]);
|
||||
|
||||
let sampled_fixture = CapacityScanFixture::new(&[DiskSpec {
|
||||
file_count: DEFAULT_SAMPLE_TRIGGER_FILE_COUNT,
|
||||
file_size: SAMPLED_FILE_SIZE,
|
||||
}]);
|
||||
|
||||
let multi_disk_fixture = CapacityScanFixture::new(&[
|
||||
DiskSpec {
|
||||
file_count: 4_000,
|
||||
file_size: 1024,
|
||||
},
|
||||
DiskSpec {
|
||||
file_count: 6_000,
|
||||
file_size: 2048,
|
||||
},
|
||||
DiskSpec {
|
||||
file_count: 8_000,
|
||||
file_size: 4096,
|
||||
},
|
||||
DiskSpec {
|
||||
file_count: 10_000,
|
||||
file_size: 1024,
|
||||
},
|
||||
]);
|
||||
|
||||
let mut exact_group = c.benchmark_group("capacity_scan_exact");
|
||||
exact_group.sample_size(10);
|
||||
exact_group.measurement_time(Duration::from_secs(10));
|
||||
exact_group.bench_function("single_disk_10k_4k", |b| {
|
||||
b.iter(|| {
|
||||
let summary = runtime
|
||||
.block_on(scan_used_capacity_disks(black_box(&exact_fixture.disks)))
|
||||
.expect("exact scan");
|
||||
black_box(summary);
|
||||
});
|
||||
});
|
||||
exact_group.finish();
|
||||
|
||||
let mut sampled_group = c.benchmark_group("capacity_scan_sampled");
|
||||
sampled_group.sample_size(10);
|
||||
sampled_group.measurement_time(Duration::from_secs(10));
|
||||
sampled_group.bench_function("single_disk_202k_1b", |b| {
|
||||
b.iter(|| {
|
||||
let summary = runtime
|
||||
.block_on(scan_used_capacity_disks(black_box(&sampled_fixture.disks)))
|
||||
.expect("sampled scan");
|
||||
black_box(summary);
|
||||
});
|
||||
});
|
||||
sampled_group.finish();
|
||||
|
||||
let mut multi_disk_group = c.benchmark_group("capacity_scan_multi_disk");
|
||||
multi_disk_group.sample_size(10);
|
||||
multi_disk_group.measurement_time(Duration::from_secs(10));
|
||||
multi_disk_group.bench_function("four_disks_mixed_exact", |b| {
|
||||
b.iter(|| {
|
||||
let summary = runtime
|
||||
.block_on(scan_used_capacity_disks(black_box(&multi_disk_fixture.disks)))
|
||||
.expect("multi-disk scan");
|
||||
black_box(summary);
|
||||
});
|
||||
});
|
||||
multi_disk_group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_capacity_scan);
|
||||
criterion_main!(benches);
|
||||
+621
-51
@@ -14,8 +14,10 @@
|
||||
|
||||
//! Hybrid Capacity Manager for efficient capacity statistics
|
||||
|
||||
use crate::app::admin_usecase::calculate_data_dir_used_capacity;
|
||||
use super::scan::refresh_capacity_with_scope;
|
||||
use super::types::CapacityDiskRef;
|
||||
use futures::FutureExt;
|
||||
use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, drain_global_dirty_scopes, take_capacity_scope};
|
||||
use rustfs_config::{
|
||||
DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH,
|
||||
DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, DEFAULT_CAPACITY_STALL_TIMEOUT_SECS,
|
||||
@@ -26,12 +28,17 @@ use rustfs_config::{
|
||||
ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL, ENV_CAPACITY_STALL_TIMEOUT, ENV_CAPACITY_STAT_TIMEOUT,
|
||||
ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY,
|
||||
};
|
||||
use rustfs_io_metrics::{record_capacity_current_bytes, record_capacity_update_completed, record_capacity_write_operation};
|
||||
use rustfs_io_metrics::capacity_metrics::{
|
||||
record_capacity_current_bytes, record_capacity_dirty_disk_count, record_capacity_refresh_inflight,
|
||||
record_capacity_refresh_joiner, record_capacity_refresh_result, record_capacity_update_completed,
|
||||
record_capacity_update_failed, record_capacity_write_operation,
|
||||
};
|
||||
use rustfs_utils::{get_env_bool, get_env_u64, get_env_usize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::{Mutex, RwLock, watch};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
@@ -297,6 +304,14 @@ pub struct CapacityUpdate {
|
||||
pub file_count: usize,
|
||||
/// Whether the value is estimated instead of exact.
|
||||
pub is_estimated: bool,
|
||||
/// Per-disk breakdown captured from a successful refresh.
|
||||
pub per_disk: Vec<DiskCapacityUpdate>,
|
||||
/// Expected disk count for a complete disk cache.
|
||||
pub expected_disk_count: Option<usize>,
|
||||
/// Whether this update should replace the current disk cache.
|
||||
pub replaces_disk_cache: bool,
|
||||
/// Dirty disks that can be cleared after the update is committed.
|
||||
pub clear_dirty_disks: Vec<CapacityScopeDisk>,
|
||||
}
|
||||
|
||||
impl CapacityUpdate {
|
||||
@@ -306,6 +321,10 @@ impl CapacityUpdate {
|
||||
total_used,
|
||||
file_count,
|
||||
is_estimated: false,
|
||||
per_disk: Vec::new(),
|
||||
expected_disk_count: None,
|
||||
replaces_disk_cache: false,
|
||||
clear_dirty_disks: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -315,6 +334,10 @@ impl CapacityUpdate {
|
||||
total_used,
|
||||
file_count,
|
||||
is_estimated: true,
|
||||
per_disk: Vec::new(),
|
||||
expected_disk_count: None,
|
||||
replaces_disk_cache: false,
|
||||
clear_dirty_disks: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -324,10 +347,27 @@ impl CapacityUpdate {
|
||||
total_used,
|
||||
file_count: 0,
|
||||
is_estimated: true,
|
||||
per_disk: Vec::new(),
|
||||
expected_disk_count: None,
|
||||
replaces_disk_cache: false,
|
||||
clear_dirty_disks: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DiskCapacityUpdate {
|
||||
pub disk: CapacityScopeDisk,
|
||||
pub used_bytes: u64,
|
||||
pub file_count: usize,
|
||||
pub is_estimated: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CachedDiskCapacity {
|
||||
used_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Copy, Eq)]
|
||||
pub enum DataSource {
|
||||
/// Real-time statistics
|
||||
@@ -342,7 +382,7 @@ pub enum DataSource {
|
||||
}
|
||||
|
||||
impl DataSource {
|
||||
fn as_metric_label(self) -> &'static str {
|
||||
pub fn as_metric_label(self) -> &'static str {
|
||||
match self {
|
||||
Self::RealTime => "realtime",
|
||||
Self::Scheduled => "scheduled",
|
||||
@@ -352,15 +392,62 @@ impl DataSource {
|
||||
}
|
||||
}
|
||||
|
||||
const WRITE_WINDOW_SECS: u64 = 60;
|
||||
const WRITE_WINDOW_BUCKETS: usize = WRITE_WINDOW_SECS as usize;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct WriteBucket {
|
||||
second: u64,
|
||||
count: usize,
|
||||
}
|
||||
|
||||
/// Write record for tracking write operations
|
||||
#[derive(Debug)]
|
||||
pub struct WriteRecord {
|
||||
/// Last write time
|
||||
pub last_write_time: Instant,
|
||||
pub last_write_time: Option<Instant>,
|
||||
/// Write count
|
||||
pub write_count: usize,
|
||||
/// Write time window (for frequency calculation)
|
||||
pub write_window: Vec<Instant>,
|
||||
/// Fixed-size time buckets for the recent write window.
|
||||
write_buckets: [WriteBucket; WRITE_WINDOW_BUCKETS],
|
||||
}
|
||||
|
||||
impl WriteRecord {
|
||||
fn current_unix_second() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or(Duration::ZERO)
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
fn recent_write_count(&self, now_second: u64) -> usize {
|
||||
self.write_buckets
|
||||
.iter()
|
||||
.filter(|bucket| {
|
||||
bucket.count > 0 && bucket.second <= now_second && now_second.saturating_sub(bucket.second) < WRITE_WINDOW_SECS
|
||||
})
|
||||
.map(|bucket| bucket.count)
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn record_write(&mut self, now: Instant) -> usize {
|
||||
let now_second = Self::current_unix_second();
|
||||
let bucket_idx = (now_second % WRITE_WINDOW_BUCKETS as u64) as usize;
|
||||
let bucket = &mut self.write_buckets[bucket_idx];
|
||||
|
||||
if bucket.second != now_second {
|
||||
*bucket = WriteBucket {
|
||||
second: now_second,
|
||||
count: 0,
|
||||
};
|
||||
}
|
||||
|
||||
bucket.count = bucket.count.saturating_add(1);
|
||||
self.last_write_time = Some(now);
|
||||
self.write_count = self.write_count.saturating_add(1);
|
||||
|
||||
self.recent_write_count(now_second)
|
||||
}
|
||||
}
|
||||
|
||||
/// Hybrid strategy configuration
|
||||
@@ -430,6 +517,12 @@ pub struct HybridCapacityManager {
|
||||
cache: Arc<RwLock<Option<CachedCapacity>>>,
|
||||
/// Write record
|
||||
write_record: Arc<RwLock<WriteRecord>>,
|
||||
/// Dirty disks recorded from write-side scope propagation.
|
||||
dirty_disks: Arc<RwLock<HashSet<CapacityScopeDisk>>>,
|
||||
/// Per-disk cache populated after a successful full refresh and updated by dirty subset refreshes.
|
||||
disk_cache: Arc<RwLock<HashMap<CapacityScopeDisk, CachedDiskCapacity>>>,
|
||||
/// Whether the per-disk cache currently covers all known disks.
|
||||
disk_cache_complete: Arc<RwLock<bool>>,
|
||||
/// Configuration
|
||||
config: HybridStrategyConfig,
|
||||
/// Shared singleflight refresh state
|
||||
@@ -437,6 +530,17 @@ pub struct HybridCapacityManager {
|
||||
}
|
||||
|
||||
impl HybridCapacityManager {
|
||||
async fn sync_global_dirty_scopes(&self) {
|
||||
let scopes = drain_global_dirty_scopes();
|
||||
if scopes.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut dirty_disks = self.dirty_disks.write().await;
|
||||
dirty_disks.extend(scopes);
|
||||
record_capacity_dirty_disk_count(dirty_disks.len());
|
||||
}
|
||||
|
||||
fn max_stale_age(&self) -> Duration {
|
||||
self.config
|
||||
.scheduled_update_interval
|
||||
@@ -448,10 +552,13 @@ impl HybridCapacityManager {
|
||||
Self {
|
||||
cache: Arc::new(RwLock::new(None)),
|
||||
write_record: Arc::new(RwLock::new(WriteRecord {
|
||||
last_write_time: Instant::now(),
|
||||
last_write_time: None,
|
||||
write_count: 0,
|
||||
write_window: Vec::new(),
|
||||
write_buckets: [WriteBucket::default(); WRITE_WINDOW_BUCKETS],
|
||||
})),
|
||||
dirty_disks: Arc::new(RwLock::new(HashSet::new())),
|
||||
disk_cache: Arc::new(RwLock::new(HashMap::new())),
|
||||
disk_cache_complete: Arc::new(RwLock::new(false)),
|
||||
config,
|
||||
refresh_state: Arc::new(Mutex::new(RefreshState::default())),
|
||||
}
|
||||
@@ -471,49 +578,97 @@ impl HybridCapacityManager {
|
||||
/// Update capacity
|
||||
pub async fn update_capacity(&self, update: CapacityUpdate, source: DataSource) {
|
||||
let start = Instant::now();
|
||||
let mut total_used = update.total_used;
|
||||
|
||||
if !update.per_disk.is_empty() {
|
||||
let mut disk_cache = self.disk_cache.write().await;
|
||||
let mut disk_cache_complete = self.disk_cache_complete.write().await;
|
||||
|
||||
if update.replaces_disk_cache && update.expected_disk_count == Some(update.per_disk.len()) {
|
||||
disk_cache.clear();
|
||||
for entry in &update.per_disk {
|
||||
disk_cache.insert(
|
||||
entry.disk.clone(),
|
||||
CachedDiskCapacity {
|
||||
used_bytes: entry.used_bytes,
|
||||
},
|
||||
);
|
||||
}
|
||||
*disk_cache_complete = true;
|
||||
total_used = disk_cache.values().map(|entry| entry.used_bytes).sum();
|
||||
} else if *disk_cache_complete {
|
||||
for entry in &update.per_disk {
|
||||
disk_cache.insert(
|
||||
entry.disk.clone(),
|
||||
CachedDiskCapacity {
|
||||
used_bytes: entry.used_bytes,
|
||||
},
|
||||
);
|
||||
}
|
||||
total_used = disk_cache.values().map(|entry| entry.used_bytes).sum();
|
||||
}
|
||||
}
|
||||
|
||||
let mut cache = self.cache.write().await;
|
||||
*cache = Some(CachedCapacity {
|
||||
total_used: update.total_used,
|
||||
total_used,
|
||||
last_update: Instant::now(),
|
||||
file_count: update.file_count,
|
||||
is_estimated: update.is_estimated,
|
||||
source,
|
||||
});
|
||||
|
||||
if !update.clear_dirty_disks.is_empty() {
|
||||
let mut dirty_disks = self.dirty_disks.write().await;
|
||||
for disk in &update.clear_dirty_disks {
|
||||
dirty_disks.remove(disk);
|
||||
}
|
||||
record_capacity_dirty_disk_count(dirty_disks.len());
|
||||
}
|
||||
|
||||
debug!(
|
||||
"Capacity updated: {} bytes, files={}, estimated={}, source: {:?}",
|
||||
update.total_used, update.file_count, update.is_estimated, source
|
||||
total_used, update.file_count, update.is_estimated, source
|
||||
);
|
||||
record_capacity_current_bytes(update.total_used);
|
||||
record_capacity_update_completed(source.as_metric_label(), start.elapsed(), update.total_used, update.is_estimated);
|
||||
record_capacity_current_bytes(total_used);
|
||||
record_capacity_update_completed(source.as_metric_label(), start.elapsed(), total_used, update.is_estimated);
|
||||
}
|
||||
|
||||
/// Record write operation
|
||||
pub async fn record_write_operation(&self) {
|
||||
let mut record = self.write_record.write().await;
|
||||
record.last_write_time = Instant::now();
|
||||
record.write_count += 1;
|
||||
|
||||
// Maintain write time window (keep last 1 minute)
|
||||
// Cap the window size to prevent unbounded memory growth at high write rates
|
||||
const MAX_WRITE_WINDOW_SIZE: usize = 10000;
|
||||
let now = Instant::now();
|
||||
record
|
||||
.write_window
|
||||
.retain(|&t| now.duration_since(t) < Duration::from_secs(60));
|
||||
// Only push if under the cap to prevent unbounded growth
|
||||
if record.write_window.len() < MAX_WRITE_WINDOW_SIZE {
|
||||
record.write_window.push(now);
|
||||
}
|
||||
let recent_write_count = record.record_write(now);
|
||||
|
||||
record_capacity_write_operation(record.write_window.len());
|
||||
record_capacity_write_operation(recent_write_count);
|
||||
debug!(
|
||||
"Write operation recorded: total writes = {}, recent writes = {}",
|
||||
record.write_count,
|
||||
record.write_window.len()
|
||||
record.write_count, recent_write_count
|
||||
);
|
||||
}
|
||||
|
||||
/// Record write scope propagated from the storage layer.
|
||||
pub async fn mark_dirty_scope(&self, scope: &CapacityScope) {
|
||||
if scope.disks.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut dirty_disks = self.dirty_disks.write().await;
|
||||
dirty_disks.extend(scope.disks.iter().cloned());
|
||||
record_capacity_dirty_disk_count(dirty_disks.len());
|
||||
}
|
||||
|
||||
/// Record a write operation and consume any propagated disk scope bound to the token.
|
||||
pub async fn record_write_operation_with_scope_token(&self, scope_token: Option<uuid::Uuid>) {
|
||||
if let Some(token) = scope_token
|
||||
&& let Some(scope) = take_capacity_scope(token)
|
||||
{
|
||||
self.mark_dirty_scope(&scope).await;
|
||||
}
|
||||
|
||||
self.record_write_operation().await;
|
||||
}
|
||||
|
||||
/// Check if fast update is needed
|
||||
pub async fn needs_fast_update(&self) -> bool {
|
||||
if !self.config.enable_smart_update {
|
||||
@@ -529,19 +684,31 @@ impl HybridCapacityManager {
|
||||
return false;
|
||||
}
|
||||
|
||||
let write_record = self.write_record.read().await;
|
||||
let time_since_write = write_record.last_write_time.elapsed();
|
||||
|
||||
// Recent write, trigger fast update
|
||||
if time_since_write < self.config.fast_update_threshold {
|
||||
debug!("Recent write detected ({:?} ago), needs fast update", time_since_write);
|
||||
return true;
|
||||
if !self.config.enable_write_trigger {
|
||||
return false;
|
||||
}
|
||||
|
||||
// High write frequency, trigger update
|
||||
let write_frequency = write_record.write_window.len();
|
||||
if write_frequency > self.config.write_frequency_threshold {
|
||||
debug!("High write frequency detected ({} writes/min), needs fast update", write_frequency);
|
||||
let write_record = self.write_record.read().await;
|
||||
let write_frequency = write_record.recent_write_count(WriteRecord::current_unix_second());
|
||||
if write_frequency <= self.config.write_frequency_threshold {
|
||||
return false;
|
||||
}
|
||||
|
||||
if let Some(last_write_time) = write_record.last_write_time {
|
||||
let time_since_write = last_write_time.elapsed();
|
||||
|
||||
if time_since_write < self.config.write_trigger_delay {
|
||||
debug!(
|
||||
"Write-triggered refresh still debounced ({:?} ago, trigger_delay={:?}, writes/min={})",
|
||||
time_since_write, self.config.write_trigger_delay, write_frequency
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
debug!(
|
||||
"Write-triggered refresh eligible after debounce ({:?} ago, trigger_delay={:?}, writes/min={})",
|
||||
time_since_write, self.config.write_trigger_delay, write_frequency
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -560,7 +727,19 @@ impl HybridCapacityManager {
|
||||
#[allow(dead_code)]
|
||||
pub async fn get_write_frequency(&self) -> usize {
|
||||
let record = self.write_record.read().await;
|
||||
record.write_window.len()
|
||||
record.recent_write_count(WriteRecord::current_unix_second())
|
||||
}
|
||||
|
||||
/// Snapshot the currently dirty disks recorded from write-side scope propagation.
|
||||
pub async fn get_dirty_disks(&self) -> Vec<CapacityScopeDisk> {
|
||||
self.sync_global_dirty_scopes().await;
|
||||
let dirty_disks = self.dirty_disks.read().await;
|
||||
dirty_disks.iter().cloned().collect()
|
||||
}
|
||||
|
||||
/// Returns true if the manager has a complete per-disk cache and can safely refresh only dirty disks.
|
||||
pub async fn can_refresh_dirty_subset(&self) -> bool {
|
||||
*self.disk_cache_complete.read().await
|
||||
}
|
||||
|
||||
/// Run a singleflight refresh. Callers either join an existing in-flight refresh or become the leader.
|
||||
@@ -577,6 +756,7 @@ impl HybridCapacityManager {
|
||||
if state.running {
|
||||
// Subscribe while holding the lock so the send that completes the current
|
||||
// refresh cycle cannot happen before we are subscribed.
|
||||
record_capacity_refresh_joiner(source.as_metric_label());
|
||||
Some(state.result_tx.subscribe())
|
||||
} else {
|
||||
// Become the leader. Create a fresh channel so that joiners from a previous
|
||||
@@ -584,6 +764,7 @@ impl HybridCapacityManager {
|
||||
let (tx, _) = watch::channel(None);
|
||||
state.result_tx = tx;
|
||||
state.running = true;
|
||||
record_capacity_refresh_inflight(1);
|
||||
None
|
||||
}
|
||||
};
|
||||
@@ -603,6 +784,7 @@ impl HybridCapacityManager {
|
||||
.unwrap_or_else(|| Err("capacity refresh completed without a result".to_string()));
|
||||
}
|
||||
|
||||
let refresh_start = Instant::now();
|
||||
let result = AssertUnwindSafe(refresh_fn()).catch_unwind().await.unwrap_or_else(|err| {
|
||||
warn!(error = ?err, "capacity refresh function panicked");
|
||||
Err("capacity refresh panicked".to_string())
|
||||
@@ -610,10 +792,20 @@ impl HybridCapacityManager {
|
||||
if let Ok(update) = &result {
|
||||
self.update_capacity(update.clone(), source).await;
|
||||
}
|
||||
let refresh_duration = refresh_start.elapsed();
|
||||
if result.is_err() {
|
||||
record_capacity_update_failed(source.as_metric_label());
|
||||
}
|
||||
record_capacity_refresh_result(
|
||||
source.as_metric_label(),
|
||||
if result.is_ok() { "success" } else { "error" },
|
||||
refresh_duration,
|
||||
);
|
||||
|
||||
{
|
||||
let mut state = self.refresh_state.lock().await;
|
||||
state.running = false;
|
||||
record_capacity_refresh_inflight(0);
|
||||
let _ = state.result_tx.send(Some(result.clone()));
|
||||
}
|
||||
|
||||
@@ -634,6 +826,7 @@ impl HybridCapacityManager {
|
||||
let (tx, _) = watch::channel(None);
|
||||
state.result_tx = tx;
|
||||
state.running = true;
|
||||
record_capacity_refresh_inflight(1);
|
||||
true
|
||||
}
|
||||
};
|
||||
@@ -643,6 +836,7 @@ impl HybridCapacityManager {
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
let refresh_start = Instant::now();
|
||||
let result = AssertUnwindSafe(refresh_fn()).catch_unwind().await.unwrap_or_else(|err| {
|
||||
warn!(error = ?err, "capacity refresh function panicked");
|
||||
Err("capacity refresh panicked".to_string())
|
||||
@@ -650,9 +844,19 @@ impl HybridCapacityManager {
|
||||
if let Ok(update) = &result {
|
||||
self.update_capacity(update.clone(), source).await;
|
||||
}
|
||||
let refresh_duration = refresh_start.elapsed();
|
||||
if result.is_err() {
|
||||
record_capacity_update_failed(source.as_metric_label());
|
||||
}
|
||||
record_capacity_refresh_result(
|
||||
source.as_metric_label(),
|
||||
if result.is_ok() { "success" } else { "error" },
|
||||
refresh_duration,
|
||||
);
|
||||
|
||||
let mut state = self.refresh_state.lock().await;
|
||||
state.running = false;
|
||||
record_capacity_refresh_inflight(0);
|
||||
let _ = state.result_tx.send(Some(result));
|
||||
});
|
||||
|
||||
@@ -670,7 +874,6 @@ impl HybridCapacityManager {
|
||||
}
|
||||
|
||||
/// Return whether a refresh is currently in flight.
|
||||
#[cfg(test)]
|
||||
pub async fn refresh_in_progress(&self) -> bool {
|
||||
self.refresh_state.lock().await.running
|
||||
}
|
||||
@@ -692,20 +895,19 @@ pub fn get_capacity_manager() -> Arc<HybridCapacityManager> {
|
||||
/// without affecting the global singleton, avoiding test pollution.
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// ```ignore
|
||||
/// let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
/// manager
|
||||
/// .update_capacity(CapacityUpdate::exact(1000, 0), DataSource::RealTime)
|
||||
/// .await;
|
||||
/// ```
|
||||
#[cfg(test)]
|
||||
#[allow(dead_code)]
|
||||
pub fn create_isolated_manager(config: HybridStrategyConfig) -> Arc<HybridCapacityManager> {
|
||||
Arc::new(HybridCapacityManager::new(config))
|
||||
}
|
||||
|
||||
/// Start background update task
|
||||
pub async fn start_background_task(disks: Vec<rustfs_madmin::Disk>) {
|
||||
pub async fn start_background_task(disks: Vec<CapacityDiskRef>) {
|
||||
let manager = get_capacity_manager();
|
||||
let mut interval = manager.get_config().scheduled_update_interval;
|
||||
|
||||
@@ -727,12 +929,10 @@ pub async fn start_background_task(disks: Vec<rustfs_madmin::Disk>) {
|
||||
let disks = disks.clone();
|
||||
let started = manager
|
||||
.clone()
|
||||
.spawn_refresh_if_needed(DataSource::Scheduled, move || async move {
|
||||
calculate_data_dir_used_capacity(&disks)
|
||||
.await
|
||||
.map(|scan| scan.to_capacity_update())
|
||||
.map_err(|e| e.to_string())
|
||||
})
|
||||
.spawn_refresh_if_needed(
|
||||
DataSource::Scheduled,
|
||||
move || async move { refresh_capacity_with_scope(disks, false).await },
|
||||
)
|
||||
.await;
|
||||
|
||||
if started {
|
||||
@@ -751,11 +951,14 @@ pub async fn start_background_task(disks: Vec<rustfs_madmin::Disk>) {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope};
|
||||
use rustfs_config::{
|
||||
ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_SAMPLE_RATE,
|
||||
ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
@@ -892,6 +1095,22 @@ mod tests {
|
||||
assert_eq!(cached.unwrap().total_used, 1000);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_update_capacity_preserves_retrieval_metadata() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(1000, 10), DataSource::RealTime)
|
||||
.await;
|
||||
|
||||
let cached = manager.get_capacity().await.unwrap();
|
||||
assert_eq!(cached.total_used, 1000);
|
||||
assert_eq!(cached.file_count, 10);
|
||||
assert_eq!(cached.source, DataSource::RealTime);
|
||||
assert!(!cached.is_estimated);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_record_write_operation() {
|
||||
@@ -903,6 +1122,37 @@ mod tests {
|
||||
assert_eq!(frequency, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_write_frequency_window() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
for _ in 0..20 {
|
||||
manager.record_write_operation().await;
|
||||
}
|
||||
|
||||
assert_eq!(manager.get_write_frequency().await, 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_recent_write_count_ignores_future_buckets() {
|
||||
let mut record = WriteRecord {
|
||||
last_write_time: None,
|
||||
write_count: 1,
|
||||
write_buckets: [WriteBucket::default(); WRITE_WINDOW_BUCKETS],
|
||||
};
|
||||
|
||||
record.write_buckets[0] = WriteBucket { second: 120, count: 3 };
|
||||
record.write_buckets[1] = WriteBucket { second: 90, count: 2 };
|
||||
|
||||
assert_eq!(
|
||||
record.recent_write_count(100),
|
||||
2,
|
||||
"buckets from future seconds should not inflate recent write frequency"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_needs_fast_update() {
|
||||
@@ -920,6 +1170,326 @@ mod tests {
|
||||
assert!(!manager.needs_fast_update().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_cache_age_tracking() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
assert!(manager.get_cache_age().await.is_none());
|
||||
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(1000, 1), DataSource::RealTime)
|
||||
.await;
|
||||
|
||||
let age = manager.get_cache_age().await.unwrap();
|
||||
assert!(age < Duration::from_secs(1));
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let age = manager.get_cache_age().await.unwrap();
|
||||
assert!(age >= Duration::from_millis(100));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_data_source_tracking() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
for source in [
|
||||
DataSource::RealTime,
|
||||
DataSource::Scheduled,
|
||||
DataSource::WriteTriggered,
|
||||
DataSource::Fallback,
|
||||
] {
|
||||
manager.update_capacity(CapacityUpdate::exact(1000, 1), source).await;
|
||||
assert_eq!(manager.get_capacity().await.unwrap().source, source);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_needs_fast_update_waits_for_write_trigger_delay() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig {
|
||||
scheduled_update_interval: Duration::from_secs(60),
|
||||
write_trigger_delay: Duration::from_millis(50),
|
||||
write_frequency_threshold: 1,
|
||||
fast_update_threshold: Duration::from_millis(10),
|
||||
enable_smart_update: true,
|
||||
enable_write_trigger: true,
|
||||
});
|
||||
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(1000, 0), DataSource::RealTime)
|
||||
.await;
|
||||
tokio::time::sleep(Duration::from_millis(15)).await;
|
||||
|
||||
manager.record_write_operation().await;
|
||||
manager.record_write_operation().await;
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
|
||||
assert!(
|
||||
!manager.needs_fast_update().await,
|
||||
"write-triggered refresh should wait for debounce delay after a qualifying burst"
|
||||
);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(60)).await;
|
||||
assert!(manager.needs_fast_update().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_needs_fast_update_respects_enable_write_trigger() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig {
|
||||
scheduled_update_interval: Duration::from_secs(60),
|
||||
write_trigger_delay: Duration::from_secs(60),
|
||||
write_frequency_threshold: 1,
|
||||
fast_update_threshold: Duration::from_millis(10),
|
||||
enable_smart_update: true,
|
||||
enable_write_trigger: false,
|
||||
});
|
||||
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(1000, 0), DataSource::RealTime)
|
||||
.await;
|
||||
tokio::time::sleep(Duration::from_millis(15)).await;
|
||||
|
||||
manager.record_write_operation().await;
|
||||
manager.record_write_operation().await;
|
||||
|
||||
assert!(
|
||||
!manager.needs_fast_update().await,
|
||||
"write-triggered refresh should be disabled when enable_write_trigger is false"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrent_access() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let mut handles = Vec::new();
|
||||
|
||||
for i in 0..10 {
|
||||
let mgr = manager.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
mgr.update_capacity(CapacityUpdate::exact(i as u64 * 100, i), DataSource::RealTime)
|
||||
.await;
|
||||
mgr.record_write_operation().await;
|
||||
}));
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
handle.await.unwrap();
|
||||
}
|
||||
|
||||
assert!(manager.get_capacity().await.is_some());
|
||||
assert_eq!(manager.get_write_frequency().await, 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_performance_overhead() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let start = Instant::now();
|
||||
|
||||
for i in 0..1000 {
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(i as u64, i), DataSource::RealTime)
|
||||
.await;
|
||||
manager.record_write_operation().await;
|
||||
let _ = manager.get_capacity().await;
|
||||
}
|
||||
|
||||
assert!(start.elapsed() < Duration::from_secs(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_refresh_or_join_singleflight() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let mgr1 = manager.clone();
|
||||
let calls1 = calls.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
mgr1.refresh_or_join(DataSource::Scheduled, move || async move {
|
||||
calls1.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
Ok(CapacityUpdate::exact(2048, 8))
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
|
||||
let mgr2 = manager.clone();
|
||||
let calls2 = calls.clone();
|
||||
let second = tokio::spawn(async move {
|
||||
mgr2.refresh_or_join(DataSource::WriteTriggered, move || async move {
|
||||
calls2.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(CapacityUpdate::exact(4096, 16))
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
let first = first.await.unwrap().unwrap();
|
||||
let second = second.await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(first.total_used, 2048);
|
||||
assert_eq!(second.total_used, 2048);
|
||||
let cached = manager.get_capacity().await.unwrap();
|
||||
assert_eq!(cached.total_used, 2048);
|
||||
assert_eq!(cached.file_count, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_spawn_refresh_if_needed_deduplicates_background_refresh() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let first_manager = manager.clone();
|
||||
let first_calls = calls.clone();
|
||||
let started = first_manager
|
||||
.clone()
|
||||
.spawn_refresh_if_needed(DataSource::Scheduled, move || async move {
|
||||
first_calls.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
Ok(CapacityUpdate::estimated(8192, 32))
|
||||
})
|
||||
.await;
|
||||
assert!(started);
|
||||
|
||||
let second_manager = manager.clone();
|
||||
let second_calls = calls.clone();
|
||||
let started = second_manager
|
||||
.clone()
|
||||
.spawn_refresh_if_needed(DataSource::Scheduled, move || async move {
|
||||
second_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(CapacityUpdate::exact(1, 1))
|
||||
})
|
||||
.await;
|
||||
assert!(!started);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert!(!manager.refresh_in_progress().await);
|
||||
let cached = manager.get_capacity().await.unwrap();
|
||||
assert_eq!(cached.total_used, 8192);
|
||||
assert!(cached.is_estimated);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_record_write_operation_with_scope_token_marks_dirty_disks() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
let token = uuid::Uuid::new_v4();
|
||||
record_capacity_scope(
|
||||
token,
|
||||
CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
}],
|
||||
},
|
||||
);
|
||||
|
||||
manager.record_write_operation_with_scope_token(Some(token)).await;
|
||||
|
||||
let dirty_disks = manager.get_dirty_disks().await;
|
||||
assert_eq!(dirty_disks.len(), 1);
|
||||
assert_eq!(dirty_disks[0].endpoint, "node-a");
|
||||
assert_eq!(dirty_disks[0].drive_path, "/tmp/disk-a");
|
||||
assert_eq!(manager.get_write_frequency().await, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_dirty_disks_drains_global_dirty_scope_registry() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
record_global_dirty_scope(CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "node-bg".to_string(),
|
||||
drive_path: "/tmp/disk-bg".to_string(),
|
||||
}],
|
||||
});
|
||||
|
||||
let dirty_disks = manager.get_dirty_disks().await;
|
||||
assert_eq!(dirty_disks.len(), 1);
|
||||
assert_eq!(dirty_disks[0].endpoint, "node-bg");
|
||||
assert_eq!(dirty_disks[0].drive_path, "/tmp/disk-bg");
|
||||
|
||||
let second_read = manager.get_dirty_disks().await;
|
||||
assert_eq!(second_read.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_update_capacity_recomputes_total_from_disk_cache_for_subset_refresh() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
|
||||
manager
|
||||
.update_capacity(
|
||||
CapacityUpdate {
|
||||
total_used: 300,
|
||||
file_count: 3,
|
||||
is_estimated: false,
|
||||
per_disk: vec![
|
||||
DiskCapacityUpdate {
|
||||
disk: CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
},
|
||||
used_bytes: 100,
|
||||
file_count: 1,
|
||||
is_estimated: false,
|
||||
},
|
||||
DiskCapacityUpdate {
|
||||
disk: CapacityScopeDisk {
|
||||
endpoint: "node-b".to_string(),
|
||||
drive_path: "/tmp/disk-b".to_string(),
|
||||
},
|
||||
used_bytes: 200,
|
||||
file_count: 2,
|
||||
is_estimated: false,
|
||||
},
|
||||
],
|
||||
expected_disk_count: Some(2),
|
||||
replaces_disk_cache: true,
|
||||
clear_dirty_disks: Vec::new(),
|
||||
},
|
||||
DataSource::RealTime,
|
||||
)
|
||||
.await;
|
||||
|
||||
manager
|
||||
.update_capacity(
|
||||
CapacityUpdate {
|
||||
total_used: 150,
|
||||
file_count: 1,
|
||||
is_estimated: false,
|
||||
per_disk: vec![DiskCapacityUpdate {
|
||||
disk: CapacityScopeDisk {
|
||||
endpoint: "node-a".to_string(),
|
||||
drive_path: "/tmp/disk-a".to_string(),
|
||||
},
|
||||
used_bytes: 150,
|
||||
file_count: 1,
|
||||
is_estimated: false,
|
||||
}],
|
||||
expected_disk_count: Some(1),
|
||||
replaces_disk_cache: false,
|
||||
clear_dirty_disks: Vec::new(),
|
||||
},
|
||||
DataSource::WriteTriggered,
|
||||
)
|
||||
.await;
|
||||
|
||||
let cached = manager.get_capacity().await.unwrap();
|
||||
assert_eq!(cached.total_used, 350);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_config_from_env() {
|
||||
@@ -12,8 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::{SelectObjectContentEventStream, SelectObjectContentOutput};
|
||||
pub mod capacity_manager;
|
||||
pub mod scan;
|
||||
pub mod types;
|
||||
|
||||
pub(crate) fn build_select_object_content_output(payload: SelectObjectContentEventStream) -> SelectObjectContentOutput {
|
||||
SelectObjectContentOutput { payload: Some(payload) }
|
||||
}
|
||||
pub use scan::scan_used_capacity_disks;
|
||||
pub use types::{CapacityDiskRef, CapacityScanSummary};
|
||||
@@ -0,0 +1,903 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::capacity_manager::{
|
||||
CapacityUpdate, DiskCapacityUpdate, HybridCapacityManager, get_enable_dynamic_timeout, get_follow_symlinks,
|
||||
get_max_files_threshold, get_max_symlink_depth, get_max_timeout, get_min_timeout, get_sample_rate, get_stall_timeout,
|
||||
get_stat_timeout,
|
||||
};
|
||||
use super::types::{CapacityDiskRef, CapacityScanResult, CapacityScanSummary};
|
||||
use futures::{StreamExt, stream};
|
||||
use rustfs_common::capacity_scope::CapacityScopeDisk;
|
||||
use rustfs_io_metrics::capacity_metrics::{
|
||||
record_capacity_dynamic_timeout, record_capacity_scan_disk, record_capacity_scan_mode, record_capacity_scan_sampling,
|
||||
record_capacity_stall_detected, record_capacity_symlink, record_capacity_timeout_fallback,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info, warn};
|
||||
use walkdir::WalkDir;
|
||||
|
||||
const MAX_CAPACITY_SCAN_CONCURRENCY: usize = 4;
|
||||
const CAPACITY_PROGRESS_CHECK_STRIDE: usize = 512;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DiskScanOutcome {
|
||||
disk_label: String,
|
||||
drive_path: String,
|
||||
duration: Duration,
|
||||
result: Result<CapacityScanResult, std::io::Error>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct DiskCapacityScanResult {
|
||||
disk: CapacityScopeDisk,
|
||||
scan: CapacityScanResult,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct CapacityScanReport {
|
||||
summary: CapacityScanResult,
|
||||
per_disk: Vec<DiskCapacityScanResult>,
|
||||
}
|
||||
|
||||
impl CapacityScanReport {
|
||||
fn into_capacity_update(self, expected_disk_count: usize, replaces_disk_cache: bool) -> CapacityUpdate {
|
||||
let mut update = if self.summary.is_estimated {
|
||||
CapacityUpdate::estimated(self.summary.used_bytes, self.summary.file_count)
|
||||
} else {
|
||||
CapacityUpdate::exact(self.summary.used_bytes, self.summary.file_count)
|
||||
};
|
||||
|
||||
if !self.summary.had_partial_errors && self.per_disk.len() == expected_disk_count {
|
||||
update.per_disk = self
|
||||
.per_disk
|
||||
.into_iter()
|
||||
.map(|entry| DiskCapacityUpdate {
|
||||
disk: entry.disk,
|
||||
used_bytes: entry.scan.used_bytes,
|
||||
file_count: entry.scan.file_count,
|
||||
is_estimated: entry.scan.is_estimated,
|
||||
})
|
||||
.collect();
|
||||
update.expected_disk_count = Some(expected_disk_count);
|
||||
update.replaces_disk_cache = replaces_disk_cache;
|
||||
update.clear_dirty_disks = update.per_disk.iter().map(|entry| entry.disk.clone()).collect();
|
||||
}
|
||||
|
||||
update
|
||||
}
|
||||
}
|
||||
|
||||
fn disk_metric_label(disk: &CapacityDiskRef) -> String {
|
||||
let mount_name = Path::new(&disk.drive_path)
|
||||
.file_name()
|
||||
.and_then(|value| value.to_str())
|
||||
.filter(|value| !value.is_empty())
|
||||
.unwrap_or(disk.drive_path.as_str());
|
||||
format!("{}:{mount_name}", disk.endpoint)
|
||||
}
|
||||
|
||||
fn disk_scope_key(disk: &CapacityDiskRef) -> CapacityScopeDisk {
|
||||
CapacityScopeDisk {
|
||||
endpoint: disk.endpoint.clone(),
|
||||
drive_path: disk.drive_path.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan_disk_used_capacity(disk: CapacityDiskRef) -> DiskScanOutcome {
|
||||
let disk_label = disk_metric_label(&disk);
|
||||
let drive_path = disk.drive_path.clone();
|
||||
let start = Instant::now();
|
||||
let result = get_dir_size_async(Path::new(&drive_path)).await;
|
||||
|
||||
DiskScanOutcome {
|
||||
disk_label,
|
||||
drive_path,
|
||||
duration: start.elapsed(),
|
||||
result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn calculate_data_dir_used_capacity_report(
|
||||
disks: &[CapacityDiskRef],
|
||||
) -> Result<CapacityScanReport, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let start = Instant::now();
|
||||
let mut total_used = 0u64;
|
||||
let mut total_files = 0usize;
|
||||
let mut total_sampled = 0usize;
|
||||
let mut has_failure = false;
|
||||
let mut has_success = false;
|
||||
let mut is_estimated = false;
|
||||
let mut per_disk = Vec::with_capacity(disks.len());
|
||||
|
||||
let concurrency_limit = disks.len().clamp(1, MAX_CAPACITY_SCAN_CONCURRENCY);
|
||||
let mut scans = stream::iter(disks.iter().cloned().map(scan_disk_used_capacity)).buffer_unordered(concurrency_limit);
|
||||
|
||||
while let Some(outcome) = scans.next().await {
|
||||
match outcome.result {
|
||||
Ok(scan) => {
|
||||
record_capacity_scan_disk(
|
||||
outcome.disk_label.as_str(),
|
||||
outcome.duration,
|
||||
scan.file_count,
|
||||
scan.sampled_count,
|
||||
scan.is_estimated,
|
||||
scan.had_partial_errors,
|
||||
);
|
||||
debug!(
|
||||
"Data directory {} size: {} bytes, files={}, sampled={}, estimated={}, duration={:?}",
|
||||
outcome.drive_path, scan.used_bytes, scan.file_count, scan.sampled_count, scan.is_estimated, outcome.duration
|
||||
);
|
||||
total_used += scan.used_bytes;
|
||||
total_files += scan.file_count;
|
||||
total_sampled += scan.sampled_count;
|
||||
is_estimated |= scan.is_estimated;
|
||||
has_failure |= scan.had_partial_errors;
|
||||
has_success = true;
|
||||
if let Some(disk) = disks
|
||||
.iter()
|
||||
.find(|disk| disk.drive_path == outcome.drive_path && disk_metric_label(disk) == outcome.disk_label)
|
||||
{
|
||||
per_disk.push(DiskCapacityScanResult {
|
||||
disk: disk_scope_key(disk),
|
||||
scan,
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
record_capacity_scan_disk(outcome.disk_label.as_str(), outcome.duration, 0, 0, false, true);
|
||||
warn!("Failed to get size for directory {}: {:?}", outcome.drive_path, e);
|
||||
has_failure = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !has_success {
|
||||
return Err("All directories failed to calculate size".into());
|
||||
}
|
||||
|
||||
if has_failure {
|
||||
warn!("Some directories failed to calculate size, result may be incomplete");
|
||||
}
|
||||
|
||||
let mut summary = CapacityScanResult {
|
||||
used_bytes: total_used,
|
||||
file_count: total_files,
|
||||
sampled_count: total_sampled,
|
||||
is_estimated,
|
||||
scan_duration: start.elapsed(),
|
||||
had_partial_errors: false,
|
||||
};
|
||||
|
||||
if has_failure {
|
||||
summary = summary.with_partial_errors();
|
||||
}
|
||||
|
||||
Ok(CapacityScanReport { summary, per_disk })
|
||||
}
|
||||
|
||||
/// Calculate actual used capacity of all data directories.
|
||||
pub(crate) async fn calculate_data_dir_used_capacity(
|
||||
disks: &[CapacityDiskRef],
|
||||
) -> Result<CapacityScanResult, Box<dyn std::error::Error + Send + Sync>> {
|
||||
Ok(calculate_data_dir_used_capacity_report(disks).await?.summary)
|
||||
}
|
||||
|
||||
pub async fn select_capacity_refresh_disks(
|
||||
capacity_manager: &HybridCapacityManager,
|
||||
disks: &[CapacityDiskRef],
|
||||
) -> (Vec<CapacityDiskRef>, bool) {
|
||||
if !capacity_manager.can_refresh_dirty_subset().await {
|
||||
return (disks.to_vec(), false);
|
||||
}
|
||||
|
||||
let dirty_disks = capacity_manager.get_dirty_disks().await;
|
||||
if dirty_disks.is_empty() {
|
||||
return (disks.to_vec(), false);
|
||||
}
|
||||
|
||||
let dirty_set: HashSet<CapacityScopeDisk> = dirty_disks.into_iter().collect();
|
||||
let selected: Vec<_> = disks
|
||||
.iter()
|
||||
.filter(|disk| dirty_set.contains(&disk_scope_key(disk)))
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
if selected.is_empty() || selected.len() >= disks.len() {
|
||||
(disks.to_vec(), false)
|
||||
} else {
|
||||
(selected, true)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn refresh_capacity_with_scope(disks: Vec<CapacityDiskRef>, dirty_subset: bool) -> Result<CapacityUpdate, String> {
|
||||
let report = calculate_data_dir_used_capacity_report(&disks)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
if dirty_subset && report.summary.had_partial_errors {
|
||||
return Err("dirty subset refresh had partial errors".to_string());
|
||||
}
|
||||
|
||||
Ok(report.into_capacity_update(disks.len(), !dirty_subset))
|
||||
}
|
||||
|
||||
/// Scan the provided local disk roots and return a summarized used-capacity result.
|
||||
///
|
||||
/// This is primarily intended for benchmarks and operational tooling that need to exercise
|
||||
/// the same scan path as admin capacity queries without going through the full admin stack.
|
||||
pub async fn scan_used_capacity_disks(
|
||||
disks: &[CapacityDiskRef],
|
||||
) -> Result<CapacityScanSummary, Box<dyn std::error::Error + Send + Sync>> {
|
||||
Ok(calculate_data_dir_used_capacity(disks).await?.into())
|
||||
}
|
||||
|
||||
/// Tracker for symlink resolution with circular reference detection.
|
||||
struct SymlinkTracker {
|
||||
visited: HashSet<PathBuf>,
|
||||
symlink_count: usize,
|
||||
symlink_size: u64,
|
||||
max_depth: u8,
|
||||
}
|
||||
|
||||
impl SymlinkTracker {
|
||||
fn new(max_depth: u8) -> Self {
|
||||
Self {
|
||||
visited: HashSet::new(),
|
||||
symlink_count: 0,
|
||||
symlink_size: 0,
|
||||
max_depth,
|
||||
}
|
||||
}
|
||||
|
||||
fn should_follow(&self, path: &Path, depth: u8) -> bool {
|
||||
if depth >= self.max_depth {
|
||||
debug!("Symlink depth limit reached: {} >= {}, not following {:?}", depth, self.max_depth, path);
|
||||
return false;
|
||||
}
|
||||
|
||||
if self.visited.contains(path) {
|
||||
warn!("Circular symlink reference detected: {:?}, skipping", path);
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
fn record_symlink(&mut self, path: PathBuf, size: u64) {
|
||||
if self.visited.insert(path) {
|
||||
self.symlink_count += 1;
|
||||
self.symlink_size += size;
|
||||
record_capacity_symlink(size);
|
||||
}
|
||||
}
|
||||
|
||||
fn get_stats(&self) -> (usize, u64) {
|
||||
(self.symlink_count, self.symlink_size)
|
||||
}
|
||||
}
|
||||
|
||||
/// Monitor for directory traversal progress with timeout and stall detection.
|
||||
struct ProgressMonitor {
|
||||
start_time: Instant,
|
||||
last_check: Instant,
|
||||
last_checkpoint_files: usize,
|
||||
timeout: Duration,
|
||||
min_timeout: Duration,
|
||||
max_timeout: Duration,
|
||||
stall_timeout: Duration,
|
||||
enable_dynamic_timeout: bool,
|
||||
used_dynamic_timeout: bool,
|
||||
}
|
||||
|
||||
impl ProgressMonitor {
|
||||
fn new(
|
||||
base_timeout: Duration,
|
||||
min_timeout: Duration,
|
||||
max_timeout: Duration,
|
||||
stall_timeout: Duration,
|
||||
enable_dynamic: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
start_time: Instant::now(),
|
||||
last_check: Instant::now(),
|
||||
last_checkpoint_files: 0,
|
||||
timeout: base_timeout,
|
||||
min_timeout,
|
||||
max_timeout,
|
||||
stall_timeout,
|
||||
enable_dynamic_timeout: enable_dynamic,
|
||||
used_dynamic_timeout: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn calculate_dynamic_timeout(&mut self, file_count: usize, avg_file_size: u64) -> Duration {
|
||||
if !self.enable_dynamic_timeout {
|
||||
return self.timeout;
|
||||
}
|
||||
|
||||
self.used_dynamic_timeout = true;
|
||||
|
||||
let file_factor = (file_count as f64).sqrt() * 0.01;
|
||||
let size_factor = if avg_file_size > 0 {
|
||||
(avg_file_size as f64).log(10.0) * 0.05
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let multiplier = 1.0 + file_factor + size_factor;
|
||||
let adjusted_timeout = self.timeout.mul_f64(multiplier.min(5.0));
|
||||
let clamped_timeout = adjusted_timeout.max(self.min_timeout).min(self.max_timeout);
|
||||
|
||||
debug!(
|
||||
"Dynamic timeout calculation: files={}, avg_size={}, multiplier={:.2}, base_timeout={:?}, adjusted_timeout={:?}, clamped_timeout={:?}",
|
||||
file_count, avg_file_size, multiplier, self.timeout, adjusted_timeout, clamped_timeout
|
||||
);
|
||||
|
||||
clamped_timeout
|
||||
}
|
||||
|
||||
fn update_and_check_timeout(&mut self, files_processed: usize, avg_file_size: u64) -> Result<(), std::io::Error> {
|
||||
let elapsed = self.start_time.elapsed();
|
||||
let dynamic_timeout = if self.enable_dynamic_timeout {
|
||||
self.calculate_dynamic_timeout(files_processed, avg_file_size)
|
||||
} else {
|
||||
self.timeout
|
||||
};
|
||||
|
||||
if elapsed >= dynamic_timeout {
|
||||
warn!(
|
||||
"Directory size calculation timeout after {} files, elapsed: {:?}, timeout: {:?}",
|
||||
files_processed, elapsed, dynamic_timeout
|
||||
);
|
||||
|
||||
if self.enable_dynamic_timeout {
|
||||
record_capacity_dynamic_timeout(dynamic_timeout);
|
||||
}
|
||||
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
format!("Timeout after {} files", files_processed),
|
||||
));
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
if now.duration_since(self.last_check) >= self.stall_timeout {
|
||||
let files_per_checkpoint = files_processed.saturating_sub(self.last_checkpoint_files);
|
||||
|
||||
if files_per_checkpoint == 0 && files_processed > 0 {
|
||||
warn!(
|
||||
"No progress detected for {:?}, possible stall at {} files",
|
||||
self.stall_timeout, files_processed
|
||||
);
|
||||
|
||||
record_capacity_stall_detected();
|
||||
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
format!("Stall detected at {} files", files_processed),
|
||||
));
|
||||
}
|
||||
|
||||
self.last_check = now;
|
||||
self.last_checkpoint_files = files_processed;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn record_timeout_fallback(&self) {
|
||||
record_capacity_timeout_fallback();
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_dir_size_async(path: &Path) -> Result<CapacityScanResult, std::io::Error> {
|
||||
let path = path.to_path_buf();
|
||||
|
||||
let max_files_threshold = get_max_files_threshold();
|
||||
let base_timeout = get_stat_timeout();
|
||||
let min_timeout = get_min_timeout();
|
||||
let max_timeout = get_max_timeout();
|
||||
let stall_timeout = get_stall_timeout();
|
||||
let sample_rate = get_sample_rate();
|
||||
let enable_dynamic_timeout = get_enable_dynamic_timeout();
|
||||
let follow_symlinks = get_follow_symlinks();
|
||||
let max_symlink_depth = get_max_symlink_depth();
|
||||
|
||||
let effective_sample_rate = if sample_rate == 0 {
|
||||
warn!("Invalid sampling configuration: sample_rate=0. Clamping to 1 to avoid panic.");
|
||||
1
|
||||
} else {
|
||||
sample_rate
|
||||
};
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
if !path.exists() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("Directory not found: {:?}", path),
|
||||
));
|
||||
}
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut exact_prefix_bytes = 0u64;
|
||||
let mut overflow_sampled_bytes = 0u64;
|
||||
let mut file_count = 0usize;
|
||||
let mut sampled_count = 0usize;
|
||||
let mut had_partial_errors = false;
|
||||
let mut last_progress_check_files = 0usize;
|
||||
|
||||
let mut symlink_tracker = SymlinkTracker::new(max_symlink_depth);
|
||||
let mut progress_monitor =
|
||||
ProgressMonitor::new(base_timeout, min_timeout, max_timeout, stall_timeout, enable_dynamic_timeout);
|
||||
|
||||
let walker = WalkDir::new(&path)
|
||||
.follow_links(follow_symlinks)
|
||||
.follow_root_links(follow_symlinks)
|
||||
.into_iter();
|
||||
|
||||
for entry_result in walker {
|
||||
let entry = match entry_result {
|
||||
Ok(entry) => entry,
|
||||
Err(err) => {
|
||||
warn!("Failed to traverse directory entry under {:?}: {}", path, err);
|
||||
had_partial_errors = true;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if follow_symlinks
|
||||
&& entry.path_is_symlink()
|
||||
&& let Ok(target) = std::fs::read_link(entry.path())
|
||||
&& symlink_tracker.should_follow(&target, entry.depth().min(u8::MAX as usize) as u8)
|
||||
{
|
||||
symlink_tracker.record_symlink(target, 0);
|
||||
}
|
||||
|
||||
let file_type = entry.file_type();
|
||||
if file_type.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if file_type.is_symlink() || !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let metadata = match entry.metadata() {
|
||||
Ok(meta) => meta,
|
||||
Err(err) => {
|
||||
warn!("Failed to get metadata for {:?}: {}", entry.path(), err);
|
||||
had_partial_errors = true;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
file_count += 1;
|
||||
let exact_count = file_count.min(max_files_threshold);
|
||||
let avg_size = if exact_count > 0 {
|
||||
exact_prefix_bytes / exact_count as u64
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let should_check_progress =
|
||||
file_count == 1 || file_count.saturating_sub(last_progress_check_files) >= CAPACITY_PROGRESS_CHECK_STRIDE;
|
||||
|
||||
if should_check_progress && let Err(e) = progress_monitor.update_and_check_timeout(file_count, avg_size) {
|
||||
if sampled_count > 0 {
|
||||
let overflow_count = file_count.saturating_sub(max_files_threshold);
|
||||
let estimated_overflow = overflow_sampled_bytes.saturating_mul(overflow_count as u64) / sampled_count as u64;
|
||||
let estimated_total = exact_prefix_bytes.saturating_add(estimated_overflow);
|
||||
info!(
|
||||
"Timeout/stall at {} files, using sampled estimate: exact_prefix={} overflow_estimate={} sampled={}",
|
||||
file_count, exact_prefix_bytes, estimated_overflow, sampled_count
|
||||
);
|
||||
progress_monitor.record_timeout_fallback();
|
||||
record_capacity_scan_sampling(sampled_count, true);
|
||||
record_capacity_scan_mode("timeout_fallback");
|
||||
return Ok(CapacityScanResult {
|
||||
used_bytes: estimated_total,
|
||||
file_count,
|
||||
sampled_count,
|
||||
is_estimated: true,
|
||||
scan_duration: start_time.elapsed(),
|
||||
had_partial_errors,
|
||||
});
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
if should_check_progress {
|
||||
last_progress_check_files = file_count;
|
||||
}
|
||||
|
||||
if file_count <= max_files_threshold {
|
||||
exact_prefix_bytes += metadata.len();
|
||||
} else {
|
||||
let overflow_index = file_count - max_files_threshold;
|
||||
if overflow_index.is_multiple_of(effective_sample_rate) {
|
||||
overflow_sampled_bytes += metadata.len();
|
||||
sampled_count += 1;
|
||||
}
|
||||
|
||||
if file_count.is_multiple_of(100_000) {
|
||||
debug!(
|
||||
"Processed {} files, exact_prefix_bytes={}, sampled_overflow={} files/{} bytes",
|
||||
file_count, exact_prefix_bytes, sampled_count, overflow_sampled_bytes
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if file_count > last_progress_check_files {
|
||||
let exact_count = file_count.min(max_files_threshold);
|
||||
let avg_size = if exact_count > 0 {
|
||||
exact_prefix_bytes / exact_count as u64
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
if let Err(e) = progress_monitor.update_and_check_timeout(file_count, avg_size) {
|
||||
if sampled_count > 0 {
|
||||
let overflow_count = file_count.saturating_sub(max_files_threshold);
|
||||
let estimated_overflow = overflow_sampled_bytes.saturating_mul(overflow_count as u64) / sampled_count as u64;
|
||||
let estimated_total = exact_prefix_bytes.saturating_add(estimated_overflow);
|
||||
info!(
|
||||
"Timeout/stall at {} files during final check, using sampled estimate: exact_prefix={} overflow_estimate={} sampled={}",
|
||||
file_count, exact_prefix_bytes, estimated_overflow, sampled_count
|
||||
);
|
||||
progress_monitor.record_timeout_fallback();
|
||||
record_capacity_scan_sampling(sampled_count, true);
|
||||
record_capacity_scan_mode("timeout_fallback");
|
||||
return Ok(CapacityScanResult {
|
||||
used_bytes: estimated_total,
|
||||
file_count,
|
||||
sampled_count,
|
||||
is_estimated: true,
|
||||
scan_duration: start_time.elapsed(),
|
||||
had_partial_errors,
|
||||
});
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
let (symlink_count, symlink_size) = symlink_tracker.get_stats();
|
||||
if symlink_count > 0 {
|
||||
info!(
|
||||
"Symlink tracking: {} symlinks processed, total tracked size: {} bytes",
|
||||
symlink_count, symlink_size
|
||||
);
|
||||
}
|
||||
|
||||
if file_count > max_files_threshold && sampled_count > 0 {
|
||||
let overflow_count = file_count - max_files_threshold;
|
||||
let estimated_overflow = overflow_sampled_bytes.saturating_mul(overflow_count as u64) / sampled_count as u64;
|
||||
let estimated_size = exact_prefix_bytes.saturating_add(estimated_overflow);
|
||||
info!(
|
||||
"Large directory detected: {} files, estimated size: {} bytes (exact prefix: {}, sampled overflow {}/{})",
|
||||
file_count, estimated_size, exact_prefix_bytes, sampled_count, overflow_count
|
||||
);
|
||||
record_capacity_scan_sampling(sampled_count, true);
|
||||
record_capacity_scan_mode("estimated");
|
||||
Ok(CapacityScanResult {
|
||||
used_bytes: estimated_size,
|
||||
file_count,
|
||||
sampled_count,
|
||||
is_estimated: true,
|
||||
scan_duration: start_time.elapsed(),
|
||||
had_partial_errors,
|
||||
})
|
||||
} else if file_count > max_files_threshold {
|
||||
let overflow_count = file_count - max_files_threshold;
|
||||
let exact_prefix_count = file_count.min(max_files_threshold) as u64;
|
||||
let avg_prefix_size = if exact_prefix_count > 0 {
|
||||
exact_prefix_bytes / exact_prefix_count
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let estimated_overflow = avg_prefix_size.saturating_mul(overflow_count as u64);
|
||||
let estimated_size = exact_prefix_bytes.saturating_add(estimated_overflow);
|
||||
info!(
|
||||
"Large directory detected: {} files, estimated size: {} bytes (no overflow samples, used prefix average {} bytes/file)",
|
||||
file_count, estimated_size, avg_prefix_size
|
||||
);
|
||||
record_capacity_scan_sampling(0, true);
|
||||
record_capacity_scan_mode("estimated");
|
||||
Ok(CapacityScanResult {
|
||||
used_bytes: estimated_size,
|
||||
file_count,
|
||||
sampled_count: 0,
|
||||
is_estimated: true,
|
||||
scan_duration: start_time.elapsed(),
|
||||
had_partial_errors,
|
||||
})
|
||||
} else {
|
||||
record_capacity_scan_sampling(0, false);
|
||||
debug!(
|
||||
"Directory size calculation completed: {} files, {} bytes, took {:?}",
|
||||
file_count,
|
||||
exact_prefix_bytes,
|
||||
start_time.elapsed()
|
||||
);
|
||||
record_capacity_scan_mode("exact");
|
||||
Ok(CapacityScanResult {
|
||||
used_bytes: exact_prefix_bytes,
|
||||
file_count,
|
||||
sampled_count,
|
||||
is_estimated: false,
|
||||
scan_duration: start_time.elapsed(),
|
||||
had_partial_errors,
|
||||
})
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(std::io::Error::other)?
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::capacity_manager::{DataSource, HybridStrategyConfig, create_isolated_manager};
|
||||
use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk};
|
||||
use rustfs_config::ENV_CAPACITY_FOLLOW_SYMLINKS;
|
||||
use serial_test::serial;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_dir_size_async_empty_directory() {
|
||||
use tempfile::TempDir;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let size = get_dir_size_async(temp_dir.path()).await.unwrap();
|
||||
assert_eq!(size.used_bytes, 0);
|
||||
assert_eq!(size.file_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_dir_size_async_single_file() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let file_path = temp_dir.path().join("test.txt");
|
||||
let mut file = File::create(&file_path).unwrap();
|
||||
file.write_all(b"Hello, World!").unwrap();
|
||||
|
||||
let size = get_dir_size_async(temp_dir.path()).await.unwrap();
|
||||
assert_eq!(size.used_bytes, 13);
|
||||
assert_eq!(size.file_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_dir_size_async_multiple_files() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
for i in 0..10 {
|
||||
let file_path = temp_dir.path().join(format!("file_{}.txt", i));
|
||||
let mut file = File::create(&file_path).unwrap();
|
||||
file.write_all(b"test").unwrap();
|
||||
}
|
||||
|
||||
let size = get_dir_size_async(temp_dir.path()).await.unwrap();
|
||||
assert_eq!(size.used_bytes, 40);
|
||||
assert_eq!(size.file_count, 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_dir_size_async_nested_directories() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let subdir = temp_dir.path().join("subdir");
|
||||
std::fs::create_dir(&subdir).unwrap();
|
||||
|
||||
let file1 = temp_dir.path().join("file1.txt");
|
||||
let mut f1 = File::create(&file1).unwrap();
|
||||
f1.write_all(b"content1").unwrap();
|
||||
|
||||
let file2 = subdir.join("file2.txt");
|
||||
let mut f2 = File::create(&file2).unwrap();
|
||||
f2.write_all(b"content2").unwrap();
|
||||
|
||||
let size = get_dir_size_async(temp_dir.path()).await.unwrap();
|
||||
assert_eq!(size.used_bytes, 16);
|
||||
assert_eq!(size.file_count, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_dir_size_async_nonexistent_directory() {
|
||||
let result = get_dir_size_async(Path::new("/nonexistent/path")).await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_calculate_data_dir_used_capacity_returns_partial_success() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let file_path = temp_dir.path().join("test.txt");
|
||||
let mut file = File::create(&file_path).unwrap();
|
||||
file.write_all(b"Hello, World!").unwrap();
|
||||
|
||||
let disks = vec![
|
||||
CapacityDiskRef {
|
||||
endpoint: "disk-1".to_string(),
|
||||
drive_path: temp_dir.path().to_string_lossy().into_owned(),
|
||||
},
|
||||
CapacityDiskRef {
|
||||
endpoint: "disk-2".to_string(),
|
||||
drive_path: "/nonexistent/path".to_string(),
|
||||
},
|
||||
];
|
||||
|
||||
let result = calculate_data_dir_used_capacity(&disks).await.unwrap();
|
||||
assert_eq!(result.used_bytes, 13);
|
||||
assert_eq!(result.file_count, 1);
|
||||
assert!(result.had_partial_errors);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_select_capacity_refresh_disks_returns_full_when_disk_cache_incomplete() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
manager
|
||||
.mark_dirty_scope(&CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "disk-1".to_string(),
|
||||
drive_path: "/tmp/disk-1".to_string(),
|
||||
}],
|
||||
})
|
||||
.await;
|
||||
|
||||
let disks = vec![
|
||||
CapacityDiskRef {
|
||||
endpoint: "disk-1".to_string(),
|
||||
drive_path: "/tmp/disk-1".to_string(),
|
||||
},
|
||||
CapacityDiskRef {
|
||||
endpoint: "disk-2".to_string(),
|
||||
drive_path: "/tmp/disk-2".to_string(),
|
||||
},
|
||||
];
|
||||
|
||||
let (selected, dirty_subset) = select_capacity_refresh_disks(manager.as_ref(), &disks).await;
|
||||
assert!(!dirty_subset);
|
||||
assert_eq!(selected.len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_select_capacity_refresh_disks_returns_dirty_subset_when_cache_complete() {
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
manager
|
||||
.update_capacity(
|
||||
CapacityUpdate {
|
||||
total_used: 300,
|
||||
file_count: 3,
|
||||
is_estimated: false,
|
||||
per_disk: vec![
|
||||
DiskCapacityUpdate {
|
||||
disk: CapacityScopeDisk {
|
||||
endpoint: "disk-1".to_string(),
|
||||
drive_path: "/tmp/disk-1".to_string(),
|
||||
},
|
||||
used_bytes: 100,
|
||||
file_count: 1,
|
||||
is_estimated: false,
|
||||
},
|
||||
DiskCapacityUpdate {
|
||||
disk: CapacityScopeDisk {
|
||||
endpoint: "disk-2".to_string(),
|
||||
drive_path: "/tmp/disk-2".to_string(),
|
||||
},
|
||||
used_bytes: 200,
|
||||
file_count: 2,
|
||||
is_estimated: false,
|
||||
},
|
||||
],
|
||||
expected_disk_count: Some(2),
|
||||
replaces_disk_cache: true,
|
||||
clear_dirty_disks: Vec::new(),
|
||||
},
|
||||
DataSource::RealTime,
|
||||
)
|
||||
.await;
|
||||
manager
|
||||
.mark_dirty_scope(&CapacityScope {
|
||||
disks: vec![CapacityScopeDisk {
|
||||
endpoint: "disk-2".to_string(),
|
||||
drive_path: "/tmp/disk-2".to_string(),
|
||||
}],
|
||||
})
|
||||
.await;
|
||||
|
||||
let disks = vec![
|
||||
CapacityDiskRef {
|
||||
endpoint: "disk-1".to_string(),
|
||||
drive_path: "/tmp/disk-1".to_string(),
|
||||
},
|
||||
CapacityDiskRef {
|
||||
endpoint: "disk-2".to_string(),
|
||||
drive_path: "/tmp/disk-2".to_string(),
|
||||
},
|
||||
];
|
||||
|
||||
let (selected, dirty_subset) = select_capacity_refresh_disks(manager.as_ref(), &disks).await;
|
||||
assert!(dirty_subset);
|
||||
assert_eq!(selected.len(), 1);
|
||||
assert_eq!(selected[0].endpoint, "disk-2");
|
||||
assert_eq!(selected[0].drive_path, "/tmp/disk-2");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_dir_size_async_ignores_symlink_targets_when_follow_disabled() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::os::unix::fs::symlink;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let scan_dir = TempDir::new().unwrap();
|
||||
let target_dir = TempDir::new().unwrap();
|
||||
let target_path = target_dir.path().join("external.txt");
|
||||
let mut file = File::create(&target_path).unwrap();
|
||||
file.write_all(b"external-bytes").unwrap();
|
||||
symlink(&target_path, scan_dir.path().join("external-link")).unwrap();
|
||||
|
||||
let size = temp_env::async_with_vars([(ENV_CAPACITY_FOLLOW_SYMLINKS, Some("false"))], async {
|
||||
get_dir_size_async(scan_dir.path()).await
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(size.used_bytes, 0);
|
||||
assert_eq!(size.file_count, 0);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_dir_size_async_counts_symlink_targets_when_follow_enabled() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::os::unix::fs::symlink;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let scan_dir = TempDir::new().unwrap();
|
||||
let target_dir = TempDir::new().unwrap();
|
||||
let target_path = target_dir.path().join("external.txt");
|
||||
let mut file = File::create(&target_path).unwrap();
|
||||
file.write_all(b"external-bytes").unwrap();
|
||||
symlink(&target_path, scan_dir.path().join("external-link")).unwrap();
|
||||
|
||||
let size = temp_env::async_with_vars([(ENV_CAPACITY_FOLLOW_SYMLINKS, Some("true"))], async {
|
||||
get_dir_size_async(scan_dir.path()).await
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(size.used_bytes, "external-bytes".len() as u64);
|
||||
assert_eq!(size.file_count, 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct CapacityDiskRef {
|
||||
pub endpoint: String,
|
||||
pub drive_path: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub(crate) struct CapacityScanResult {
|
||||
pub used_bytes: u64,
|
||||
pub file_count: usize,
|
||||
pub sampled_count: usize,
|
||||
pub is_estimated: bool,
|
||||
pub scan_duration: Duration,
|
||||
pub had_partial_errors: bool,
|
||||
}
|
||||
|
||||
impl CapacityScanResult {
|
||||
pub(crate) fn with_partial_errors(mut self) -> Self {
|
||||
self.had_partial_errors = true;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Public summary type for external tooling such as benches.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct CapacityScanSummary {
|
||||
pub used_bytes: u64,
|
||||
pub file_count: usize,
|
||||
pub sampled_count: usize,
|
||||
pub is_estimated: bool,
|
||||
pub had_partial_errors: bool,
|
||||
pub scan_duration: Duration,
|
||||
}
|
||||
|
||||
impl From<CapacityScanResult> for CapacityScanSummary {
|
||||
fn from(scan: CapacityScanResult) -> Self {
|
||||
Self {
|
||||
used_bytes: scan.used_bytes,
|
||||
file_count: scan.file_count,
|
||||
sampled_count: scan.sampled_count,
|
||||
is_estimated: scan.is_estimated,
|
||||
had_partial_errors: scan.had_partial_errors,
|
||||
scan_duration: scan.scan_duration,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -143,6 +143,14 @@ pub fn chunk_body_data_plane_labels(
|
||||
)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn get_object_chunk_path_label(path: GetObjectChunkPath) -> &'static str {
|
||||
match path {
|
||||
GetObjectChunkPath::Direct => "direct",
|
||||
GetObjectChunkPath::Bridge => "bridge",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_object_chunk_fast_path_guard(
|
||||
has_sse_customer_key: bool,
|
||||
has_sse_customer_key_md5: bool,
|
||||
|
||||
+2
-2
@@ -91,6 +91,7 @@ rustfs-zip = { workspace = true }
|
||||
rustfs-io-core = { workspace = true }
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
rustfs-object-io = { workspace = true }
|
||||
rustfs-object-capacity = { workspace = true }
|
||||
rustfs-concurrency = { workspace = true }
|
||||
rustfs-scanner = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
@@ -119,7 +120,6 @@ tower-http = { workspace = true, features = ["trace", "compression-full", "cors"
|
||||
# Serialization and Data Formats
|
||||
bytes = { workspace = true }
|
||||
flatbuffers.workspace = true
|
||||
walkdir = { workspace = true }
|
||||
rmp-serde.workspace = true
|
||||
rustfs-signer.workspace = true
|
||||
serde.workspace = true
|
||||
@@ -197,7 +197,7 @@ tempfile = { workspace = true }
|
||||
aws-config = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
temp-env = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
|
||||
[build-dependencies]
|
||||
http.workspace = true
|
||||
|
||||
+1
-1
@@ -81,7 +81,7 @@ To get started with RustFS, follow these steps:
|
||||
podman run -d -p 9000:9000 -v /data:/data rustfs/rustfs:latest
|
||||
```
|
||||
|
||||
3. **Access the Console**: Open your web browser and navigate to `http://localhost:9000` to access the RustFS console,
|
||||
3. **Access the Console**: Open your web browser and navigate to `http://localhost:9001` to access the RustFS console,
|
||||
default username and password is `rustfsadmin` .
|
||||
4. **Create a Bucket**: Use the console to create a new bucket for your objects.
|
||||
5. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs to interact with your
|
||||
|
||||
@@ -15,10 +15,7 @@
|
||||
//! Admin application use-case contracts.
|
||||
|
||||
use crate::app::context::{AppContext, get_global_app_context};
|
||||
use crate::capacity::capacity_manager::{
|
||||
CapacityUpdate, DataSource, get_capacity_manager, get_enable_dynamic_timeout, get_follow_symlinks, get_max_files_threshold,
|
||||
get_max_symlink_depth, get_max_timeout, get_min_timeout, get_sample_rate, get_stall_timeout, get_stat_timeout,
|
||||
};
|
||||
use crate::capacity::resolve_admin_used_capacity;
|
||||
use crate::error::ApiError;
|
||||
use rustfs_common::data_usage::DataUsageInfo;
|
||||
use rustfs_ecstore::admin_server_info::get_server_info;
|
||||
@@ -27,18 +24,10 @@ use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
use rustfs_ecstore::new_object_layer_fn;
|
||||
use rustfs_ecstore::pools::{PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free};
|
||||
use rustfs_ecstore::store_api::StorageAPI;
|
||||
use rustfs_io_metrics::{
|
||||
record_capacity_dynamic_timeout, record_capacity_scan_sampling, record_capacity_stall_detected, record_capacity_symlink,
|
||||
record_capacity_timeout_fallback,
|
||||
};
|
||||
use rustfs_madmin::{InfoMessage, StorageInfo};
|
||||
use s3s::S3ErrorCode;
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, error, info, warn};
|
||||
use walkdir::WalkDir;
|
||||
|
||||
pub type AdminUsecaseResult<T> = Result<T, ApiError>;
|
||||
|
||||
@@ -47,31 +36,6 @@ pub struct QueryServerInfoRequest {
|
||||
pub include_pools: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub(crate) struct CapacityScanResult {
|
||||
pub used_bytes: u64,
|
||||
pub file_count: usize,
|
||||
pub sampled_count: usize,
|
||||
pub is_estimated: bool,
|
||||
pub scan_duration: Duration,
|
||||
pub had_partial_errors: bool,
|
||||
}
|
||||
|
||||
impl CapacityScanResult {
|
||||
fn with_partial_errors(mut self) -> Self {
|
||||
self.had_partial_errors = true;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn to_capacity_update(self) -> CapacityUpdate {
|
||||
if self.is_estimated {
|
||||
CapacityUpdate::estimated(self.used_bytes, self.file_count)
|
||||
} else {
|
||||
CapacityUpdate::exact(self.used_bytes, self.file_count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct QueryServerInfoResponse {
|
||||
pub info: InfoMessage,
|
||||
}
|
||||
@@ -94,475 +58,6 @@ pub struct QueryPoolStatusRequest {
|
||||
pub by_id: bool,
|
||||
}
|
||||
|
||||
/// Calculate actual used capacity of all data directories
|
||||
pub(crate) async fn calculate_data_dir_used_capacity(
|
||||
disks: &[rustfs_madmin::Disk],
|
||||
) -> Result<CapacityScanResult, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let start = Instant::now();
|
||||
let mut total_used = 0u64;
|
||||
let mut total_files = 0usize;
|
||||
let mut total_sampled = 0usize;
|
||||
let mut has_failure = false;
|
||||
let mut has_success = false;
|
||||
let mut is_estimated = false;
|
||||
|
||||
for disk in disks {
|
||||
let path = Path::new(&disk.drive_path);
|
||||
|
||||
if !path.exists() {
|
||||
warn!("Data directory does not exist: {}", disk.drive_path);
|
||||
has_failure = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
match get_dir_size_async(path).await {
|
||||
Ok(scan) => {
|
||||
debug!(
|
||||
"Data directory {} size: {} bytes, files={}, sampled={}, estimated={}",
|
||||
disk.drive_path, scan.used_bytes, scan.file_count, scan.sampled_count, scan.is_estimated
|
||||
);
|
||||
total_used += scan.used_bytes;
|
||||
total_files += scan.file_count;
|
||||
total_sampled += scan.sampled_count;
|
||||
is_estimated |= scan.is_estimated;
|
||||
has_failure |= scan.had_partial_errors;
|
||||
has_success = true;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to get size for directory {}: {:?}", disk.drive_path, e);
|
||||
has_failure = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !has_success {
|
||||
return Err("All directories failed to calculate size".into());
|
||||
}
|
||||
|
||||
if has_failure {
|
||||
warn!("Some directories failed to calculate size, result may be incomplete");
|
||||
}
|
||||
|
||||
let mut result = CapacityScanResult {
|
||||
used_bytes: total_used,
|
||||
file_count: total_files,
|
||||
sampled_count: total_sampled,
|
||||
is_estimated,
|
||||
scan_duration: start.elapsed(),
|
||||
had_partial_errors: false,
|
||||
};
|
||||
|
||||
if has_failure {
|
||||
result = result.with_partial_errors();
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Symlink Tracker for Circular Reference Detection
|
||||
// ============================================================================
|
||||
|
||||
/// Tracker for symlink resolution with circular reference detection
|
||||
struct SymlinkTracker {
|
||||
/// Set of visited symlink paths to detect circular references
|
||||
visited: HashSet<PathBuf>,
|
||||
/// Count of symlinks encountered
|
||||
symlink_count: usize,
|
||||
/// Total size of symlink targets
|
||||
symlink_size: u64,
|
||||
/// Maximum symlink depth to follow
|
||||
max_depth: u8,
|
||||
}
|
||||
|
||||
impl SymlinkTracker {
|
||||
/// Create a new symlink tracker
|
||||
fn new(max_depth: u8) -> Self {
|
||||
Self {
|
||||
visited: HashSet::new(),
|
||||
symlink_count: 0,
|
||||
symlink_size: 0,
|
||||
max_depth,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if we should follow a symlink at the given depth
|
||||
fn should_follow(&self, path: &Path, depth: u8) -> bool {
|
||||
if depth >= self.max_depth {
|
||||
debug!("Symlink depth limit reached: {} >= {}, not following {:?}", depth, self.max_depth, path);
|
||||
return false;
|
||||
}
|
||||
|
||||
if self.visited.contains(path) {
|
||||
warn!("Circular symlink reference detected: {:?}, skipping", path);
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
/// Record a visited symlink path and update metrics
|
||||
fn record_symlink(&mut self, path: PathBuf, size: u64) {
|
||||
self.visited.insert(path);
|
||||
self.symlink_count += 1;
|
||||
self.symlink_size += size;
|
||||
record_capacity_symlink(size);
|
||||
}
|
||||
|
||||
/// Get symlink statistics
|
||||
fn get_stats(&self) -> (usize, u64) {
|
||||
(self.symlink_count, self.symlink_size)
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Progress Monitor for Timeout and Stall Detection
|
||||
// ============================================================================
|
||||
|
||||
/// Monitor for directory traversal progress with timeout and stall detection
|
||||
struct ProgressMonitor {
|
||||
/// Start time of the operation
|
||||
start_time: Instant,
|
||||
/// Last check time for stall detection
|
||||
last_check: Instant,
|
||||
/// Number of files processed at last checkpoint
|
||||
last_checkpoint_files: usize,
|
||||
/// Base timeout for this operation
|
||||
timeout: Duration,
|
||||
/// Minimum allowed timeout
|
||||
min_timeout: Duration,
|
||||
/// Maximum allowed timeout
|
||||
max_timeout: Duration,
|
||||
/// Stall detection timeout
|
||||
stall_timeout: Duration,
|
||||
/// Enable dynamic timeout calculation
|
||||
enable_dynamic_timeout: bool,
|
||||
/// Track if dynamic timeout was used
|
||||
used_dynamic_timeout: bool,
|
||||
}
|
||||
|
||||
impl ProgressMonitor {
|
||||
/// Create a new progress monitor
|
||||
fn new(
|
||||
base_timeout: Duration,
|
||||
min_timeout: Duration,
|
||||
max_timeout: Duration,
|
||||
stall_timeout: Duration,
|
||||
enable_dynamic: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
start_time: Instant::now(),
|
||||
last_check: Instant::now(),
|
||||
last_checkpoint_files: 0,
|
||||
timeout: base_timeout,
|
||||
min_timeout,
|
||||
max_timeout,
|
||||
stall_timeout,
|
||||
enable_dynamic_timeout: enable_dynamic,
|
||||
used_dynamic_timeout: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Calculate dynamic timeout based on directory characteristics
|
||||
fn calculate_dynamic_timeout(&mut self, file_count: usize, avg_file_size: u64) -> Duration {
|
||||
if !self.enable_dynamic_timeout {
|
||||
return self.timeout;
|
||||
}
|
||||
|
||||
// Mark that we're using dynamic timeout
|
||||
self.used_dynamic_timeout = true;
|
||||
|
||||
// Calculate multipliers based on directory characteristics
|
||||
let file_factor = (file_count as f64).sqrt() * 0.01; // File count influence
|
||||
let size_factor = if avg_file_size > 0 {
|
||||
(avg_file_size as f64).log(10.0) * 0.05 // File size influence
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let multiplier = 1.0 + file_factor + size_factor;
|
||||
let adjusted_timeout = self.timeout.mul_f64(multiplier.min(5.0)); // Max 5x multiplier
|
||||
|
||||
// Clamp to min/max bounds
|
||||
let clamped_timeout = adjusted_timeout.max(self.min_timeout).min(self.max_timeout);
|
||||
|
||||
debug!(
|
||||
"Dynamic timeout calculation: files={}, avg_size={}, multiplier={:.2}, base_timeout={:?}, adjusted_timeout={:?}, clamped_timeout={:?}",
|
||||
file_count, avg_file_size, multiplier, self.timeout, adjusted_timeout, clamped_timeout
|
||||
);
|
||||
|
||||
clamped_timeout
|
||||
}
|
||||
|
||||
/// Update and check for timeout or stall
|
||||
fn update_and_check_timeout(&mut self, files_processed: usize, avg_file_size: u64) -> Result<(), std::io::Error> {
|
||||
let elapsed = self.start_time.elapsed();
|
||||
|
||||
// Calculate dynamic timeout based on current state
|
||||
let dynamic_timeout = if self.enable_dynamic_timeout {
|
||||
self.calculate_dynamic_timeout(files_processed, avg_file_size)
|
||||
} else {
|
||||
self.timeout
|
||||
};
|
||||
|
||||
// Check for hard timeout
|
||||
if elapsed >= dynamic_timeout {
|
||||
warn!(
|
||||
"Directory size calculation timeout after {} files, elapsed: {:?}, timeout: {:?}",
|
||||
files_processed, elapsed, dynamic_timeout
|
||||
);
|
||||
|
||||
if self.enable_dynamic_timeout {
|
||||
record_capacity_dynamic_timeout(dynamic_timeout);
|
||||
}
|
||||
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
format!("Timeout after {} files", files_processed),
|
||||
));
|
||||
}
|
||||
|
||||
// Check for stall (no progress)
|
||||
let now = Instant::now();
|
||||
if now.duration_since(self.last_check) >= self.stall_timeout {
|
||||
let files_per_checkpoint = files_processed.saturating_sub(self.last_checkpoint_files);
|
||||
|
||||
if files_per_checkpoint == 0 && files_processed > 0 {
|
||||
// No progress for stall_timeout duration
|
||||
warn!(
|
||||
"No progress detected for {:?}, possible stall at {} files",
|
||||
self.stall_timeout, files_processed
|
||||
);
|
||||
|
||||
record_capacity_stall_detected();
|
||||
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
format!("Stall detected at {} files", files_processed),
|
||||
));
|
||||
}
|
||||
|
||||
self.last_check = now;
|
||||
self.last_checkpoint_files = files_processed;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Record timeout fallback to sampling
|
||||
fn record_timeout_fallback(&self) {
|
||||
record_capacity_timeout_fallback();
|
||||
}
|
||||
}
|
||||
|
||||
/// Asynchronously get directory size with enhanced symlink handling and dynamic timeout
|
||||
async fn get_dir_size_async(path: &Path) -> Result<CapacityScanResult, std::io::Error> {
|
||||
let path = path.to_path_buf();
|
||||
|
||||
let max_files_threshold = get_max_files_threshold();
|
||||
let base_timeout = get_stat_timeout();
|
||||
let min_timeout = get_min_timeout();
|
||||
let max_timeout = get_max_timeout();
|
||||
let stall_timeout = get_stall_timeout();
|
||||
let sample_rate = get_sample_rate();
|
||||
let enable_dynamic_timeout = get_enable_dynamic_timeout();
|
||||
let follow_symlinks = get_follow_symlinks();
|
||||
let max_symlink_depth = get_max_symlink_depth();
|
||||
|
||||
let effective_sample_rate = if sample_rate == 0 {
|
||||
warn!("Invalid sampling configuration: sample_rate=0. Clamping to 1 to avoid panic.");
|
||||
1
|
||||
} else {
|
||||
sample_rate
|
||||
};
|
||||
|
||||
if !path.exists() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("Directory not found: {:?}", path),
|
||||
));
|
||||
}
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let start_time = Instant::now();
|
||||
let mut exact_prefix_bytes = 0u64;
|
||||
let mut overflow_sampled_bytes = 0u64;
|
||||
let mut file_count = 0usize;
|
||||
let mut sampled_count = 0usize;
|
||||
let mut had_partial_errors = false;
|
||||
|
||||
let mut symlink_tracker = if follow_symlinks {
|
||||
Some(SymlinkTracker::new(max_symlink_depth))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut progress_monitor =
|
||||
ProgressMonitor::new(base_timeout, min_timeout, max_timeout, stall_timeout, enable_dynamic_timeout);
|
||||
|
||||
let mut walker_builder = WalkDir::new(&path);
|
||||
if !follow_symlinks {
|
||||
walker_builder = walker_builder.follow_links(false);
|
||||
}
|
||||
let walker = walker_builder.into_iter();
|
||||
|
||||
for entry_result in walker {
|
||||
let entry = match entry_result {
|
||||
Ok(entry) => entry,
|
||||
Err(err) => {
|
||||
warn!("Failed to traverse directory entry under {:?}: {}", path, err);
|
||||
had_partial_errors = true;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let metadata = match entry.metadata() {
|
||||
Ok(meta) => meta,
|
||||
Err(err) => {
|
||||
warn!("Failed to get metadata for {:?}: {}", entry.path(), err);
|
||||
had_partial_errors = true;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if metadata.is_symlink() {
|
||||
if let Some(ref mut tracker) = symlink_tracker
|
||||
&& let Ok(target) = std::fs::read_link(entry.path())
|
||||
&& tracker.should_follow(&target, 0)
|
||||
{
|
||||
tracker.record_symlink(target, metadata.len());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if !metadata.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
file_count += 1;
|
||||
let exact_count = file_count.min(max_files_threshold);
|
||||
let avg_size = if exact_count > 0 {
|
||||
exact_prefix_bytes / exact_count as u64
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
if let Err(e) = progress_monitor.update_and_check_timeout(file_count, avg_size) {
|
||||
if sampled_count > 0 {
|
||||
let overflow_count = file_count.saturating_sub(max_files_threshold);
|
||||
let estimated_overflow = overflow_sampled_bytes.saturating_mul(overflow_count as u64) / sampled_count as u64;
|
||||
let estimated_total = exact_prefix_bytes.saturating_add(estimated_overflow);
|
||||
info!(
|
||||
"Timeout/stall at {} files, using sampled estimate: exact_prefix={} overflow_estimate={} sampled={}",
|
||||
file_count, exact_prefix_bytes, estimated_overflow, sampled_count
|
||||
);
|
||||
progress_monitor.record_timeout_fallback();
|
||||
record_capacity_scan_sampling(sampled_count, true);
|
||||
return Ok(CapacityScanResult {
|
||||
used_bytes: estimated_total,
|
||||
file_count,
|
||||
sampled_count,
|
||||
is_estimated: true,
|
||||
scan_duration: start_time.elapsed(),
|
||||
had_partial_errors,
|
||||
});
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if file_count <= max_files_threshold {
|
||||
exact_prefix_bytes += metadata.len();
|
||||
} else {
|
||||
let overflow_index = file_count - max_files_threshold;
|
||||
if overflow_index.is_multiple_of(effective_sample_rate) {
|
||||
overflow_sampled_bytes += metadata.len();
|
||||
sampled_count += 1;
|
||||
}
|
||||
|
||||
if file_count.is_multiple_of(100_000) {
|
||||
debug!(
|
||||
"Processed {} files, exact_prefix_bytes={}, sampled_overflow={} files/{} bytes",
|
||||
file_count, exact_prefix_bytes, sampled_count, overflow_sampled_bytes
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(tracker) = symlink_tracker {
|
||||
let (count, size) = tracker.get_stats();
|
||||
if count > 0 {
|
||||
info!("Symlink tracking: {} symlinks processed, total target size: {} bytes", count, size);
|
||||
}
|
||||
}
|
||||
|
||||
if file_count > max_files_threshold && sampled_count > 0 {
|
||||
let overflow_count = file_count - max_files_threshold;
|
||||
let estimated_overflow = overflow_sampled_bytes.saturating_mul(overflow_count as u64) / sampled_count as u64;
|
||||
let estimated_size = exact_prefix_bytes.saturating_add(estimated_overflow);
|
||||
info!(
|
||||
"Large directory detected: {} files, estimated size: {} bytes (exact prefix: {}, sampled overflow {}/{})",
|
||||
file_count, estimated_size, exact_prefix_bytes, sampled_count, overflow_count
|
||||
);
|
||||
record_capacity_scan_sampling(sampled_count, true);
|
||||
Ok(CapacityScanResult {
|
||||
used_bytes: estimated_size,
|
||||
file_count,
|
||||
sampled_count,
|
||||
is_estimated: true,
|
||||
scan_duration: start_time.elapsed(),
|
||||
had_partial_errors,
|
||||
})
|
||||
} else if file_count > max_files_threshold {
|
||||
// sampled_count == 0: too few overflow files to reach the sample rate threshold.
|
||||
// Fall back to estimating the overflow using the average file size from the exact
|
||||
// prefix so that overflow files are not silently dropped from the total.
|
||||
let overflow_count = file_count - max_files_threshold;
|
||||
// Use the actual number of files counted in the exact prefix, not the threshold
|
||||
// value, to avoid a divide-by-zero or incorrect average when fewer files were
|
||||
// processed than max_files_threshold.
|
||||
let exact_prefix_count = file_count.min(max_files_threshold) as u64;
|
||||
let avg_prefix_size = if exact_prefix_count > 0 {
|
||||
exact_prefix_bytes / exact_prefix_count
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let estimated_overflow = avg_prefix_size.saturating_mul(overflow_count as u64);
|
||||
let estimated_size = exact_prefix_bytes.saturating_add(estimated_overflow);
|
||||
info!(
|
||||
"Large directory detected: {} files, estimated size: {} bytes (no overflow samples, used prefix average {} bytes/file)",
|
||||
file_count, estimated_size, avg_prefix_size
|
||||
);
|
||||
record_capacity_scan_sampling(0, true);
|
||||
Ok(CapacityScanResult {
|
||||
used_bytes: estimated_size,
|
||||
file_count,
|
||||
sampled_count: 0,
|
||||
is_estimated: true,
|
||||
scan_duration: start_time.elapsed(),
|
||||
had_partial_errors,
|
||||
})
|
||||
} else {
|
||||
record_capacity_scan_sampling(0, false);
|
||||
debug!(
|
||||
"Directory size calculation completed: {} files, {} bytes, took {:?}",
|
||||
file_count,
|
||||
exact_prefix_bytes,
|
||||
start_time.elapsed()
|
||||
);
|
||||
Ok(CapacityScanResult {
|
||||
used_bytes: exact_prefix_bytes,
|
||||
file_count,
|
||||
sampled_count,
|
||||
is_estimated: false,
|
||||
scan_duration: start_time.elapsed(),
|
||||
had_partial_errors,
|
||||
})
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(std::io::Error::other)?
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct DefaultAdminUsecase {
|
||||
context: Option<Arc<AppContext>>,
|
||||
@@ -688,115 +183,8 @@ impl DefaultAdminUsecase {
|
||||
info.total_free_capacity = free_u64;
|
||||
}
|
||||
|
||||
// Use hybrid strategy for capacity calculation
|
||||
let capacity_manager = get_capacity_manager();
|
||||
|
||||
// Check if we have a valid cache
|
||||
if let Some(cached) = capacity_manager.get_capacity().await {
|
||||
let cache_age = cached.last_update.elapsed();
|
||||
let fast_update_threshold = capacity_manager.get_config().fast_update_threshold;
|
||||
|
||||
// If cache is fresh (< fast_update_threshold), use it directly
|
||||
if cache_age < fast_update_threshold {
|
||||
info.total_used_capacity = cached.total_used;
|
||||
debug!(
|
||||
"Using cached capacity: {} bytes (age: {:?}, source: {:?}, files={}, estimated={})",
|
||||
cached.total_used, cache_age, cached.source, cached.file_count, cached.is_estimated
|
||||
);
|
||||
} else {
|
||||
// Cache is stale, check if we need fast update
|
||||
let needs_update = capacity_manager.needs_fast_update().await;
|
||||
let should_block = capacity_manager.should_block_on_refresh(cache_age);
|
||||
|
||||
if needs_update && should_block {
|
||||
let start = Instant::now();
|
||||
match capacity_manager
|
||||
.refresh_or_join(DataSource::WriteTriggered, || async {
|
||||
calculate_data_dir_used_capacity(&storage_info.disks)
|
||||
.await
|
||||
.map(|scan| scan.to_capacity_update())
|
||||
.map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(update) => {
|
||||
info.total_used_capacity = update.total_used;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
debug!(
|
||||
"Foreground capacity refresh completed in {:?} (files={}, estimated={})",
|
||||
elapsed, update.file_count, update.is_estimated
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Foreground capacity refresh failed: {}, using cached value", e);
|
||||
info.total_used_capacity = cached.total_used;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
info.total_used_capacity = cached.total_used;
|
||||
debug!(
|
||||
"Using stale cached capacity: {} bytes (age: {:?}, source: {:?}, files={}, estimated={}, needs_update={}, blocking={})",
|
||||
cached.total_used,
|
||||
cache_age,
|
||||
cached.source,
|
||||
cached.file_count,
|
||||
cached.is_estimated,
|
||||
needs_update,
|
||||
should_block
|
||||
);
|
||||
|
||||
let disks = storage_info.disks.clone();
|
||||
let manager = capacity_manager.clone();
|
||||
if manager
|
||||
.clone()
|
||||
.spawn_refresh_if_needed(DataSource::Scheduled, move || async move {
|
||||
calculate_data_dir_used_capacity(&disks)
|
||||
.await
|
||||
.map(|scan| scan.to_capacity_update())
|
||||
.map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
{
|
||||
debug!("Background capacity update started");
|
||||
} else {
|
||||
debug!("Background update already in progress, skipping spawn");
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No cache, perform initial calculation
|
||||
let start = Instant::now();
|
||||
match capacity_manager
|
||||
.refresh_or_join(DataSource::RealTime, || async {
|
||||
calculate_data_dir_used_capacity(&storage_info.disks)
|
||||
.await
|
||||
.map(|scan| scan.to_capacity_update())
|
||||
.map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(update) => {
|
||||
info.total_used_capacity = update.total_used;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
info!(
|
||||
"Initial capacity calculation completed: {} bytes in {:?} (files={}, estimated={})",
|
||||
update.total_used, elapsed, update.file_count, update.is_estimated
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"Failed to calculate data directory used capacity: {}, falling back to disk used capacity",
|
||||
e
|
||||
);
|
||||
info.total_used_capacity = info.total_capacity.saturating_sub(info.total_free_capacity);
|
||||
capacity_manager
|
||||
.update_capacity(CapacityUpdate::fallback(info.total_used_capacity), DataSource::Fallback)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
info.total_used_capacity =
|
||||
resolve_admin_used_capacity(&storage_info.disks, info.total_capacity.saturating_sub(info.total_free_capacity)).await;
|
||||
debug!(
|
||||
"Capacity statistics: total={:.2} TiB, free={:.2} TiB, used={:.2} TiB",
|
||||
info.total_capacity as f64 / (1024.0_f64.powi(4)),
|
||||
@@ -885,7 +273,6 @@ impl DefaultAdminUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serial_test::serial;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_query_storage_info_returns_internal_error_when_store_uninitialized() {
|
||||
@@ -911,83 +298,4 @@ mod tests {
|
||||
let _ = readiness.storage_ready;
|
||||
let _ = readiness.iam_ready;
|
||||
}
|
||||
|
||||
// Tests for directory size calculation functions
|
||||
#[tokio::test]
|
||||
async fn test_get_dir_size_async_empty_directory() {
|
||||
use tempfile::TempDir;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let size = get_dir_size_async(temp_dir.path()).await.unwrap();
|
||||
assert_eq!(size.used_bytes, 0);
|
||||
assert_eq!(size.file_count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_dir_size_async_single_file() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let file_path = temp_dir.path().join("test.txt");
|
||||
let mut file = File::create(&file_path).unwrap();
|
||||
file.write_all(b"Hello, World!").unwrap();
|
||||
|
||||
let size = get_dir_size_async(temp_dir.path()).await.unwrap();
|
||||
assert_eq!(size.used_bytes, 13);
|
||||
assert_eq!(size.file_count, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_dir_size_async_multiple_files() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
// Create multiple files
|
||||
for i in 0..10 {
|
||||
let file_path = temp_dir.path().join(format!("file_{}.txt", i));
|
||||
let mut file = File::create(&file_path).unwrap();
|
||||
file.write_all(b"test").unwrap();
|
||||
}
|
||||
|
||||
let size = get_dir_size_async(temp_dir.path()).await.unwrap();
|
||||
assert_eq!(size.used_bytes, 40); // 10 files * 4 bytes
|
||||
assert_eq!(size.file_count, 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_dir_size_async_nested_directories() {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
// Create nested directories and files
|
||||
let subdir = temp_dir.path().join("subdir");
|
||||
std::fs::create_dir(&subdir).unwrap();
|
||||
|
||||
let file1 = temp_dir.path().join("file1.txt");
|
||||
let mut f1 = File::create(&file1).unwrap();
|
||||
f1.write_all(b"content1").unwrap();
|
||||
|
||||
let file2 = subdir.join("file2.txt");
|
||||
let mut f2 = File::create(&file2).unwrap();
|
||||
f2.write_all(b"content2").unwrap();
|
||||
|
||||
let size = get_dir_size_async(temp_dir.path()).await.unwrap();
|
||||
assert_eq!(size.used_bytes, 16); // "content1" (8) + "content2" (8)
|
||||
assert_eq!(size.file_count, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_get_dir_size_async_nonexistent_directory() {
|
||||
let result = get_dir_size_async(Path::new("/nonexistent/path")).await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
+176
-136
@@ -23,9 +23,12 @@ use crate::error::ApiError;
|
||||
use crate::server::RemoteAddr;
|
||||
use crate::storage::access::{ReqInfo, authorize_request, req_info_ref};
|
||||
use crate::storage::helper::{OperationHelper, spawn_background_with_context};
|
||||
use crate::storage::s3_api::bucket::{build_list_buckets_output, build_list_objects_v2_output};
|
||||
use crate::storage::s3_api::acl;
|
||||
use crate::storage::s3_api::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
};
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::s3_api::{acl, encryption, replication, tagging};
|
||||
use crate::storage::*;
|
||||
use futures::StreamExt;
|
||||
use http::StatusCode;
|
||||
@@ -970,7 +973,7 @@ impl DefaultBucketUsecase {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket tagging delete hook failed");
|
||||
}
|
||||
|
||||
Ok(S3Response::new(tagging::build_delete_bucket_tagging_output()))
|
||||
Ok(S3Response::new(DeleteBucketTaggingOutput {}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
@@ -1030,9 +1033,9 @@ impl DefaultBucketUsecase {
|
||||
}
|
||||
};
|
||||
|
||||
Ok(S3Response::new(encryption::build_get_bucket_encryption_output(
|
||||
Ok(S3Response::new(GetBucketEncryptionOutput {
|
||||
server_side_encryption_configuration,
|
||||
)))
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
@@ -1294,9 +1297,9 @@ impl DefaultBucketUsecase {
|
||||
}
|
||||
};
|
||||
|
||||
Ok(S3Response::new(replication::build_get_bucket_replication_output(
|
||||
replication_configuration,
|
||||
)))
|
||||
Ok(S3Response::new(GetBucketReplicationOutput {
|
||||
replication_configuration: Some(replication_configuration),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
@@ -1330,7 +1333,7 @@ impl DefaultBucketUsecase {
|
||||
}
|
||||
};
|
||||
|
||||
Ok(S3Response::new(tagging::build_get_bucket_tagging_output(tag_set)))
|
||||
Ok(S3Response::new(GetBucketTaggingOutput { tag_set }))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
@@ -1436,7 +1439,7 @@ impl DefaultBucketUsecase {
|
||||
if let Err(err) = site_replication_bucket_meta_hook(item).await {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket encryption hook failed");
|
||||
}
|
||||
Ok(S3Response::new(encryption::build_put_bucket_encryption_output()))
|
||||
Ok(S3Response::new(PutBucketEncryptionOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
@@ -1494,10 +1497,7 @@ impl DefaultBucketUsecase {
|
||||
&& let Some(store) = new_object_layer_fn()
|
||||
{
|
||||
let bucket_name = bucket.clone();
|
||||
let request_context = req
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.cloned();
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
spawn_background_with_context(request_context, async move {
|
||||
if let Err(err) = enqueue_transition_for_existing_objects(store, &bucket_name).await {
|
||||
warn!(bucket = %bucket_name, error = ?err, "failed to enqueue transition for existing objects");
|
||||
@@ -1721,7 +1721,7 @@ impl DefaultBucketUsecase {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket replication-config hook failed");
|
||||
}
|
||||
|
||||
Ok(S3Response::new(replication::build_put_bucket_replication_output()))
|
||||
Ok(S3Response::new(PutBucketReplicationOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
@@ -1788,7 +1788,7 @@ impl DefaultBucketUsecase {
|
||||
warn!(bucket = %bucket, error = ?err, "site replication bucket tagging hook failed");
|
||||
}
|
||||
|
||||
Ok(S3Response::new(tagging::build_put_bucket_tagging_output()))
|
||||
Ok(S3Response::new(PutBucketTaggingOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
@@ -1844,46 +1844,21 @@ impl DefaultBucketUsecase {
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
let prefix = prefix.unwrap_or_default();
|
||||
|
||||
// Log debug info for prefixes with special characters to help diagnose encoding issues
|
||||
if prefix.contains([' ', '+', '%', '\n', '\r', '\0']) {
|
||||
debug!("LIST objects with special characters in prefix: {:?}", prefix);
|
||||
}
|
||||
|
||||
let max_keys = max_keys.unwrap_or(1000);
|
||||
if max_keys < 0 {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid max keys".to_string()));
|
||||
}
|
||||
|
||||
let delimiter = delimiter.filter(|v| !v.is_empty());
|
||||
let ListObjectsV2Params {
|
||||
prefix,
|
||||
max_keys,
|
||||
delimiter,
|
||||
response_start_after,
|
||||
start_after_for_query,
|
||||
response_continuation_token,
|
||||
decoded_continuation_token,
|
||||
} = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
|
||||
|
||||
validate_list_object_unordered_with_delimiter(delimiter.as_ref(), req.uri.query())?;
|
||||
|
||||
// Save original start_after for response (per S3 API spec, must echo back if provided)
|
||||
let response_start_after = start_after.clone();
|
||||
let start_after_for_query = start_after.filter(|v| !v.is_empty());
|
||||
|
||||
// Save original continuation_token for response (per S3 API spec, must echo back if provided)
|
||||
// Note: empty string should still be echoed back in the response
|
||||
let response_continuation_token = continuation_token.clone();
|
||||
let continuation_token_for_query = continuation_token.filter(|v| !v.is_empty());
|
||||
|
||||
// Decode continuation_token from base64 for internal use
|
||||
let decoded_continuation_token = continuation_token_for_query
|
||||
.map(|token| {
|
||||
base64_simd::STANDARD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(|_| s3_error!(InvalidArgument, "Invalid continuation token"))
|
||||
.and_then(|bytes| {
|
||||
String::from_utf8(bytes).map_err(|_| s3_error!(InvalidArgument, "Invalid continuation token"))
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
|
||||
let incl_deleted = rustfs_utils::http::get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
let incl_deleted = get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
.map(|v| v.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
@@ -1937,33 +1912,20 @@ impl DefaultBucketUsecase {
|
||||
..
|
||||
} = input;
|
||||
|
||||
let prefix = prefix.unwrap_or_default();
|
||||
let max_keys = max_keys.unwrap_or(1000);
|
||||
if max_keys < 0 {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid max keys".to_string()));
|
||||
}
|
||||
let ListObjectsV2Params {
|
||||
prefix,
|
||||
max_keys,
|
||||
delimiter,
|
||||
response_start_after,
|
||||
start_after_for_query,
|
||||
response_continuation_token,
|
||||
decoded_continuation_token,
|
||||
} = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
|
||||
|
||||
let delimiter = delimiter.filter(|value| !value.is_empty());
|
||||
validate_list_object_unordered_with_delimiter(delimiter.as_ref(), req.uri.query())?;
|
||||
|
||||
let response_start_after = start_after.clone();
|
||||
let start_after_for_query = start_after.filter(|value| !value.is_empty());
|
||||
let response_continuation_token = continuation_token.clone();
|
||||
let continuation_token_for_query = continuation_token.filter(|value| !value.is_empty());
|
||||
|
||||
let decoded_continuation_token = continuation_token_for_query
|
||||
.map(|token| {
|
||||
base64_simd::STANDARD
|
||||
.decode_to_vec(token.as_bytes())
|
||||
.map_err(|_| s3_error!(InvalidArgument, "Invalid continuation token"))
|
||||
.and_then(|bytes| {
|
||||
String::from_utf8(bytes).map_err(|_| s3_error!(InvalidArgument, "Invalid continuation token"))
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
let incl_deleted = rustfs_utils::http::get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
let incl_deleted = get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
.map(|value| value.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
@@ -2015,12 +1977,13 @@ impl DefaultBucketUsecase {
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
let prefix = prefix.unwrap_or_default();
|
||||
let max_keys = max_keys.unwrap_or(1000);
|
||||
|
||||
let key_marker = key_marker.filter(|v| !v.is_empty());
|
||||
let version_id_marker = version_id_marker.filter(|v| !v.is_empty());
|
||||
let delimiter = delimiter.filter(|v| !v.is_empty());
|
||||
let ListObjectVersionsParams {
|
||||
prefix,
|
||||
delimiter,
|
||||
key_marker,
|
||||
version_id_marker,
|
||||
max_keys,
|
||||
} = parse_list_object_versions_params(prefix, delimiter, key_marker, version_id_marker, max_keys)?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
|
||||
@@ -2029,57 +1992,7 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let objects: Vec<ObjectVersion> = object_infos
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|v| !v.name.is_empty() && !v.delete_marker)
|
||||
.map(|v| ObjectVersion {
|
||||
key: Some(v.name.to_owned()),
|
||||
last_modified: v.mod_time.map(Timestamp::from),
|
||||
size: Some(v.size),
|
||||
version_id: Some(v.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string())),
|
||||
is_latest: Some(v.is_latest),
|
||||
e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
storage_class: v.storage_class.clone().map(ObjectVersionStorageClass::from),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let common_prefixes = object_infos
|
||||
.prefixes
|
||||
.into_iter()
|
||||
.map(|v| CommonPrefix { prefix: Some(v) })
|
||||
.collect();
|
||||
|
||||
let delete_markers = object_infos
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|o| o.delete_marker)
|
||||
.map(|o| DeleteMarkerEntry {
|
||||
key: Some(o.name.clone()),
|
||||
version_id: Some(o.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string())),
|
||||
is_latest: Some(o.is_latest),
|
||||
last_modified: o.mod_time.map(Timestamp::from),
|
||||
..Default::default()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let next_key_marker = object_infos.next_marker.filter(|v| !v.is_empty());
|
||||
let next_version_id_marker = object_infos.next_version_idmarker.filter(|v| !v.is_empty());
|
||||
|
||||
let output = ListObjectVersionsOutput {
|
||||
is_truncated: Some(object_infos.is_truncated),
|
||||
max_keys: Some(max_keys),
|
||||
delimiter,
|
||||
name: Some(bucket),
|
||||
prefix: Some(prefix),
|
||||
common_prefixes: Some(common_prefixes),
|
||||
versions: Some(objects),
|
||||
delete_markers: Some(delete_markers),
|
||||
next_key_marker,
|
||||
next_version_id_marker,
|
||||
..Default::default()
|
||||
};
|
||||
let output = build_list_object_versions_output(object_infos, bucket, prefix, delimiter, max_keys);
|
||||
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
@@ -2104,11 +2017,13 @@ impl DefaultBucketUsecase {
|
||||
..
|
||||
} = input;
|
||||
|
||||
let prefix = prefix.unwrap_or_default();
|
||||
let max_keys = max_keys.unwrap_or(1000);
|
||||
let key_marker = key_marker.filter(|value| !value.is_empty());
|
||||
let version_id_marker = version_id_marker.filter(|value| !value.is_empty());
|
||||
let delimiter = delimiter.filter(|value| !value.is_empty());
|
||||
let ListObjectVersionsParams {
|
||||
prefix,
|
||||
delimiter,
|
||||
key_marker,
|
||||
version_id_marker,
|
||||
max_keys,
|
||||
} = parse_list_object_versions_params(prefix, delimiter, key_marker, version_id_marker, max_keys)?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
let object_infos = store
|
||||
@@ -2440,6 +2355,20 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_bucket_encryption_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetBucketEncryptionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_get_bucket_encryption(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_bucket_replication_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetBucketReplicationInput::builder()
|
||||
@@ -2468,6 +2397,20 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_bucket_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetBucketTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_get_bucket_tagging(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_bucket_versioning_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetBucketVersioningInput::builder()
|
||||
@@ -2926,6 +2869,41 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_bucket_encryption_returns_internal_error_when_store_uninitialized() {
|
||||
let input = PutBucketEncryptionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.server_side_encryption_configuration(ServerSideEncryptionConfiguration::default())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_put_bucket_encryption(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_bucket_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
let input = PutBucketTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("prod".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_put_bucket_tagging(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_public_access_block_returns_internal_error_when_store_uninitialized() {
|
||||
let input = PutPublicAccessBlockInput::builder()
|
||||
@@ -2956,6 +2934,22 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_objects_v2_rejects_invalid_continuation_token_before_store_lookup() {
|
||||
let input = ListObjectsV2Input::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.continuation_token(Some("%%%".to_string()))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_list_objects_v2(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("Invalid continuation token"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_objects_v2m_rejects_negative_max_keys() {
|
||||
let input = ListObjectsV2Input::builder()
|
||||
@@ -2970,4 +2964,50 @@ mod tests {
|
||||
let err = usecase.execute_list_objects_v2m(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_objects_v2m_rejects_invalid_continuation_token_before_store_lookup() {
|
||||
let input = ListObjectsV2Input::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.continuation_token(Some("%%%".to_string()))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_list_objects_v2m(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("Invalid continuation token"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_object_versions_rejects_negative_max_keys_before_store_lookup() {
|
||||
let input = ListObjectVersionsInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.max_keys(Some(-1))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_list_object_versions(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_object_versions_m_rejects_negative_max_keys_before_store_lookup() {
|
||||
let input = ListObjectVersionsInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.max_keys(Some(-1))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_list_object_versions_m(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_ecstore::{
|
||||
bucket::metadata_sys,
|
||||
disk::endpoint::Endpoint,
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
store::ECStore,
|
||||
store_api::{
|
||||
BucketOperations, BucketOptions, ChunkNativePutData, HealOperations, MakeBucketOptions, ObjectIO, ObjectOptions,
|
||||
},
|
||||
};
|
||||
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
fs as stdfs,
|
||||
path::Path,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Once, OnceLock},
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
use tokio::fs;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
static CAPACITY_DIRTY_SCOPE_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>, TempDir)> = OnceLock::new();
|
||||
static CAPACITY_DIRTY_SCOPE_INIT: Once = Once::new();
|
||||
|
||||
fn init_capacity_dirty_scope_tracing() {
|
||||
CAPACITY_DIRTY_SCOPE_INIT.call_once(|| {});
|
||||
}
|
||||
|
||||
async fn setup_capacity_dirty_scope_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
init_capacity_dirty_scope_tracing();
|
||||
|
||||
if let Some((paths, store, _)) = CAPACITY_DIRTY_SCOPE_ENV.get() {
|
||||
return (paths.clone(), store.clone());
|
||||
}
|
||||
|
||||
let temp_dir = TempDir::new().expect("create temp dir for capacity dirty scope test");
|
||||
let temp_path = temp_dir.path().to_path_buf();
|
||||
|
||||
let disk_paths = vec![
|
||||
temp_path.join("disk1"),
|
||||
temp_path.join("disk2"),
|
||||
temp_path.join("disk3"),
|
||||
temp_path.join("disk4"),
|
||||
];
|
||||
for disk_path in &disk_paths {
|
||||
fs::create_dir_all(disk_path).await.unwrap();
|
||||
}
|
||||
|
||||
let mut endpoints = Vec::new();
|
||||
for (i, disk_path) in disk_paths.iter().enumerate() {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
let pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "capacity-dirty-scope-test".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
};
|
||||
|
||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||
rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let buckets_list = ecstore
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
let _ = CAPACITY_DIRTY_SCOPE_ENV.set((disk_paths.clone(), ecstore.clone(), temp_dir));
|
||||
(disk_paths, ecstore)
|
||||
}
|
||||
|
||||
fn find_part_file(root: &Path, part_name: &str) -> Option<PathBuf> {
|
||||
let entries = stdfs::read_dir(root).ok()?;
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
if let Some(found) = find_part_file(&path, part_name) {
|
||||
return Some(found);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if path.file_name().and_then(|name| name.to_str()) == Some(part_name) {
|
||||
return Some(path);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn data_movement_put_object_marks_dirty_disks_for_capacity_manager() {
|
||||
let (disk_paths, ecstore) = setup_capacity_dirty_scope_env().await;
|
||||
let bucket_name = format!("dirty-scope-{}", Uuid::new_v4());
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket_name, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create test bucket");
|
||||
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
let _ = manager.get_dirty_disks().await;
|
||||
|
||||
let payload = b"data-movement-dirty-scope".to_vec();
|
||||
let mut reader = ChunkNativePutData::from_vec(payload);
|
||||
let opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx: 0,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
ecstore
|
||||
.put_object(&bucket_name, "object.bin", &mut reader, &opts)
|
||||
.await
|
||||
.expect("data movement put_object should succeed");
|
||||
|
||||
let dirty_disks = manager.get_dirty_disks().await;
|
||||
assert_eq!(dirty_disks.len(), disk_paths.len());
|
||||
|
||||
let actual_paths: HashSet<_> = dirty_disks
|
||||
.into_iter()
|
||||
.map(|disk| stdfs::canonicalize(&disk.drive_path).unwrap().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let expected_paths: HashSet<_> = disk_paths
|
||||
.iter()
|
||||
.map(|path| stdfs::canonicalize(path).unwrap().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(actual_paths, expected_paths);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn heal_object_marks_missing_shard_disk_dirty_for_capacity_manager() {
|
||||
let (disk_paths, ecstore) = setup_capacity_dirty_scope_env().await;
|
||||
let bucket_name = format!("dirty-heal-{}", Uuid::new_v4());
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket_name, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create test bucket");
|
||||
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
let _ = manager.get_dirty_disks().await;
|
||||
|
||||
let payload_len = 3 * 1024 * 1024 + 137;
|
||||
let payload: Vec<u8> = (0..payload_len).map(|idx| (idx % 251) as u8).collect();
|
||||
let mut reader = ChunkNativePutData::from_vec(payload);
|
||||
let object_name = "test/heal.bin";
|
||||
let put_info = ecstore
|
||||
.put_object(&bucket_name, object_name, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("put object for heal test");
|
||||
assert!(put_info.data_blocks > 1, "expected multi-shard object for heal test");
|
||||
|
||||
let _ = manager.get_dirty_disks().await;
|
||||
|
||||
let object_root = disk_paths[0].join(&bucket_name).join("test").join("heal.bin");
|
||||
let missing_part = find_part_file(&object_root, "part.1").expect("part file on first disk");
|
||||
fs::remove_file(&missing_part).await.expect("remove shard to force heal");
|
||||
|
||||
let heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: false,
|
||||
remove: false,
|
||||
recreate: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: true,
|
||||
no_lock: false,
|
||||
pool: None,
|
||||
set: None,
|
||||
};
|
||||
|
||||
let (_result, error) = ecstore
|
||||
.heal_object(&bucket_name, object_name, "", &heal_opts)
|
||||
.await
|
||||
.expect("heal_object call should succeed");
|
||||
|
||||
let dirty_disks = manager.get_dirty_disks().await;
|
||||
let actual_paths: HashSet<_> = dirty_disks
|
||||
.into_iter()
|
||||
.map(|disk| stdfs::canonicalize(&disk.drive_path).unwrap().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let expected_missing_disk = stdfs::canonicalize(&disk_paths[0]).unwrap().to_string_lossy().into_owned();
|
||||
|
||||
assert!(
|
||||
error.is_none() || actual_paths.contains(&expected_missing_disk),
|
||||
"heal returned {error:?} and did not mark the repaired shard disk dirty: {actual_paths:?}"
|
||||
);
|
||||
}
|
||||
@@ -35,12 +35,14 @@ use rustfs_ecstore::{
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
},
|
||||
};
|
||||
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
|
||||
use rustfs_utils::http::{SUFFIX_FORCE_DELETE, insert_header};
|
||||
use s3s::{S3Request, dto::*};
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
convert::Infallible,
|
||||
fs as stdfs,
|
||||
io::Cursor,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Once, OnceLock},
|
||||
@@ -539,3 +541,49 @@ async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
|
||||
"transitioned object should be removed from remote tier after delete usecase"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "requires isolated global object layer state"]
|
||||
async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
let _ = manager.get_dirty_disks().await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let _backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-capacity-transition-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
let payload = b"transition should mark dirty scope";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
|
||||
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
bucket.as_str(),
|
||||
)
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned object");
|
||||
|
||||
let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before dirty scope assertion");
|
||||
|
||||
let dirty_disks = manager.get_dirty_disks().await;
|
||||
assert_eq!(dirty_disks.len(), disk_paths.len());
|
||||
|
||||
let actual_paths: std::collections::HashSet<_> = dirty_disks
|
||||
.into_iter()
|
||||
.map(|disk| stdfs::canonicalize(&disk.drive_path).unwrap().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let expected_paths: std::collections::HashSet<_> = disk_paths
|
||||
.iter()
|
||||
.map(|path| stdfs::canonicalize(path).unwrap().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(actual_paths, expected_paths);
|
||||
}
|
||||
|
||||
@@ -21,5 +21,7 @@ pub mod context;
|
||||
pub mod multipart_usecase;
|
||||
pub mod object_usecase;
|
||||
|
||||
#[cfg(test)]
|
||||
mod capacity_dirty_scope_test;
|
||||
#[cfg(test)]
|
||||
mod lifecycle_transition_api_test;
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
use crate::app::context::{AppContext, get_global_app_context};
|
||||
use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
|
||||
use crate::capacity::record_capacity_write;
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::access::has_bypass_governance_header;
|
||||
use crate::storage::entity;
|
||||
@@ -24,7 +25,10 @@ use crate::storage::options::{
|
||||
copy_src_opts, extract_metadata, get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts,
|
||||
parse_copy_source_range, put_opts, validate_archive_content_encoding,
|
||||
};
|
||||
use crate::storage::s3_api::multipart::build_list_parts_output;
|
||||
use crate::storage::s3_api::multipart::{
|
||||
ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output,
|
||||
parse_list_multipart_uploads_params, parse_list_parts_params,
|
||||
};
|
||||
use crate::storage::*;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
@@ -41,7 +45,7 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use rustfs_ecstore::compress::is_compressible;
|
||||
use rustfs_ecstore::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
use rustfs_ecstore::new_object_layer_fn;
|
||||
use rustfs_ecstore::set_disk::{MAX_PARTS_COUNT, is_valid_storage_class};
|
||||
use rustfs_ecstore::set_disk::is_valid_storage_class;
|
||||
use rustfs_ecstore::store_api::{
|
||||
ChunkNativePutData, CompletePart, HTTPRangeSpec, MultipartUploadResult, ObjectIO, ObjectOptions,
|
||||
};
|
||||
@@ -64,12 +68,9 @@ use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
use tokio_util::io::StreamReader;
|
||||
use tracing::{info, instrument, warn};
|
||||
use tracing::{instrument, warn};
|
||||
use urlencoding::encode;
|
||||
|
||||
async fn maybe_enqueue_transition_immediate(obj_info: &rustfs_ecstore::store_api::ObjectInfo, src: LcEventSrc) {
|
||||
enqueue_transition_immediate(obj_info, src).await;
|
||||
}
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Returns InvalidRange error if CopySourceRange end exceeds the source object size.
|
||||
/// Used by execute_upload_part_copy to reject out-of-bounds ranges per S3 spec.
|
||||
@@ -286,7 +287,9 @@ impl DefaultMultipartUsecase {
|
||||
|
||||
let Some(multipart_upload) = multipart_upload else { return Err(s3_error!(InvalidPart)) };
|
||||
|
||||
let opts = &get_complete_multipart_upload_opts(&req.headers).map_err(ApiError::from)?;
|
||||
let mut opts = get_complete_multipart_upload_opts(&req.headers).map_err(ApiError::from)?;
|
||||
let capacity_scope_token = Uuid::new_v4();
|
||||
opts.capacity_scope_token = Some(capacity_scope_token);
|
||||
|
||||
let uploaded_parts_vec = multipart_upload
|
||||
.parts
|
||||
@@ -320,30 +323,15 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
// TDD: Get multipart info to extract encryption configuration before completing
|
||||
info!(
|
||||
"TDD: Attempting to get multipart info for bucket={}, key={}, upload_id={}",
|
||||
bucket, key, upload_id
|
||||
);
|
||||
|
||||
let multipart_info = store
|
||||
.get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
info!("TDD: Got multipart info successfully");
|
||||
info!("TDD: Multipart info metadata: {:?}", multipart_info.user_defined);
|
||||
|
||||
// TDD: Extract encryption information from multipart upload metadata
|
||||
let server_side_encryption = multipart_info
|
||||
.user_defined
|
||||
.get("x-amz-server-side-encryption")
|
||||
.map(|s| ServerSideEncryption::from(s.clone()));
|
||||
info!(
|
||||
"TDD: Raw encryption from metadata: {:?} -> parsed: {:?}",
|
||||
multipart_info.user_defined.get("x-amz-server-side-encryption"),
|
||||
server_side_encryption
|
||||
);
|
||||
|
||||
let ssekms_key_id = match server_side_encryption.as_ref() {
|
||||
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => multipart_info
|
||||
@@ -353,16 +341,12 @@ impl DefaultMultipartUsecase {
|
||||
_ => None,
|
||||
};
|
||||
|
||||
info!(
|
||||
"TDD: Extracted encryption info - SSE: {:?}, KMS Key: {:?}",
|
||||
server_side_encryption, ssekms_key_id
|
||||
);
|
||||
|
||||
let obj_info = store
|
||||
.clone()
|
||||
.complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts)
|
||||
.complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
record_capacity_write(Some(capacity_scope_token)).await;
|
||||
|
||||
// check quota after completing multipart upload
|
||||
if let Some(metadata_sys) = self.bucket_metadata_sys() {
|
||||
@@ -394,7 +378,7 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
|
||||
enqueue_transition_immediate(&obj_info, LcEventSrc::S3CompleteMultipartUpload).await;
|
||||
|
||||
let raw_mpu_version = obj_info.version_id.map(|v| v.to_string());
|
||||
let mpu_version = if BucketVersioningSys::prefix_enabled(&bucket, &key).await {
|
||||
@@ -403,11 +387,6 @@ impl DefaultMultipartUsecase {
|
||||
None
|
||||
};
|
||||
let mpu_version_for_event = mpu_version.clone();
|
||||
info!(
|
||||
"TDD: Creating output with SSE: {:?}, KMS Key: {:?}",
|
||||
server_side_encryption, ssekms_key_id
|
||||
);
|
||||
|
||||
let mut checksum_crc32 = input.checksum_crc32;
|
||||
let mut checksum_crc32c = input.checksum_crc32c;
|
||||
let mut checksum_sha1 = input.checksum_sha1;
|
||||
@@ -468,11 +447,6 @@ impl DefaultMultipartUsecase {
|
||||
checksum_type,
|
||||
..Default::default()
|
||||
};
|
||||
info!(
|
||||
"TDD: Created output: SSE={:?}, KMS={:?}",
|
||||
output.server_side_encryption, output.ssekms_key_id
|
||||
);
|
||||
|
||||
let mt2 = HashMap::new();
|
||||
let replicate_options =
|
||||
get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts.clone());
|
||||
@@ -483,10 +457,6 @@ impl DefaultMultipartUsecase {
|
||||
warn!("need multipart replication");
|
||||
schedule_replication(obj_info.clone(), store, dsc, ReplicationType::Object).await;
|
||||
}
|
||||
info!(
|
||||
"TDD: About to return S3Response with output: SSE={:?}, KMS={:?}",
|
||||
output.server_side_encryption, output.ssekms_key_id
|
||||
);
|
||||
|
||||
// Set object info for event notification
|
||||
helper = helper.object(obj_info);
|
||||
@@ -912,55 +882,22 @@ impl DefaultMultipartUsecase {
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
let ListMultipartUploadsParams {
|
||||
prefix,
|
||||
key_marker,
|
||||
max_uploads,
|
||||
} = parse_list_multipart_uploads_params(prefix, key_marker, max_uploads)?;
|
||||
|
||||
let Some(store) = new_object_layer_fn() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let prefix = prefix.unwrap_or_default();
|
||||
let max_uploads = max_uploads.map(|x| x as usize).unwrap_or(MAX_PARTS_COUNT);
|
||||
|
||||
if let Some(key_marker) = &key_marker
|
||||
&& !key_marker.starts_with(prefix.as_str())
|
||||
{
|
||||
return Err(s3_error!(NotImplemented, "Invalid key marker"));
|
||||
}
|
||||
|
||||
let result = store
|
||||
.list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let output = ListMultipartUploadsOutput {
|
||||
bucket: Some(bucket),
|
||||
prefix: Some(prefix),
|
||||
delimiter: result.delimiter,
|
||||
key_marker: result.key_marker,
|
||||
upload_id_marker: result.upload_id_marker,
|
||||
max_uploads: Some(result.max_uploads as i32),
|
||||
is_truncated: Some(result.is_truncated),
|
||||
uploads: Some(
|
||||
result
|
||||
.uploads
|
||||
.into_iter()
|
||||
.map(|u| MultipartUpload {
|
||||
key: Some(u.object),
|
||||
upload_id: Some(u.upload_id),
|
||||
initiated: u.initiated.map(Timestamp::from),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
common_prefixes: Some(
|
||||
result
|
||||
.common_prefixes
|
||||
.into_iter()
|
||||
.map(|c| CommonPrefix { prefix: Some(c) })
|
||||
.collect(),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
Ok(S3Response::new(output))
|
||||
Ok(S3Response::new(build_list_multipart_uploads_output(bucket, prefix, result)))
|
||||
}
|
||||
|
||||
pub async fn execute_list_parts(&self, req: S3Request<ListPartsInput>) -> S3Result<S3Response<ListPartsOutput>> {
|
||||
@@ -977,23 +914,21 @@ impl DefaultMultipartUsecase {
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
let params = parse_list_parts_params(part_number_marker, max_parts)?;
|
||||
|
||||
let Some(store) = new_object_layer_fn() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let part_number_marker = part_number_marker.map(|x| x as usize);
|
||||
let max_parts = match max_parts {
|
||||
Some(parts) => {
|
||||
if !(1..=1000).contains(&parts) {
|
||||
return Err(s3_error!(InvalidArgument, "max-parts must be between 1 and 1000"));
|
||||
}
|
||||
parts as usize
|
||||
}
|
||||
None => 1000,
|
||||
};
|
||||
|
||||
let res = store
|
||||
.list_object_parts(&bucket, &key, &upload_id, part_number_marker, max_parts, &ObjectOptions::default())
|
||||
.list_object_parts(
|
||||
&bucket,
|
||||
&key,
|
||||
&upload_id,
|
||||
params.part_number_marker,
|
||||
params.max_parts,
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
@@ -1056,7 +991,7 @@ impl DefaultMultipartUsecase {
|
||||
let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(ApiError::from)?;
|
||||
src_opts.version_id = src_version_id.clone();
|
||||
|
||||
let h = http::HeaderMap::new();
|
||||
let h = HeaderMap::new();
|
||||
let get_opts = ObjectOptions {
|
||||
version_id: src_opts.version_id.clone(),
|
||||
versioned: src_opts.versioned,
|
||||
@@ -1108,7 +1043,7 @@ impl DefaultMultipartUsecase {
|
||||
(0, src_info.size)
|
||||
};
|
||||
|
||||
let h = http::HeaderMap::new();
|
||||
let h = HeaderMap::new();
|
||||
let get_opts = ObjectOptions {
|
||||
version_id: src_opts.version_id.clone(),
|
||||
versioned: src_opts.versioned,
|
||||
@@ -1453,6 +1388,36 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_multipart_uploads_rejects_invalid_key_marker_before_store_lookup() {
|
||||
let input = ListMultipartUploadsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.prefix(Some("prefix/".to_string()))
|
||||
.key_marker(Some("other/key".to_string()))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_multipart_uploads(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
|
||||
assert_eq!(err.message(), Some("Invalid key marker"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_multipart_uploads_rejects_invalid_max_uploads_before_store_lookup() {
|
||||
let input = ListMultipartUploadsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.max_uploads(Some(0))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
let expected = "max-uploads must be between 1 and 1000";
|
||||
|
||||
let err = make_usecase().execute_list_multipart_uploads(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some(expected));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_parts_returns_internal_error_when_store_uninitialized() {
|
||||
let input = ListPartsInput::builder()
|
||||
@@ -1467,6 +1432,38 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_parts_rejects_negative_part_number_marker_before_store_lookup() {
|
||||
let input = ListPartsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.part_number_marker(Some(-1))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("part-number-marker must be non-negative"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_parts_rejects_invalid_max_parts_before_store_lookup() {
|
||||
let input = ListPartsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.max_parts(Some(1001))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("max-parts must be between 1 and 1000"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_copy_returns_internal_error_when_store_uninitialized() {
|
||||
let input = UploadPartCopyInput::builder()
|
||||
|
||||
+269
-116
@@ -14,34 +14,30 @@
|
||||
|
||||
//! Object application use-case contracts.
|
||||
|
||||
mod app_adapters;
|
||||
mod get_object_flow;
|
||||
mod get_object_zero_copy;
|
||||
mod put_object_extract;
|
||||
mod put_object_flow;
|
||||
mod types;
|
||||
#[cfg(test)]
|
||||
mod zero_copy_tests;
|
||||
use self::app_adapters::*;
|
||||
use self::get_object_flow::{GetObjectBootstrap, GetObjectFlowRuntime};
|
||||
use self::types::*;
|
||||
use self::get_object_flow::GetObjectBootstrap;
|
||||
|
||||
use crate::app::context::{AppContext, default_notify_interface, get_global_app_context};
|
||||
use crate::capacity::capacity_manager::get_capacity_manager;
|
||||
use crate::capacity::record_capacity_write;
|
||||
use crate::config::RustFSBufferConfig;
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut};
|
||||
use crate::storage::concurrency::{GetObjectGuard, get_concurrency_manager};
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
|
||||
use crate::storage::helper::OperationHelper;
|
||||
use crate::storage::helper::{OperationHelper, spawn_background};
|
||||
use crate::storage::options::{
|
||||
copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name,
|
||||
filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts,
|
||||
validate_archive_content_encoding,
|
||||
};
|
||||
use crate::storage::s3_api::acl;
|
||||
use crate::storage::s3_api::multipart::parse_list_parts_params;
|
||||
use crate::storage::s3_api::{acl, restore, select};
|
||||
use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig};
|
||||
use crate::storage::*;
|
||||
use bytes::Bytes;
|
||||
@@ -143,6 +139,34 @@ struct DeadlockRequestGuard {
|
||||
request_id: String,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct GetObjectRequestContext {
|
||||
pub(super) bucket: String,
|
||||
pub(super) key: String,
|
||||
pub(super) part_number: Option<usize>,
|
||||
pub(super) rs: Option<HTTPRangeSpec>,
|
||||
pub(super) opts: ObjectOptions,
|
||||
pub(super) headers: HeaderMap,
|
||||
pub(super) sse_customer_key: Option<String>,
|
||||
pub(super) sse_customer_key_md5: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) type PutObjectChecksums = rustfs_object_io::put::PutObjectChecksums;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct PutObjectRequestContext {
|
||||
pub(super) headers: HeaderMap,
|
||||
pub(super) trailing_headers: Option<s3s::TrailingHeaders>,
|
||||
pub(super) uri_query: Option<String>,
|
||||
pub(super) is_post_object: bool,
|
||||
pub(super) method: hyper::Method,
|
||||
pub(super) uri: hyper::Uri,
|
||||
pub(super) extensions: http::Extensions,
|
||||
pub(super) credentials: Option<s3s::auth::Credentials>,
|
||||
pub(super) region: Option<s3s::region::Region>,
|
||||
pub(super) service: Option<String>,
|
||||
}
|
||||
|
||||
impl DeadlockRequestGuard {
|
||||
fn new(deadlock_detector: Arc<deadlock_detector::DeadlockDetector>, request_id: String) -> Self {
|
||||
Self {
|
||||
@@ -158,6 +182,26 @@ impl Drop for DeadlockRequestGuard {
|
||||
}
|
||||
}
|
||||
|
||||
async fn resolve_bucket_default_server_side_encryption(bucket: &str) -> (Option<ServerSideEncryption>, Option<String>) {
|
||||
let Some((config, _timestamp)) = metadata_sys::get_sse_config(bucket).await.ok() else {
|
||||
return (None, None);
|
||||
};
|
||||
let Some(default_sse) = config
|
||||
.rules
|
||||
.first()
|
||||
.and_then(|rule| rule.apply_server_side_encryption_by_default.as_ref())
|
||||
else {
|
||||
return (None, None);
|
||||
};
|
||||
|
||||
let server_side_encryption = Some(match default_sse.sse_algorithm.as_str() {
|
||||
"aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
|
||||
_ => ServerSideEncryption::from_static(ServerSideEncryption::AES256),
|
||||
});
|
||||
|
||||
(server_side_encryption, default_sse.kms_master_key_id.clone())
|
||||
}
|
||||
|
||||
async fn enqueue_transitioned_delete_cleanup(bucket: &str, object: &str, opts: &ObjectOptions, existing: Option<&ObjectInfo>) {
|
||||
let Some(existing) = existing else {
|
||||
return;
|
||||
@@ -267,10 +311,6 @@ mod deadlock_request_guard_tests {
|
||||
assert_eq!(detector.tracked_count(), 0);
|
||||
}
|
||||
}
|
||||
async fn maybe_enqueue_transition_immediate(obj_info: &ObjectInfo, src: LcEventSrc) {
|
||||
enqueue_transition_immediate(obj_info, src).await;
|
||||
}
|
||||
|
||||
fn normalize_delete_objects_version_id(version_id: Option<String>) -> Result<(Option<String>, Option<Uuid>), String> {
|
||||
let version_id = version_id.map(|v| v.trim().to_string()).filter(|v| !v.is_empty());
|
||||
match version_id {
|
||||
@@ -301,6 +341,58 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option<String
|
||||
Some(format!("expiry-date=\"{}\", rule-id=\"{}\"", expiry_date, event.rule_id))
|
||||
}
|
||||
|
||||
async fn prepare_get_object_request_context(req: &S3Request<GetObjectInput>) -> S3Result<GetObjectRequestContext> {
|
||||
let GetObjectInput {
|
||||
bucket,
|
||||
key,
|
||||
version_id,
|
||||
part_number,
|
||||
range,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
validate_object_key(&key, "GET")?;
|
||||
|
||||
let part_number = part_number.map(|value| value as usize);
|
||||
if let Some(part_number) = part_number
|
||||
&& part_number == 0
|
||||
{
|
||||
return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0"));
|
||||
}
|
||||
|
||||
let rs = range.map(|value| match value {
|
||||
Range::Int { first, last } => HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: first as i64,
|
||||
end: last.map_or(-1, |last| last as i64),
|
||||
},
|
||||
Range::Suffix { length } => HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: length as i64,
|
||||
end: -1,
|
||||
},
|
||||
});
|
||||
|
||||
if rs.is_some() && part_number.is_some() {
|
||||
return Err(s3_error!(InvalidArgument, "range and part_number invalid"));
|
||||
}
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, part_number, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
Ok(GetObjectRequestContext {
|
||||
bucket,
|
||||
key,
|
||||
part_number,
|
||||
rs,
|
||||
opts,
|
||||
headers: req.headers.clone(),
|
||||
sse_customer_key: req.input.sse_customer_key.clone(),
|
||||
sse_customer_key_md5: req.input.sse_customer_key_md5.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn apply_put_request_metadata(
|
||||
metadata: &mut HashMap<String, String>,
|
||||
@@ -605,10 +697,23 @@ impl DefaultObjectUsecase {
|
||||
let _ = context.object_store();
|
||||
}
|
||||
|
||||
let request_context = prepare_put_object_request_context(&req);
|
||||
let (event_name, quota_operation, request_method_name) = put_object_execution_context(&req);
|
||||
let helper = new_operation_helper(&req, event_name, S3Operation::PutObject, false);
|
||||
|
||||
let request_context = PutObjectRequestContext {
|
||||
headers: req.headers.clone(),
|
||||
trailing_headers: req.trailing_headers.clone(),
|
||||
uri_query: req.uri.query().map(str::to_string),
|
||||
is_post_object: req.extensions.get::<PostObjectRequestMarker>().is_some(),
|
||||
method: req.method.clone(),
|
||||
uri: req.uri.clone(),
|
||||
extensions: req.extensions.clone(),
|
||||
credentials: req.credentials.clone(),
|
||||
region: req.region.clone(),
|
||||
service: req.service.clone(),
|
||||
};
|
||||
let (event_name, quota_operation) = if request_context.is_post_object {
|
||||
(EventName::ObjectCreatedPost, QuotaOperation::PostObject)
|
||||
} else {
|
||||
(EventName::ObjectCreatedPut, QuotaOperation::PutObject)
|
||||
};
|
||||
if request_context.is_post_object && is_post_object_sse_kms_requested(&req.input, &request_context.headers) {
|
||||
return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for POST object uploads"));
|
||||
}
|
||||
@@ -618,18 +723,27 @@ impl DefaultObjectUsecase {
|
||||
return Err(s3_error!(InvalidStorageClass));
|
||||
}
|
||||
if is_put_object_extract_requested(&request_context.headers) {
|
||||
return self.execute_put_object_extract(req).await;
|
||||
return self.execute_put_object_extract(req, request_context).await;
|
||||
}
|
||||
|
||||
let helper = OperationHelper::new(&req, event_name, S3Operation::PutObject);
|
||||
|
||||
let resolved_size = resolve_put_body_size(req.input.content_length, &request_context.headers)?;
|
||||
self.check_bucket_quota(&req.input.bucket, quota_operation, resolved_size as u64)
|
||||
.await?;
|
||||
|
||||
let input = req.input;
|
||||
let flow_result =
|
||||
DefaultObjectUsecase::run_put_object_flow(input, request_context, request_method_name, resolved_size).await?;
|
||||
let helper = bind_helper_object(helper, flow_result.helper_object, flow_result.helper_version_id);
|
||||
complete_put_response(helper, flow_result.output)
|
||||
let (output, helper_object) = DefaultObjectUsecase::run_put_object_flow(input, request_context, resolved_size).await?;
|
||||
let helper_version_id = helper_object.version_id.map(|version_id| version_id.to_string());
|
||||
let helper = helper.object(helper_object);
|
||||
let helper = if let Some(version_id) = helper_version_id {
|
||||
helper.version_id(version_id)
|
||||
} else {
|
||||
helper
|
||||
};
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
|
||||
pub async fn execute_put_object_acl(&self, req: S3Request<PutObjectAclInput>) -> S3Result<S3Response<PutObjectAclOutput>> {
|
||||
@@ -914,43 +1028,12 @@ impl DefaultObjectUsecase {
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
if tagging.tag_set.len() > 10 {
|
||||
error!("Tag set exceeds maximum of 10 tags: {}", tagging.tag_set.len());
|
||||
return Err(s3_error!(InvalidTag, "Cannot have more than 10 tags per object"));
|
||||
}
|
||||
crate::storage::s3_api::tagging::validate_object_tag_set(&tagging.tag_set)?;
|
||||
|
||||
let Some(store) = new_object_layer_fn() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let mut tag_keys = std::collections::HashSet::with_capacity(tagging.tag_set.len());
|
||||
for tag in &tagging.tag_set {
|
||||
let key = tag.key.as_ref().filter(|k| !k.is_empty()).ok_or_else(|| {
|
||||
error!("Empty tag key");
|
||||
s3_error!(InvalidTag, "Tag key cannot be empty")
|
||||
})?;
|
||||
|
||||
if key.len() > 128 {
|
||||
error!("Tag key too long: {} bytes", key.len());
|
||||
return Err(s3_error!(InvalidTag, "Tag key is too long, maximum allowed length is 128 characters"));
|
||||
}
|
||||
|
||||
let value = tag.value.as_ref().ok_or_else(|| {
|
||||
error!("Null tag value");
|
||||
s3_error!(InvalidTag, "Tag value cannot be null")
|
||||
})?;
|
||||
|
||||
if value.len() > 256 {
|
||||
error!("Tag value too long: {} bytes", value.len());
|
||||
return Err(s3_error!(InvalidTag, "Tag value is too long, maximum allowed length is 256 characters"));
|
||||
}
|
||||
|
||||
if !tag_keys.insert(key) {
|
||||
error!("Duplicate tag key: {}", key);
|
||||
return Err(s3_error!(InvalidTag, "Cannot provide multiple Tags with the same key"));
|
||||
}
|
||||
}
|
||||
|
||||
let tags = encode_tags(tagging.tag_set);
|
||||
debug!("Encoded tags: {}", tags);
|
||||
|
||||
@@ -1020,20 +1103,57 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.get::<request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.clone())
|
||||
.unwrap_or_else(|| crate::storage::request_context::RequestContext::fallback().request_id);
|
||||
let bootstrap = init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?;
|
||||
.unwrap_or_else(|| request_context::RequestContext::fallback().request_id);
|
||||
let bootstrap = {
|
||||
let timeout_config = TimeoutConfig::from_env();
|
||||
let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.clone());
|
||||
let request_start = std::time::Instant::now();
|
||||
let request_guard = crate::storage::concurrency::ConcurrencyManager::track_request();
|
||||
let concurrent_requests = GetObjectGuard::concurrent_requests();
|
||||
|
||||
let deadlock_detector = deadlock_detector::get_deadlock_detector();
|
||||
deadlock_detector.register_request(&request_id, format!("GetObject {}/{}", req.input.bucket, req.input.key));
|
||||
let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id);
|
||||
|
||||
if wrapper.is_timeout() {
|
||||
warn!(
|
||||
bucket = %req.input.bucket,
|
||||
key = %req.input.key,
|
||||
timeout_secs = timeout_config.get_object_timeout.as_secs(),
|
||||
elapsed_ms = wrapper.elapsed().as_millis(),
|
||||
"GetObject request timed out before processing"
|
||||
);
|
||||
return Err(s3_error!(InternalError, "Request timeout before processing"));
|
||||
}
|
||||
|
||||
rustfs_io_metrics::record_get_object_request_start(concurrent_requests);
|
||||
|
||||
debug!(
|
||||
"GetObject request started with {} concurrent requests, timeout={:?}",
|
||||
concurrent_requests, timeout_config.get_object_timeout
|
||||
);
|
||||
|
||||
GetObjectBootstrap {
|
||||
timeout_config,
|
||||
wrapper,
|
||||
request_start,
|
||||
request_guard,
|
||||
_deadlock_request_guard: deadlock_request_guard,
|
||||
}
|
||||
};
|
||||
let version_id_for_event = req.input.version_id.clone().unwrap_or_default();
|
||||
let request_context = prepare_get_object_request_context(&req).await?;
|
||||
let base_buffer_size = self.base_buffer_size();
|
||||
let manager = get_concurrency_manager();
|
||||
let flow_runtime = GetObjectFlowRuntime {
|
||||
manager,
|
||||
bootstrap: &bootstrap,
|
||||
base_buffer_size,
|
||||
};
|
||||
let helper = new_operation_helper(&req, EventName::ObjectAccessedGet, S3Operation::GetObject, true);
|
||||
let flow_result = get_object_flow::run_get_object_flow(request_context.clone(), flow_runtime).await;
|
||||
let cors_bucket = request_context.bucket.clone();
|
||||
let cors_method = req.method.clone();
|
||||
let cors_headers = request_context.headers.clone();
|
||||
let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).suppress_event();
|
||||
let flow_result =
|
||||
get_object_flow::run_get_object_flow(request_context, version_id_for_event, manager, &bootstrap, base_buffer_size)
|
||||
.await;
|
||||
|
||||
let GetObjectBootstrap {
|
||||
mut request_guard,
|
||||
@@ -1042,7 +1162,15 @@ impl DefaultObjectUsecase {
|
||||
} = bootstrap;
|
||||
|
||||
let result = match flow_result {
|
||||
Ok(flow_result) => complete_get_flow_result(helper, &request_context, flow_result).await,
|
||||
Ok(flow_result) => {
|
||||
let helper = helper
|
||||
.object(flow_result.event_info)
|
||||
.version_id(flow_result.version_id_for_event);
|
||||
let response = wrap_response_with_cors(&cors_bucket, &cors_method, &cors_headers, flow_result.output).await;
|
||||
let result = Ok(response);
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
|
||||
@@ -1766,7 +1894,7 @@ impl DefaultObjectUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await;
|
||||
enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await;
|
||||
|
||||
// Update quota tracking after successful copy
|
||||
if has_bucket_metadata {
|
||||
@@ -1848,6 +1976,7 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let version_cfg = BucketVersioningSys::get(&bucket).await.unwrap_or_default();
|
||||
let bypass_governance = has_bypass_governance_header(&req.headers);
|
||||
let capacity_scope_token = Uuid::new_v4();
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
struct DeleteResult {
|
||||
@@ -1970,6 +2099,7 @@ impl DefaultObjectUsecase {
|
||||
object_to_delete.clone(),
|
||||
ObjectOptions {
|
||||
version_suspended: version_cfg.suspended(),
|
||||
capacity_scope_token: Some(capacity_scope_token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -2076,7 +2206,7 @@ impl DefaultObjectUsecase {
|
||||
.as_ref()
|
||||
.map(|context| context.notify())
|
||||
.unwrap_or_else(default_notify_interface);
|
||||
crate::storage::helper::spawn_background(async move {
|
||||
spawn_background(async move {
|
||||
for res in delete_results {
|
||||
if let Some(dobj) = res.delete_object {
|
||||
let event_name = if dobj.delete_marker {
|
||||
@@ -2108,8 +2238,7 @@ impl DefaultObjectUsecase {
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
// Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead)
|
||||
let manager = get_capacity_manager();
|
||||
manager.record_write_operation().await;
|
||||
record_capacity_write(Some(capacity_scope_token)).await;
|
||||
result
|
||||
}
|
||||
|
||||
@@ -2144,6 +2273,8 @@ impl DefaultObjectUsecase {
|
||||
let mut opts: ObjectOptions = del_opts(&bucket, &key, version_id, &req.headers, metadata)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let capacity_scope_token = Uuid::new_v4();
|
||||
opts.capacity_scope_token = Some(capacity_scope_token);
|
||||
let force_delete = opts.delete_prefix;
|
||||
|
||||
let lock_cfg = BucketObjectLockSys::get(&bucket).await;
|
||||
@@ -2255,8 +2386,7 @@ impl DefaultObjectUsecase {
|
||||
})
|
||||
.version_id(String::new());
|
||||
let result = Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT));
|
||||
let manager = get_capacity_manager();
|
||||
manager.record_write_operation().await;
|
||||
record_capacity_write(Some(capacity_scope_token)).await;
|
||||
let _ = helper.complete(&result);
|
||||
return result;
|
||||
}
|
||||
@@ -2304,8 +2434,7 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
// Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead)
|
||||
let manager = get_capacity_manager();
|
||||
manager.record_write_operation().await;
|
||||
record_capacity_write(Some(capacity_scope_token)).await;
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
@@ -2830,8 +2959,10 @@ impl DefaultObjectUsecase {
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrCopyObject".into()), "restore object failed."))?;
|
||||
|
||||
if already_restored {
|
||||
let output =
|
||||
restore::build_restore_object_output(Some(RequestCharged::from_static(RequestCharged::REQUESTER)), None);
|
||||
let output = RestoreObjectOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
restore_output_path: None,
|
||||
};
|
||||
helper = helper
|
||||
.object(event_object_info.clone())
|
||||
.version_id(version_id_str.clone())
|
||||
@@ -2860,7 +2991,7 @@ impl DefaultObjectUsecase {
|
||||
let rreq_clone = rreq.clone();
|
||||
let version_id_clone = version_id.clone();
|
||||
|
||||
crate::storage::request_context::spawn_traced(async move {
|
||||
request_context::spawn_traced(async move {
|
||||
let opts = ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
restore_request: rreq_clone,
|
||||
@@ -2886,7 +3017,10 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
});
|
||||
|
||||
let output = restore::build_restore_object_output(Some(RequestCharged::from_static(RequestCharged::REQUESTER)), None);
|
||||
let output = RestoreObjectOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
restore_output_path: None,
|
||||
};
|
||||
helper = helper.object(event_object_info).version_id(version_id_str);
|
||||
let result = Ok(S3Response::with_headers(output, header));
|
||||
let _ = helper.complete(&result);
|
||||
@@ -2953,7 +3087,7 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let (tx, rx) = mpsc::channel::<S3Result<SelectObjectContentEvent>>(2);
|
||||
let stream = ReceiverStream::new(rx);
|
||||
crate::storage::request_context::spawn_traced(async move {
|
||||
request_context::spawn_traced(async move {
|
||||
let _ = tx
|
||||
.send(Ok(SelectObjectContentEvent::Cont(ContinuationEvent::default())))
|
||||
.await;
|
||||
@@ -2967,15 +3101,18 @@ impl DefaultObjectUsecase {
|
||||
drop(tx);
|
||||
});
|
||||
|
||||
Ok(S3Response::new(select::build_select_object_content_output(
|
||||
SelectObjectContentEventStream::new(stream),
|
||||
)))
|
||||
Ok(S3Response::new(SelectObjectContentOutput {
|
||||
payload: Some(SelectObjectContentEventStream::new(stream)),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_put_object_extract(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
let request_context = prepare_put_object_request_context(&req);
|
||||
let helper = new_operation_helper(&req, EventName::ObjectCreatedPut, S3Operation::PutObject, true);
|
||||
#[instrument(level = "debug", skip(self, req, request_context))]
|
||||
async fn execute_put_object_extract(
|
||||
&self,
|
||||
req: S3Request<PutObjectInput>,
|
||||
request_context: PutObjectRequestContext,
|
||||
) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject).suppress_event();
|
||||
if is_sse_kms_requested(&req.input, &request_context.headers) {
|
||||
return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads"));
|
||||
}
|
||||
@@ -2989,7 +3126,9 @@ impl DefaultObjectUsecase {
|
||||
.unwrap_or_else(default_notify_interface);
|
||||
let input = req.input;
|
||||
let output = DefaultObjectUsecase::run_put_object_extract_flow(input, request_context, notify, resolved_size).await?;
|
||||
complete_put_response(helper, output)
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3021,37 +3160,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_execution_context_defaults_to_put() {
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let (event_name, quota_operation, method_name) = put_object_execution_context(&req);
|
||||
assert_eq!(event_name, EventName::ObjectCreatedPut);
|
||||
assert!(matches!(quota_operation, QuotaOperation::PutObject));
|
||||
assert_eq!(method_name, "PUT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_object_execution_context_uses_post_marker() {
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(PostObjectRequestMarker);
|
||||
|
||||
let (event_name, quota_operation, method_name) = put_object_execution_context(&req);
|
||||
assert_eq!(event_name, EventName::ObjectCreatedPost);
|
||||
assert!(matches!(quota_operation, QuotaOperation::PostObject));
|
||||
assert_eq!(method_name, "POST");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_rejects_invalid_storage_class() {
|
||||
let input = PutObjectInput::builder()
|
||||
@@ -3069,6 +3177,51 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_tagging_rejects_too_many_tags() {
|
||||
let tag_set = (0..11)
|
||||
.map(|index| Tag {
|
||||
key: Some(format!("k{index}")),
|
||||
value: Some(format!("v{index}")),
|
||||
})
|
||||
.collect();
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging { tag_set })
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_put_object_tagging(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Cannot have more than 10 tags per object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_tagging_rejects_empty_tag_key_before_store_lookup() {
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some(String::new()),
|
||||
value: Some("v1".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::PUT);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_put_object_tagging(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Tag key cannot be empty"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_copy_object_rejects_self_copy_without_replace_directive() {
|
||||
let input = CopyObjectInput::builder()
|
||||
|
||||
@@ -1,461 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::get_object_flow::GetObjectBootstrap;
|
||||
use super::*;
|
||||
use crate::app::context::NotifyInterface;
|
||||
use crate::storage::concurrency::{self, ConcurrencyManager, get_buffer_size_opt_in};
|
||||
use hashbrown::HashMap;
|
||||
use rustfs_object_io::get::{
|
||||
GetObjectBodyPlan as ObjectIoGetObjectBodyPlan, GetObjectBodyPlanningInputs as ObjectIoGetObjectBodyPlanningInputs,
|
||||
GetObjectDataPlaneMetricContract as ObjectIoGetObjectDataPlaneMetricContract, GetObjectFlowResult,
|
||||
MaterializeGetObjectBodyError as ObjectIoMaterializeGetObjectBodyError,
|
||||
materialize_get_object_body as object_io_materialize_get_object_body, plan_get_object_body as object_io_plan_get_object_body,
|
||||
plan_get_object_strategy_layout as object_io_plan_get_object_strategy_layout,
|
||||
};
|
||||
|
||||
pub(super) async fn prepare_get_object_request_context(req: &S3Request<GetObjectInput>) -> S3Result<GetObjectRequestContext> {
|
||||
let GetObjectInput {
|
||||
bucket,
|
||||
key,
|
||||
version_id,
|
||||
part_number,
|
||||
range,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
validate_object_key(&key, "GET")?;
|
||||
|
||||
let part_number = part_number.map(|v| v as usize);
|
||||
|
||||
if let Some(part_num) = part_number
|
||||
&& part_num == 0
|
||||
{
|
||||
return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0"));
|
||||
}
|
||||
|
||||
let rs = range.map(|v| match v {
|
||||
Range::Int { first, last } => HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: first as i64,
|
||||
end: if let Some(last) = last { last as i64 } else { -1 },
|
||||
},
|
||||
Range::Suffix { length } => HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: length as i64,
|
||||
end: -1,
|
||||
},
|
||||
});
|
||||
|
||||
if rs.is_some() && part_number.is_some() {
|
||||
return Err(s3_error!(InvalidArgument, "range and part_number invalid"));
|
||||
}
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), part_number, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
Ok(GetObjectRequestContext {
|
||||
version_id_for_event: version_id.unwrap_or_default(),
|
||||
bucket,
|
||||
key,
|
||||
part_number,
|
||||
rs,
|
||||
opts,
|
||||
headers: req.headers.clone(),
|
||||
method: req.method.clone(),
|
||||
sse_customer_key: req.input.sse_customer_key.clone(),
|
||||
sse_customer_key_md5: req.input.sse_customer_key_md5.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
|
||||
let timeout_config = TimeoutConfig::from_env();
|
||||
let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string());
|
||||
let request_start = std::time::Instant::now();
|
||||
let request_guard = ConcurrencyManager::track_request();
|
||||
let concurrent_requests = GetObjectGuard::concurrent_requests();
|
||||
|
||||
let deadlock_detector = deadlock_detector::get_deadlock_detector();
|
||||
deadlock_detector.register_request(request_id, format!("GetObject {bucket}/{key}"));
|
||||
let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id.to_string());
|
||||
|
||||
if wrapper.is_timeout() {
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
timeout_secs = timeout_config.get_object_timeout.as_secs(),
|
||||
elapsed_ms = wrapper.elapsed().as_millis(),
|
||||
"GetObject request timed out before processing"
|
||||
);
|
||||
return Err(s3_error!(InternalError, "Request timeout before processing"));
|
||||
}
|
||||
|
||||
rustfs_io_metrics::record_get_object_request_start(concurrent_requests);
|
||||
|
||||
debug!(
|
||||
"GetObject request started with {} concurrent requests, timeout={:?}",
|
||||
concurrent_requests, timeout_config.get_object_timeout
|
||||
);
|
||||
|
||||
Ok(GetObjectBootstrap {
|
||||
timeout_config,
|
||||
wrapper,
|
||||
request_start,
|
||||
request_guard,
|
||||
_deadlock_request_guard: deadlock_request_guard,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn build_get_object_body_adapter<R>(
|
||||
final_stream: R,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
response_content_length: i64,
|
||||
optimal_buffer_size: usize,
|
||||
planning_inputs: ObjectIoGetObjectBodyPlanningInputs,
|
||||
) -> S3Result<Option<StreamingBlob>>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
let body_plan = object_io_plan_get_object_body(planning_inputs, rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD);
|
||||
|
||||
match body_plan {
|
||||
ObjectIoGetObjectBodyPlan::BufferSeekable => {
|
||||
debug!(
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
size = response_content_length,
|
||||
"reading object into memory for seek support"
|
||||
);
|
||||
}
|
||||
ObjectIoGetObjectBodyPlan::Stream if planning_inputs.encryption_applied => {
|
||||
info!(
|
||||
"Encrypted object: Using unlimited stream for decryption with buffer size {}",
|
||||
optimal_buffer_size
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let materialized =
|
||||
object_io_materialize_get_object_body(final_stream, body_plan, response_content_length, optimal_buffer_size)
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
ObjectIoMaterializeGetObjectBodyError::EncryptedRead(err) => {
|
||||
error!("Failed to read decrypted object into memory: {}", err);
|
||||
ApiError::from(StorageError::other(format!("Failed to read decrypted object: {err}")))
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(materialized.body)
|
||||
}
|
||||
|
||||
pub(super) struct GetObjectCompletionInputs<'a> {
|
||||
pub(super) bucket: &'a str,
|
||||
pub(super) key: &'a str,
|
||||
pub(super) wrapper: &'a RequestTimeoutWrapper,
|
||||
pub(super) timeout_config: &'a TimeoutConfig,
|
||||
pub(super) total_duration: Duration,
|
||||
pub(super) response_content_length: i64,
|
||||
pub(super) optimal_buffer_size: usize,
|
||||
pub(super) metric_contract: ObjectIoGetObjectDataPlaneMetricContract,
|
||||
}
|
||||
|
||||
pub(super) struct GetObjectStrategyRuntimeInputs<'a> {
|
||||
pub(super) base_buffer_size: usize,
|
||||
pub(super) manager: &'a ConcurrencyManager,
|
||||
pub(super) bucket: &'a str,
|
||||
pub(super) key: &'a str,
|
||||
pub(super) info: &'a ObjectInfo,
|
||||
pub(super) rs: Option<&'a HTTPRangeSpec>,
|
||||
pub(super) response_content_length: i64,
|
||||
pub(super) permit_wait_duration: Duration,
|
||||
pub(super) queue_utilization: f64,
|
||||
pub(super) queue_status: &'a concurrency::IoQueueStatus,
|
||||
}
|
||||
|
||||
pub(super) fn finalize_get_object_completion(inputs: GetObjectCompletionInputs<'_>) {
|
||||
let GetObjectCompletionInputs {
|
||||
bucket,
|
||||
key,
|
||||
wrapper,
|
||||
timeout_config,
|
||||
total_duration,
|
||||
response_content_length,
|
||||
optimal_buffer_size,
|
||||
metric_contract,
|
||||
} = inputs;
|
||||
|
||||
rustfs_io_metrics::record_get_object_completion(total_duration.as_secs_f64(), response_content_length, optimal_buffer_size);
|
||||
|
||||
rustfs_io_metrics::record_get_object(total_duration.as_millis() as f64, response_content_length);
|
||||
rustfs_io_metrics::record_io_copy_mode("get", metric_contract.copy_mode, response_content_length.max(0) as usize);
|
||||
|
||||
if wrapper.is_timeout() {
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
elapsed = ?wrapper.elapsed(),
|
||||
timeout = ?timeout_config.get_object_timeout,
|
||||
"GetObject request exceeded timeout"
|
||||
);
|
||||
rustfs_io_metrics::record_get_object_timeout(None, Some(wrapper.elapsed().as_secs_f64()));
|
||||
}
|
||||
|
||||
debug!(
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
size = response_content_length,
|
||||
duration = ?total_duration,
|
||||
buffer = optimal_buffer_size,
|
||||
"GetObject completed"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn finalize_get_object_strategy_runtime(inputs: GetObjectStrategyRuntimeInputs<'_>) -> usize {
|
||||
let GetObjectStrategyRuntimeInputs {
|
||||
base_buffer_size,
|
||||
manager,
|
||||
bucket,
|
||||
key,
|
||||
info,
|
||||
rs,
|
||||
response_content_length,
|
||||
permit_wait_duration,
|
||||
queue_utilization,
|
||||
queue_status,
|
||||
} = inputs;
|
||||
|
||||
let strategy_layout = object_io_plan_get_object_strategy_layout(
|
||||
rs,
|
||||
response_content_length,
|
||||
0,
|
||||
get_buffer_size_opt_in(response_content_length),
|
||||
);
|
||||
|
||||
if let Some(range_spec) = rs
|
||||
&& range_spec.start >= 0
|
||||
{
|
||||
manager.record_access(range_spec.start as u64, response_content_length as u64);
|
||||
}
|
||||
|
||||
if response_content_length > 0 {
|
||||
manager.record_transfer(response_content_length as u64, permit_wait_duration);
|
||||
}
|
||||
|
||||
let io_strategy = manager.calculate_io_strategy_with_context(
|
||||
info.size,
|
||||
base_buffer_size,
|
||||
permit_wait_duration,
|
||||
strategy_layout.is_sequential_hint,
|
||||
);
|
||||
|
||||
debug!(
|
||||
wait_ms = permit_wait_duration.as_millis() as u64,
|
||||
load_level = ?io_strategy.load_level,
|
||||
buffer_size = io_strategy.buffer_size,
|
||||
buffer_multiplier = io_strategy.buffer_multiplier,
|
||||
readahead = io_strategy.enable_readahead,
|
||||
storage_media = ?io_strategy.storage_media,
|
||||
access_pattern = ?io_strategy.access_pattern,
|
||||
bandwidth_tier = ?io_strategy.bandwidth_tier,
|
||||
concurrent_requests = io_strategy.concurrent_requests,
|
||||
file_size = info.size,
|
||||
is_sequential = strategy_layout.is_sequential_hint,
|
||||
"Enhanced multi-factor I/O strategy calculated"
|
||||
);
|
||||
|
||||
let io_priority = manager.get_io_priority(response_content_length);
|
||||
|
||||
if manager.is_priority_scheduling_enabled() {
|
||||
debug!(
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
priority = %io_priority,
|
||||
request_size = response_content_length,
|
||||
"I/O priority assigned (based on actual request size)"
|
||||
);
|
||||
|
||||
rustfs_io_metrics::record_io_priority_assignment(io_priority.as_str());
|
||||
}
|
||||
|
||||
rustfs_io_metrics::record_get_object_io_state(
|
||||
permit_wait_duration.as_secs_f64(),
|
||||
queue_utilization,
|
||||
queue_status.permits_in_use,
|
||||
queue_status.total_permits.saturating_sub(queue_status.permits_in_use),
|
||||
io_strategy.load_level.as_str(),
|
||||
io_strategy.buffer_multiplier,
|
||||
);
|
||||
|
||||
let strategy_layout = object_io_plan_get_object_strategy_layout(
|
||||
rs,
|
||||
response_content_length,
|
||||
io_strategy.buffer_size,
|
||||
get_buffer_size_opt_in(response_content_length),
|
||||
);
|
||||
|
||||
debug!(
|
||||
actual_request_size = response_content_length,
|
||||
priority = %io_priority.as_str(),
|
||||
"I/O priority finalized with actual request size"
|
||||
);
|
||||
|
||||
debug!(
|
||||
"GetObject buffer sizing: file_size={}, base={}, optimal={}, concurrent_requests={}, io_strategy={:?}",
|
||||
response_content_length,
|
||||
get_buffer_size_opt_in(response_content_length),
|
||||
strategy_layout.optimal_buffer_size,
|
||||
io_strategy.concurrent_requests,
|
||||
io_strategy.load_level
|
||||
);
|
||||
|
||||
strategy_layout.optimal_buffer_size
|
||||
}
|
||||
|
||||
pub(super) fn prepare_put_object_request_context(req: &S3Request<PutObjectInput>) -> PutObjectRequestContext {
|
||||
PutObjectRequestContext {
|
||||
headers: req.headers.clone(),
|
||||
trailing_headers: req.trailing_headers.clone(),
|
||||
uri_query: req.uri.query().map(str::to_string),
|
||||
is_post_object: req.extensions.get::<PostObjectRequestMarker>().is_some(),
|
||||
method: req.method.clone(),
|
||||
uri: req.uri.clone(),
|
||||
extensions: req.extensions.clone(),
|
||||
credentials: req.credentials.clone(),
|
||||
region: req.region.clone(),
|
||||
service: req.service.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn put_object_execution_context(req: &S3Request<PutObjectInput>) -> (EventName, QuotaOperation, &'static str) {
|
||||
if req.extensions.get::<PostObjectRequestMarker>().is_some() {
|
||||
(EventName::ObjectCreatedPost, QuotaOperation::PostObject, "POST")
|
||||
} else {
|
||||
(EventName::ObjectCreatedPut, QuotaOperation::PutObject, "PUT")
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn new_operation_helper<T: Send + Sync>(
|
||||
req: &S3Request<T>,
|
||||
event_name: EventName,
|
||||
operation: S3Operation,
|
||||
suppress_event: bool,
|
||||
) -> OperationHelper {
|
||||
let helper = OperationHelper::new(req, event_name, operation);
|
||||
if suppress_event { helper.suppress_event() } else { helper }
|
||||
}
|
||||
|
||||
pub(super) fn bind_helper_object(
|
||||
helper: OperationHelper,
|
||||
object_info: ObjectInfo,
|
||||
version_id: Option<String>,
|
||||
) -> OperationHelper {
|
||||
let helper = helper.object(object_info);
|
||||
if let Some(version_id) = version_id {
|
||||
helper.version_id(version_id)
|
||||
} else {
|
||||
helper
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn complete_get_flow_result(
|
||||
helper: OperationHelper,
|
||||
request_context: &GetObjectRequestContext,
|
||||
flow_result: GetObjectFlowResult,
|
||||
) -> S3Result<S3Response<GetObjectOutput>> {
|
||||
let helper = helper
|
||||
.object(flow_result.event_info)
|
||||
.version_id(flow_result.version_id_for_event);
|
||||
let response = wrap_response_with_cors(
|
||||
&request_context.bucket,
|
||||
&request_context.method,
|
||||
&request_context.headers,
|
||||
flow_result.output,
|
||||
)
|
||||
.await;
|
||||
let result = Ok(response);
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) fn complete_put_response(helper: OperationHelper, output: PutObjectOutput) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn spawn_put_extract_notification(
|
||||
notify: Arc<dyn NotifyInterface>,
|
||||
request_context: Option<crate::storage::request_context::RequestContext>,
|
||||
bucket: String,
|
||||
req_params: HashMap<String, String>,
|
||||
version_id: String,
|
||||
host: String,
|
||||
port: u16,
|
||||
user_agent: String,
|
||||
obj_info: ObjectInfo,
|
||||
output: PutObjectOutput,
|
||||
) {
|
||||
let event_args = rustfs_notify::EventArgs {
|
||||
event_name: EventName::ObjectCreatedPut,
|
||||
bucket_name: bucket,
|
||||
object: obj_info,
|
||||
req_params,
|
||||
resp_elements: extract_resp_elements(&S3Response::new(output)),
|
||||
version_id,
|
||||
host,
|
||||
port,
|
||||
user_agent,
|
||||
};
|
||||
|
||||
crate::storage::helper::spawn_background_with_context(request_context, async move {
|
||||
notify.notify(event_args).await;
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) async fn get_validated_store_adapter(bucket: &str) -> S3Result<Arc<rustfs_ecstore::store::ECStore>> {
|
||||
get_validated_store(bucket).await
|
||||
}
|
||||
|
||||
pub(super) async fn bucket_prefix_versioning_enabled(bucket: &str, key: &str) -> bool {
|
||||
BucketVersioningSys::prefix_enabled(bucket, key).await
|
||||
}
|
||||
|
||||
pub(super) async fn authorize_extract_put_target(
|
||||
request_context: &PutObjectRequestContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> S3Result<()> {
|
||||
let mut auth_req = S3Request {
|
||||
input: PutObjectInput::default(),
|
||||
method: request_context.method.clone(),
|
||||
uri: request_context.uri.clone(),
|
||||
headers: request_context.headers.clone(),
|
||||
extensions: request_context.extensions.clone(),
|
||||
credentials: request_context.credentials.clone(),
|
||||
region: request_context.region.clone(),
|
||||
service: request_context.service.clone(),
|
||||
trailing_headers: request_context.trailing_headers.clone(),
|
||||
};
|
||||
{
|
||||
let req_info = req_info_mut(&mut auth_req)?;
|
||||
req_info.bucket = Some(bucket.to_string());
|
||||
req_info.object = Some(object.to_string());
|
||||
req_info.version_id = None;
|
||||
}
|
||||
authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await
|
||||
}
|
||||
@@ -13,29 +13,35 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::DeadlockRequestGuard;
|
||||
use super::app_adapters::{
|
||||
GetObjectCompletionInputs, GetObjectStrategyRuntimeInputs, bucket_prefix_versioning_enabled, build_get_object_body_adapter,
|
||||
finalize_get_object_completion, finalize_get_object_strategy_runtime,
|
||||
};
|
||||
use super::get_object_zero_copy::{GetObjectPreparedRead, prepare_get_object_read_execution};
|
||||
use super::types::GetObjectRequestContext;
|
||||
use super::GetObjectRequestContext;
|
||||
use super::get_object_zero_copy::{GetObjectIoPlanning, GetObjectPreparedRead, prepare_get_object_read_execution};
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::concurrency::{self, ConcurrencyManager, GetObjectGuard};
|
||||
use crate::storage::concurrency::{ConcurrencyManager, GetObjectGuard, get_buffer_size_opt_in};
|
||||
use crate::storage::options::filter_object_metadata;
|
||||
use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig};
|
||||
use rustfs_ecstore::store_api::{HTTPRangeSpec, ObjectInfo};
|
||||
use futures_util::StreamExt;
|
||||
use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
|
||||
use rustfs_ecstore::error::StorageError;
|
||||
use rustfs_ecstore::store_api::{GetObjectChunkPath, HTTPRangeSpec, ObjectInfo};
|
||||
use rustfs_io_core::BoxChunkStream;
|
||||
use rustfs_object_io::get::{
|
||||
GetObjectBodyPlanningInputs as ObjectIoGetObjectBodyPlanningInputs, GetObjectBodySource,
|
||||
GetObjectDataPlaneMetricContract as ObjectIoGetObjectDataPlaneMetricContract, GetObjectFlowResult, GetObjectOutputContext,
|
||||
GetObjectReadSetup, build_chunk_blob as object_io_build_chunk_blob,
|
||||
GetObjectBodyPlan as ObjectIoGetObjectBodyPlan, GetObjectBodyPlanningInputs as ObjectIoGetObjectBodyPlanningInputs,
|
||||
GetObjectBodySource, GetObjectDataPlaneMetricContract as ObjectIoGetObjectDataPlaneMetricContract, GetObjectFlowResult,
|
||||
GetObjectOutputContext, GetObjectReadSetup, MaterializeGetObjectBodyError as ObjectIoMaterializeGetObjectBodyError,
|
||||
build_chunk_blob as object_io_build_chunk_blob,
|
||||
build_cors_wrapped_get_object_flow_result as object_io_build_cors_wrapped_get_object_flow_result,
|
||||
build_get_object_checksums as object_io_build_get_object_checksums,
|
||||
build_get_object_output_context as object_io_build_get_object_output_context,
|
||||
chunk_body_data_plane_labels as object_io_chunk_body_data_plane_labels,
|
||||
get_object_chunk_path_label as object_io_get_object_chunk_path_label,
|
||||
materialize_get_object_body as object_io_materialize_get_object_body, plan_get_object_body as object_io_plan_get_object_body,
|
||||
plan_get_object_strategy_layout as object_io_plan_get_object_strategy_layout,
|
||||
};
|
||||
use s3s::S3Result;
|
||||
use s3s::dto::{ContentType, SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryption, Timestamp};
|
||||
use s3s::dto::StreamingBlob;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
pub(super) struct GetObjectBootstrap {
|
||||
pub(super) timeout_config: TimeoutConfig,
|
||||
@@ -45,51 +51,287 @@ pub(super) struct GetObjectBootstrap {
|
||||
pub(super) _deadlock_request_guard: DeadlockRequestGuard,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct GetObjectFlowRuntime<'a> {
|
||||
pub(super) manager: &'a ConcurrencyManager,
|
||||
pub(super) bootstrap: &'a GetObjectBootstrap,
|
||||
pub(super) base_buffer_size: usize,
|
||||
fn classify_get_object_midstream_error(err: &std::io::Error) -> &'static str {
|
||||
let lower = err.to_string().to_ascii_lowercase();
|
||||
if lower.contains("bitrot") {
|
||||
"bitrot"
|
||||
} else if lower.contains("decode") {
|
||||
"decode"
|
||||
} else {
|
||||
match err.kind() {
|
||||
std::io::ErrorKind::UnexpectedEof => "unexpected_eof",
|
||||
std::io::ErrorKind::BrokenPipe | std::io::ErrorKind::ConnectionAborted => "channel_closed",
|
||||
_ => "io_other",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn instrument_get_object_chunk_stream(
|
||||
request_context: &GetObjectRequestContext,
|
||||
chunk_stream: BoxChunkStream,
|
||||
path: GetObjectChunkPath,
|
||||
copy_mode: rustfs_io_metrics::CopyMode,
|
||||
response_content_length: i64,
|
||||
) -> BoxChunkStream {
|
||||
let bucket = request_context.bucket.clone();
|
||||
let key = request_context.key.clone();
|
||||
let version_id = request_context.opts.version_id.clone();
|
||||
let path_label = object_io_get_object_chunk_path_label(path);
|
||||
let mut sent_bytes = 0usize;
|
||||
|
||||
Box::pin(chunk_stream.map(move |result| match result {
|
||||
Ok(chunk) => {
|
||||
sent_bytes = sent_bytes.saturating_add(chunk.len());
|
||||
Ok(chunk)
|
||||
}
|
||||
Err(err) => {
|
||||
let error_kind = classify_get_object_midstream_error(&err);
|
||||
rustfs_io_metrics::record_get_object_fast_path_midstream_error(
|
||||
path_label,
|
||||
copy_mode,
|
||||
error_kind,
|
||||
sent_bytes,
|
||||
response_content_length,
|
||||
);
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
version_id = ?version_id,
|
||||
path = path_label,
|
||||
copy_mode = copy_mode.as_str(),
|
||||
promised_bytes = response_content_length,
|
||||
sent_bytes,
|
||||
error_kind,
|
||||
error = %err,
|
||||
"GetObject chunk fast path failed mid-stream after response commit"
|
||||
);
|
||||
Err(err)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
async fn build_get_object_body_adapter<R>(
|
||||
final_stream: R,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
response_content_length: i64,
|
||||
optimal_buffer_size: usize,
|
||||
planning_inputs: ObjectIoGetObjectBodyPlanningInputs,
|
||||
) -> S3Result<Option<StreamingBlob>>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
let body_plan = object_io_plan_get_object_body(planning_inputs, rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD);
|
||||
|
||||
match body_plan {
|
||||
ObjectIoGetObjectBodyPlan::BufferSeekable => {
|
||||
debug!(
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
size = response_content_length,
|
||||
"reading object into memory for seek support"
|
||||
);
|
||||
}
|
||||
ObjectIoGetObjectBodyPlan::Stream if planning_inputs.encryption_applied => {
|
||||
info!(
|
||||
"Encrypted object: Using unlimited stream for decryption with buffer size {}",
|
||||
optimal_buffer_size
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let materialized =
|
||||
object_io_materialize_get_object_body(final_stream, body_plan, response_content_length, optimal_buffer_size)
|
||||
.await
|
||||
.map_err(|err| match err {
|
||||
ObjectIoMaterializeGetObjectBodyError::EncryptedRead(err) => {
|
||||
error!("Failed to read decrypted object into memory: {}", err);
|
||||
ApiError::from(StorageError::other(format!("Failed to read decrypted object: {err}")))
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(materialized.body)
|
||||
}
|
||||
|
||||
fn finalize_get_object_completion(
|
||||
request_context: &GetObjectRequestContext,
|
||||
wrapper: &RequestTimeoutWrapper,
|
||||
timeout_config: &TimeoutConfig,
|
||||
total_duration: Duration,
|
||||
response_content_length: i64,
|
||||
optimal_buffer_size: usize,
|
||||
metric_contract: ObjectIoGetObjectDataPlaneMetricContract,
|
||||
) {
|
||||
rustfs_io_metrics::record_get_object_completion(total_duration.as_secs_f64(), response_content_length, optimal_buffer_size);
|
||||
|
||||
rustfs_io_metrics::record_get_object(total_duration.as_millis() as f64, response_content_length);
|
||||
rustfs_io_metrics::record_io_copy_mode("get", metric_contract.copy_mode, response_content_length.max(0) as usize);
|
||||
|
||||
if wrapper.is_timeout() {
|
||||
warn!(
|
||||
bucket = %request_context.bucket,
|
||||
key = %request_context.key,
|
||||
elapsed = ?wrapper.elapsed(),
|
||||
timeout = ?timeout_config.get_object_timeout,
|
||||
"GetObject request exceeded timeout"
|
||||
);
|
||||
rustfs_io_metrics::record_get_object_timeout(None, Some(wrapper.elapsed().as_secs_f64()));
|
||||
}
|
||||
|
||||
debug!(
|
||||
bucket = %request_context.bucket,
|
||||
key = %request_context.key,
|
||||
size = response_content_length,
|
||||
duration = ?total_duration,
|
||||
buffer = optimal_buffer_size,
|
||||
"GetObject completed"
|
||||
);
|
||||
}
|
||||
|
||||
fn get_object_strategy_range<'a>(
|
||||
request_context: &'a GetObjectRequestContext,
|
||||
resolved_range: Option<&'a HTTPRangeSpec>,
|
||||
) -> Option<&'a HTTPRangeSpec> {
|
||||
resolved_range.or(request_context.rs.as_ref())
|
||||
}
|
||||
|
||||
fn finalize_get_object_strategy_runtime(
|
||||
request_context: &GetObjectRequestContext,
|
||||
resolved_range: Option<&HTTPRangeSpec>,
|
||||
manager: &ConcurrencyManager,
|
||||
base_buffer_size: usize,
|
||||
info: &ObjectInfo,
|
||||
response_content_length: i64,
|
||||
io_planning: &GetObjectIoPlanning<'_>,
|
||||
) -> usize {
|
||||
let strategy_range = get_object_strategy_range(request_context, resolved_range);
|
||||
let strategy_layout = object_io_plan_get_object_strategy_layout(
|
||||
strategy_range,
|
||||
response_content_length,
|
||||
0,
|
||||
get_buffer_size_opt_in(response_content_length),
|
||||
);
|
||||
|
||||
if let Some(range_spec) = strategy_range
|
||||
&& range_spec.start >= 0
|
||||
{
|
||||
manager.record_access(range_spec.start as u64, response_content_length as u64);
|
||||
}
|
||||
|
||||
if response_content_length > 0 {
|
||||
manager.record_transfer(response_content_length as u64, io_planning.permit_wait_duration);
|
||||
}
|
||||
|
||||
let io_strategy = manager.calculate_io_strategy_with_context(
|
||||
info.size,
|
||||
base_buffer_size,
|
||||
io_planning.permit_wait_duration,
|
||||
strategy_layout.is_sequential_hint,
|
||||
);
|
||||
|
||||
debug!(
|
||||
wait_ms = io_planning.permit_wait_duration.as_millis() as u64,
|
||||
load_level = ?io_strategy.load_level,
|
||||
buffer_size = io_strategy.buffer_size,
|
||||
buffer_multiplier = io_strategy.buffer_multiplier,
|
||||
readahead = io_strategy.enable_readahead,
|
||||
storage_media = ?io_strategy.storage_media,
|
||||
access_pattern = ?io_strategy.access_pattern,
|
||||
bandwidth_tier = ?io_strategy.bandwidth_tier,
|
||||
concurrent_requests = io_strategy.concurrent_requests,
|
||||
file_size = info.size,
|
||||
is_sequential = strategy_layout.is_sequential_hint,
|
||||
"Enhanced multi-factor I/O strategy calculated"
|
||||
);
|
||||
|
||||
let io_priority = manager.get_io_priority(response_content_length);
|
||||
|
||||
if manager.is_priority_scheduling_enabled() {
|
||||
debug!(
|
||||
bucket = %request_context.bucket,
|
||||
key = %request_context.key,
|
||||
priority = %io_priority,
|
||||
request_size = response_content_length,
|
||||
"I/O priority assigned (based on actual request size)"
|
||||
);
|
||||
|
||||
rustfs_io_metrics::record_io_priority_assignment(io_priority.as_str());
|
||||
}
|
||||
|
||||
rustfs_io_metrics::record_get_object_io_state(
|
||||
io_planning.permit_wait_duration.as_secs_f64(),
|
||||
io_planning.queue_utilization,
|
||||
io_planning.queue_status.permits_in_use,
|
||||
io_planning
|
||||
.queue_status
|
||||
.total_permits
|
||||
.saturating_sub(io_planning.queue_status.permits_in_use),
|
||||
io_strategy.load_level.as_str(),
|
||||
io_strategy.buffer_multiplier,
|
||||
);
|
||||
|
||||
let strategy_layout = object_io_plan_get_object_strategy_layout(
|
||||
strategy_range,
|
||||
response_content_length,
|
||||
io_strategy.buffer_size,
|
||||
get_buffer_size_opt_in(response_content_length),
|
||||
);
|
||||
|
||||
debug!(
|
||||
actual_request_size = response_content_length,
|
||||
priority = %io_priority.as_str(),
|
||||
"I/O priority finalized with actual request size"
|
||||
);
|
||||
|
||||
debug!(
|
||||
"GetObject buffer sizing: file_size={}, base={}, optimal={}, concurrent_requests={}, io_strategy={:?}",
|
||||
response_content_length,
|
||||
get_buffer_size_opt_in(response_content_length),
|
||||
strategy_layout.optimal_buffer_size,
|
||||
io_strategy.concurrent_requests,
|
||||
io_strategy.load_level
|
||||
);
|
||||
|
||||
strategy_layout.optimal_buffer_size
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn build_get_object_output_context(
|
||||
request_context: &GetObjectRequestContext,
|
||||
manager: &ConcurrencyManager,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
info: ObjectInfo,
|
||||
event_info: ObjectInfo,
|
||||
body_source: GetObjectBodySource,
|
||||
rs: Option<HTTPRangeSpec>,
|
||||
content_type: Option<ContentType>,
|
||||
last_modified: Option<Timestamp>,
|
||||
response_content_length: i64,
|
||||
content_range: Option<String>,
|
||||
server_side_encryption: Option<ServerSideEncryption>,
|
||||
sse_customer_algorithm: Option<SSECustomerAlgorithm>,
|
||||
sse_customer_key_md5: Option<SSECustomerKeyMD5>,
|
||||
ssekms_key_id: Option<SSEKMSKeyId>,
|
||||
encryption_applied: bool,
|
||||
permit_wait_duration: Duration,
|
||||
queue_utilization: f64,
|
||||
queue_status: &concurrency::IoQueueStatus,
|
||||
read_setup: GetObjectReadSetup,
|
||||
io_planning: &GetObjectIoPlanning<'_>,
|
||||
base_buffer_size: usize,
|
||||
part_number: Option<usize>,
|
||||
versioned: bool,
|
||||
) -> S3Result<(GetObjectOutputContext, ObjectIoGetObjectDataPlaneMetricContract)> {
|
||||
let optimal_buffer_size = finalize_get_object_strategy_runtime(GetObjectStrategyRuntimeInputs {
|
||||
base_buffer_size,
|
||||
manager,
|
||||
bucket,
|
||||
key,
|
||||
info: &info,
|
||||
rs: rs.as_ref(),
|
||||
let bucket = &request_context.bucket;
|
||||
let key = &request_context.key;
|
||||
let part_number = request_context.part_number;
|
||||
let GetObjectReadSetup {
|
||||
info,
|
||||
event_info,
|
||||
body_source,
|
||||
rs,
|
||||
content_type,
|
||||
last_modified,
|
||||
response_content_length,
|
||||
permit_wait_duration,
|
||||
queue_utilization,
|
||||
queue_status,
|
||||
});
|
||||
content_range,
|
||||
server_side_encryption,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
ssekms_key_id,
|
||||
encryption_applied,
|
||||
} = read_setup;
|
||||
|
||||
let optimal_buffer_size = finalize_get_object_strategy_runtime(
|
||||
request_context,
|
||||
rs.as_ref(),
|
||||
manager,
|
||||
base_buffer_size,
|
||||
&info,
|
||||
response_content_length,
|
||||
io_planning,
|
||||
);
|
||||
|
||||
let (body, metric_contract) = match body_source {
|
||||
GetObjectBodySource::Reader(final_stream) => {
|
||||
@@ -120,6 +362,8 @@ pub(super) async fn build_get_object_output_context(
|
||||
copy_mode,
|
||||
} => {
|
||||
let (io_path, copy_mode) = object_io_chunk_body_data_plane_labels(path, copy_mode);
|
||||
let chunk_stream =
|
||||
instrument_get_object_chunk_stream(request_context, chunk_stream, path, copy_mode, response_content_length);
|
||||
(
|
||||
object_io_build_chunk_blob(chunk_stream),
|
||||
ObjectIoGetObjectDataPlaneMetricContract::disk(io_path, copy_mode),
|
||||
@@ -156,97 +400,130 @@ pub(super) async fn build_get_object_output_context(
|
||||
|
||||
pub(super) async fn run_get_object_flow(
|
||||
request_context: GetObjectRequestContext,
|
||||
runtime: GetObjectFlowRuntime<'_>,
|
||||
version_id_for_event: String,
|
||||
manager: &ConcurrencyManager,
|
||||
bootstrap: &GetObjectBootstrap,
|
||||
base_buffer_size: usize,
|
||||
) -> S3Result<GetObjectFlowResult> {
|
||||
let GetObjectFlowRuntime {
|
||||
manager,
|
||||
bootstrap,
|
||||
base_buffer_size,
|
||||
} = runtime;
|
||||
let timeout_config = &bootstrap.timeout_config;
|
||||
let wrapper = &bootstrap.wrapper;
|
||||
let request_start = bootstrap.request_start;
|
||||
let bucket = request_context.bucket.clone();
|
||||
let key = request_context.key.clone();
|
||||
let version_id_for_event = request_context.version_id_for_event.clone();
|
||||
let part_number = request_context.part_number;
|
||||
let rs = request_context.rs.clone();
|
||||
let opts = request_context.opts.clone();
|
||||
|
||||
let prepared_read = prepare_get_object_read_execution(
|
||||
&request_context,
|
||||
manager,
|
||||
wrapper,
|
||||
timeout_config,
|
||||
&bucket,
|
||||
&key,
|
||||
rs,
|
||||
&opts,
|
||||
part_number,
|
||||
)
|
||||
.await?;
|
||||
let prepared_read = prepare_get_object_read_execution(&request_context, manager, wrapper, timeout_config).await?;
|
||||
let GetObjectPreparedRead { io_planning, read_setup } = prepared_read;
|
||||
let permit_wait_duration = io_planning.permit_wait_duration;
|
||||
let queue_status = io_planning.queue_status;
|
||||
let queue_utilization = io_planning.queue_utilization;
|
||||
|
||||
let GetObjectReadSetup {
|
||||
info,
|
||||
event_info,
|
||||
body_source,
|
||||
rs,
|
||||
content_type,
|
||||
last_modified,
|
||||
response_content_length,
|
||||
content_range,
|
||||
server_side_encryption,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
ssekms_key_id,
|
||||
encryption_applied,
|
||||
} = read_setup;
|
||||
|
||||
let versioned = bucket_prefix_versioning_enabled(&bucket, &key).await;
|
||||
let (output_context, metric_contract) = build_get_object_output_context(
|
||||
&request_context,
|
||||
manager,
|
||||
&bucket,
|
||||
&key,
|
||||
info,
|
||||
event_info,
|
||||
body_source,
|
||||
rs,
|
||||
content_type,
|
||||
last_modified,
|
||||
response_content_length,
|
||||
content_range,
|
||||
server_side_encryption,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
ssekms_key_id,
|
||||
encryption_applied,
|
||||
permit_wait_duration,
|
||||
queue_utilization,
|
||||
&queue_status,
|
||||
base_buffer_size,
|
||||
part_number,
|
||||
versioned,
|
||||
)
|
||||
.await?;
|
||||
let versioned = BucketVersioningSys::prefix_enabled(&request_context.bucket, &request_context.key).await;
|
||||
let (output_context, metric_contract) =
|
||||
build_get_object_output_context(&request_context, manager, read_setup, &io_planning, base_buffer_size, versioned).await?;
|
||||
let response_content_length = output_context.response_content_length;
|
||||
let optimal_buffer_size = output_context.optimal_buffer_size;
|
||||
|
||||
let total_duration = request_start.elapsed();
|
||||
finalize_get_object_completion(GetObjectCompletionInputs {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
finalize_get_object_completion(
|
||||
&request_context,
|
||||
wrapper,
|
||||
timeout_config,
|
||||
total_duration,
|
||||
response_content_length,
|
||||
optimal_buffer_size,
|
||||
metric_contract,
|
||||
});
|
||||
);
|
||||
|
||||
Ok(object_io_build_cors_wrapped_get_object_flow_result(output_context, version_id_for_event))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::get_object_strategy_range;
|
||||
use super::*;
|
||||
use http::HeaderMap;
|
||||
use rustfs_ecstore::store_api::ObjectOptions;
|
||||
use rustfs_io_core::IoChunk;
|
||||
|
||||
fn sample_range(start: i64, end: i64) -> HTTPRangeSpec {
|
||||
HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start,
|
||||
end,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_request_context() -> GetObjectRequestContext {
|
||||
GetObjectRequestContext {
|
||||
bucket: "bucket".to_string(),
|
||||
key: "key".to_string(),
|
||||
part_number: None,
|
||||
rs: None,
|
||||
opts: ObjectOptions::default(),
|
||||
headers: HeaderMap::new(),
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strategy_range_prefers_resolved_range_for_part_reads() {
|
||||
let request_context = sample_request_context();
|
||||
let resolved_range = sample_range(1024, 2047);
|
||||
|
||||
let strategy_range = get_object_strategy_range(&request_context, Some(&resolved_range)).unwrap();
|
||||
|
||||
assert_eq!(strategy_range.start, 1024);
|
||||
assert_eq!(strategy_range.end, 2047);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strategy_range_falls_back_to_raw_request_range() {
|
||||
let mut request_context = sample_request_context();
|
||||
request_context.rs = Some(sample_range(0, 511));
|
||||
|
||||
let strategy_range = get_object_strategy_range(&request_context, None).unwrap();
|
||||
|
||||
assert_eq!(strategy_range.start, 0);
|
||||
assert_eq!(strategy_range.end, 511);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_get_object_midstream_error_maps_expected_variants() {
|
||||
assert_eq!(
|
||||
classify_get_object_midstream_error(&std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "eof")),
|
||||
"unexpected_eof"
|
||||
);
|
||||
assert_eq!(
|
||||
classify_get_object_midstream_error(&std::io::Error::new(std::io::ErrorKind::BrokenPipe, "closed")),
|
||||
"channel_closed"
|
||||
);
|
||||
assert_eq!(
|
||||
classify_get_object_midstream_error(&std::io::Error::other("bitrot verification failed")),
|
||||
"bitrot"
|
||||
);
|
||||
assert_eq!(
|
||||
classify_get_object_midstream_error(&std::io::Error::other("decode chunk failed")),
|
||||
"decode"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn instrument_get_object_chunk_stream_preserves_payload() {
|
||||
let request_context = sample_request_context();
|
||||
let chunk_stream: BoxChunkStream = Box::pin(futures_util::stream::iter(vec![
|
||||
Ok(IoChunk::Shared(bytes::Bytes::from_static(b"hello"))),
|
||||
Ok(IoChunk::Shared(bytes::Bytes::from_static(b" world"))),
|
||||
]));
|
||||
|
||||
let mut instrumented = instrument_get_object_chunk_stream(
|
||||
&request_context,
|
||||
chunk_stream,
|
||||
GetObjectChunkPath::Direct,
|
||||
rustfs_io_metrics::CopyMode::SharedBytes,
|
||||
11,
|
||||
);
|
||||
|
||||
let mut collected = Vec::new();
|
||||
while let Some(chunk) = instrumented.next().await {
|
||||
collected.extend_from_slice(chunk.unwrap().as_bytes().as_ref());
|
||||
}
|
||||
|
||||
assert_eq!(collected, b"hello world");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,29 +12,64 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::app_adapters::get_validated_store_adapter;
|
||||
use super::types::GetObjectRequestContext;
|
||||
use super::GetObjectRequestContext;
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::concurrency::{self, ConcurrencyManager};
|
||||
use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig};
|
||||
use crate::storage::{
|
||||
DecryptionRequest, check_preconditions, sse_decryption, validate_sse_headers_for_read, validate_ssec_for_read,
|
||||
DecryptionRequest, check_preconditions, get_validated_store, sse_decryption, validate_sse_headers_for_read,
|
||||
validate_ssec_for_read,
|
||||
};
|
||||
use futures_util::{StreamExt, stream};
|
||||
use http::HeaderMap;
|
||||
use rustfs_concurrency::GetObjectQueueSnapshot;
|
||||
use rustfs_ecstore::store_api::{HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectOptions};
|
||||
use rustfs_ecstore::store_api::{ObjectIO, ObjectOperations};
|
||||
use rustfs_io_core::{BoxChunkStream, IoChunk};
|
||||
use rustfs_object_io::get::{
|
||||
ChunkReadDecision, ChunkReadPlanError, GetObjectEncryptionState as ObjectIoGetObjectEncryptionState, GetObjectReadSetup,
|
||||
build_reader_read_setup as object_io_build_reader_read_setup,
|
||||
finalize_chunk_read_setup as object_io_finalize_chunk_read_setup,
|
||||
get_object_chunk_fast_path_guard as object_io_get_object_chunk_fast_path_guard, plan_chunk_read as object_io_plan_chunk_read,
|
||||
plan_legacy_read as object_io_plan_legacy_read,
|
||||
get_object_chunk_fast_path_guard as object_io_get_object_chunk_fast_path_guard,
|
||||
get_object_chunk_path_label as object_io_get_object_chunk_path_label, map_chunk_copy_mode as object_io_map_chunk_copy_mode,
|
||||
plan_chunk_read as object_io_plan_chunk_read, plan_legacy_read as object_io_plan_legacy_read,
|
||||
};
|
||||
use rustfs_rio::{Reader, WarpReader};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
fn get_object_chunk_fast_path_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_OBJECT_GET_CHUNK_FAST_PATH_ENABLE,
|
||||
rustfs_config::DEFAULT_OBJECT_GET_CHUNK_FAST_PATH_ENABLE,
|
||||
)
|
||||
}
|
||||
|
||||
async fn probe_chunk_stream_before_commit(
|
||||
mut chunk_stream: BoxChunkStream,
|
||||
response_content_length: i64,
|
||||
) -> Result<BoxChunkStream, rustfs_io_metrics::FallbackReason> {
|
||||
if response_content_length <= 0 {
|
||||
return Ok(chunk_stream);
|
||||
}
|
||||
|
||||
let mut prefetched = Vec::new();
|
||||
|
||||
loop {
|
||||
match chunk_stream.next().await {
|
||||
Some(Ok(chunk)) => {
|
||||
let chunk_len = chunk.len();
|
||||
prefetched.push(chunk);
|
||||
if chunk_len > 0 {
|
||||
let prefix = stream::iter(prefetched.into_iter().map(Ok::<IoChunk, std::io::Error>));
|
||||
return Ok(Box::pin(prefix.chain(chunk_stream)));
|
||||
}
|
||||
}
|
||||
Some(Err(_)) | None => return Err(rustfs_io_metrics::FallbackReason::ProbeFailed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct GetObjectIoPlanning<'a> {
|
||||
pub(super) _disk_permit: tokio::sync::SemaphorePermit<'a>,
|
||||
pub(super) permit_wait_duration: Duration,
|
||||
@@ -115,21 +150,20 @@ pub(super) async fn acquire_get_object_io_planning<'a>(
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn prepare_get_object_read(
|
||||
request_context: &GetObjectRequestContext,
|
||||
store: &rustfs_ecstore::store::ECStore,
|
||||
manager: &ConcurrencyManager,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
rs: Option<HTTPRangeSpec>,
|
||||
h: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
part_number: Option<usize>,
|
||||
read_start: std::time::Instant,
|
||||
) -> S3Result<GetObjectReadSetup> {
|
||||
let reader = store
|
||||
.get_object_reader(bucket, key, rs.clone(), h, opts)
|
||||
.get_object_reader(
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
request_context.rs.clone(),
|
||||
HeaderMap::new(),
|
||||
&request_context.opts,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
@@ -159,7 +193,8 @@ pub(super) async fn prepare_get_object_read(
|
||||
request_context.sse_customer_key.as_ref(),
|
||||
request_context.sse_customer_key_md5.as_ref(),
|
||||
)?;
|
||||
let read_plan = object_io_plan_legacy_read(&info, rs, part_number).map_err(ApiError::from)?;
|
||||
let read_plan =
|
||||
object_io_plan_legacy_read(&info, request_context.rs.clone(), request_context.part_number).map_err(ApiError::from)?;
|
||||
|
||||
debug!(
|
||||
"GET object metadata check: parts={}, provided_sse_key={:?}",
|
||||
@@ -168,8 +203,8 @@ pub(super) async fn prepare_get_object_read(
|
||||
);
|
||||
|
||||
let decryption_request = DecryptionRequest {
|
||||
bucket,
|
||||
key,
|
||||
bucket: &request_context.bucket,
|
||||
key: &request_context.key,
|
||||
metadata: &info.user_defined,
|
||||
sse_customer_key: request_context.sse_customer_key.as_ref(),
|
||||
sse_customer_key_md5: request_context.sse_customer_key_md5.as_ref(),
|
||||
@@ -218,68 +253,52 @@ pub(super) async fn prepare_get_object_read(
|
||||
))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn prepare_get_object_read_execution<'a>(
|
||||
request_context: &GetObjectRequestContext,
|
||||
manager: &'a ConcurrencyManager,
|
||||
wrapper: &RequestTimeoutWrapper,
|
||||
timeout_config: &TimeoutConfig,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
rs: Option<HTTPRangeSpec>,
|
||||
opts: &ObjectOptions,
|
||||
part_number: Option<usize>,
|
||||
) -> S3Result<GetObjectPreparedRead<'a>> {
|
||||
let h = HeaderMap::new();
|
||||
let io_planning = acquire_get_object_io_planning(manager, wrapper, timeout_config, bucket, key).await?;
|
||||
let store = get_validated_store_adapter(bucket).await?;
|
||||
let io_planning =
|
||||
acquire_get_object_io_planning(manager, wrapper, timeout_config, &request_context.bucket, &request_context.key).await?;
|
||||
let store = get_validated_store(&request_context.bucket).await?;
|
||||
|
||||
let read_start = std::time::Instant::now();
|
||||
let read_setup = match object_io_get_object_chunk_fast_path_guard(
|
||||
request_context.sse_customer_key.is_some(),
|
||||
request_context.sse_customer_key_md5.is_some(),
|
||||
) {
|
||||
Ok(()) => match prepare_get_object_chunk_read(
|
||||
request_context,
|
||||
&store,
|
||||
manager,
|
||||
bucket,
|
||||
key,
|
||||
rs.clone(),
|
||||
part_number,
|
||||
opts,
|
||||
read_start,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Some(read_setup) => read_setup,
|
||||
None => {
|
||||
prepare_get_object_read(request_context, &store, manager, bucket, key, rs, h, opts, part_number, read_start)
|
||||
.await?
|
||||
let read_setup = if !get_object_chunk_fast_path_enabled() {
|
||||
rustfs_io_metrics::record_io_fallback(
|
||||
rustfs_io_metrics::IoStage::ReadSetup,
|
||||
rustfs_io_metrics::FallbackReason::FeatureDisabled,
|
||||
);
|
||||
prepare_get_object_read(request_context, &store, manager, read_start).await?
|
||||
} else {
|
||||
match object_io_get_object_chunk_fast_path_guard(
|
||||
request_context.sse_customer_key.is_some(),
|
||||
request_context.sse_customer_key_md5.is_some(),
|
||||
) {
|
||||
Ok(()) => match prepare_get_object_chunk_read(request_context, &store, manager, read_start).await? {
|
||||
Some(read_setup) => read_setup,
|
||||
None => prepare_get_object_read(request_context, &store, manager, read_start).await?,
|
||||
},
|
||||
Err(fallback) => {
|
||||
rustfs_io_metrics::record_io_fallback(fallback.stage, fallback.reason);
|
||||
prepare_get_object_read(request_context, &store, manager, read_start).await?
|
||||
}
|
||||
},
|
||||
Err(fallback) => {
|
||||
rustfs_io_metrics::record_io_fallback(fallback.stage, fallback.reason);
|
||||
prepare_get_object_read(request_context, &store, manager, bucket, key, rs, h, opts, part_number, read_start).await?
|
||||
}
|
||||
};
|
||||
|
||||
Ok(GetObjectPreparedRead { io_planning, read_setup })
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn prepare_get_object_chunk_read(
|
||||
request_context: &GetObjectRequestContext,
|
||||
store: &rustfs_ecstore::store::ECStore,
|
||||
manager: &ConcurrencyManager,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
mut rs: Option<HTTPRangeSpec>,
|
||||
part_number: Option<usize>,
|
||||
opts: &ObjectOptions,
|
||||
read_start: std::time::Instant,
|
||||
) -> S3Result<Option<GetObjectReadSetup>> {
|
||||
let info = store.get_object_info(bucket, key, opts).await.map_err(ApiError::from)?;
|
||||
let info = store
|
||||
.get_object_info(&request_context.bucket, &request_context.key, &request_context.opts)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
validate_sse_headers_for_read(&info.user_defined, &request_context.headers)?;
|
||||
validate_ssec_for_read(
|
||||
@@ -301,7 +320,12 @@ pub(super) async fn prepare_get_object_chunk_read(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let plan = match object_io_plan_chunk_read(&info, opts.version_id.is_none(), rs.clone(), part_number) {
|
||||
let plan = match object_io_plan_chunk_read(
|
||||
&info,
|
||||
request_context.opts.version_id.is_none(),
|
||||
request_context.rs.clone(),
|
||||
request_context.part_number,
|
||||
) {
|
||||
Ok(ChunkReadDecision::Eligible(plan)) => plan,
|
||||
Ok(ChunkReadDecision::Fallback(fallback)) => {
|
||||
rustfs_io_metrics::record_io_fallback(fallback.stage, fallback.reason);
|
||||
@@ -311,14 +335,21 @@ pub(super) async fn prepare_get_object_chunk_read(
|
||||
Err(ChunkReadPlanError::MethodNotAllowed) => return Err(S3Error::new(S3ErrorCode::MethodNotAllowed)),
|
||||
Err(ChunkReadPlanError::Io(err)) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
rs = plan.rs.clone();
|
||||
let rs = plan.rs.clone();
|
||||
let response_content_length = plan.response_content_length;
|
||||
|
||||
let read_duration = read_start.elapsed();
|
||||
manager.record_disk_operation(info.size as u64, read_duration, true).await;
|
||||
let event_info = info.clone();
|
||||
|
||||
let chunk_result = match store
|
||||
.get_object_chunks(bucket, key, rs.clone(), HeaderMap::new(), opts)
|
||||
.get_object_chunks(
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
rs.clone(),
|
||||
HeaderMap::new(),
|
||||
&request_context.opts,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)
|
||||
{
|
||||
@@ -331,8 +362,94 @@ pub(super) async fn prepare_get_object_chunk_read(
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
let path_label = object_io_get_object_chunk_path_label(chunk_result.path);
|
||||
let copy_mode = object_io_map_chunk_copy_mode(chunk_result.copy_mode);
|
||||
let chunk_result = match probe_chunk_stream_before_commit(chunk_result.stream, response_content_length).await {
|
||||
Ok(stream) => rustfs_ecstore::store_api::GetObjectChunkResult {
|
||||
stream,
|
||||
path: chunk_result.path,
|
||||
copy_mode: chunk_result.copy_mode,
|
||||
},
|
||||
Err(reason) => {
|
||||
rustfs_io_metrics::record_io_fallback(rustfs_io_metrics::IoStage::ReadSetup, reason);
|
||||
rustfs_io_metrics::record_get_object_fast_path_probe_failed(path_label, copy_mode, response_content_length);
|
||||
warn!(
|
||||
bucket = %request_context.bucket,
|
||||
key = %request_context.key,
|
||||
version_id = ?request_context.opts.version_id,
|
||||
path = path_label,
|
||||
copy_mode = copy_mode.as_str(),
|
||||
promised_bytes = response_content_length,
|
||||
fallback_reason = reason.as_str(),
|
||||
"GetObject chunk fast path probe failed before response commit"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
|
||||
let setup_result = object_io_finalize_chunk_read_setup(info, event_info, chunk_result, plan);
|
||||
rustfs_io_metrics::record_get_object_fast_path_selected(path_label, copy_mode, response_content_length);
|
||||
rustfs_io_metrics::record_io_path_selected("get", setup_result.io_path);
|
||||
|
||||
Ok(Some(setup_result.read_setup))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{get_object_chunk_fast_path_enabled, probe_chunk_stream_before_commit};
|
||||
use bytes::Bytes;
|
||||
use futures_util::{StreamExt, stream};
|
||||
use rustfs_io_core::{BoxChunkStream, IoChunk};
|
||||
|
||||
#[test]
|
||||
fn get_object_chunk_fast_path_defaults_to_disabled() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_OBJECT_GET_CHUNK_FAST_PATH_ENABLE, || {
|
||||
assert!(!get_object_chunk_fast_path_enabled());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_object_chunk_fast_path_can_be_explicitly_enabled() {
|
||||
temp_env::with_var(rustfs_config::ENV_OBJECT_GET_CHUNK_FAST_PATH_ENABLE, Some("true"), || {
|
||||
assert!(get_object_chunk_fast_path_enabled());
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_chunk_stream_before_commit_preserves_prefetched_payload() {
|
||||
let stream: BoxChunkStream = Box::pin(stream::iter(vec![
|
||||
Ok(IoChunk::Shared(Bytes::from_static(b"hello "))),
|
||||
Ok(IoChunk::Shared(Bytes::from_static(b"world"))),
|
||||
]));
|
||||
|
||||
let mut probed = probe_chunk_stream_before_commit(stream, 11).await.unwrap();
|
||||
let mut collected = Vec::new();
|
||||
while let Some(chunk) = probed.next().await {
|
||||
collected.extend_from_slice(chunk.unwrap().as_bytes().as_ref());
|
||||
}
|
||||
|
||||
assert_eq!(collected, b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_chunk_stream_before_commit_rejects_midstream_failure_before_first_chunk() {
|
||||
let stream: BoxChunkStream = Box::pin(stream::iter(vec![Err(std::io::Error::other("probe failed"))]));
|
||||
|
||||
let err = match probe_chunk_stream_before_commit(stream, 1).await {
|
||||
Ok(_) => panic!("expected probe failure"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert_eq!(err, rustfs_io_metrics::FallbackReason::ProbeFailed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_chunk_stream_before_commit_rejects_unexpected_empty_stream() {
|
||||
let stream: BoxChunkStream = Box::pin(stream::empty());
|
||||
|
||||
let err = match probe_chunk_stream_before_commit(stream, 1).await {
|
||||
Ok(_) => panic!("expected probe failure"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert_eq!(err, rustfs_io_metrics::FallbackReason::ProbeFailed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,29 +64,9 @@ impl DefaultObjectUsecase {
|
||||
let sse_customer_key_md5 = sse_customer_key_md5.or(h_md5);
|
||||
|
||||
let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&request_context.headers)?);
|
||||
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
|
||||
let mut effective_sse = original_sse.or_else(|| {
|
||||
bucket_sse_config.as_ref().and_then(|(config, _timestamp)| {
|
||||
config.rules.first().and_then(|rule| {
|
||||
rule.apply_server_side_encryption_by_default
|
||||
.as_ref()
|
||||
.map(|sse| match sse.sse_algorithm.as_str() {
|
||||
"AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256),
|
||||
"aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
|
||||
_ => ServerSideEncryption::from_static(ServerSideEncryption::AES256),
|
||||
})
|
||||
})
|
||||
})
|
||||
});
|
||||
let mut effective_kms_key_id = ssekms_key_id.or_else(|| {
|
||||
bucket_sse_config.as_ref().and_then(|(config, _timestamp)| {
|
||||
config.rules.first().and_then(|rule| {
|
||||
rule.apply_server_side_encryption_by_default
|
||||
.as_ref()
|
||||
.and_then(|sse| sse.kms_master_key_id.clone())
|
||||
})
|
||||
})
|
||||
});
|
||||
let (default_sse, default_kms_key_id) = resolve_bucket_default_server_side_encryption(&bucket).await;
|
||||
let mut effective_sse = original_sse.or(default_sse);
|
||||
let mut effective_kms_key_id = ssekms_key_id.or(default_kms_key_id);
|
||||
if effective_sse
|
||||
.as_ref()
|
||||
.is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS))
|
||||
@@ -167,10 +147,7 @@ impl DefaultObjectUsecase {
|
||||
let host = get_request_host(&request_context.headers);
|
||||
let port = get_request_port(&request_context.headers);
|
||||
let user_agent = get_request_user_agent(&request_context.headers);
|
||||
let tracing_context = request_context
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.cloned();
|
||||
let tracing_context = request_context.extensions.get::<request_context::RequestContext>().cloned();
|
||||
|
||||
while let Some(entry) = entries.next().await {
|
||||
let mut f = match entry {
|
||||
@@ -199,7 +176,24 @@ impl DefaultObjectUsecase {
|
||||
let is_dir = f.header().entry_type().is_dir();
|
||||
let fpath = normalize_extract_entry_key(&fpath.to_string_lossy(), extract_options.prefix.as_deref(), is_dir);
|
||||
|
||||
authorize_extract_put_target(&request_context, &bucket, &fpath).await?;
|
||||
let mut auth_req = S3Request {
|
||||
input: PutObjectInput::default(),
|
||||
method: request_context.method.clone(),
|
||||
uri: request_context.uri.clone(),
|
||||
headers: request_context.headers.clone(),
|
||||
extensions: request_context.extensions.clone(),
|
||||
credentials: request_context.credentials.clone(),
|
||||
region: request_context.region.clone(),
|
||||
service: request_context.service.clone(),
|
||||
trailing_headers: request_context.trailing_headers.clone(),
|
||||
};
|
||||
{
|
||||
let req_info = req_info_mut(&mut auth_req)?;
|
||||
req_info.bucket = Some(bucket.clone());
|
||||
req_info.object = Some(fpath.clone());
|
||||
req_info.version_id = None;
|
||||
}
|
||||
authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await?;
|
||||
|
||||
let mut size = f.header().size().unwrap_or_default() as i64;
|
||||
let archive_entry_mod_time = f
|
||||
@@ -299,7 +293,9 @@ impl DefaultObjectUsecase {
|
||||
opts.user_defined.extend(encryption_metadata);
|
||||
}
|
||||
opts.user_defined.extend(metadata);
|
||||
let mut reader = rustfs_ecstore::store_api::ChunkNativePutData::new(hrd);
|
||||
let capacity_scope_token = Uuid::new_v4();
|
||||
opts.capacity_scope_token = Some(capacity_scope_token);
|
||||
let mut reader = ChunkNativePutData::new(hrd);
|
||||
|
||||
let obj_info = match store.put_object(&bucket, &fpath, &mut reader, &opts).await {
|
||||
Ok(info) => info,
|
||||
@@ -311,6 +307,7 @@ impl DefaultObjectUsecase {
|
||||
return Err(ApiError::from(e).into());
|
||||
}
|
||||
};
|
||||
record_capacity_write(Some(capacity_scope_token)).await;
|
||||
|
||||
let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag));
|
||||
|
||||
@@ -319,18 +316,23 @@ impl DefaultObjectUsecase {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
spawn_put_extract_notification(
|
||||
notify.clone(),
|
||||
tracing_context.clone(),
|
||||
bucket.clone(),
|
||||
req_params.clone(),
|
||||
version_id.clone(),
|
||||
host.clone(),
|
||||
let event_args = rustfs_notify::EventArgs {
|
||||
event_name: EventName::ObjectCreatedPut,
|
||||
bucket_name: bucket.clone(),
|
||||
object: obj_info.clone(),
|
||||
req_params: req_params.clone(),
|
||||
resp_elements: extract_resp_elements(&S3Response::new(output)),
|
||||
version_id: version_id.clone(),
|
||||
host: host.clone(),
|
||||
port,
|
||||
user_agent.clone(),
|
||||
obj_info.clone(),
|
||||
output,
|
||||
);
|
||||
user_agent: user_agent.clone(),
|
||||
};
|
||||
crate::storage::helper::spawn_background_with_context(tracing_context.clone(), {
|
||||
let notify = notify.clone();
|
||||
async move {
|
||||
notify.notify(event_args).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let mut checksums = PutObjectChecksums {
|
||||
|
||||
@@ -32,7 +32,7 @@ const SLOW_PUT_PHASE_DEBUG_THRESHOLD_MS: u64 = 100;
|
||||
const SLOW_PUT_PHASE_WARN_THRESHOLD_MS: u64 = 1_000;
|
||||
const SLOW_PUT_PHASE_ERROR_THRESHOLD_MS: u64 = 5_000;
|
||||
|
||||
fn resolved_checksum_bytes(checksums: &PutObjectChecksums) -> Option<bytes::Bytes> {
|
||||
fn resolved_checksum_bytes(checksums: &PutObjectChecksums) -> Option<Bytes> {
|
||||
[
|
||||
(rustfs_rio::ChecksumType::CRC32, checksums.crc32.as_deref()),
|
||||
(rustfs_rio::ChecksumType::CRC32C, checksums.crc32c.as_deref()),
|
||||
@@ -52,14 +52,6 @@ fn clamp_small_put_eager_max_bytes(inline_object_limit_bytes: Option<usize>) ->
|
||||
.min(DEFAULT_SMALL_PUT_EAGER_MAX_BYTES as usize) as i64
|
||||
}
|
||||
|
||||
fn env_flag_enabled(name: &str) -> bool {
|
||||
rustfs_utils::get_env_bool(name, false)
|
||||
}
|
||||
|
||||
fn env_non_negative_i64(name: &str) -> Option<i64> {
|
||||
rustfs_utils::get_env_opt_i64(name).filter(|value| *value >= 0)
|
||||
}
|
||||
|
||||
fn topology_aware_small_put_eager_max_bytes(store: &rustfs_ecstore::store::ECStore, versioned: bool) -> i64 {
|
||||
let Some(first_pool) = store.pools.first() else {
|
||||
return DEFAULT_SMALL_PUT_EAGER_MAX_BYTES;
|
||||
@@ -78,11 +70,12 @@ fn topology_aware_small_put_eager_max_bytes(store: &rustfs_ecstore::store::ECSto
|
||||
}
|
||||
|
||||
fn resolved_small_put_eager_max_bytes(default_max_bytes: i64) -> i64 {
|
||||
if env_flag_enabled(ENV_RUSTFS_PUT_FORCE_DISABLE_SMALL_EAGER) {
|
||||
if rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_FORCE_DISABLE_SMALL_EAGER, false) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
env_non_negative_i64(ENV_RUSTFS_PUT_SMALL_EAGER_MAX_BYTES)
|
||||
rustfs_utils::get_env_opt_i64(ENV_RUSTFS_PUT_SMALL_EAGER_MAX_BYTES)
|
||||
.filter(|value| *value >= 0)
|
||||
.map(|value| value.min(DEFAULT_SMALL_PUT_EAGER_MAX_BYTES).min(default_max_bytes))
|
||||
.unwrap_or(default_max_bytes)
|
||||
}
|
||||
@@ -91,37 +84,13 @@ fn should_use_small_put_eager_path(size: i64, eager_max_bytes: i64, compression_
|
||||
size > 0 && size <= eager_max_bytes && !compression_enabled && !encryption_enabled
|
||||
}
|
||||
|
||||
fn request_uses_trailing_checksum(headers: &HeaderMap, trailing_headers: &Option<s3s::TrailingHeaders>) -> bool {
|
||||
trailing_headers.is_some()
|
||||
|| headers.contains_key(AMZ_TRAILER)
|
||||
|| matches!(
|
||||
rustfs_rio::get_content_checksum(headers),
|
||||
Ok(Some(checksum)) if checksum.checksum_type.trailing()
|
||||
)
|
||||
}
|
||||
|
||||
fn put_path_label(small_eager: bool, reduced_copy: bool, compressed: bool) -> &'static str {
|
||||
if small_eager {
|
||||
"small_eager"
|
||||
} else if compressed {
|
||||
"compressed"
|
||||
} else if reduced_copy {
|
||||
"reduced_copy"
|
||||
} else {
|
||||
"legacy_plain"
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn log_put_flow_phase(
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
phase: &str,
|
||||
elapsed: std::time::Duration,
|
||||
elapsed: Duration,
|
||||
object_size: i64,
|
||||
small_eager: bool,
|
||||
reduced_copy: bool,
|
||||
compressed: bool,
|
||||
put_path: &'static str,
|
||||
encrypted: bool,
|
||||
) {
|
||||
let duration_ms = elapsed.as_millis() as u64;
|
||||
@@ -129,21 +98,20 @@ fn log_put_flow_phase(
|
||||
return;
|
||||
}
|
||||
|
||||
let put_path = put_path_label(small_eager, reduced_copy, compressed);
|
||||
if duration_ms >= SLOW_PUT_PHASE_ERROR_THRESHOLD_MS {
|
||||
error!(
|
||||
phase,
|
||||
duration_ms, object_size, put_path, compressed, encrypted, bucket, key, "Small PUT phase is critically slow"
|
||||
duration_ms, object_size, put_path, encrypted, bucket, key, "Small PUT phase is critically slow"
|
||||
);
|
||||
} else if duration_ms >= SLOW_PUT_PHASE_WARN_THRESHOLD_MS {
|
||||
warn!(
|
||||
phase,
|
||||
duration_ms, object_size, put_path, compressed, encrypted, bucket, key, "Small PUT phase is slow"
|
||||
duration_ms, object_size, put_path, encrypted, bucket, key, "Small PUT phase is slow"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
phase,
|
||||
duration_ms, object_size, put_path, compressed, encrypted, bucket, key, "Small PUT phase exceeded debug threshold"
|
||||
duration_ms, object_size, put_path, encrypted, bucket, key, "Small PUT phase exceeded debug threshold"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -159,11 +127,11 @@ impl PooledBufferReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl tokio::io::AsyncRead for PooledBufferReader {
|
||||
impl AsyncRead for PooledBufferReader {
|
||||
fn poll_read(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
_cx: &mut std::task::Context<'_>,
|
||||
buf: &mut tokio::io::ReadBuf<'_>,
|
||||
buf: &mut ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
let remaining = &self.buffer[self.position..];
|
||||
if remaining.is_empty() {
|
||||
@@ -199,7 +167,7 @@ impl HashReaderDetector for PooledBufferReader {}
|
||||
|
||||
impl TryGetIndex for PooledBufferReader {}
|
||||
|
||||
async fn read_small_put_body_eager<S, B, E>(body: S, size: i64, pool: std::sync::Arc<BytesPool>) -> S3Result<PooledBuffer>
|
||||
async fn read_small_put_body_eager<S, B, E>(body: S, size: i64, pool: Arc<BytesPool>) -> S3Result<PooledBuffer>
|
||||
where
|
||||
S: Stream<Item = Result<B, E>>,
|
||||
B: Buf,
|
||||
@@ -243,7 +211,7 @@ where
|
||||
async fn build_small_put_eager_hash_stage<S, B, E>(
|
||||
body: S,
|
||||
size: i64,
|
||||
pool: std::sync::Arc<BytesPool>,
|
||||
pool: Arc<BytesPool>,
|
||||
hash_values: PutObjectLegacyHashValues,
|
||||
headers: &HeaderMap,
|
||||
trailing_headers: Option<s3s::TrailingHeaders>,
|
||||
@@ -274,9 +242,8 @@ impl DefaultObjectUsecase {
|
||||
pub(super) async fn run_put_object_flow(
|
||||
input: PutObjectInput,
|
||||
request_context: PutObjectRequestContext,
|
||||
request_method_name: &'static str,
|
||||
resolved_size: i64,
|
||||
) -> S3Result<PutObjectFlowResult> {
|
||||
) -> S3Result<(PutObjectOutput, ObjectInfo)> {
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
let PutObjectInput {
|
||||
@@ -315,7 +282,7 @@ impl DefaultObjectUsecase {
|
||||
let server_side_encryption =
|
||||
server_side_encryption.or(extract_server_side_encryption_from_headers(&request_context.headers)?);
|
||||
|
||||
validate_object_key(&key, request_method_name)?;
|
||||
validate_object_key(&key, if request_context.is_post_object { "POST" } else { "PUT" })?;
|
||||
|
||||
let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
|
||||
|
||||
@@ -325,33 +292,11 @@ impl DefaultObjectUsecase {
|
||||
let mut small_object_eager_stage = false;
|
||||
let bytes_pool = get_concurrency_manager().bytes_pool();
|
||||
|
||||
let store = get_validated_store_adapter(&bucket).await?;
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
|
||||
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
|
||||
|
||||
let mut effective_sse = server_side_encryption.or_else(|| {
|
||||
bucket_sse_config.as_ref().and_then(|(config, _timestamp)| {
|
||||
config.rules.first().and_then(|rule| {
|
||||
rule.apply_server_side_encryption_by_default
|
||||
.as_ref()
|
||||
.map(|sse| match sse.sse_algorithm.as_str() {
|
||||
"AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256),
|
||||
"aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
|
||||
_ => ServerSideEncryption::from_static(ServerSideEncryption::AES256),
|
||||
})
|
||||
})
|
||||
})
|
||||
});
|
||||
|
||||
let mut effective_kms_key_id = ssekms_key_id.or_else(|| {
|
||||
bucket_sse_config.as_ref().and_then(|(config, _timestamp)| {
|
||||
config.rules.first().and_then(|rule| {
|
||||
rule.apply_server_side_encryption_by_default
|
||||
.as_ref()
|
||||
.and_then(|sse| sse.kms_master_key_id.clone())
|
||||
})
|
||||
})
|
||||
});
|
||||
let (default_sse, default_kms_key_id) = resolve_bucket_default_server_side_encryption(&bucket).await;
|
||||
let mut effective_sse = server_side_encryption.or(default_sse);
|
||||
let mut effective_kms_key_id = ssekms_key_id.or(default_kms_key_id);
|
||||
|
||||
validate_sse_headers_for_write(
|
||||
effective_sse.as_ref(),
|
||||
@@ -423,8 +368,12 @@ impl DefaultObjectUsecase {
|
||||
.await?;
|
||||
let eager_max_bytes =
|
||||
resolved_small_put_eager_max_bytes(topology_aware_small_put_eager_max_bytes(&store, opts.versioned));
|
||||
let can_use_small_put_eager =
|
||||
!request_uses_trailing_checksum(&request_context.headers, &request_context.trailing_headers);
|
||||
let can_use_small_put_eager = request_context.trailing_headers.is_none()
|
||||
&& !request_context.headers.contains_key(AMZ_TRAILER)
|
||||
&& !matches!(
|
||||
rustfs_rio::get_content_checksum(&request_context.headers),
|
||||
Ok(Some(checksum)) if checksum.checksum_type.trailing()
|
||||
);
|
||||
|
||||
let current_opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &request_context.headers)
|
||||
.await
|
||||
@@ -539,15 +488,22 @@ impl DefaultObjectUsecase {
|
||||
|
||||
stage
|
||||
};
|
||||
let put_path = if small_object_eager_stage {
|
||||
"small_eager"
|
||||
} else if transform_stage.compression_applied() {
|
||||
"compressed"
|
||||
} else if plain_reduced_copy_stage {
|
||||
"reduced_copy"
|
||||
} else {
|
||||
"legacy_plain"
|
||||
};
|
||||
log_put_flow_phase(
|
||||
&bucket,
|
||||
&key,
|
||||
"build_hash_stage",
|
||||
reader_stage_start.elapsed(),
|
||||
actual_size,
|
||||
small_object_eager_stage,
|
||||
plain_reduced_copy_stage,
|
||||
transform_stage.compression_applied(),
|
||||
put_path,
|
||||
false,
|
||||
);
|
||||
let mut reader = stage.reader;
|
||||
@@ -587,6 +543,8 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let mt2 = metadata.clone();
|
||||
opts.user_defined.extend(metadata);
|
||||
let capacity_scope_token = Uuid::new_v4();
|
||||
opts.capacity_scope_token = Some(capacity_scope_token);
|
||||
|
||||
let repoptions =
|
||||
get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts.clone());
|
||||
@@ -613,19 +571,17 @@ impl DefaultObjectUsecase {
|
||||
"store_put_object",
|
||||
store_put_start.elapsed(),
|
||||
actual_size,
|
||||
small_object_eager_stage,
|
||||
plain_reduced_copy_stage,
|
||||
transform_stage.compression_applied(),
|
||||
put_path,
|
||||
transform_stage.encryption_applied(),
|
||||
);
|
||||
|
||||
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
|
||||
enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
|
||||
|
||||
rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, obj_info.size as u64).await;
|
||||
|
||||
let raw_version = obj_info.version_id.map(|v| v.to_string());
|
||||
|
||||
let put_version = if bucket_prefix_versioning_enabled(&bucket, &key).await {
|
||||
let put_version = if BucketVersioningSys::prefix_enabled(&bucket, &key).await {
|
||||
raw_version.clone()
|
||||
} else {
|
||||
None
|
||||
@@ -689,8 +645,7 @@ impl DefaultObjectUsecase {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let manager = get_capacity_manager();
|
||||
manager.record_write_operation().await;
|
||||
record_capacity_write(Some(capacity_scope_token)).await;
|
||||
|
||||
{
|
||||
let duration_ms = start_time.elapsed().as_millis() as f64;
|
||||
@@ -711,11 +666,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
Ok(PutObjectFlowResult {
|
||||
output,
|
||||
helper_object: obj_info,
|
||||
helper_version_id: raw_version,
|
||||
})
|
||||
Ok((output, obj_info))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct GetObjectRequestContext {
|
||||
pub(super) bucket: String,
|
||||
pub(super) key: String,
|
||||
pub(super) version_id_for_event: String,
|
||||
pub(super) part_number: Option<usize>,
|
||||
pub(super) rs: Option<HTTPRangeSpec>,
|
||||
pub(super) opts: ObjectOptions,
|
||||
pub(super) headers: HeaderMap,
|
||||
pub(super) method: hyper::Method,
|
||||
pub(super) sse_customer_key: Option<String>,
|
||||
pub(super) sse_customer_key_md5: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) type PutObjectChecksums = rustfs_object_io::put::PutObjectChecksums;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct PutObjectRequestContext {
|
||||
pub(super) headers: HeaderMap,
|
||||
pub(super) trailing_headers: Option<s3s::TrailingHeaders>,
|
||||
pub(super) uri_query: Option<String>,
|
||||
pub(super) is_post_object: bool,
|
||||
pub(super) method: hyper::Method,
|
||||
pub(super) uri: hyper::Uri,
|
||||
pub(super) extensions: http::Extensions,
|
||||
pub(super) credentials: Option<s3s::auth::Credentials>,
|
||||
pub(super) region: Option<s3s::region::Region>,
|
||||
pub(super) service: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) struct PutObjectFlowResult {
|
||||
pub(super) output: PutObjectOutput,
|
||||
pub(super) helper_object: ObjectInfo,
|
||||
pub(super) helper_version_id: Option<String>,
|
||||
}
|
||||
@@ -85,7 +85,7 @@ async fn setup_direct_chunk_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
.unwrap();
|
||||
|
||||
let buckets_list = ecstore
|
||||
.list_bucket(&rustfs_ecstore::store_api::BucketOptions {
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
@@ -151,7 +151,7 @@ async fn setup_direct_chunk_multi_disk_test_env() -> (Vec<PathBuf>, Arc<ECStore>
|
||||
.unwrap();
|
||||
|
||||
let buckets_list = ecstore
|
||||
.list_bucket(&rustfs_ecstore::store_api::BucketOptions {
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
@@ -212,7 +212,7 @@ async fn create_direct_chunk_test_multipart_object(
|
||||
parts
|
||||
}
|
||||
|
||||
fn find_part_file(root: &std::path::Path, part_name: &str) -> Option<PathBuf> {
|
||||
fn find_part_file(root: &Path, part_name: &str) -> Option<PathBuf> {
|
||||
let entries = std::fs::read_dir(root).ok()?;
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
@@ -231,7 +231,7 @@ fn find_part_file(root: &std::path::Path, part_name: &str) -> Option<PathBuf> {
|
||||
None
|
||||
}
|
||||
|
||||
fn find_part_files(root: &std::path::Path, part_name: &str, out: &mut Vec<PathBuf>) {
|
||||
fn find_part_files(root: &Path, part_name: &str, out: &mut Vec<PathBuf>) {
|
||||
let Ok(entries) = std::fs::read_dir(root) else {
|
||||
return;
|
||||
};
|
||||
@@ -248,7 +248,7 @@ fn find_part_files(root: &std::path::Path, part_name: &str, out: &mut Vec<PathBu
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_part_files(root: &std::path::Path, part_name: &str) -> Vec<(PathBuf, Vec<u8>)> {
|
||||
async fn remove_part_files(root: &Path, part_name: &str) -> Vec<(PathBuf, Vec<u8>)> {
|
||||
let mut paths = Vec::new();
|
||||
find_part_files(root, part_name, &mut paths);
|
||||
|
||||
@@ -287,20 +287,11 @@ async fn select_reconstructed_chunk_read(
|
||||
continue;
|
||||
}
|
||||
|
||||
let candidate = get_object_zero_copy::prepare_get_object_chunk_read(
|
||||
request_context,
|
||||
ecstore,
|
||||
manager,
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
request_context.rs.clone(),
|
||||
request_context.part_number,
|
||||
&request_context.opts,
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
let candidate =
|
||||
get_object_zero_copy::prepare_get_object_chunk_read(request_context, ecstore, manager, std::time::Instant::now())
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
|
||||
let is_reconstructed = matches!(
|
||||
&candidate.body_source,
|
||||
@@ -334,6 +325,16 @@ fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_get_object_with_fast_path_enabled(
|
||||
usecase: &DefaultObjectUsecase,
|
||||
req: S3Request<GetObjectInput>,
|
||||
) -> S3Result<S3Response<GetObjectOutput>> {
|
||||
temp_env::with_var(rustfs_config::ENV_OBJECT_GET_CHUNK_FAST_PATH_ENABLE, Some("true"), || async move {
|
||||
usecase.execute_get_object(req).await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_object_rejects_zero_part_number() {
|
||||
let input = GetObjectInput::builder()
|
||||
@@ -377,20 +378,11 @@ async fn prepare_get_object_chunk_read_marks_direct_path_for_single_disk_store()
|
||||
let request_context = prepare_get_object_request_context(&req).await.unwrap();
|
||||
let manager = get_concurrency_manager();
|
||||
|
||||
let read_setup = get_object_zero_copy::prepare_get_object_chunk_read(
|
||||
&request_context,
|
||||
&ecstore,
|
||||
manager,
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
request_context.rs.clone(),
|
||||
request_context.part_number,
|
||||
&request_context.opts,
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
let read_setup =
|
||||
get_object_zero_copy::prepare_get_object_chunk_read(&request_context, &ecstore, manager, std::time::Instant::now())
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
|
||||
match read_setup.body_source {
|
||||
GetObjectBodySource::Chunk { path, copy_mode, .. } => {
|
||||
@@ -401,7 +393,7 @@ async fn prepare_get_object_chunk_read_marks_direct_path_for_single_disk_store()
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some(payload.len() as i64));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -441,19 +433,10 @@ async fn prepare_get_object_chunk_read_falls_back_to_legacy_when_chunk_bridge_fa
|
||||
let request_context = prepare_get_object_request_context(&req).await.unwrap();
|
||||
let manager = get_concurrency_manager();
|
||||
|
||||
let read_setup = get_object_zero_copy::prepare_get_object_chunk_read(
|
||||
&request_context,
|
||||
&ecstore,
|
||||
manager,
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
request_context.rs.clone(),
|
||||
request_context.part_number,
|
||||
&request_context.opts,
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let read_setup =
|
||||
get_object_zero_copy::prepare_get_object_chunk_read(&request_context, &ecstore, manager, std::time::Instant::now())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(read_setup.is_none(), "chunk bridge failure should fall back to legacy reader");
|
||||
}
|
||||
@@ -489,20 +472,11 @@ async fn execute_get_object_range_marks_direct_path_for_single_disk_store() {
|
||||
let request_context = prepare_get_object_request_context(&req).await.unwrap();
|
||||
let manager = get_concurrency_manager();
|
||||
|
||||
let read_setup = get_object_zero_copy::prepare_get_object_chunk_read(
|
||||
&request_context,
|
||||
&ecstore,
|
||||
manager,
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
request_context.rs.clone(),
|
||||
request_context.part_number,
|
||||
&request_context.opts,
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
let read_setup =
|
||||
get_object_zero_copy::prepare_get_object_chunk_read(&request_context, &ecstore, manager, std::time::Instant::now())
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
|
||||
match read_setup.body_source {
|
||||
GetObjectBodySource::Chunk { path, .. } => {
|
||||
@@ -512,7 +486,7 @@ async fn execute_get_object_range_marks_direct_path_for_single_disk_store() {
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some(63 * 1024 + 1));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -562,20 +536,11 @@ async fn execute_get_object_range_marks_direct_path_for_multi_disk_store_without
|
||||
let request_context = prepare_get_object_request_context(&req).await.unwrap();
|
||||
let manager = get_concurrency_manager();
|
||||
|
||||
let read_setup = get_object_zero_copy::prepare_get_object_chunk_read(
|
||||
&request_context,
|
||||
&ecstore,
|
||||
manager,
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
request_context.rs.clone(),
|
||||
request_context.part_number,
|
||||
&request_context.opts,
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
let read_setup =
|
||||
get_object_zero_copy::prepare_get_object_chunk_read(&request_context, &ecstore, manager, std::time::Instant::now())
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
|
||||
match read_setup.body_source {
|
||||
GetObjectBodySource::Chunk { path, copy_mode, .. } => {
|
||||
@@ -597,7 +562,7 @@ async fn execute_get_object_range_marks_direct_path_for_multi_disk_store_without
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some((range_end - range_start + 1) as i64));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -646,20 +611,11 @@ async fn execute_get_object_range_marks_reconstructed_path_for_multi_disk_store_
|
||||
let request_context = prepare_get_object_request_context(&req).await.unwrap();
|
||||
let manager = get_concurrency_manager();
|
||||
|
||||
let read_setup = get_object_zero_copy::prepare_get_object_chunk_read(
|
||||
&request_context,
|
||||
&ecstore,
|
||||
manager,
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
request_context.rs.clone(),
|
||||
request_context.part_number,
|
||||
&request_context.opts,
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
let read_setup =
|
||||
get_object_zero_copy::prepare_get_object_chunk_read(&request_context, &ecstore, manager, std::time::Instant::now())
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
|
||||
match read_setup.body_source {
|
||||
GetObjectBodySource::Chunk { path, copy_mode, .. } => {
|
||||
@@ -674,7 +630,7 @@ async fn execute_get_object_range_marks_reconstructed_path_for_multi_disk_store_
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some((range_end - range_start + 1) as i64));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -708,20 +664,11 @@ async fn execute_get_object_part_number_marks_direct_path_for_single_disk_store(
|
||||
let request_context = prepare_get_object_request_context(&req).await.unwrap();
|
||||
let manager = get_concurrency_manager();
|
||||
|
||||
let read_setup = get_object_zero_copy::prepare_get_object_chunk_read(
|
||||
&request_context,
|
||||
&ecstore,
|
||||
manager,
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
request_context.rs.clone(),
|
||||
request_context.part_number,
|
||||
&request_context.opts,
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
let read_setup =
|
||||
get_object_zero_copy::prepare_get_object_chunk_read(&request_context, &ecstore, manager, std::time::Instant::now())
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
|
||||
match read_setup.body_source {
|
||||
GetObjectBodySource::Chunk { path, copy_mode, .. } => {
|
||||
@@ -732,7 +679,7 @@ async fn execute_get_object_part_number_marks_direct_path_for_single_disk_store(
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some(parts[0].len() as i64));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -765,20 +712,11 @@ async fn execute_get_object_whole_multipart_marks_direct_path_for_single_disk_st
|
||||
let request_context = prepare_get_object_request_context(&req).await.unwrap();
|
||||
let manager = get_concurrency_manager();
|
||||
|
||||
let read_setup = get_object_zero_copy::prepare_get_object_chunk_read(
|
||||
&request_context,
|
||||
&ecstore,
|
||||
manager,
|
||||
&request_context.bucket,
|
||||
&request_context.key,
|
||||
request_context.rs.clone(),
|
||||
request_context.part_number,
|
||||
&request_context.opts,
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
let read_setup =
|
||||
get_object_zero_copy::prepare_get_object_chunk_read(&request_context, &ecstore, manager, std::time::Instant::now())
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expected chunk fast path");
|
||||
|
||||
match read_setup.body_source {
|
||||
GetObjectBodySource::Chunk { path, copy_mode, .. } => {
|
||||
@@ -789,7 +727,7 @@ async fn execute_get_object_whole_multipart_marks_direct_path_for_single_disk_st
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some(parts.iter().map(|part| part.len() as i64).sum()));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -846,7 +784,7 @@ async fn execute_get_object_part_number_marks_reconstructed_path_for_multi_disk_
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some(parts[0].len() as i64));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -897,7 +835,7 @@ async fn execute_get_object_part_number_marks_reconstructed_path_for_second_mult
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some(parts[1].len() as i64));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -948,7 +886,7 @@ async fn execute_get_object_part_number_marks_reconstructed_path_for_final_multi
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some(parts[2].len() as i64));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -1025,7 +963,7 @@ async fn execute_get_object_whole_multipart_marks_reconstructed_path_for_missing
|
||||
);
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some(expected.len() as i64));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
@@ -1090,7 +1028,7 @@ async fn execute_get_object_multipart_range_marks_reconstructed_path_for_missing
|
||||
}
|
||||
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
let response = usecase.execute_get_object(req).await.unwrap();
|
||||
let response = execute_get_object_with_fast_path_enabled(&usecase, req).await.unwrap();
|
||||
assert_eq!(response.output.content_length, Some((range_end - range_start + 1) as i64));
|
||||
let mut body = response.output.body.expect("expected body");
|
||||
let mut collected = Vec::new();
|
||||
|
||||
@@ -14,71 +14,26 @@
|
||||
|
||||
//! Capacity management integration for application startup
|
||||
|
||||
use crate::capacity::capacity_manager::{DataSource, get_capacity_manager, start_background_task};
|
||||
use rustfs_ecstore::disk::DiskAPI;
|
||||
use rustfs_io_metrics::{record_capacity_cache_hit, record_capacity_cache_miss};
|
||||
use tracing::{info, warn};
|
||||
use crate::capacity::{get_cached_capacity_with_metrics, init_capacity_management_for_local_disks};
|
||||
|
||||
/// Initialize capacity management system
|
||||
/// This should be called during application startup after local disks are initialized
|
||||
pub async fn init_capacity_management() {
|
||||
info!("Initializing capacity management system...");
|
||||
|
||||
// Get all local disks
|
||||
let disks = rustfs_ecstore::store::all_local_disk().await;
|
||||
|
||||
if disks.is_empty() {
|
||||
warn!("No local disks found, capacity management will not run");
|
||||
return;
|
||||
}
|
||||
|
||||
info!("Found {} local disk(s)", disks.len());
|
||||
|
||||
// Convert DiskStore to Disk (for compatibility with capacity_manager)
|
||||
let disk_refs: Vec<rustfs_madmin::Disk> = disks
|
||||
.iter()
|
||||
.map(|ds| rustfs_madmin::Disk {
|
||||
endpoint: ds.endpoint().to_string(),
|
||||
drive_path: ds.to_string(),
|
||||
root_disk: true,
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Start background update task
|
||||
info!("Starting background capacity update task...");
|
||||
start_background_task(disk_refs).await;
|
||||
|
||||
info!("Capacity management system initialized successfully");
|
||||
init_capacity_management_for_local_disks().await;
|
||||
}
|
||||
|
||||
/// Get capacity statistics with metrics
|
||||
#[allow(dead_code)]
|
||||
pub async fn get_capacity_with_metrics() -> Option<(u64, String)> {
|
||||
let manager = get_capacity_manager();
|
||||
|
||||
// Check cache
|
||||
if let Some(cached) = manager.get_capacity().await {
|
||||
record_capacity_cache_hit();
|
||||
|
||||
let source = match cached.source {
|
||||
DataSource::RealTime => "real-time",
|
||||
DataSource::Scheduled => "scheduled",
|
||||
DataSource::WriteTriggered => "write-triggered",
|
||||
DataSource::Fallback => "fallback",
|
||||
};
|
||||
|
||||
return Some((cached.total_used, source.to_string()));
|
||||
}
|
||||
|
||||
record_capacity_cache_miss();
|
||||
None
|
||||
get_cached_capacity_with_metrics()
|
||||
.await
|
||||
.map(|(capacity, source)| (capacity, source.to_string()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::capacity::capacity_manager::{CapacityUpdate, DataSource, get_capacity_manager};
|
||||
use rustfs_object_capacity::capacity_manager::{CapacityUpdate, DataSource, get_capacity_manager};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_capacity_with_metrics() {
|
||||
|
||||
@@ -1,267 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Comprehensive tests for Hybrid Capacity Manager
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::capacity::capacity_manager::{CapacityUpdate, DataSource, HybridCapacityManager, HybridStrategyConfig};
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_capacity_manager_initialization() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
assert!(manager.get_capacity().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_capacity_update_and_retrieval() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
assert!(manager.get_capacity().await.is_none());
|
||||
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(1000, 10), DataSource::RealTime)
|
||||
.await;
|
||||
|
||||
let cached = manager.get_capacity().await;
|
||||
assert!(cached.is_some());
|
||||
let cached = cached.unwrap();
|
||||
assert_eq!(cached.total_used, 1000);
|
||||
assert_eq!(cached.file_count, 10);
|
||||
assert_eq!(cached.source, DataSource::RealTime);
|
||||
assert!(!cached.is_estimated);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_write_operation_recording() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
manager.record_write_operation().await;
|
||||
manager.record_write_operation().await;
|
||||
manager.record_write_operation().await;
|
||||
|
||||
let frequency = manager.get_write_frequency().await;
|
||||
assert_eq!(frequency, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_fast_update_detection() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
assert!(!manager.needs_fast_update().await);
|
||||
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(1000, 1), DataSource::RealTime)
|
||||
.await;
|
||||
|
||||
assert!(!manager.needs_fast_update().await);
|
||||
|
||||
manager.record_write_operation().await;
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let _needs_update = manager.needs_fast_update().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cache_age_tracking() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
assert!(manager.get_cache_age().await.is_none());
|
||||
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(1000, 1), DataSource::RealTime)
|
||||
.await;
|
||||
|
||||
let age = manager.get_cache_age().await;
|
||||
assert!(age.is_some());
|
||||
let age = age.unwrap();
|
||||
assert!(age < Duration::from_secs(1));
|
||||
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let age = manager.get_cache_age().await.unwrap();
|
||||
assert!(age >= Duration::from_millis(100));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_data_source_tracking() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
let sources = vec![
|
||||
DataSource::RealTime,
|
||||
DataSource::Scheduled,
|
||||
DataSource::WriteTriggered,
|
||||
DataSource::Fallback,
|
||||
];
|
||||
|
||||
for source in sources {
|
||||
manager.update_capacity(CapacityUpdate::exact(1000, 1), source).await;
|
||||
let cached = manager.get_capacity().await.unwrap();
|
||||
assert_eq!(cached.source, source);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_config_from_env() {
|
||||
let config = HybridStrategyConfig::from_env();
|
||||
|
||||
assert_eq!(config.scheduled_update_interval, Duration::from_secs(120));
|
||||
assert_eq!(config.write_trigger_delay, Duration::from_secs(5));
|
||||
assert_eq!(config.write_frequency_threshold, 5);
|
||||
assert_eq!(config.fast_update_threshold, Duration::from_secs(30));
|
||||
assert!(config.enable_smart_update);
|
||||
assert!(config.enable_write_trigger);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_write_frequency_window() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
for _ in 0..20 {
|
||||
manager.record_write_operation().await;
|
||||
}
|
||||
|
||||
let frequency = manager.get_write_frequency().await;
|
||||
assert_eq!(frequency, 20);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrent_access() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let mut handles = vec![];
|
||||
|
||||
for i in 0..10 {
|
||||
let mgr = manager.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
mgr.update_capacity(CapacityUpdate::exact(i as u64 * 100, i), DataSource::RealTime)
|
||||
.await;
|
||||
mgr.record_write_operation().await;
|
||||
});
|
||||
handles.push(handle);
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
handle.await.unwrap();
|
||||
}
|
||||
|
||||
let cached = manager.get_capacity().await;
|
||||
assert!(cached.is_some());
|
||||
|
||||
let frequency = manager.get_write_frequency().await;
|
||||
assert_eq!(frequency, 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_performance_overhead() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
for i in 0..1000 {
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(i as u64, i), DataSource::RealTime)
|
||||
.await;
|
||||
manager.record_write_operation().await;
|
||||
let _ = manager.get_capacity().await;
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
assert!(elapsed < Duration::from_secs(1));
|
||||
|
||||
println!("1000 operations completed in {:?}", elapsed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_refresh_or_join_singleflight() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let mgr1 = manager.clone();
|
||||
let calls1 = calls.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
mgr1.refresh_or_join(DataSource::Scheduled, move || async move {
|
||||
calls1.fetch_add(1, Ordering::SeqCst);
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
Ok(CapacityUpdate::exact(2048, 8))
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
|
||||
let mgr2 = manager.clone();
|
||||
let calls2 = calls.clone();
|
||||
let second = tokio::spawn(async move {
|
||||
mgr2.refresh_or_join(DataSource::WriteTriggered, move || async move {
|
||||
calls2.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(CapacityUpdate::exact(4096, 16))
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
let first = first.await.unwrap().unwrap();
|
||||
let second = second.await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(first.total_used, 2048);
|
||||
assert_eq!(second.total_used, 2048);
|
||||
let cached = manager.get_capacity().await.unwrap();
|
||||
assert_eq!(cached.total_used, 2048);
|
||||
assert_eq!(cached.file_count, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_spawn_refresh_if_needed_deduplicates_background_refresh() {
|
||||
let manager = Arc::new(HybridCapacityManager::from_env());
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let first_manager = manager.clone();
|
||||
let first_calls = calls.clone();
|
||||
let started = first_manager
|
||||
.clone()
|
||||
.spawn_refresh_if_needed(DataSource::Scheduled, move || async move {
|
||||
first_calls.fetch_add(1, Ordering::SeqCst);
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
Ok(CapacityUpdate::estimated(8192, 32))
|
||||
})
|
||||
.await;
|
||||
assert!(started);
|
||||
|
||||
let second_manager = manager.clone();
|
||||
let second_calls = calls.clone();
|
||||
let started = second_manager
|
||||
.clone()
|
||||
.spawn_refresh_if_needed(DataSource::Scheduled, move || async move {
|
||||
second_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(CapacityUpdate::exact(1, 1))
|
||||
})
|
||||
.await;
|
||||
assert!(!started);
|
||||
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
assert!(!manager.refresh_in_progress().await);
|
||||
let cached = manager.get_capacity().await.unwrap();
|
||||
assert_eq!(cached.total_used, 8192);
|
||||
assert!(cached.is_estimated);
|
||||
}
|
||||
}
|
||||
@@ -48,22 +48,11 @@
|
||||
//! Capacity metrics flow through the existing observability pipeline via the `metrics`
|
||||
//! crate and `rustfs-io-metrics`; this module does not expose a Prometheus HTTP endpoint.
|
||||
//!
|
||||
//! ## Testing
|
||||
//!
|
||||
//! For isolated tests, use `create_isolated_manager()` to create independent
|
||||
//! instances instead of the global singleton:
|
||||
//!
|
||||
//! ```ignore
|
||||
//! use crate::capacity::create_isolated_manager;
|
||||
//!
|
||||
//! let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
//! // Test without affecting global state
|
||||
//! ```
|
||||
//!
|
||||
|
||||
pub mod capacity_integration;
|
||||
pub mod capacity_manager;
|
||||
#[cfg(test)]
|
||||
mod capacity_manager_test;
|
||||
#[cfg(test)]
|
||||
mod write_trigger_test;
|
||||
pub mod service;
|
||||
|
||||
pub use service::{
|
||||
capacity_disk_ref, get_cached_capacity_with_metrics, init_capacity_management_for_local_disks, record_capacity_write,
|
||||
refresh_or_join_admin_disks, resolve_admin_used_capacity, spawn_refresh_if_needed_admin_disks,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_ecstore::disk::DiskAPI;
|
||||
use rustfs_io_metrics::capacity_metrics::{
|
||||
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request,
|
||||
record_capacity_scan_mode,
|
||||
};
|
||||
use rustfs_object_capacity::{CapacityDiskRef, capacity_manager, scan};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
pub fn capacity_disk_ref(endpoint: impl Into<String>, drive_path: impl Into<String>) -> CapacityDiskRef {
|
||||
CapacityDiskRef {
|
||||
endpoint: endpoint.into(),
|
||||
drive_path: drive_path.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn capacity_disk_refs(disks: &[rustfs_madmin::Disk]) -> Vec<CapacityDiskRef> {
|
||||
disks
|
||||
.iter()
|
||||
.map(|disk| capacity_disk_ref(disk.endpoint.clone(), disk.drive_path.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn refresh_admin_disks_with_subset_fallback(
|
||||
capacity_manager: &capacity_manager::HybridCapacityManager,
|
||||
all_disks: Vec<CapacityDiskRef>,
|
||||
allow_dirty_subset: bool,
|
||||
) -> Result<capacity_manager::CapacityUpdate, String> {
|
||||
let (refresh_disks, dirty_subset) = if allow_dirty_subset {
|
||||
scan::select_capacity_refresh_disks(capacity_manager, &all_disks).await
|
||||
} else {
|
||||
(all_disks.clone(), false)
|
||||
};
|
||||
|
||||
match scan::refresh_capacity_with_scope(refresh_disks.clone(), dirty_subset).await {
|
||||
Ok(update) => Ok(update),
|
||||
Err(err) if dirty_subset => {
|
||||
warn!("Dirty-subset capacity refresh failed: {}. Retrying full-disk refresh for recovery", err);
|
||||
scan::refresh_capacity_with_scope(all_disks, false).await
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn refresh_or_join_admin_disks(
|
||||
capacity_manager: Arc<capacity_manager::HybridCapacityManager>,
|
||||
source: capacity_manager::DataSource,
|
||||
disks: &[rustfs_madmin::Disk],
|
||||
allow_dirty_subset: bool,
|
||||
) -> Result<capacity_manager::CapacityUpdate, String> {
|
||||
let all_disks = capacity_disk_refs(disks);
|
||||
let refresh_manager = capacity_manager.clone();
|
||||
|
||||
capacity_manager
|
||||
.refresh_or_join(source, move || {
|
||||
let capacity_manager = refresh_manager.clone();
|
||||
let all_disks = all_disks.clone();
|
||||
async move {
|
||||
refresh_admin_disks_with_subset_fallback(capacity_manager.as_ref(), all_disks, allow_dirty_subset).await
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn spawn_refresh_if_needed_admin_disks(
|
||||
capacity_manager: Arc<capacity_manager::HybridCapacityManager>,
|
||||
source: capacity_manager::DataSource,
|
||||
disks: &[rustfs_madmin::Disk],
|
||||
allow_dirty_subset: bool,
|
||||
) -> bool {
|
||||
let all_disks = capacity_disk_refs(disks);
|
||||
let refresh_manager = capacity_manager.clone();
|
||||
|
||||
capacity_manager
|
||||
.spawn_refresh_if_needed(source, move || async move {
|
||||
refresh_admin_disks_with_subset_fallback(refresh_manager.as_ref(), all_disks, allow_dirty_subset).await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn record_capacity_write(scope_token: Option<uuid::Uuid>) {
|
||||
capacity_manager::get_capacity_manager()
|
||||
.record_write_operation_with_scope_token(scope_token)
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn resolve_admin_used_capacity(disks: &[rustfs_madmin::Disk], fallback_used_capacity: u64) -> u64 {
|
||||
let capacity_manager = capacity_manager::get_capacity_manager();
|
||||
|
||||
if let Some(cached) = capacity_manager.get_capacity().await {
|
||||
record_capacity_cache_hit();
|
||||
let cache_age = cached.last_update.elapsed();
|
||||
let fast_update_threshold = capacity_manager.get_config().fast_update_threshold;
|
||||
|
||||
if cache_age < fast_update_threshold {
|
||||
record_capacity_cache_served("fresh");
|
||||
debug!(
|
||||
"Using cached capacity: {} bytes (age: {:?}, source: {:?}, files={}, estimated={})",
|
||||
cached.total_used, cache_age, cached.source, cached.file_count, cached.is_estimated
|
||||
);
|
||||
return cached.total_used;
|
||||
}
|
||||
|
||||
let needs_update = capacity_manager.needs_fast_update().await;
|
||||
let should_block = capacity_manager.should_block_on_refresh(cache_age);
|
||||
|
||||
if needs_update && should_block {
|
||||
let start = Instant::now();
|
||||
record_capacity_refresh_request("blocking", capacity_manager::DataSource::WriteTriggered.as_metric_label());
|
||||
return match refresh_or_join_admin_disks(
|
||||
capacity_manager.clone(),
|
||||
capacity_manager::DataSource::WriteTriggered,
|
||||
disks,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(update) => {
|
||||
let elapsed = start.elapsed();
|
||||
debug!(
|
||||
"Foreground capacity refresh completed in {:?} (files={}, estimated={})",
|
||||
elapsed, update.file_count, update.is_estimated
|
||||
);
|
||||
update.total_used
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("Foreground capacity refresh failed: {}, using cached value", err);
|
||||
record_capacity_cache_served("stale");
|
||||
cached.total_used
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
record_capacity_cache_served("stale");
|
||||
debug!(
|
||||
"Using stale cached capacity: {} bytes (age: {:?}, source: {:?}, files={}, estimated={}, needs_update={}, blocking={})",
|
||||
cached.total_used, cache_age, cached.source, cached.file_count, cached.is_estimated, needs_update, should_block
|
||||
);
|
||||
|
||||
record_capacity_refresh_request("background", capacity_manager::DataSource::Scheduled.as_metric_label());
|
||||
if spawn_refresh_if_needed_admin_disks(capacity_manager.clone(), capacity_manager::DataSource::Scheduled, disks, true)
|
||||
.await
|
||||
{
|
||||
debug!("Background capacity update started");
|
||||
} else {
|
||||
debug!("Background update already in progress, skipping spawn");
|
||||
}
|
||||
|
||||
return cached.total_used;
|
||||
}
|
||||
|
||||
let start = Instant::now();
|
||||
record_capacity_cache_miss();
|
||||
record_capacity_refresh_request("initial", capacity_manager::DataSource::RealTime.as_metric_label());
|
||||
match refresh_or_join_admin_disks(capacity_manager.clone(), capacity_manager::DataSource::RealTime, disks, false).await {
|
||||
Ok(update) => {
|
||||
let elapsed = start.elapsed();
|
||||
info!(
|
||||
"Initial capacity calculation completed: {} bytes in {:?} (files={}, estimated={})",
|
||||
update.total_used, elapsed, update.file_count, update.is_estimated
|
||||
);
|
||||
update.total_used
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
"Failed to calculate data directory used capacity: {}, falling back to disk used capacity",
|
||||
err
|
||||
);
|
||||
record_capacity_cache_served("fallback");
|
||||
record_capacity_scan_mode("fallback");
|
||||
capacity_manager
|
||||
.update_capacity(
|
||||
capacity_manager::CapacityUpdate::fallback(fallback_used_capacity),
|
||||
capacity_manager::DataSource::Fallback,
|
||||
)
|
||||
.await;
|
||||
fallback_used_capacity
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init_capacity_management_for_local_disks() {
|
||||
info!("Initializing capacity management system...");
|
||||
|
||||
let disks = rustfs_ecstore::store::all_local_disk().await;
|
||||
if disks.is_empty() {
|
||||
warn!("No local disks found, capacity management will not run");
|
||||
return;
|
||||
}
|
||||
|
||||
info!("Found {} local disk(s)", disks.len());
|
||||
|
||||
let disk_refs = disks
|
||||
.iter()
|
||||
.map(|ds| capacity_disk_ref(ds.endpoint().to_string(), ds.to_string()))
|
||||
.collect();
|
||||
|
||||
info!("Starting background capacity update task...");
|
||||
capacity_manager::start_background_task(disk_refs).await;
|
||||
|
||||
info!("Capacity management system initialized successfully");
|
||||
}
|
||||
|
||||
pub async fn get_cached_capacity_with_metrics() -> Option<(u64, &'static str)> {
|
||||
let manager = capacity_manager::get_capacity_manager();
|
||||
|
||||
if let Some(cached) = manager.get_capacity().await {
|
||||
record_capacity_cache_hit();
|
||||
return Some((cached.total_used, capacity_source_label(cached.source)));
|
||||
}
|
||||
|
||||
record_capacity_cache_miss();
|
||||
None
|
||||
}
|
||||
|
||||
fn capacity_source_label(source: capacity_manager::DataSource) -> &'static str {
|
||||
match source {
|
||||
capacity_manager::DataSource::RealTime => "real-time",
|
||||
capacity_manager::DataSource::Scheduled => "scheduled",
|
||||
capacity_manager::DataSource::WriteTriggered => "write-triggered",
|
||||
capacity_manager::DataSource::Fallback => "fallback",
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Write trigger integration tests
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::capacity::capacity_manager::{CapacityUpdate, DataSource, HybridCapacityManager};
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_write_trigger_integration() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
manager.record_write_operation().await;
|
||||
manager.record_write_operation().await;
|
||||
manager.record_write_operation().await;
|
||||
|
||||
let frequency = manager.get_write_frequency().await;
|
||||
assert_eq!(frequency, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_write_trigger_with_capacity_update() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(1000, 4), DataSource::WriteTriggered)
|
||||
.await;
|
||||
|
||||
let cached = manager.get_capacity().await;
|
||||
assert!(cached.is_some());
|
||||
let cached = cached.unwrap();
|
||||
assert_eq!(cached.total_used, 1000);
|
||||
assert_eq!(cached.file_count, 4);
|
||||
assert_eq!(cached.source, DataSource::WriteTriggered);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_write_frequency_tracking() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
assert_eq!(manager.get_write_frequency().await, 0);
|
||||
|
||||
for _ in 0..5 {
|
||||
manager.record_write_operation().await;
|
||||
}
|
||||
|
||||
assert_eq!(manager.get_write_frequency().await, 5);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
assert_eq!(manager.get_write_frequency().await, 5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_needs_fast_update() {
|
||||
let manager = HybridCapacityManager::from_env();
|
||||
|
||||
assert!(!manager.needs_fast_update().await);
|
||||
|
||||
manager
|
||||
.update_capacity(CapacityUpdate::exact(1000, 1), DataSource::Scheduled)
|
||||
.await;
|
||||
|
||||
assert!(!manager.needs_fast_update().await);
|
||||
|
||||
manager.record_write_operation().await;
|
||||
|
||||
let needs_update = manager.needs_fast_update().await;
|
||||
#[allow(clippy::overly_complex_bool_expr)]
|
||||
let _ = needs_update || !needs_update;
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::{GetBucketEncryptionOutput, PutBucketEncryptionOutput, ServerSideEncryptionConfiguration};
|
||||
|
||||
pub(crate) fn build_get_bucket_encryption_output(
|
||||
server_side_encryption_configuration: Option<ServerSideEncryptionConfiguration>,
|
||||
) -> GetBucketEncryptionOutput {
|
||||
GetBucketEncryptionOutput {
|
||||
server_side_encryption_configuration,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_put_bucket_encryption_output() -> PutBucketEncryptionOutput {
|
||||
PutBucketEncryptionOutput::default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{build_get_bucket_encryption_output, build_put_bucket_encryption_output};
|
||||
use s3s::dto::ServerSideEncryptionConfiguration;
|
||||
|
||||
#[test]
|
||||
fn test_build_get_bucket_encryption_output_preserves_configuration() {
|
||||
let config = Some(ServerSideEncryptionConfiguration::default());
|
||||
let output = build_get_bucket_encryption_output(config.clone());
|
||||
|
||||
assert_eq!(output.server_side_encryption_configuration, config);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_put_bucket_encryption_output_is_default() {
|
||||
let output = build_put_bucket_encryption_output();
|
||||
assert_eq!(output, Default::default());
|
||||
}
|
||||
}
|
||||
@@ -22,18 +22,5 @@
|
||||
pub(crate) mod acl;
|
||||
pub(crate) mod bucket;
|
||||
pub(crate) mod common;
|
||||
pub(crate) mod encryption;
|
||||
pub(crate) mod multipart;
|
||||
pub(crate) mod object_lock;
|
||||
/// Object helper facade placeholder.
|
||||
///
|
||||
/// Read-path helpers shared across storage components should live in neutral
|
||||
/// modules (for example, `storage::readers`) and be consumed from there.
|
||||
/// Object-specific extraction steps can be added here incrementally.
|
||||
pub(crate) mod object {}
|
||||
pub(crate) mod replication;
|
||||
pub(crate) mod response;
|
||||
pub(crate) mod restore;
|
||||
pub(crate) mod select;
|
||||
pub(crate) mod tagging;
|
||||
pub(crate) mod validation {}
|
||||
|
||||
@@ -14,11 +14,12 @@
|
||||
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use rustfs_ecstore::set_disk::MAX_PARTS_COUNT;
|
||||
use rustfs_ecstore::store_api::{ListMultipartsInfo, ListPartsInfo};
|
||||
use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
|
||||
const MAX_MULTIPART_UPLOADS_LIST: i32 = 1000;
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub(crate) struct ListPartsParams {
|
||||
pub part_number_marker: Option<usize>,
|
||||
@@ -110,23 +111,16 @@ pub(crate) fn parse_list_multipart_uploads_params(
|
||||
let prefix = prefix.unwrap_or_default();
|
||||
let max_uploads = match max_uploads {
|
||||
Some(value) => {
|
||||
let value = usize::try_from(value).map_err(|_| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InvalidArgument,
|
||||
format!("max-uploads must be between 1 and {}", MAX_PARTS_COUNT),
|
||||
)
|
||||
})?;
|
||||
|
||||
if value == 0 || value > MAX_PARTS_COUNT {
|
||||
if !(1..=MAX_MULTIPART_UPLOADS_LIST).contains(&value) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidArgument,
|
||||
format!("max-uploads must be between 1 and {}", MAX_PARTS_COUNT),
|
||||
format!("max-uploads must be between 1 and {}", MAX_MULTIPART_UPLOADS_LIST),
|
||||
));
|
||||
}
|
||||
|
||||
value
|
||||
value as usize
|
||||
}
|
||||
None => MAX_PARTS_COUNT,
|
||||
None => MAX_MULTIPART_UPLOADS_LIST as usize,
|
||||
};
|
||||
|
||||
if let Some(key_marker) = &key_marker
|
||||
@@ -181,12 +175,11 @@ pub(crate) fn build_list_multipart_uploads_output(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params,
|
||||
parse_list_parts_params,
|
||||
MAX_MULTIPART_UPLOADS_LIST, build_list_multipart_uploads_output, build_list_parts_output,
|
||||
parse_list_multipart_uploads_params, parse_list_parts_params,
|
||||
};
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use rustfs_ecstore::set_disk::MAX_PARTS_COUNT;
|
||||
use rustfs_ecstore::store_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::Timestamp;
|
||||
@@ -319,6 +312,13 @@ mod tests {
|
||||
assert_eq!(*err.code(), S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_list_parts_params_rejects_negative_part_number_marker() {
|
||||
let err = parse_list_parts_params(Some(-1), None).expect_err("expected invalid part_number_marker");
|
||||
assert_eq!(*err.code(), S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("part-number-marker must be non-negative"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_list_multipart_uploads_params_defaults_and_valid_values() {
|
||||
let parsed =
|
||||
@@ -331,7 +331,7 @@ mod tests {
|
||||
let parsed = parse_list_multipart_uploads_params(None, None, None).expect("expected default params");
|
||||
assert_eq!(parsed.prefix, "");
|
||||
assert_eq!(parsed.key_marker, None);
|
||||
assert_eq!(parsed.max_uploads, MAX_PARTS_COUNT);
|
||||
assert_eq!(parsed.max_uploads, MAX_MULTIPART_UPLOADS_LIST as usize);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -353,7 +353,7 @@ mod tests {
|
||||
.expect_err("expected invalid max_uploads");
|
||||
assert_eq!(*err.code(), S3ErrorCode::InvalidArgument);
|
||||
|
||||
let err = parse_list_multipart_uploads_params(Some("prefix/".to_string()), None, Some((MAX_PARTS_COUNT + 1) as i32))
|
||||
let err = parse_list_multipart_uploads_params(Some("prefix/".to_string()), None, Some(MAX_MULTIPART_UPLOADS_LIST + 1))
|
||||
.expect_err("expected invalid max_uploads");
|
||||
assert_eq!(*err.code(), S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::{
|
||||
GetObjectLegalHoldOutput, GetObjectLockConfigurationOutput, GetObjectRetentionOutput, ObjectLockConfiguration,
|
||||
ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode, PutObjectLegalHoldOutput,
|
||||
PutObjectRetentionOutput, RequestCharged, Timestamp,
|
||||
};
|
||||
|
||||
pub(crate) fn build_get_object_legal_hold_output(legal_hold_status: Option<String>) -> GetObjectLegalHoldOutput {
|
||||
let status = legal_hold_status.unwrap_or_else(|| ObjectLockLegalHoldStatus::OFF.to_string());
|
||||
GetObjectLegalHoldOutput {
|
||||
legal_hold: Some(ObjectLockLegalHold {
|
||||
status: Some(ObjectLockLegalHoldStatus::from(status)),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_get_object_lock_configuration_output(
|
||||
object_lock_configuration: Option<ObjectLockConfiguration>,
|
||||
) -> GetObjectLockConfigurationOutput {
|
||||
GetObjectLockConfigurationOutput {
|
||||
object_lock_configuration,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_get_object_retention_output(
|
||||
mode: Option<ObjectLockRetentionMode>,
|
||||
retain_until_date: Option<Timestamp>,
|
||||
) -> GetObjectRetentionOutput {
|
||||
GetObjectRetentionOutput {
|
||||
retention: Some(ObjectLockRetention { mode, retain_until_date }),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_put_object_legal_hold_output() -> PutObjectLegalHoldOutput {
|
||||
PutObjectLegalHoldOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_put_object_retention_output() -> PutObjectRetentionOutput {
|
||||
PutObjectRetentionOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
build_get_object_legal_hold_output, build_get_object_lock_configuration_output, build_get_object_retention_output,
|
||||
build_put_object_legal_hold_output, build_put_object_retention_output,
|
||||
};
|
||||
use s3s::dto::{
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode, RequestCharged,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[test]
|
||||
fn test_build_get_object_legal_hold_output_defaults_to_off_when_missing() {
|
||||
let output = build_get_object_legal_hold_output(None);
|
||||
let status = output
|
||||
.legal_hold
|
||||
.as_ref()
|
||||
.and_then(|hold| hold.status.as_ref())
|
||||
.map(ObjectLockLegalHoldStatus::as_str);
|
||||
assert_eq!(status, Some(ObjectLockLegalHoldStatus::OFF));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_get_object_legal_hold_output_uses_input_status() {
|
||||
let output = build_get_object_legal_hold_output(Some(ObjectLockLegalHoldStatus::ON.to_string()));
|
||||
let status = output
|
||||
.legal_hold
|
||||
.as_ref()
|
||||
.and_then(|hold| hold.status.as_ref())
|
||||
.map(ObjectLockLegalHoldStatus::as_str);
|
||||
assert_eq!(status, Some(ObjectLockLegalHoldStatus::ON));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_get_object_lock_configuration_output_preserves_field() {
|
||||
let cfg = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
..Default::default()
|
||||
};
|
||||
let output = build_get_object_lock_configuration_output(Some(cfg.clone()));
|
||||
assert_eq!(output.object_lock_configuration, Some(cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_get_object_retention_output_preserves_fields() {
|
||||
let mode = Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE));
|
||||
let retain_until_date = Some(OffsetDateTime::UNIX_EPOCH.into());
|
||||
let output = build_get_object_retention_output(mode.clone(), retain_until_date.clone());
|
||||
|
||||
let retention = output.retention.expect("retention should be present");
|
||||
assert_eq!(retention.mode, mode);
|
||||
assert_eq!(retention.retain_until_date, retain_until_date);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_put_object_legal_hold_output_sets_request_charged() {
|
||||
let output = build_put_object_legal_hold_output();
|
||||
assert_eq!(
|
||||
output.request_charged.as_ref().map(RequestCharged::as_str),
|
||||
Some(RequestCharged::REQUESTER)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_put_object_retention_output_sets_request_charged() {
|
||||
let output = build_put_object_retention_output();
|
||||
assert_eq!(
|
||||
output.request_charged.as_ref().map(RequestCharged::as_str),
|
||||
Some(RequestCharged::REQUESTER)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::{GetBucketReplicationOutput, PutBucketReplicationOutput, ReplicationConfiguration};
|
||||
|
||||
pub(crate) fn build_get_bucket_replication_output(
|
||||
replication_configuration: ReplicationConfiguration,
|
||||
) -> GetBucketReplicationOutput {
|
||||
GetBucketReplicationOutput {
|
||||
replication_configuration: Some(replication_configuration),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_put_bucket_replication_output() -> PutBucketReplicationOutput {
|
||||
PutBucketReplicationOutput::default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{build_get_bucket_replication_output, build_put_bucket_replication_output};
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
|
||||
#[test]
|
||||
fn test_build_get_bucket_replication_output_sets_configuration() {
|
||||
let config = ReplicationConfiguration::default();
|
||||
let output = build_get_bucket_replication_output(config.clone());
|
||||
|
||||
assert_eq!(output.replication_configuration, Some(config));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_put_bucket_replication_output_is_default() {
|
||||
let output = build_put_bucket_replication_output();
|
||||
assert_eq!(output, Default::default());
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::ApiError;
|
||||
use rustfs_ecstore::error::StorageError;
|
||||
use s3s::{S3Error, S3ErrorCode, S3Response};
|
||||
|
||||
pub(crate) fn s3_response<T>(output: T) -> S3Response<T> {
|
||||
S3Response::new(output)
|
||||
}
|
||||
|
||||
pub(crate) fn not_initialized_error() -> S3Error {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, "Not init")
|
||||
}
|
||||
|
||||
pub(crate) fn access_denied_error() -> S3Error {
|
||||
S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied")
|
||||
}
|
||||
|
||||
pub(crate) fn map_abort_multipart_upload_error(err: StorageError) -> S3Error {
|
||||
// For abort multipart upload, malformed upload IDs should be hidden as NoSuchUpload
|
||||
// to match S3 API compatibility expectations.
|
||||
if matches!(err, StorageError::MalformedUploadID(_)) {
|
||||
return S3Error::new(S3ErrorCode::NoSuchUpload);
|
||||
}
|
||||
|
||||
ApiError::from(err).into()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{access_denied_error, map_abort_multipart_upload_error, not_initialized_error, s3_response};
|
||||
use rustfs_ecstore::error::StorageError;
|
||||
use s3s::{S3ErrorCode, S3Response};
|
||||
|
||||
#[test]
|
||||
fn test_s3_response_wraps_output() {
|
||||
let response: S3Response<i32> = s3_response(7);
|
||||
assert_eq!(response.output, 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_initialized_error_shape() {
|
||||
let err = not_initialized_error();
|
||||
assert_eq!(*err.code(), S3ErrorCode::InternalError);
|
||||
assert_eq!(err.message(), Some("Not init"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_access_denied_error_shape() {
|
||||
let err = access_denied_error();
|
||||
assert_eq!(*err.code(), S3ErrorCode::AccessDenied);
|
||||
assert_eq!(err.message(), Some("Access Denied"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_map_abort_multipart_upload_error_for_malformed_id() {
|
||||
let err = map_abort_multipart_upload_error(StorageError::MalformedUploadID("bad-id".to_string()));
|
||||
assert_eq!(*err.code(), S3ErrorCode::NoSuchUpload);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_map_abort_multipart_upload_error_for_unexpected_error() {
|
||||
let err = map_abort_multipart_upload_error(StorageError::Unexpected);
|
||||
assert_eq!(*err.code(), S3ErrorCode::InternalError);
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::{RequestCharged, RestoreObjectOutput};
|
||||
|
||||
pub(crate) fn build_restore_object_output(
|
||||
request_charged: Option<RequestCharged>,
|
||||
restore_output_path: Option<String>,
|
||||
) -> RestoreObjectOutput {
|
||||
RestoreObjectOutput {
|
||||
request_charged,
|
||||
restore_output_path,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::build_restore_object_output;
|
||||
use s3s::dto::RequestCharged;
|
||||
|
||||
#[test]
|
||||
fn test_build_restore_object_output_preserves_fields() {
|
||||
let output = build_restore_object_output(
|
||||
Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
Some("s3://bucket/prefix/id".to_string()),
|
||||
);
|
||||
|
||||
assert_eq!(output.request_charged, Some(RequestCharged::from_static(RequestCharged::REQUESTER)));
|
||||
assert_eq!(output.restore_output_path, Some("s3://bucket/prefix/id".to_string()));
|
||||
}
|
||||
}
|
||||
@@ -12,10 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::{
|
||||
DeleteBucketTaggingOutput, DeleteObjectTaggingOutput, GetBucketTaggingOutput, GetObjectTaggingOutput, PutBucketTaggingOutput,
|
||||
PutObjectTaggingOutput, Tag,
|
||||
};
|
||||
use s3s::dto::Tag;
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::collections::HashSet;
|
||||
|
||||
@@ -57,39 +54,11 @@ pub(crate) fn validate_object_tag_set(tag_set: &[Tag]) -> S3Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn build_get_bucket_tagging_output(tag_set: Vec<Tag>) -> GetBucketTaggingOutput {
|
||||
GetBucketTaggingOutput { tag_set }
|
||||
}
|
||||
|
||||
pub(crate) fn build_get_object_tagging_output(tag_set: Vec<Tag>, version_id: Option<String>) -> GetObjectTaggingOutput {
|
||||
GetObjectTaggingOutput { tag_set, version_id }
|
||||
}
|
||||
|
||||
pub(crate) fn build_put_object_tagging_output(version_id: Option<String>) -> PutObjectTaggingOutput {
|
||||
PutObjectTaggingOutput { version_id }
|
||||
}
|
||||
|
||||
pub(crate) fn build_delete_object_tagging_output(version_id: Option<String>) -> DeleteObjectTaggingOutput {
|
||||
DeleteObjectTaggingOutput { version_id }
|
||||
}
|
||||
|
||||
pub(crate) fn build_put_bucket_tagging_output() -> PutBucketTaggingOutput {
|
||||
PutBucketTaggingOutput::default()
|
||||
}
|
||||
|
||||
pub(crate) fn build_delete_bucket_tagging_output() -> DeleteBucketTaggingOutput {
|
||||
DeleteBucketTaggingOutput {}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
build_delete_bucket_tagging_output, build_delete_object_tagging_output, build_get_bucket_tagging_output,
|
||||
build_get_object_tagging_output, build_put_bucket_tagging_output, build_put_object_tagging_output,
|
||||
validate_object_tag_set,
|
||||
};
|
||||
use super::validate_object_tag_set;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::{DeleteBucketTaggingOutput, Tag};
|
||||
use s3s::dto::Tag;
|
||||
|
||||
fn tag(key: Option<&str>, value: Option<&str>) -> Tag {
|
||||
Tag {
|
||||
@@ -158,30 +127,4 @@ mod tests {
|
||||
assert_eq!(*err.code(), S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Cannot provide multiple Tags with the same key"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_tagging_outputs_preserve_fields() {
|
||||
let tag_set = vec![tag(Some("k1"), Some("v1"))];
|
||||
let version_id = Some("vid-1".to_string());
|
||||
|
||||
let bucket_output = build_get_bucket_tagging_output(tag_set.clone());
|
||||
let get_object_output = build_get_object_tagging_output(tag_set.clone(), version_id.clone());
|
||||
let put_object_output = build_put_object_tagging_output(version_id.clone());
|
||||
let delete_object_output = build_delete_object_tagging_output(version_id.clone());
|
||||
|
||||
assert_eq!(bucket_output.tag_set, tag_set);
|
||||
assert_eq!(get_object_output.tag_set, vec![tag(Some("k1"), Some("v1"))]);
|
||||
assert_eq!(get_object_output.version_id, version_id);
|
||||
assert_eq!(put_object_output.version_id, Some("vid-1".to_string()));
|
||||
assert_eq!(delete_object_output.version_id, Some("vid-1".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_bucket_tagging_outputs_are_default_shape() {
|
||||
let put_output = build_put_bucket_tagging_output();
|
||||
let delete_output = build_delete_bucket_tagging_output();
|
||||
|
||||
assert_eq!(put_output, Default::default());
|
||||
assert_eq!(delete_output, DeleteBucketTaggingOutput {});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user