gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improve legacy metadata and admin compatibility (#2202)

Browse Source
This commit is contained in:
weisd
2026-03-18 21:05:09 +08:00
committed by GitHub
parent 84077adf17
commit b9b7d86ae4
133 changed files with 11707 additions and 1945 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -16,6 +16,7 @@ vendor
cli/rustfs-gui/embedded-rustfs/rustfs
*.log
deploy/certs/*
deploy/data/*
*jsonl
.env
.rustfs.sys
Cargo.lock
Generated
+140 -39
View File
@@ -69,6 +69,17 @@ dependencies = [
"subtle",
]
[[package]]
name = "ahash"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
dependencies = [
"getrandom 0.2.17",
"once_cell",
"version_check",
]
[[package]]
name = "ahash"
version = "0.8.12"
@@ -285,7 +296,7 @@ version = "57.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef"
dependencies = [
"ahash",
"ahash 0.8.12",
"arrow-buffer",
"arrow-data",
"arrow-schema",
@@ -442,7 +453,7 @@ version = "57.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b"
dependencies = [
"ahash",
"ahash 0.8.12",
"arrow-array",
"arrow-buffer",
"arrow-data",
@@ -731,7 +742,7 @@ dependencies = [
"http 0.2.12",
"http 1.4.0",
"http-body 1.0.1",
"lru",
"lru 0.16.3",
"percent-encoding",
"regex-lite",
"sha2 0.10.9",
@@ -2192,7 +2203,7 @@ dependencies = [
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
"parking_lot_core 0.9.12",
]
[[package]]
@@ -2244,7 +2255,7 @@ dependencies = [
"liblzma",
"log",
"object_store",
"parking_lot",
"parking_lot 0.12.5",
"parquet",
"rand 0.9.2",
"regex",
@@ -2277,7 +2288,7 @@ dependencies = [
"itertools 0.14.0",
"log",
"object_store",
"parking_lot",
"parking_lot 0.12.5",
"tokio",
]
@@ -2310,7 +2321,7 @@ version = "52.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea2df29b9592a5d55b8238eaf67d2f21963d5a08cd1a8b7670134405206caabd"
dependencies = [
"ahash",
"ahash 0.8.12",
"arrow",
"arrow-ipc",
"chrono",
@@ -2468,7 +2479,7 @@ dependencies = [
"itertools 0.14.0",
"log",
"object_store",
"parking_lot",
"parking_lot 0.12.5",
"parquet",
"tokio",
]
@@ -2494,7 +2505,7 @@ dependencies = [
"futures",
"log",
"object_store",
"parking_lot",
"parking_lot 0.12.5",
"rand 0.9.2",
"tempfile",
"url",
@@ -2573,7 +2584,7 @@ version = "52.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "325a00081898945d48d6194d9ca26120e523c993be3bb7c084061a5a2a72e787"
dependencies = [
"ahash",
"ahash 0.8.12",
"arrow",
"datafusion-common",
"datafusion-doc",
@@ -2594,7 +2605,7 @@ version = "52.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "809bbcb1e0dbec5d0ce30d493d135aea7564f1ba4550395f7f94321223df2dae"
dependencies = [
"ahash",
"ahash 0.8.12",
"arrow",
"datafusion-common",
"datafusion-expr-common",
@@ -2636,7 +2647,7 @@ dependencies = [
"datafusion-common",
"datafusion-expr",
"datafusion-physical-plan",
"parking_lot",
"parking_lot 0.12.5",
"paste",
]
@@ -2705,7 +2716,7 @@ version = "52.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d3a86264bb9163e7360b6622e789bc7fcbb43672e78a8493f0bc369a41a57c6"
dependencies = [
"ahash",
"ahash 0.8.12",
"arrow",
"datafusion-common",
"datafusion-expr",
@@ -2716,7 +2727,7 @@ dependencies = [
"hashbrown 0.16.1",
"indexmap 2.13.0",
"itertools 0.14.0",
"parking_lot",
"parking_lot 0.12.5",
"paste",
"petgraph",
"recursive",
@@ -2744,7 +2755,7 @@ version = "52.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ae769ea5d688b4e74e9be5cad6f9d9f295b540825355868a3ab942380dd97ce"
dependencies = [
"ahash",
"ahash 0.8.12",
"arrow",
"chrono",
"datafusion-common",
@@ -2752,7 +2763,7 @@ dependencies = [
"hashbrown 0.16.1",
"indexmap 2.13.0",
"itertools 0.14.0",
"parking_lot",
"parking_lot 0.12.5",
]
[[package]]
@@ -2780,7 +2791,7 @@ version = "52.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79949cbb109c2a45c527bfe0d956b9f2916807c05d4d2e66f3fd0af827ac2b61"
dependencies = [
"ahash",
"ahash 0.8.12",
"arrow",
"arrow-ord",
"arrow-schema",
@@ -2800,7 +2811,7 @@ dependencies = [
"indexmap 2.13.0",
"itertools 0.14.0",
"log",
"parking_lot",
"parking_lot 0.12.5",
"pin-project-lite",
"tokio",
]
@@ -2833,7 +2844,7 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-plan",
"parking_lot",
"parking_lot 0.12.5",
]
[[package]]
@@ -2873,9 +2884,9 @@ dependencies = [
"http-body-util",
"libc",
"log",
"lru",
"lru 0.16.3",
"mime_guess",
"parking_lot",
"parking_lot 0.12.5",
"percent-encoding",
"pin-project-lite",
"reflink-copy",
@@ -4032,6 +4043,9 @@ name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash 0.7.8",
]
[[package]]
name = "hashbrown"
@@ -4501,7 +4515,7 @@ version = "0.11.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
dependencies = [
"ahash",
"ahash 0.8.12",
"indexmap 2.13.0",
"is-terminal",
"itoa",
@@ -4519,7 +4533,7 @@ version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90807d610575744524d9bdc69f3885d96f0e6c3354565b0828354a7ff2a262b8"
dependencies = [
"ahash",
"ahash 0.8.12",
"clap",
"crossbeam-channel",
"crossbeam-utils",
@@ -4553,6 +4567,15 @@ dependencies = [
"hybrid-array",
]
[[package]]
name = "instant"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
dependencies = [
"cfg-if",
]
[[package]]
name = "integer-encoding"
version = "3.0.4"
@@ -5040,6 +5063,15 @@ version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "lru"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a"
dependencies = [
"hashbrown 0.12.3",
]
[[package]]
name = "lru"
version = "0.16.3"
@@ -5202,7 +5234,7 @@ version = "0.24.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d5312e9ba3771cfa961b585728215e3d972c950a3eed9252aa093d6301277e8"
dependencies = [
"ahash",
"ahash 0.8.12",
"portable-atomic",
]
@@ -5271,7 +5303,7 @@ dependencies = [
"equivalent",
"event-listener",
"futures-util",
"parking_lot",
"parking_lot 0.12.5",
"portable-atomic",
"smallvec",
"tagptr",
@@ -5306,7 +5338,7 @@ dependencies = [
"libc",
"log",
"neli-proc-macros",
"parking_lot",
"parking_lot 0.12.5",
]
[[package]]
@@ -5629,7 +5661,7 @@ dependencies = [
"http 1.4.0",
"humantime",
"itertools 0.14.0",
"parking_lot",
"parking_lot 0.12.5",
"percent-encoding",
"thiserror 2.0.18",
"tokio",
@@ -5882,6 +5914,17 @@ version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]]
name = "parking_lot"
version = "0.12.5"
@@ -5889,7 +5932,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
dependencies = [
"lock_api",
"parking_lot_core",
"parking_lot_core 0.9.12",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
]
[[package]]
@@ -5911,7 +5968,7 @@ version = "57.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb"
dependencies = [
"ahash",
"ahash 0.8.12",
"arrow-array",
"arrow-buffer",
"arrow-cast",
@@ -6403,7 +6460,7 @@ dependencies = [
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"parking_lot 0.12.5",
"thiserror 2.0.18",
]
@@ -6763,7 +6820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36ea961700fd7260e7fa3701c8287d901b2172c51f9c1421fa0f21d7f7e184b7"
dependencies = [
"clocksource",
"parking_lot",
"parking_lot 0.12.5",
"thiserror 1.0.69",
]
@@ -6827,6 +6884,15 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -6856,6 +6922,21 @@ dependencies = [
"thiserror 2.0.18",
]
[[package]]
name = "reed-solomon-erasure"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f"
dependencies = [
"cc",
"libc",
"libm",
"lru 0.7.8",
"parking_lot 0.11.2",
"smallvec",
"spin 0.9.8",
]
[[package]]
name = "reed-solomon-simd"
version = "3.1.0"
@@ -7292,6 +7373,7 @@ dependencies = [
"rustfs-common",
"rustfs-config",
"rustfs-credentials",
"rustfs-crypto",
"rustfs-ecstore",
"rustfs-filemeta",
"rustfs-heal",
@@ -7326,6 +7408,7 @@ dependencies = [
"starshard",
"subtle",
"sysinfo",
"temp-env",
"tempfile",
"thiserror 2.0.18",
"tikv-jemalloc-ctl",
@@ -7488,12 +7571,13 @@ dependencies = [
"md-5 0.11.0-rc.5",
"moka",
"num_cpus",
"parking_lot",
"parking_lot 0.12.5",
"path-absolutize",
"pin-project-lite",
"quick-xml 0.39.2",
"rand 0.10.0",
"ratelimit",
"reed-solomon-erasure",
"reed-solomon-simd",
"regex",
"reqwest 0.13.2",
@@ -7516,6 +7600,7 @@ dependencies = [
"rustls",
"s3s",
"serde",
"serde_bytes",
"serde_json",
"serde_urlencoded",
"serial_test",
@@ -7633,8 +7718,10 @@ dependencies = [
"reqwest 0.13.2",
"rustfs-credentials",
"rustfs-policy",
"rustfs-utils",
"serde",
"serde_json",
"temp-env",
"thiserror 2.0.18",
"time",
"tokio",
@@ -7656,9 +7743,11 @@ dependencies = [
"moka",
"rand 0.10.0",
"reqwest 0.13.2",
"rustfs-utils",
"serde",
"serde_json",
"sha2 0.11.0-rc.5",
"temp-env",
"tempfile",
"thiserror 2.0.18",
"tokio",
@@ -7676,7 +7765,7 @@ dependencies = [
"async-trait",
"crossbeam-queue",
"futures",
"parking_lot",
"parking_lot 0.12.5",
"rustfs-utils",
"serde",
"serde_json",
@@ -7941,7 +8030,7 @@ dependencies = [
"futures-core",
"http 1.4.0",
"object_store",
"parking_lot",
"parking_lot 0.12.5",
"pin-project-lite",
"rustfs-common",
"rustfs-ecstore",
@@ -7964,7 +8053,7 @@ dependencies = [
"datafusion",
"derive_builder 0.20.2",
"futures",
"parking_lot",
"parking_lot 0.12.5",
"rustfs-s3select-api",
"s3s",
"snafu 0.9.0",
@@ -8066,6 +8155,7 @@ name = "rustfs-utils"
version = "0.0.5"
dependencies = [
"base64-simd",
"blake2 0.11.0-rc.5",
"blake3",
"brotli",
"bytes",
@@ -8097,6 +8187,7 @@ dependencies = [
"siphasher",
"snap",
"sysinfo",
"temp-env",
"tempfile",
"thiserror 2.0.18",
"tokio",
@@ -8306,7 +8397,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.14.0-dev"
source = "git+https://github.com/s3s-project/s3s?rev=c2dc7b16535659904d4efff52c558fc039be1ef3#c2dc7b16535659904d4efff52c558fc039be1ef3"
source = "git+https://github.com/rustfs/s3s?rev=d9556e3c0036bd3f2b330966009cbaa5aebf19a3#d9556e3c0036bd3f2b330966009cbaa5aebf19a3"
dependencies = [
"arc-swap",
"arrayvec",
@@ -8518,6 +8609,16 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_bytes"
version = "0.11.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
dependencies = [
"serde",
"serde_core",
]
[[package]]
name = "serde_core"
version = "1.0.228"
@@ -8645,7 +8746,7 @@ dependencies = [
"futures-util",
"log",
"once_cell",
"parking_lot",
"parking_lot 0.12.5",
"scc",
"serial_test_derive",
]
@@ -9241,7 +9342,7 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050"
dependencies = [
"parking_lot",
"parking_lot 0.12.5",
]
[[package]]
@@ -9468,7 +9569,7 @@ dependencies = [
"bytes",
"libc",
"mio",
"parking_lot",
"parking_lot 0.12.5",
"pin-project-lite",
"signal-hook-registry",
"socket2",
Cargo.toml
+5 -2
View File
@@ -148,6 +148,7 @@ rmcp = { version = "1.2.0" }
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
serde = { version = "1.0.228", features = ["derive"] }
serde_bytes = "0.11"
serde_json = { version = "1.0.149", features = ["raw_value"] }
serde_urlencoded = "0.7.1"
schemars = "1.2.1"
@@ -155,6 +156,7 @@ schemars = "1.2.1"
# Cryptography and Security
aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] }
argon2 = { version = "0.6.0-rc.7" }
blake2 = "0.11.0-rc.5"
blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
chacha20poly1305 = { version = "0.11.0-rc.3" }
crc-fast = "1.9.0"
@@ -236,13 +238,14 @@ pretty_assertions = "1.4.1"
rand = { version = "0.10.0", features = ["serde"] }
ratelimit = "0.10.0"
rayon = "1.11.0"
reed-solomon-simd = { version = "3.1.0" }
reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
reed-solomon-simd = "3.1.0"
regex = { version = "1.12.3" }
rumqttc = { version = "0.25.1" }
rustix = { version = "1.1.4", features = ["fs"] }
rust-embed = { version = "8.11.0" }
rustc-hash = { version = "2.1.1" }
s3s = { git = "https://github.com/s3s-project/s3s", rev = "c2dc7b16535659904d4efff52c558fc039be1ef3", features = ["minio"] }
s3s = { git = "https://github.com/rustfs/s3s", rev = "d9556e3c0036bd3f2b330966009cbaa5aebf19a3", features = ["minio"] }
serial_test = "3.4.0"
shadow-rs = { version = "1.7.1", default-features = false }
siphasher = "1.0.2"
Dockerfile
+1 -6
View File
@@ -86,12 +86,7 @@ RUN addgroup -g 10001 -S rustfs && \
chown -R rustfs:rustfs /data /logs && \
chmod 0750 /data /logs
ENV RUSTFS_ADDRESS=":9000" \
RUSTFS_CONSOLE_ADDRESS=":9001" \
RUSTFS_ACCESS_KEY="rustfsadmin" \
RUSTFS_SECRET_KEY="rustfsadmin" \
RUSTFS_CONSOLE_ENABLE="true" \
RUSTFS_CORS_ALLOWED_ORIGINS="*" \
ENV RUSTFS_CORS_ALLOWED_ORIGINS="*" \
RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS="*" \
RUSTFS_VOLUMES="/data" \
RUSTFS_OBS_LOGGER_LEVEL=warn \
Dockerfile.decommission-local
+65
View File
@@ -0,0 +1,65 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM rust:1.91-trixie
RUN set -eux; \
export DEBIAN_FRONTEND=noninteractive; \
apt-get update; \
apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
curl \
git \
pkg-config \
libssl-dev \
lld \
protobuf-compiler \
flatbuffers-compiler; \
rm -rf /var/lib/apt/lists/*
WORKDIR /usr/src/rustfs
COPY . .
RUN ./scripts/static.sh
RUN cargo run --bin gproto
RUN cargo build --release --locked --bin rustfs
RUN set -eux; \
groupadd -g 10001 rustfs; \
useradd -u 10001 -g rustfs -M -s /usr/sbin/nologin rustfs
WORKDIR /app
RUN set -eux; \
mkdir -p /data /logs; \
chown -R rustfs:rustfs /data /logs /app; \
chmod 0750 /data /logs
COPY entrypoint.sh /entrypoint.sh
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
ENV RUSTFS_VOLUMES="/data" \
RUST_LOG="warn" \
RUSTFS_OBS_LOG_DIRECTORY="/logs" \
RUSTFS_USERNAME="rustfs" \
RUSTFS_GROUPNAME="rustfs" \
RUSTFS_UID="10001" \
RUSTFS_GID="10001"
EXPOSE 9000 9001
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/usr/bin/rustfs"]
Dockerfile.source
+2 -10
View File
@@ -157,11 +157,7 @@ WORKDIR /app
ENV CARGO_INCREMENTAL=1
# Ensure we have the same default env vars available
ENV RUSTFS_ADDRESS=":9000" \
RUSTFS_ACCESS_KEY="rustfsadmin" \
RUSTFS_SECRET_KEY="rustfsadmin" \
RUSTFS_CONSOLE_ENABLE="true" \
RUSTFS_VOLUMES="/data" \
ENV RUSTFS_VOLUMES="/data" \
RUST_LOG="warn" \
RUSTFS_OBS_LOG_DIRECTORY="/logs" \
RUSTFS_USERNAME="rustfs" \
@@ -222,11 +218,7 @@ COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
# Default environment (override in docker run/compose as needed)
ENV RUSTFS_ADDRESS=":9000" \
RUSTFS_ACCESS_KEY="rustfsadmin" \
RUSTFS_SECRET_KEY="rustfsadmin" \
RUSTFS_CONSOLE_ENABLE="true" \
RUSTFS_VOLUMES="/data" \
ENV RUSTFS_VOLUMES="/data" \
RUST_LOG="warn" \
RUSTFS_USERNAME="rustfs" \
RUSTFS_GROUPNAME="rustfs" \
crates/credentials/Cargo.toml
+1 -1
View File
@@ -15,7 +15,7 @@ base64-simd = { workspace = true }
rand = { workspace = true }
serde = { workspace = true }
serde_json.workspace = true
time = { workspace = true, features = ["serde-human-readable"] }
time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] }
[lints]
workspace = true
crates/credentials/src/credentials.rs
+37
View File
@@ -273,15 +273,25 @@ impl<'a> fmt::Display for Masked<'a> {
///
#[derive(Serialize, Deserialize, Clone, Default)]
pub struct Credentials {
#[serde(rename = "accessKey", alias = "access_key", default)]
pub access_key: String,
#[serde(rename = "secretKey", alias = "secret_key", default)]
pub secret_key: String,
#[serde(rename = "sessionToken", alias = "session_token", default)]
pub session_token: String,
#[serde(default, with = "crate::serde_datetime::option")]
pub expiration: Option<OffsetDateTime>,
#[serde(default)]
pub status: String,
#[serde(rename = "parentUser", alias = "parent_user", default)]
pub parent_user: String,
#[serde(default)]
pub groups: Option<Vec<String>>,
#[serde(default)]
pub claims: Option<HashMap<String, Value>>,
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub description: Option<String>,
}
@@ -491,4 +501,31 @@ mod tests {
assert_eq!(format!("{:?}", Masked(Some("中文"))), "中***|2");
assert_eq!(format!("{:?}", Masked(Some("中文测试"))), "中***试|4");
}
#[test]
fn test_credentials_expiration_serialize_as_rfc3339() {
use time::OffsetDateTime;
let c = Credentials {
access_key: "ak".to_string(),
secret_key: "sk12345678".to_string(),
expiration: Some(OffsetDateTime::now_utc()),
..Default::default()
};
let json = serde_json::to_string(&c).expect("serialize");
assert!(
json.contains('T') && (json.contains('Z') || json.contains("+00:00")),
"Credentials expiration should be RFC3339; got: {}",
json
);
}
#[test]
fn test_credentials_deserialize_minio_style_rfc3339_expiration() {
let minio_style = r#"{"accessKey":"ak","secretKey":"sk12345678","expiration":"2025-03-07T12:00:00Z"}"#;
let c: Credentials = serde_json::from_str(minio_style).expect("deserialize");
assert_eq!(c.access_key, "ak");
assert!(c.expiration.is_some());
}
}
crates/credentials/src/lib.rs
+1
View File
@@ -14,6 +14,7 @@
mod constants;
mod credentials;
mod serde_datetime;
pub use constants::*;
pub use credentials::*;
crates/credentials/src/serde_datetime.rs
+68
View File
@@ -0,0 +1,68 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Serde helpers for expiration timestamp: serialize as RFC3339 (MinIO-compatible),
//! deserialize from RFC3339 or legacy RustFS human-readable format.
use time::OffsetDateTime;
use time::format_description;
use time::format_description::well_known::Rfc3339;
static LEGACY_FORMAT: std::sync::OnceLock<time::format_description::OwnedFormatItem> = std::sync::OnceLock::new();
fn legacy_format() -> &'static time::format_description::OwnedFormatItem {
LEGACY_FORMAT.get_or_init(|| {
format_description::parse_owned::<2>(
"[year]-[month]-[day] [hour]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]",
)
.expect("legacy format description is valid")
});
LEGACY_FORMAT.get().expect("initialized above")
}
fn parse_rfc3339_or_legacy(s: &str) -> Result<OffsetDateTime, time::Error> {
OffsetDateTime::parse(s, &Rfc3339).or_else(|_| OffsetDateTime::parse(s, legacy_format()).map_err(Into::into))
}
/// Option<OffsetDateTime>: serialize as RFC3339; deserialize from RFC3339 or legacy.
pub mod option {
use serde::{Deserialize, Deserializer, Serializer};
use time::OffsetDateTime;
use super::{Rfc3339, parse_rfc3339_or_legacy};
pub fn serialize<S>(opt: &Option<OffsetDateTime>, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match opt {
Some(dt) => {
let s = dt.format(&Rfc3339).map_err(serde::ser::Error::custom)?;
serializer.serialize_some(&s)
}
None => serializer.serialize_none(),
}
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<OffsetDateTime>, D::Error>
where
D: Deserializer<'de>,
{
let opt: Option<&str> = Option::deserialize(deserializer)?;
match opt {
None => Ok(None),
Some(s) => parse_rfc3339_or_legacy(s).map(Some).map_err(serde::de::Error::custom),
}
}
}
crates/crypto/src/encdec.rs
+3
View File
@@ -21,5 +21,8 @@ pub(crate) mod id;
pub(crate) mod decrypt;
pub(crate) mod encrypt;
#[cfg(any(test, feature = "crypto"))]
pub(crate) mod stream_io;
#[cfg(test)]
mod tests;
crates/crypto/src/encdec/stream_io.rs
+219
View File
@@ -0,0 +1,219 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! sio-go compatible stream encryption for IAM config.
//! Header: salt(32) + alg_id(1) + nonce_prefix(8) = 41 bytes.
//! Body: DARE-style fragmented AEAD (bufSize=16384, per-fragment nonce).
#![allow(deprecated)] // AeadInPlace deprecated in favor of AeadInOut; keep for aead 0.6 compatibility
use crate::encdec::id::ID;
use crate::error::Error;
use aes_gcm::{
Aes256Gcm,
aead::{AeadCore, AeadInPlace, KeyInit as _, array::Array},
};
use chacha20poly1305::ChaCha20Poly1305;
const STREAM_IO_HEADER_LEN: usize = 41;
const SIO_BUF_SIZE: usize = 16384;
const SIO_NONCE_PREFIX_LEN: usize = 8;
const AES_GCM_OVERHEAD: usize = 16;
const CHACHA_OVERHEAD: usize = 16;
/// Decrypt data in stream_io (sio-go) format.
pub fn decrypt_stream_io(password: &[u8], data: &[u8]) -> Result<Vec<u8>, Error> {
if data.len() < STREAM_IO_HEADER_LEN {
return Err(Error::ErrUnexpectedHeader);
}
let salt = &data[0..32];
let id = ID::try_from(data[32])?;
let nonce_prefix = &data[33..41];
let body = &data[STREAM_IO_HEADER_LEN..];
let key = id.get_key(password, salt)?;
match id {
ID::Argon2idChaCHa20Poly1305 => decrypt_stream(
ChaCha20Poly1305::new_from_slice(&key).map_err(|e| Error::ErrInvalidInput(e.to_string()))?,
nonce_prefix,
body,
CHACHA_OVERHEAD,
),
_ => decrypt_stream(
Aes256Gcm::new_from_slice(&key).map_err(|e| Error::ErrInvalidInput(e.to_string()))?,
nonce_prefix,
body,
AES_GCM_OVERHEAD,
),
}
}
fn decrypt_stream<A>(aead: A, nonce_prefix: &[u8], body: &[u8], overhead: usize) -> Result<Vec<u8>, Error>
where
A: AeadInPlace,
{
let ciphertext_len = SIO_BUF_SIZE + overhead;
let ad = build_associated_data(&aead, nonce_prefix)?;
let mut plain = Vec::with_capacity(body.len());
let mut seq_num: u32 = 1;
let mut pos = 0;
while pos < body.len() {
let remaining = body.len() - pos;
let frag_len = remaining.min(ciphertext_len);
let is_last = (pos + frag_len) == body.len();
if frag_len < overhead {
return Err(Error::ErrDecryptFailed(aes_gcm::aead::Error));
}
let mut nonce = [0u8; 12];
nonce[0..SIO_NONCE_PREFIX_LEN].copy_from_slice(nonce_prefix);
nonce[8..12].copy_from_slice(&seq_num.to_le_bytes());
let mut ad_mut = ad.clone();
ad_mut[0] = if is_last { 0x80 } else { 0x00 };
let fragment = &body[pos..pos + frag_len];
let tag_len = overhead;
let (ct, tag) = fragment.split_at(frag_len - tag_len);
let mut buffer = ct.to_vec();
let nonce_arr = Array::<u8, <A as AeadCore>::NonceSize>::try_from(&nonce[..])
.map_err(|_| Error::ErrDecryptFailed(aes_gcm::aead::Error))?;
let tag_arr =
Array::<u8, <A as AeadCore>::TagSize>::try_from(tag).map_err(|_| Error::ErrDecryptFailed(aes_gcm::aead::Error))?;
aead.decrypt_in_place_detached(&nonce_arr, &ad_mut, &mut buffer, &tag_arr)
.map_err(|_| Error::ErrDecryptFailed(aes_gcm::aead::Error))?;
plain.extend_from_slice(&buffer);
pos += frag_len;
seq_num += 1;
if is_last {
break;
}
}
Ok(plain)
}
fn build_associated_data<A>(aead: &A, nonce_prefix: &[u8]) -> Result<Vec<u8>, Error>
where
A: AeadInPlace,
{
let mut nonce = [0u8; 12];
nonce[0..SIO_NONCE_PREFIX_LEN].copy_from_slice(nonce_prefix);
nonce[8..12].copy_from_slice(&0u32.to_le_bytes());
let nonce_arr = Array::<u8, <A as AeadCore>::NonceSize>::try_from(&nonce[..])
.map_err(|_| Error::ErrEncryptFailed(aes_gcm::aead::Error))?;
let mut empty: [u8; 0] = [];
let tag = aead
.encrypt_in_place_detached(&nonce_arr, &[] as &[u8], &mut empty)
.map_err(Error::ErrEncryptFailed)?;
let mut ad = vec![0u8; 1 + tag.len()];
ad[0] = 0x00;
ad[1..].copy_from_slice(tag.as_slice());
Ok(ad)
}
/// Encrypt data in stream_io (sio-go) format.
pub fn encrypt_stream_io(password: &[u8], data: &[u8]) -> Result<Vec<u8>, Error> {
let salt: [u8; 32] = rand::random();
#[cfg(feature = "fips")]
let id = ID::Pbkdf2AESGCM;
#[cfg(not(feature = "fips"))]
let id = if crate::encdec::encrypt::native_aes() {
ID::Argon2idAESGCM
} else {
ID::Argon2idChaCHa20Poly1305
};
let key = id.get_key(password, &salt)?;
let nonce_prefix: [u8; SIO_NONCE_PREFIX_LEN] = rand::random();
let mut out = Vec::with_capacity(STREAM_IO_HEADER_LEN + data.len() + 32);
out.extend_from_slice(&salt);
out.push(id as u8);
out.extend_from_slice(&nonce_prefix);
match id {
ID::Argon2idChaCHa20Poly1305 => encrypt_stream(
ChaCha20Poly1305::new_from_slice(&key).map_err(|e| Error::ErrInvalidInput(e.to_string()))?,
&nonce_prefix,
data,
&mut out,
CHACHA_OVERHEAD,
)?,
_ => encrypt_stream(
Aes256Gcm::new_from_slice(&key).map_err(|e| Error::ErrInvalidInput(e.to_string()))?,
&nonce_prefix,
data,
&mut out,
AES_GCM_OVERHEAD,
)?,
}
Ok(out)
}
fn encrypt_stream<A>(
aead: A,
nonce_prefix: &[u8; SIO_NONCE_PREFIX_LEN],
data: &[u8],
out: &mut Vec<u8>,
_overhead: usize,
) -> Result<(), Error>
where
A: AeadInPlace,
{
let ad = build_associated_data(&aead, nonce_prefix)?;
let mut seq_num: u32 = 1;
let mut pos = 0;
while pos < data.len() {
let remaining = data.len() - pos;
let is_last = remaining <= SIO_BUF_SIZE;
let chunk_len = if is_last { remaining } else { SIO_BUF_SIZE };
let chunk = &data[pos..pos + chunk_len];
let mut nonce = [0u8; 12];
nonce[0..SIO_NONCE_PREFIX_LEN].copy_from_slice(nonce_prefix);
nonce[8..12].copy_from_slice(&seq_num.to_le_bytes());
let mut ad_mut = ad.clone();
ad_mut[0] = if is_last { 0x80 } else { 0x00 };
let mut buffer = chunk.to_vec();
let nonce_arr = Array::<u8, <A as AeadCore>::NonceSize>::try_from(&nonce[..])
.map_err(|_| Error::ErrEncryptFailed(aes_gcm::aead::Error))?;
let tag = aead
.encrypt_in_place_detached(&nonce_arr, &ad_mut, &mut buffer)
.map_err(Error::ErrEncryptFailed)?;
out.extend_from_slice(&buffer);
out.extend_from_slice(tag.as_slice());
pos += chunk_len;
seq_num += 1;
}
Ok(())
}
crates/crypto/src/encdec/tests.rs
+65 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{decrypt_data, encrypt_data};
use crate::{decrypt_data, decrypt_stream_io, encrypt_data, encrypt_stream_io};
const PASSWORD: &[u8] = "test_password".as_bytes();
const LONG_PASSWORD: &[u8] = "very_long_password_with_many_characters_for_testing_purposes_123456789".as_bytes();
@@ -317,3 +317,67 @@ fn test_concurrent_encryption_safety() -> Result<(), crate::Error> {
Ok(())
}
#[test]
fn test_stream_io_roundtrip() -> Result<(), crate::Error> {
let password = b"access:secret";
let data = br#"{"Version":1,"policy":"readonly"}"#;
let encrypted = encrypt_stream_io(password, data)?;
let decrypted = decrypt_stream_io(password, &encrypted)?;
assert_eq!(data, decrypted.as_slice());
Ok(())
}
#[test]
fn test_stream_io_large_data_roundtrip() -> Result<(), crate::Error> {
let password = b"access:secret";
let data = vec![0xAB; 32 * 1024]; // > SIO_BUF_SIZE to test fragmentation
let encrypted = encrypt_stream_io(password, &data)?;
let decrypted = decrypt_stream_io(password, &encrypted)?;
assert_eq!(data, decrypted);
Ok(())
}
#[test]
fn test_stream_io_wrong_password_fails() {
let password = b"access:secret";
let data = br#"{"Version":1}"#;
let encrypted = encrypt_stream_io(password, data).expect("encrypt should succeed");
let result = decrypt_stream_io(b"wrong:password", &encrypted);
assert!(result.is_err(), "decrypt with wrong password should fail");
}
#[test]
fn test_stream_io_empty_data() -> Result<(), crate::Error> {
let password = b"access:secret";
let data: &[u8] = &[];
let encrypted = encrypt_stream_io(password, data)?;
let decrypted = decrypt_stream_io(password, &encrypted)?;
assert!(decrypted.is_empty());
Ok(())
}
#[test]
fn test_stream_io_header_format() -> Result<(), crate::Error> {
let password = b"access:secret";
let data = b"test";
let encrypted = encrypt_stream_io(password, data)?;
// stream_io header: salt(32) + alg_id(1) + nonce_prefix(8) = 41 bytes
const STREAM_IO_HEADER_LEN: usize = 41;
assert!(encrypted.len() >= STREAM_IO_HEADER_LEN, "encrypted should have at least 41-byte header");
assert!(
encrypted[32] == 0x00 || encrypted[32] == 0x01 || encrypted[32] == 0x02,
"alg_id should be 0x00, 0x01, or 0x02"
);
Ok(())
}
#[test]
fn test_stream_io_truncated_data_fails() {
let password = b"access:secret";
let data = b"test";
let encrypted = encrypt_stream_io(password, data).expect("encrypt should succeed");
let truncated = &encrypted[..40]; // less than 41-byte header
let result = decrypt_stream_io(password, truncated);
assert!(result.is_err(), "truncated data should fail decrypt");
}
crates/crypto/src/lib.rs
+3
View File
@@ -19,6 +19,9 @@ mod jwt;
pub use encdec::decrypt::decrypt_data;
pub use encdec::encrypt::encrypt_data;
#[cfg(feature = "crypto")]
pub use encdec::stream_io::{decrypt_stream_io, encrypt_stream_io};
pub use error::Error;
pub use jwt::decode::decode as jwt_decode;
pub use jwt::encode::encode as jwt_encode;
crates/ecstore/Cargo.toml
+2
View File
@@ -66,6 +66,7 @@ http-body = { workspace = true }
http-body-util.workspace = true
url.workspace = true
uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
reed-solomon-erasure = { workspace = true }
reed-solomon-simd = { workspace = true }
lazy_static.workspace = true
rustfs-lock.workspace = true
@@ -73,6 +74,7 @@ regex = { workspace = true }
path-absolutize = { workspace = true }
rmp.workspace = true
rmp-serde.workspace = true
serde_bytes.workspace = true
tokio-util = { workspace = true, features = ["io", "compat"] }
base64 = { workspace = true }
hmac = { workspace = true }
crates/ecstore/benches/erasure_benchmark.rs
+51 -68
View File
@@ -118,41 +118,38 @@ fn bench_encode_performance(c: &mut Criterion) {
});
group.finish();
// Test direct SIMD implementation for large shards (>= 512 bytes)
// Test direct reed-solomon-erasure implementation for large shards (>= 512 bytes)
let shard_size = calc_shard_size(config.data_size, config.data_shards);
if shard_size >= 512 {
let mut simd_group = c.benchmark_group("encode_simd_direct");
simd_group.throughput(Throughput::Bytes(config.data_size as u64));
simd_group.sample_size(10);
simd_group.measurement_time(Duration::from_secs(5));
if shard_size >= 512 && config.parity_shards > 0 {
use reed_solomon_erasure::galois_8::ReedSolomon;
simd_group.bench_with_input(BenchmarkId::new("simd_direct", &config.name), &(&data, &config), |b, (data, config)| {
b.iter(|| {
// Direct SIMD implementation
let per_shard_size = calc_shard_size(data.len(), config.data_shards);
match reed_solomon_simd::ReedSolomonEncoder::new(config.data_shards, config.parity_shards, per_shard_size) {
Ok(mut encoder) => {
// Create properly sized buffer and fill with data
let mut buffer = vec![0u8; per_shard_size * config.data_shards];
let mut rse_group = c.benchmark_group("encode_rse_direct");
rse_group.throughput(Throughput::Bytes(config.data_size as u64));
rse_group.sample_size(10);
rse_group.measurement_time(Duration::from_secs(5));
if let Ok(rs) = ReedSolomon::new(config.data_shards, config.parity_shards) {
let total_shards = config.data_shards + config.parity_shards;
let per_shard_size = calc_shard_size(config.data_size, config.data_shards);
let need_total = per_shard_size * total_shards;
rse_group.bench_with_input(
BenchmarkId::new("rse_direct", &config.name),
&(&data, need_total, per_shard_size),
|b, (data, need_total, per_shard_size)| {
b.iter(|| {
let mut buffer = vec![0u8; *need_total];
let copy_len = data.len().min(buffer.len());
buffer[..copy_len].copy_from_slice(&data[..copy_len]);
// Add data shards with correct shard size
for chunk in buffer.chunks_exact(per_shard_size) {
encoder.add_original_shard(black_box(chunk)).unwrap();
}
let result = encoder.encode().unwrap();
black_box(result);
}
Err(_) => {
// SIMD doesn't support this configuration, skip
black_box(());
}
}
});
});
simd_group.finish();
let mut slices: Vec<&mut [u8]> = buffer.chunks_exact_mut(*per_shard_size).collect();
rs.encode(&mut slices).unwrap();
black_box(buffer);
});
},
);
}
rse_group.finish();
}
}
}
@@ -203,47 +200,33 @@ fn bench_decode_performance(c: &mut Criterion) {
);
group.finish();
// Test direct SIMD decoding for large shards
// Test direct reed-solomon-erasure decoding for large shards
let shard_size = calc_shard_size(config.data_size, config.data_shards);
if shard_size >= 512 {
let mut simd_group = c.benchmark_group("decode_simd_direct");
simd_group.throughput(Throughput::Bytes(config.data_size as u64));
simd_group.sample_size(10);
simd_group.measurement_time(Duration::from_secs(5));
if shard_size >= 512 && config.parity_shards > 0 {
use reed_solomon_erasure::galois_8::ReedSolomon;
simd_group.bench_with_input(
BenchmarkId::new("simd_direct", &config.name),
&(&encoded_shards, &config),
|b, (shards, config)| {
b.iter(|| {
let per_shard_size = calc_shard_size(config.data_size, config.data_shards);
match reed_solomon_simd::ReedSolomonDecoder::new(config.data_shards, config.parity_shards, per_shard_size)
{
Ok(mut decoder) => {
// Add available shards (except lost ones)
for (i, shard) in shards.iter().enumerate() {
if i != config.data_shards - 1 && i != config.data_shards {
if i < config.data_shards {
decoder.add_original_shard(i, black_box(shard)).unwrap();
} else {
let recovery_idx = i - config.data_shards;
decoder.add_recovery_shard(recovery_idx, black_box(shard)).unwrap();
}
}
}
if let Ok(rs) = ReedSolomon::new(config.data_shards, config.parity_shards) {
let mut rse_group = c.benchmark_group("decode_rse_direct");
rse_group.throughput(Throughput::Bytes(config.data_size as u64));
rse_group.sample_size(10);
rse_group.measurement_time(Duration::from_secs(5));
let result = decoder.decode().unwrap();
black_box(result);
}
Err(_) => {
// SIMD doesn't support this configuration, skip
black_box(());
}
}
});
},
);
simd_group.finish();
rse_group.bench_with_input(
BenchmarkId::new("rse_direct", &config.name),
&(&encoded_shards, &config),
|b, (shards, config)| {
b.iter(|| {
let mut shards_opt: Vec<Option<Vec<u8>>> = shards.iter().map(|s| Some(s.to_vec())).collect();
shards_opt[config.data_shards - 1] = None;
shards_opt[config.data_shards] = None;
rs.reconstruct_data(&mut shards_opt).unwrap();
black_box(shards_opt);
});
},
);
rse_group.finish();
}
}
}
}
crates/ecstore/src/bitrot.rs
+4 -4
View File
@@ -119,7 +119,7 @@ mod tests {
async fn test_create_bitrot_reader_with_inline_data() {
let test_data = b"hello world test data";
let shard_size = 16;
let checksum_algo = HashAlgorithm::HighwayHash256;
let checksum_algo = HashAlgorithm::HighwayHash256S;
let result =
create_bitrot_reader(Some(test_data), None, "test-bucket", "test-path", 0, 0, shard_size, checksum_algo, false).await;
@@ -131,7 +131,7 @@ mod tests {
#[tokio::test]
async fn test_create_bitrot_reader_without_data_or_disk() {
let shard_size = 16;
let checksum_algo = HashAlgorithm::HighwayHash256;
let checksum_algo = HashAlgorithm::HighwayHash256S;
let result =
create_bitrot_reader(None, None, "test-bucket", "test-path", 0, 1024, shard_size, checksum_algo, false).await;
@@ -151,7 +151,7 @@ mod tests {
"test-path",
1024, // length
1024, // shard_size
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await;
@@ -183,7 +183,7 @@ mod tests {
"test-path",
1024, // length
1024, // shard_size
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await;
crates/ecstore/src/bucket/bucket_target_sys.rs
+22 -31
View File
@@ -43,11 +43,13 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU
use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, RUSTFS_BUCKET_REPLICATION_CHECK,
RUSTFS_BUCKET_REPLICATION_DELETE_MARKER, RUSTFS_BUCKET_REPLICATION_REQUEST, RUSTFS_BUCKET_SOURCE_ETAG,
RUSTFS_BUCKET_SOURCE_MTIME, RUSTFS_BUCKET_SOURCE_VERSION_ID, RUSTFS_FORCE_DELETE, is_amz_header, is_minio_header,
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header,
is_rustfs_header, is_standard_header, is_storageclass_header,
};
use rustfs_utils::http::{
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::error::Error;
@@ -68,8 +70,6 @@ use uuid::Uuid;
const DEFAULT_HEALTH_CHECK_DURATION: Duration = Duration::from_secs(5);
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
const REPLICATION_REQUEST_TRUE: HeaderValue = HeaderValue::from_static("true");
pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
#[derive(Debug, Clone)]
@@ -1081,23 +1081,21 @@ impl PutObjectOptions {
}
if !self.internal.source_version_id.is_empty() {
header.insert(
RUSTFS_BUCKET_SOURCE_VERSION_ID,
HeaderValue::from_str(&self.internal.source_version_id).expect("err"),
);
insert_header(&mut header, SUFFIX_SOURCE_VERSION_ID, &self.internal.source_version_id);
}
if self.internal.source_etag.is_empty() {
header.insert(RUSTFS_BUCKET_SOURCE_ETAG, HeaderValue::from_str(&self.internal.source_etag).expect("err"));
insert_header(&mut header, SUFFIX_SOURCE_ETAG, &self.internal.source_etag);
}
if self.internal.source_mtime.unix_timestamp() != 0 {
header.insert(
RUSTFS_BUCKET_SOURCE_MTIME,
HeaderValue::from_str(&self.internal.source_mtime.format(&Rfc3339).unwrap_or_default()).expect("err"),
insert_header(
&mut header,
SUFFIX_SOURCE_MTIME,
self.internal.source_mtime.format(&Rfc3339).unwrap_or_default(),
);
}
if self.internal.replication_request {
header.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
}
header
@@ -1266,10 +1264,8 @@ impl TargetClient {
let builder = self.client.put_object();
let version_id = opts.internal.source_version_id.clone();
if !version_id.is_empty()
&& let Ok(header_value) = HeaderValue::from_str(&version_id)
{
headers.insert(RUSTFS_BUCKET_SOURCE_VERSION_ID, header_value);
if !version_id.is_empty() {
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &version_id);
}
match builder
@@ -1303,13 +1299,11 @@ impl TargetClient {
) -> Result<String, S3ClientError> {
let mut headers = HeaderMap::new();
let version_id = opts.internal.source_version_id.clone();
if !version_id.is_empty()
&& let Ok(header_value) = HeaderValue::from_str(&version_id)
{
headers.insert(RUSTFS_BUCKET_SOURCE_VERSION_ID, header_value);
if !version_id.is_empty() {
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &version_id);
}
if opts.internal.replication_request {
headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, REPLICATION_REQUEST_TRUE);
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
}
match self
@@ -1418,21 +1412,18 @@ impl TargetClient {
) -> Result<(), S3ClientError> {
let mut headers = HeaderMap::new();
if opts.force_delete {
headers.insert(RUSTFS_FORCE_DELETE, "true".parse().unwrap());
insert_header(&mut headers, SUFFIX_FORCE_DELETE, "true");
}
if opts.governance_bypass {
headers.insert(AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, "true".parse().unwrap());
}
if opts.replication_delete_marker {
headers.insert(RUSTFS_BUCKET_REPLICATION_DELETE_MARKER, "true".parse().unwrap());
insert_header(&mut headers, SUFFIX_SOURCE_DELETEMARKER, "true");
}
if let Some(t) = opts.replication_mtime {
headers.insert(
RUSTFS_BUCKET_SOURCE_MTIME,
t.format(&Rfc3339).unwrap_or_default().as_str().parse().unwrap(),
);
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, t.format(&Rfc3339).unwrap_or_default());
}
if !opts.replication_status.is_empty() {
@@ -1440,10 +1431,10 @@ impl TargetClient {
}
if opts.replication_request {
headers.insert(RUSTFS_BUCKET_REPLICATION_REQUEST, "true".parse().unwrap());
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
}
if opts.replication_validity_check {
headers.insert(RUSTFS_BUCKET_REPLICATION_CHECK, "true".parse().unwrap());
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
}
match self
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+14 -24
View File
@@ -45,8 +45,7 @@ use rustfs_common::heal_channel::rep_has_active_rules;
use rustfs_common::metrics::{IlmAction, Metrics};
use rustfs_filemeta::{NULL_VERSION_ID, RestoreStatusOps, is_restored_object_on_disk};
use rustfs_s3_common::EventName;
use rustfs_utils::path::encode_dir_object;
use rustfs_utils::string::strings_has_prefix_fold;
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
use s3s::Body;
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
@@ -97,8 +96,14 @@ impl LifecycleSys {
}
pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
let lc = get_lifecycle_config(bucket).await.expect("get_lifecycle_config err!").0;
Some(lc)
match get_lifecycle_config(bucket).await {
Ok((lc, _)) => Some(lc),
Err(err) if err == Error::ConfigNotFound => None,
Err(err) => {
warn!(bucket, error = ?err, "failed to load lifecycle config");
None
}
}
}
pub fn trace(_oi: &ObjectInfo) -> TraceFn {
@@ -471,10 +476,7 @@ impl TransitionState {
}
pub async fn init(api: Arc<ECStore>) {
let max_workers = env::var("RUSTFS_MAX_TRANSITION_WORKERS")
.ok()
.and_then(|s| s.parse::<i64>().ok())
.unwrap_or_else(|| std::cmp::min(num_cpus::get() as i64, 16));
let max_workers = get_env_i64("RUSTFS_MAX_TRANSITION_WORKERS", std::cmp::min(num_cpus::get() as i64, 16));
let mut n = max_workers;
let tw = 8; //globalILMConfig.getTransitionWorkers();
if tw > 0 {
@@ -569,17 +571,11 @@ impl TransitionState {
pub async fn update_workers_inner(api: Arc<ECStore>, n: i64) {
let mut n = n;
if n == 0 {
let max_workers = env::var("RUSTFS_MAX_TRANSITION_WORKERS")
.ok()
.and_then(|s| s.parse::<i64>().ok())
.unwrap_or_else(|| std::cmp::min(num_cpus::get() as i64, 16));
let max_workers = get_env_i64("RUSTFS_MAX_TRANSITION_WORKERS", std::cmp::min(num_cpus::get() as i64, 16));
n = max_workers;
}
// Allow environment override of maximum workers
let absolute_max = env::var("RUSTFS_ABSOLUTE_MAX_WORKERS")
.ok()
.and_then(|s| s.parse::<i64>().ok())
.unwrap_or(32);
let absolute_max = get_env_i64("RUSTFS_ABSOLUTE_MAX_WORKERS", 32);
n = std::cmp::min(n, absolute_max);
let mut num_workers = GLOBAL_TransitionState.num_workers.load(Ordering::SeqCst);
@@ -603,10 +599,7 @@ impl TransitionState {
}
pub async fn init_background_expiry(api: Arc<ECStore>) {
let mut workers = env::var("RUSTFS_MAX_EXPIRY_WORKERS")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or_else(|| std::cmp::min(num_cpus::get(), 16));
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
//globalILMConfig.getExpirationWorkers()
if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS") {
if let Ok(num_expirations) = env_expiration_workers.parse::<usize>() {
@@ -615,10 +608,7 @@ pub async fn init_background_expiry(api: Arc<ECStore>) {
}
if workers == 0 {
workers = env::var("RUSTFS_DEFAULT_EXPIRY_WORKERS")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(8);
workers = get_env_usize("RUSTFS_DEFAULT_EXPIRY_WORKERS", 8);
}
//let expiry_state = GLOBAL_ExpiryStSate.write().await;
crates/ecstore/src/bucket/lifecycle/lifecycle.rs
+128 -12
View File
@@ -49,6 +49,8 @@ const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days m
const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering";
const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
pub use rustfs_common::metrics::IlmAction;
@@ -117,19 +119,43 @@ impl RuleValidate for LifecycleRule {
}*/
fn validate(&self) -> Result<(), std::io::Error> {
/*self.validate_id()?;
self.validate_status()?;
self.validate_expiration()?;
self.validate_noncurrent_expiration()?;
self.validate_prefix_and_filter()?;
self.validate_transition()?;
self.validate_noncurrent_transition()?;
if (!self.Filter.Tag.IsEmpty() || len(self.Filter.And.Tags) != 0) && !self.delmarker_expiration.Empty() {
return errInvalidRuleDelMarkerExpiration
// Rule with DelMarkerExpiration cannot have tags based filtering
let has_tag_filter = self
.filter
.as_ref()
.map_or(false, |f| f.tag.is_some() || f.and.as_ref().and_then(|a| a.tags.as_ref()).is_some());
if has_tag_filter && self.del_marker_expiration.is_some() {
return Err(std::io::Error::other(ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS));
}
// Rule must have at least one action
let has_expiration = self.expiration.is_some();
let has_transition = self.transitions.as_ref().map_or(false, |t| !t.is_empty());
let has_noncurrent_expiration = self
.noncurrent_version_expiration
.as_ref()
.and_then(|e| e.noncurrent_days)
.map_or(false, |d| d != 0);
let has_noncurrent_transition = self
.noncurrent_version_transitions
.as_ref()
.and_then(|t| t.first())
.and_then(|t| t.storage_class.as_ref())
.is_some();
let has_abort_incomplete_multipart_upload = self.abort_incomplete_multipart_upload.is_some();
let has_del_marker_expiration = self
.del_marker_expiration
.as_ref()
.and_then(|d| d.days)
.map_or(false, |d| d > 0);
if !has_expiration
&& !has_transition
&& !has_noncurrent_expiration
&& !has_noncurrent_transition
&& !has_abort_incomplete_multipart_upload
&& !has_del_marker_expiration
{
return Err(std::io::Error::other(ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION));
}
if !self.expiration.set && !self.transition.set && !self.noncurrent_version_expiration.set && !self.noncurrent_version_transitions.unwrap()[0].set && self.delmarker_expiration.Empty() {
return errXMLNotWellFormed
}*/
Ok(())
}
}
@@ -456,6 +482,27 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
}
}
// DelMarkerExpiration: expire delete marker after N days from mod_time
if obj.delete_marker {
if let Some(ref dme) = rule.del_marker_expiration {
if let Some(days) = dme.days {
if days > 0 {
let due = expected_expiry_time(mod_time, days);
if now.unix_timestamp() >= due.unix_timestamp() {
events.push(Event {
action: IlmAction::DelMarkerDeleteAllVersionsAction,
rule_id: rule.id.clone().unwrap_or_default(),
due: Some(due),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
}
continue;
}
}
}
}
}
if !obj.is_latest {
@@ -866,6 +913,7 @@ mod tests {
#[serial]
async fn validate_rejects_non_positive_expiration_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -873,6 +921,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -894,6 +943,7 @@ mod tests {
#[serial]
async fn validate_accepts_positive_expiration_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -901,6 +951,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -915,10 +966,37 @@ mod tests {
.expect("expected validation to pass");
}
#[tokio::test]
#[serial]
async fn validate_accepts_abort_incomplete_multipart_upload_only_rule() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: Some(s3s::dto::AbortIncompleteMultipartUpload {
days_after_initiation: Some(2),
}),
del_marker_expiration: None,
filter: None,
id: Some("abort-only".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: Some("test/".to_string()),
transitions: None,
}],
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("expected validation to pass");
}
#[tokio::test]
#[serial]
async fn validate_rejects_non_midnight_expiration_date() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -926,6 +1004,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -945,6 +1024,7 @@ mod tests {
async fn predict_expiration_selects_closest_expiry_for_put_object() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![
LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
@@ -953,6 +1033,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("rule-days".to_string()),
noncurrent_version_expiration: None,
@@ -967,6 +1048,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("rule-date".to_string()),
noncurrent_version_expiration: None,
@@ -996,6 +1078,7 @@ mod tests {
#[serial]
async fn validate_accepts_multiple_rules_without_ids() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![
LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
@@ -1004,6 +1087,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -1018,6 +1102,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -1037,6 +1122,7 @@ mod tests {
#[serial]
async fn validate_rejects_rule_id_too_long() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1044,6 +1130,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("a".repeat(256)),
noncurrent_version_expiration: None,
@@ -1062,6 +1149,7 @@ mod tests {
#[serial]
async fn validate_rejects_duplicate_rule_ids() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![
LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
@@ -1070,6 +1158,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("dup-rule".to_string()),
noncurrent_version_expiration: None,
@@ -1090,6 +1179,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
},
],
};
@@ -1103,6 +1193,7 @@ mod tests {
async fn eval_inner_expires_latest_object_after_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1110,6 +1201,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expire-days".to_string()),
noncurrent_version_expiration: None,
@@ -1137,6 +1229,7 @@ mod tests {
async fn eval_inner_keeps_latest_object_before_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1144,6 +1237,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("expire-days".to_string()),
noncurrent_version_expiration: None,
@@ -1169,10 +1263,12 @@ mod tests {
async fn eval_inner_transitions_latest_object_after_days_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("transition-days".to_string()),
noncurrent_version_expiration: None,
@@ -1205,10 +1301,12 @@ mod tests {
async fn eval_inner_expires_noncurrent_version_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("noncurrent-expire".to_string()),
noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration {
@@ -1241,10 +1339,12 @@ mod tests {
async fn eval_inner_transitions_noncurrent_version_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("noncurrent-transition".to_string()),
noncurrent_version_expiration: None,
@@ -1278,10 +1378,12 @@ mod tests {
#[serial]
async fn noncurrent_versions_expiration_limit_returns_configured_limits() {
let lc = Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("noncurrent-limit".to_string()),
noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration {
@@ -1313,6 +1415,7 @@ mod tests {
#[serial]
async fn validate_rejects_invalid_status_case_sensitive() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static("enabled"),
expiration: Some(LifecycleExpiration {
@@ -1320,6 +1423,7 @@ mod tests {
..Default::default()
}),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
@@ -1340,6 +1444,7 @@ mod tests {
let mut filter = LifecycleRuleFilter::default();
filter.prefix = Some("prefix".to_string());
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1353,6 +1458,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1385,6 +1491,7 @@ mod tests {
filter.and = Some(and);
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1398,6 +1505,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1425,6 +1533,7 @@ mod tests {
async fn expired_object_delete_marker_requires_single_version() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1439,6 +1548,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1462,6 +1572,7 @@ mod tests {
async fn expired_object_delete_marker_deletes_only_delete_marker_after_due() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1476,6 +1587,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1500,6 +1612,7 @@ mod tests {
async fn expired_object_delete_marker_without_date_or_days_deletes_immediately() {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1513,6 +1626,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
@@ -1539,6 +1653,7 @@ mod tests {
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).unwrap();
let future_date = base_time + Duration::days(10);
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
@@ -1553,6 +1668,7 @@ mod tests {
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
del_marker_expiration: None,
}],
};
crates/ecstore/src/bucket/metadata.rs
+218 -56
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
use super::object_lock::ObjectLockApi;
use super::versioning::VersioningApi;
use super::{quota::BucketQuota, target::BucketTargets};
@@ -22,7 +23,6 @@ use crate::error::{Error, Result};
use crate::new_object_layer_fn;
use crate::store::ECStore;
use byteorder::{BigEndian, ByteOrder, LittleEndian};
use rmp_serde::Serializer as rmpSerializer;
use rustfs_policy::policy::BucketPolicy;
use s3s::dto::{
BucketLifecycleConfiguration, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration,
@@ -30,12 +30,41 @@ use s3s::dto::{
VersioningConfiguration,
};
use serde::Serializer;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::{Read, Write};
use std::sync::Arc;
use time::OffsetDateTime;
use tracing::error;
fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
let len = rmp::decode::read_str_len(rd)? as usize;
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
Ok(String::from_utf8(buf)?)
}
fn read_msgp_time_value<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
match marker {
rmp::Marker::Null => Ok(OffsetDateTime::UNIX_EPOCH),
rmp::Marker::Ext8 => read_msgp_ext8_time(rd),
_ => Err(Error::other(format!("expected time ext or nil, got marker: {marker:?}"))),
}
}
fn read_msgp_bin<R: Read>(rd: &mut R) -> Result<Vec<u8>> {
let len = rmp::decode::read_bin_len(rd)? as usize;
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
Ok(buf)
}
fn write_bin_field<W: Write>(wr: &mut W, key: &str, val: &[u8]) -> Result<()> {
rmp::encode::write_str(wr, key)?;
rmp::encode::write_bin(wr, val)?;
Ok(())
}
pub const BUCKET_METADATA_FILE: &str = ".metadata.bin";
pub const BUCKET_METADATA_FORMAT: u16 = 1;
pub const BUCKET_METADATA_VERSION: u16 = 1;
@@ -54,8 +83,7 @@ pub const BUCKET_CORS_CONFIG: &str = "cors.xml";
pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
#[derive(Debug, Deserialize, Serialize, Clone)]
#[serde(rename_all = "PascalCase", default)]
#[derive(Debug, Clone)]
pub struct BucketMetadata {
pub name: String,
pub created: OffsetDateTime,
@@ -90,36 +118,21 @@ pub struct BucketMetadata {
pub public_access_block_config_updated_at: OffsetDateTime,
pub bucket_acl_config_updated_at: OffsetDateTime,
#[serde(skip)]
pub new_field_updated_at: OffsetDateTime,
#[serde(skip)]
pub policy_config: Option<BucketPolicy>,
#[serde(skip)]
pub notification_config: Option<NotificationConfiguration>,
#[serde(skip)]
pub lifecycle_config: Option<BucketLifecycleConfiguration>,
#[serde(skip)]
pub object_lock_config: Option<ObjectLockConfiguration>,
#[serde(skip)]
pub versioning_config: Option<VersioningConfiguration>,
#[serde(skip)]
pub sse_config: Option<ServerSideEncryptionConfiguration>,
#[serde(skip)]
pub tagging_config: Option<Tagging>,
#[serde(skip)]
pub quota_config: Option<BucketQuota>,
#[serde(skip)]
pub replication_config: Option<ReplicationConfiguration>,
#[serde(skip)]
pub bucket_target_config: Option<BucketTargets>,
#[serde(skip)]
pub bucket_target_config_meta: Option<HashMap<String, String>>,
#[serde(skip)]
pub cors_config: Option<CORSConfiguration>,
#[serde(skip)]
pub public_access_block_config: Option<PublicAccessBlockConfiguration>,
#[serde(skip)]
pub bucket_acl_config: Option<String>,
}
@@ -198,17 +211,137 @@ impl BucketMetadata {
self.lock_enabled || (self.versioning_config.as_ref().is_some_and(|v| v.enabled()))
}
/// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
pub fn decode_from<R: Read>(&mut self, rd: &mut R) -> Result<()> {
let mut fields = rmp::decode::read_map_len(rd)?;
*self = Self::default();
while fields > 0 {
fields -= 1;
let key_len = rmp::decode::read_str_len(rd)?;
let mut key_buf = vec![0u8; key_len as usize];
rd.read_exact(&mut key_buf)?;
let key = String::from_utf8(key_buf)?;
match key.as_str() {
"Name" => self.name = read_msgp_str(rd)?,
"Created" => self.created = read_msgp_time_value(rd)?,
"LockEnabled" => self.lock_enabled = rmp::decode::read_bool(rd)?,
"PolicyConfigJSON" => self.policy_config_json = read_msgp_bin(rd)?,
"NotificationConfigXML" => self.notification_config_xml = read_msgp_bin(rd)?,
"LifecycleConfigXML" => self.lifecycle_config_xml = read_msgp_bin(rd)?,
"ObjectLockConfigXML" => self.object_lock_config_xml = read_msgp_bin(rd)?,
"VersioningConfigXML" => self.versioning_config_xml = read_msgp_bin(rd)?,
"EncryptionConfigXML" => self.encryption_config_xml = read_msgp_bin(rd)?,
"TaggingConfigXML" => self.tagging_config_xml = read_msgp_bin(rd)?,
"QuotaConfigJSON" => self.quota_config_json = read_msgp_bin(rd)?,
"ReplicationConfigXML" => self.replication_config_xml = read_msgp_bin(rd)?,
"BucketTargetsConfigJSON" => self.bucket_targets_config_json = read_msgp_bin(rd)?,
"BucketTargetsConfigMetaJSON" => self.bucket_targets_config_meta_json = read_msgp_bin(rd)?,
"PolicyConfigUpdatedAt" => self.policy_config_updated_at = read_msgp_time_value(rd)?,
"ObjectLockConfigUpdatedAt" => self.object_lock_config_updated_at = read_msgp_time_value(rd)?,
"EncryptionConfigUpdatedAt" => self.encryption_config_updated_at = read_msgp_time_value(rd)?,
"TaggingConfigUpdatedAt" => self.tagging_config_updated_at = read_msgp_time_value(rd)?,
"QuotaConfigUpdatedAt" => self.quota_config_updated_at = read_msgp_time_value(rd)?,
"ReplicationConfigUpdatedAt" => self.replication_config_updated_at = read_msgp_time_value(rd)?,
"VersioningConfigUpdatedAt" => self.versioning_config_updated_at = read_msgp_time_value(rd)?,
"LifecycleConfigUpdatedAt" => self.lifecycle_config_updated_at = read_msgp_time_value(rd)?,
"NotificationConfigUpdatedAt" => self.notification_config_updated_at = read_msgp_time_value(rd)?,
"BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?,
"BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?,
"CorsConfigXML" => self.cors_config_xml = read_msgp_bin(rd)?,
"PublicAccessBlockConfigXML" => self.public_access_block_config_xml = read_msgp_bin(rd)?,
"BucketAclConfigJSON" => self.bucket_acl_config_json = read_msgp_bin(rd)?,
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
"PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?,
"BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?,
other => {
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
skip_msgp_value(rd)?;
}
}
}
Ok(())
}
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
// Map size: MinIO fields (25) + RustFS extensions (6)
let map_len: u32 = 31;
rmp::encode::write_map_len(wr, map_len)?;
// MinIO field order (same as Go struct)
rmp::encode::write_str(wr, "Name")?;
rmp::encode::write_str(wr, &self.name)?;
rmp::encode::write_str(wr, "Created")?;
write_msgp_time(wr, self.created)?;
rmp::encode::write_str(wr, "LockEnabled")?;
rmp::encode::write_bool(wr, self.lock_enabled)?;
write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?;
write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?;
write_bin_field(wr, "LifecycleConfigXML", &self.lifecycle_config_xml)?;
write_bin_field(wr, "ObjectLockConfigXML", &self.object_lock_config_xml)?;
write_bin_field(wr, "VersioningConfigXML", &self.versioning_config_xml)?;
write_bin_field(wr, "EncryptionConfigXML", &self.encryption_config_xml)?;
write_bin_field(wr, "TaggingConfigXML", &self.tagging_config_xml)?;
write_bin_field(wr, "QuotaConfigJSON", &self.quota_config_json)?;
write_bin_field(wr, "ReplicationConfigXML", &self.replication_config_xml)?;
write_bin_field(wr, "BucketTargetsConfigJSON", &self.bucket_targets_config_json)?;
write_bin_field(wr, "BucketTargetsConfigMetaJSON", &self.bucket_targets_config_meta_json)?;
rmp::encode::write_str(wr, "PolicyConfigUpdatedAt")?;
write_msgp_time(wr, self.policy_config_updated_at)?;
rmp::encode::write_str(wr, "ObjectLockConfigUpdatedAt")?;
write_msgp_time(wr, self.object_lock_config_updated_at)?;
rmp::encode::write_str(wr, "EncryptionConfigUpdatedAt")?;
write_msgp_time(wr, self.encryption_config_updated_at)?;
rmp::encode::write_str(wr, "TaggingConfigUpdatedAt")?;
write_msgp_time(wr, self.tagging_config_updated_at)?;
rmp::encode::write_str(wr, "QuotaConfigUpdatedAt")?;
write_msgp_time(wr, self.quota_config_updated_at)?;
rmp::encode::write_str(wr, "ReplicationConfigUpdatedAt")?;
write_msgp_time(wr, self.replication_config_updated_at)?;
rmp::encode::write_str(wr, "VersioningConfigUpdatedAt")?;
write_msgp_time(wr, self.versioning_config_updated_at)?;
rmp::encode::write_str(wr, "LifecycleConfigUpdatedAt")?;
write_msgp_time(wr, self.lifecycle_config_updated_at)?;
rmp::encode::write_str(wr, "NotificationConfigUpdatedAt")?;
write_msgp_time(wr, self.notification_config_updated_at)?;
rmp::encode::write_str(wr, "BucketTargetsConfigUpdatedAt")?;
write_msgp_time(wr, self.bucket_targets_config_updated_at)?;
rmp::encode::write_str(wr, "BucketTargetsConfigMetaUpdatedAt")?;
write_msgp_time(wr, self.bucket_targets_config_meta_updated_at)?;
// RustFS extensions
write_bin_field(wr, "CorsConfigXML", &self.cors_config_xml)?;
write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?;
write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?;
rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
write_msgp_time(wr, self.cors_config_updated_at)?;
rmp::encode::write_str(wr, "PublicAccessBlockConfigUpdatedAt")?;
write_msgp_time(wr, self.public_access_block_config_updated_at)?;
rmp::encode::write_str(wr, "BucketAclConfigUpdatedAt")?;
write_msgp_time(wr, self.bucket_acl_config_updated_at)?;
Ok(())
}
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
self.encode_to(&mut buf)?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: BucketMetadata = rmp_serde::from_slice(buf)?;
Ok(t)
let mut bm = Self::default();
let mut cur = std::io::Cursor::new(buf);
bm.decode_from(&mut cur)?;
Ok(bm)
}
pub fn check_header(buf: &[u8]) -> Result<()> {
@@ -382,50 +515,80 @@ impl BucketMetadata {
}
fn parse_all_configs(&mut self, _api: Arc<ECStore>) -> Result<()> {
self.parse_policy_config()?;
if !self.notification_config_xml.is_empty() {
self.notification_config = Some(deserialize::<NotificationConfiguration>(&self.notification_config_xml)?);
if let Err(e) = self.parse_policy_config() {
tracing::warn!(bucket = %self.name, config = "policy", error = %e, "parse_all_configs: failed to parse");
}
if !self.lifecycle_config_xml.is_empty() {
self.lifecycle_config = Some(deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml)?);
if !self.notification_config_xml.is_empty()
&& let Err(e) = deserialize::<NotificationConfiguration>(&self.notification_config_xml)
.map(|c| self.notification_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "notification", error = %e, "parse_all_configs: failed to parse");
}
if !self.object_lock_config_xml.is_empty() {
self.object_lock_config = Some(deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml)?);
if !self.lifecycle_config_xml.is_empty()
&& let Err(e) =
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "lifecycle", error = %e, "parse_all_configs: failed to parse");
}
if !self.versioning_config_xml.is_empty() {
self.versioning_config = Some(deserialize::<VersioningConfiguration>(&self.versioning_config_xml)?);
if !self.object_lock_config_xml.is_empty()
&& let Err(e) =
deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "object_lock", error = %e, "parse_all_configs: failed to parse");
}
if !self.encryption_config_xml.is_empty() {
self.sse_config = Some(deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml)?);
if !self.versioning_config_xml.is_empty()
&& let Err(e) =
deserialize::<VersioningConfiguration>(&self.versioning_config_xml).map(|c| self.versioning_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "versioning", error = %e, "parse_all_configs: failed to parse");
}
if !self.tagging_config_xml.is_empty() {
self.tagging_config = Some(deserialize::<Tagging>(&self.tagging_config_xml)?);
if !self.encryption_config_xml.is_empty()
&& let Err(e) =
deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml).map(|c| self.sse_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "encryption", error = %e, "parse_all_configs: failed to parse");
}
if !self.quota_config_json.is_empty() {
self.quota_config = Some(serde_json::from_slice(&self.quota_config_json)?);
if !self.tagging_config_xml.is_empty()
&& let Err(e) = deserialize::<Tagging>(&self.tagging_config_xml).map(|c| self.tagging_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "tagging", error = %e, "parse_all_configs: failed to parse");
}
if !self.replication_config_xml.is_empty() {
self.replication_config = Some(deserialize::<ReplicationConfiguration>(&self.replication_config_xml)?);
if !self.quota_config_json.is_empty()
&& let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "quota", error = %e, "parse_all_configs: failed to parse");
}
if !self.replication_config_xml.is_empty()
&& let Err(e) =
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "replication", error = %e, "parse_all_configs: failed to parse");
}
//let temp = self.bucket_targets_config_json.clone();
if !self.bucket_targets_config_json.is_empty() {
let bucket_targets: BucketTargets = serde_json::from_slice(&self.bucket_targets_config_json)?;
self.bucket_target_config = Some(bucket_targets);
if let Err(e) = serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json)
.map(|t| self.bucket_target_config = Some(t))
{
tracing::warn!(bucket = %self.name, config = "bucket_targets", error = %e, "parse_all_configs: failed to parse");
self.bucket_target_config = Some(BucketTargets::default());
}
} else {
self.bucket_target_config = Some(BucketTargets::default())
self.bucket_target_config = Some(BucketTargets::default());
}
if !self.cors_config_xml.is_empty() {
self.cors_config = Some(deserialize::<CORSConfiguration>(&self.cors_config_xml)?);
if !self.cors_config_xml.is_empty()
&& let Err(e) = deserialize::<CORSConfiguration>(&self.cors_config_xml).map(|c| self.cors_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "cors", error = %e, "parse_all_configs: failed to parse");
}
if !self.public_access_block_config_xml.is_empty() {
self.public_access_block_config =
Some(deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)?);
if !self.public_access_block_config_xml.is_empty()
&& let Err(e) = deserialize::<PublicAccessBlockConfiguration>(&self.public_access_block_config_xml)
.map(|c| self.public_access_block_config = Some(c))
{
tracing::warn!(bucket = %self.name, config = "public_access_block", error = %e, "parse_all_configs: failed to parse");
}
if !self.bucket_acl_config_json.is_empty() {
let acl = String::from_utf8(self.bucket_acl_config_json.clone())
.map_err(|e| Error::other(format!("invalid UTF-8 in bucket ACL: {}", e)))?;
self.bucket_acl_config = Some(acl);
if !self.bucket_acl_config_json.is_empty()
&& let Err(e) = String::from_utf8(self.bucket_acl_config_json.clone()).map(|acl| self.bucket_acl_config = Some(acl))
{
tracing::warn!(bucket = %self.name, config = "bucket_acl", error = %e, "parse_all_configs: failed to parse");
}
Ok(())
@@ -478,7 +641,6 @@ async fn read_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<BucketM
Ok(bm)
}
fn _write_time<S>(t: &OffsetDateTime, s: S) -> std::result::Result<S::Ok, S::Error>
where
S: Serializer,
crates/ecstore/src/bucket/metadata_sys.rs
+8 -6
View File
@@ -360,12 +360,14 @@ impl BucketMetadataSys {
};
if !meta.lifecycle_config_xml.is_empty() {
let cfg = deserialize::<BucketLifecycleConfiguration>(&meta.lifecycle_config_xml)?;
// TODO: FIXME:
// for _v in cfg.rules.iter() {
// break;
// }
if let Some(_v) = cfg.rules.first() {}
if let Ok(cfg) = deserialize::<BucketLifecycleConfiguration>(&meta.lifecycle_config_xml) {
if let Some(_v) = cfg.rules.first() {}
} else {
tracing::warn!(
bucket = %bucket,
"delete: failed to parse lifecycle config XML"
);
}
}
// TODO: other lifecycle handle
crates/ecstore/src/bucket/metadata_test.rs
+261
View File
@@ -0,0 +1,261 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::metadata::BucketMetadata;
use time::OffsetDateTime;
/// Full BucketMetadata hex (all fields populated).
const TEST_BUCKET_METADATA_HEX: &str = "de0019a44e616d65b27275737466732d636f6d7061742d74657374a743726561746564c70c050000000065920080075bcd15ab4c6f636b456e61626c6564c3b0506f6c696379436f6e6669674a534f4ec4907b2256657273696f6e223a22323031322d31302d3137222c2253746174656d656e74223a5b7b22456666656374223a22416c6c6f77222c225072696e636970616c223a222a222c22416374696f6e223a2273333a4765744f626a656374222c225265736f75726365223a2261726e3a6177733a73333a3a3a7275737466732d636f6d7061742d746573742f2a227d5d7db54e6f74696669636174696f6e436f6e666967584d4cc4963c4e6f74696669636174696f6e436f6e66696775726174696f6e3e3c436c6f75645761746368436f6e66696775726174696f6e3e3c49643e6e313c2f49643e3c4576656e743e73333a4f626a656374437265617465643a2a3c2f4576656e743e3c2f436c6f75645761746368436f6e66696775726174696f6e3e3c2f4e6f74696669636174696f6e436f6e66696775726174696f6e3eb24c6966656379636c65436f6e666967584d4cc48c3c4c6966656379636c65436f6e66696775726174696f6e3e3c52756c653e3c49443e72756c65313c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c45787069726174696f6e3e3c446179733e33303c2f446179733e3c2f45787069726174696f6e3e3c2f52756c653e3c2f4c6966656379636c65436f6e66696775726174696f6e3eb34f626a6563744c6f636b436f6e666967584d4cc4b83c4f626a6563744c6f636b436f6e66696775726174696f6e3e3c4f626a6563744c6f636b456e61626c65643e456e61626c65643c2f4f626a6563744c6f636b456e61626c65643e3c52756c653e3c44656661756c74526574656e74696f6e3e3c4d6f64653e474f5645524e414e43453c2f4d6f64653e3c446179733e373c2f446179733e3c2f44656661756c74526574656e74696f6e3e3c2f52756c653e3c2f4f626a6563744c6f636b436f6e66696775726174696f6e3eb356657273696f6e696e67436f6e666967584d4cc44b3c56657273696f6e696e67436f6e66696775726174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f56657273696f6e696e67436f6e66696775726174696f6e3eb3456e6372797074696f6e436f6e666967584d4cc4c03c53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3e3c52756c653e3c4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c535345416c676f726974686d3e4145533235363c2f535345416c676f726974686d3e3c2f4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c2f52756c653e3c2f53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3eb054616767696e67436f6e666967584d4cc4503c54616767696e673e3c5461675365743e3c5461673e3c4b65793e456e763c2f4b65793e3c56616c75653e546573743c2f56616c75653e3c2f5461673e3c2f5461675365743e3c2f54616767696e673eaf51756f7461436f6e6669674a534f4ec4707b2271756f7461223a313037333734313832342c2271756f74615f74797065223a2248617264222c22637265617465645f6174223a22323032342d30312d30315430303a30303a30305a222c22757064617465645f6174223a22323032342d30312d30315430303a30303a30305a227db45265706c69636174696f6e436f6e666967584d4cc4e73c5265706c69636174696f6e436f6e66696775726174696f6e3e3c526f6c653e61726e3a6177733a69616d3a3a3132333435363738393031323a726f6c652f7265706c3c2f526f6c653e3c52756c653e3c49443e72313c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c5072656669783e646f632f3c2f5072656669783e3c44657374696e6174696f6e3e3c4275636b65743e61726e3a6177733a73333a3a3a646573743c2f4275636b65743e3c2f44657374696e6174696f6e3e3c2f52756c653e3c2f5265706c69636174696f6e436f6e66696775726174696f6e3eb74275636b657454617267657473436f6e6669674a534f4ec4535b7b22656e64706f696e74223a22687474703a2f2f7461726765742e6578616d706c652e636f6d222c227461726765744275636b6574223a227462222c22726567696f6e223a2275732d656173742d31227d5dbb4275636b657454617267657473436f6e6669674d6574614a534f4ec42d7b227265706c69636174696f6e4964223a227265706c2d31222c2273796e634d6f6465223a226173796e63227db5506f6c696379436f6e666967557064617465644174c70c050000000065a5022000000000b94f626a6563744c6f636b436f6e666967557064617465644174c70c050000000065a5022000000000b9456e6372797074696f6e436f6e666967557064617465644174c70c050000000065a5022000000000b654616767696e67436f6e666967557064617465644174c70c050000000065a5022000000000b451756f7461436f6e666967557064617465644174c70c050000000065a5022000000000ba5265706c69636174696f6e436f6e666967557064617465644174c70c050000000065a5022000000000b956657273696f6e696e67436f6e666967557064617465644174c70c050000000065a5022000000000b84c6966656379636c65436f6e666967557064617465644174c70c050000000065a5022000000000bb4e6f74696669636174696f6e436f6e666967557064617465644174c70c050000000065a5022000000000bc4275636b657454617267657473436f6e666967557064617465644174c70c050000000065a5022000000000d9204275636b657454617267657473436f6e6669674d657461557064617465644174c70c050000000065a5022000000000";
#[tokio::test]
async fn marshal_msg() {
let bm = BucketMetadata::new("dada");
let buf = bm.marshal_msg().unwrap();
let new = BucketMetadata::unmarshal(&buf).unwrap();
assert_eq!(bm.name, new.name);
}
/// Verifies that serialized time uses msgp ext type 5.
#[tokio::test]
async fn marshal_msg_uses_time_format() {
let mut bm = BucketMetadata::new("test-bucket");
bm.created = OffsetDateTime::from_unix_timestamp(1704067200).unwrap(); // 2024-01-01 00:00:00 UTC
let buf = bm.marshal_msg().unwrap();
// msgp uses ext8 (0xc7), len 12, type 5 for time
assert!(
buf.windows(3).any(|w| w == [0xc7, 0x0c, 0x05]),
"serialized data should contain msgp time ext (0xc7 0x0c 0x05)"
);
}
#[tokio::test]
async fn unmarshal_test_bucket_metadata() {
use faster_hex::hex_decode;
let mut bytes = vec![0u8; TEST_BUCKET_METADATA_HEX.len() / 2];
hex_decode(TEST_BUCKET_METADATA_HEX.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("RustFS must unmarshal MinIO format");
assert_eq!(bm.name, "rustfs-compat-test");
assert_eq!(bm.created.unix_timestamp(), 1704067200);
assert_eq!(bm.created.nanosecond(), 123456789);
assert!(bm.lock_enabled);
assert!(!bm.policy_config_json.is_empty());
assert!(bm.policy_config_json.starts_with(b"{\"Version\""));
assert!(!bm.notification_config_xml.is_empty());
assert!(bm.notification_config_xml.starts_with(b"<Notification"));
assert!(!bm.lifecycle_config_xml.is_empty());
assert!(bm.lifecycle_config_xml.starts_with(b"<Lifecycle"));
assert!(!bm.object_lock_config_xml.is_empty());
assert!(bm.object_lock_config_xml.starts_with(b"<ObjectLock"));
assert!(!bm.versioning_config_xml.is_empty());
assert!(bm.versioning_config_xml.starts_with(b"<Versioning"));
assert!(!bm.encryption_config_xml.is_empty());
assert!(bm.encryption_config_xml.starts_with(b"<ServerSide"));
assert!(!bm.tagging_config_xml.is_empty());
assert!(bm.tagging_config_xml.starts_with(b"<Tagging"));
assert!(!bm.quota_config_json.is_empty());
assert!(bm.quota_config_json.starts_with(b"{\"quota\""));
assert!(!bm.replication_config_xml.is_empty());
assert!(bm.replication_config_xml.starts_with(b"<Replication"));
assert!(!bm.bucket_targets_config_json.is_empty());
assert!(bm.bucket_targets_config_json.starts_with(b"[{"));
assert!(!bm.bucket_targets_config_meta_json.is_empty());
assert!(bm.bucket_targets_config_meta_json.starts_with(b"{\"replication"));
let updated_sec = 1705312800; // 2024-01-15 12:00:00 UTC
assert_eq!(bm.policy_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.object_lock_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.encryption_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.tagging_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.quota_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.replication_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.versioning_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.lifecycle_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.notification_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.bucket_targets_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.bucket_targets_config_meta_updated_at.unix_timestamp(), updated_sec);
assert!(bm.cors_config_xml.is_empty());
assert!(bm.public_access_block_config_xml.is_empty());
assert!(bm.bucket_acl_config_json.is_empty());
}
#[tokio::test]
async fn marshal_msg_complete_example() {
// Create a complete BucketMetadata with various configurations
let mut bm = BucketMetadata::new("test-bucket");
// Set creation time to current time
bm.created = OffsetDateTime::now_utc();
bm.lock_enabled = true;
// Add policy configuration
let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}"#;
bm.policy_config_json = policy_json.as_bytes().to_vec();
bm.policy_config_updated_at = OffsetDateTime::now_utc();
// Add lifecycle configuration
let lifecycle_xml = r#"<LifecycleConfiguration><Rule><ID>rule1</ID><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#;
bm.lifecycle_config_xml = lifecycle_xml.as_bytes().to_vec();
bm.lifecycle_config_updated_at = OffsetDateTime::now_utc();
// Add versioning configuration
let versioning_xml = r#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#;
bm.versioning_config_xml = versioning_xml.as_bytes().to_vec();
bm.versioning_config_updated_at = OffsetDateTime::now_utc();
// Add encryption configuration
let encryption_xml = r#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#;
bm.encryption_config_xml = encryption_xml.as_bytes().to_vec();
bm.encryption_config_updated_at = OffsetDateTime::now_utc();
// Add tagging configuration
let tagging_xml = r#"<Tagging><TagSet><Tag><Key>Environment</Key><Value>Test</Value></Tag><Tag><Key>Owner</Key><Value>RustFS</Value></Tag></TagSet></Tagging>"#;
bm.tagging_config_xml = tagging_xml.as_bytes().to_vec();
bm.tagging_config_updated_at = OffsetDateTime::now_utc();
// Add quota configuration
let quota_json =
r#"{"quota":1073741824,"quota_type":"Hard","created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#; // 1GB quota
bm.quota_config_json = quota_json.as_bytes().to_vec();
bm.quota_config_updated_at = OffsetDateTime::now_utc();
// Add object lock configuration
let object_lock_xml = r#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#;
bm.object_lock_config_xml = object_lock_xml.as_bytes().to_vec();
bm.object_lock_config_updated_at = OffsetDateTime::now_utc();
// Add notification configuration
let notification_xml = r#"<NotificationConfiguration><CloudWatchConfiguration><Id>notification1</Id><Event>s3:ObjectCreated:*</Event><CloudWatchConfiguration><LogGroupName>test-log-group</LogGroupName></CloudWatchConfiguration></CloudWatchConfiguration></NotificationConfiguration>"#;
bm.notification_config_xml = notification_xml.as_bytes().to_vec();
bm.notification_config_updated_at = OffsetDateTime::now_utc();
// Add replication configuration
let replication_xml = r#"<ReplicationConfiguration><Role>arn:aws:iam::123456789012:role/replication-role</Role><Rule><ID>rule1</ID><Status>Enabled</Status><Prefix>documents/</Prefix><Destination><Bucket>arn:aws:s3:::destination-bucket</Bucket></Destination></Rule></ReplicationConfiguration>"#;
bm.replication_config_xml = replication_xml.as_bytes().to_vec();
bm.replication_config_updated_at = OffsetDateTime::now_utc();
// Add bucket targets configuration
let bucket_targets_json = r#"[{"endpoint":"http://target1.example.com","credentials":{"accessKey":"key1","secretKey":"secret1"},"targetBucket":"target-bucket-1","region":"us-east-1"},{"endpoint":"http://target2.example.com","credentials":{"accessKey":"key2","secretKey":"secret2"},"targetBucket":"target-bucket-2","region":"us-west-2"}]"#;
bm.bucket_targets_config_json = bucket_targets_json.as_bytes().to_vec();
bm.bucket_targets_config_updated_at = OffsetDateTime::now_utc();
// Add bucket targets meta configuration
let bucket_targets_meta_json = r#"{"replicationId":"repl-123","syncMode":"async","bandwidth":"100MB"}"#;
bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec();
bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc();
// Add public access block configuration
let public_access_block_xml = r#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#;
bm.public_access_block_config_xml = public_access_block_xml.as_bytes().to_vec();
bm.public_access_block_config_updated_at = OffsetDateTime::now_utc();
let bucket_acl = r#"{"owner":{"id":"rustfsadmin","display_name":"RustFS Tester"},"grants":[{"grantee":{"grantee_type":"CanonicalUser","id":"rustfsadmin","display_name":"RustFS Tester","uri":null,"email_address":null},"permission":"FULL_CONTROL"}]}"#;
bm.bucket_acl_config_json = bucket_acl.as_bytes().to_vec();
bm.bucket_acl_config_updated_at = OffsetDateTime::now_utc();
// Test serialization
let buf = bm.marshal_msg().unwrap();
assert!(!buf.is_empty(), "Serialized buffer should not be empty");
// Test deserialization
let deserialized_bm = BucketMetadata::unmarshal(&buf).unwrap();
// Verify all fields are correctly serialized and deserialized
assert_eq!(bm.name, deserialized_bm.name);
assert_eq!(bm.created.unix_timestamp(), deserialized_bm.created.unix_timestamp());
assert_eq!(bm.lock_enabled, deserialized_bm.lock_enabled);
// Verify configuration data
assert_eq!(bm.policy_config_json, deserialized_bm.policy_config_json);
assert_eq!(bm.lifecycle_config_xml, deserialized_bm.lifecycle_config_xml);
assert_eq!(bm.versioning_config_xml, deserialized_bm.versioning_config_xml);
assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml);
assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml);
assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json);
assert_eq!(bm.public_access_block_config_xml, deserialized_bm.public_access_block_config_xml);
assert_eq!(bm.bucket_acl_config_json, deserialized_bm.bucket_acl_config_json);
assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml);
assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml);
assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml);
assert_eq!(bm.bucket_targets_config_json, deserialized_bm.bucket_targets_config_json);
assert_eq!(bm.bucket_targets_config_meta_json, deserialized_bm.bucket_targets_config_meta_json);
// Verify timestamps (comparing unix timestamps to avoid precision issues)
assert_eq!(
bm.policy_config_updated_at.unix_timestamp(),
deserialized_bm.policy_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.lifecycle_config_updated_at.unix_timestamp(),
deserialized_bm.lifecycle_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.versioning_config_updated_at.unix_timestamp(),
deserialized_bm.versioning_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.encryption_config_updated_at.unix_timestamp(),
deserialized_bm.encryption_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.tagging_config_updated_at.unix_timestamp(),
deserialized_bm.tagging_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.quota_config_updated_at.unix_timestamp(),
deserialized_bm.quota_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.object_lock_config_updated_at.unix_timestamp(),
deserialized_bm.object_lock_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.notification_config_updated_at.unix_timestamp(),
deserialized_bm.notification_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.replication_config_updated_at.unix_timestamp(),
deserialized_bm.replication_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.bucket_targets_config_updated_at.unix_timestamp(),
deserialized_bm.bucket_targets_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.bucket_targets_config_meta_updated_at.unix_timestamp(),
deserialized_bm.bucket_targets_config_meta_updated_at.unix_timestamp()
);
// Test that the serialized data contains expected content
let buf_str = String::from_utf8_lossy(&buf);
assert!(buf_str.contains("test-bucket"), "Serialized data should contain bucket name");
// Verify the buffer size is reasonable (should be larger due to all the config data)
assert!(buf.len() > 1000, "Buffer should be substantial in size due to all configurations");
println!("✅ Complete BucketMetadata serialization test passed");
println!(" - Bucket name: {}", deserialized_bm.name);
println!(" - Lock enabled: {}", deserialized_bm.lock_enabled);
println!(" - Policy config size: {} bytes", deserialized_bm.policy_config_json.len());
println!(" - Lifecycle config size: {} bytes", deserialized_bm.lifecycle_config_xml.len());
println!(" - Serialized buffer size: {} bytes", buf.len());
}
crates/ecstore/src/bucket/migration.rs
+420
View File
@@ -0,0 +1,420 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Migration of bucket metadata and IAM config from legacy format to RustFS format.
use crate::bucket::metadata::BUCKET_METADATA_FILE;
use crate::bucket::replication::{decode_resync_file, encode_resync_file};
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::store_api::{BucketOptions, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_policy::auth::UserIdentity;
use rustfs_policy::policy::PolicyDoc;
use rustfs_utils::path::SLASH_SEPARATOR;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use time::OffsetDateTime;
use tracing::{debug, info, warn};
/// IAM config prefix under meta bucket (e.g. config/iam/).
const IAM_CONFIG_PREFIX: &str = "config/iam";
const IAM_FORMAT_FILE_PATH: &str = "config/iam/format.json";
const IAM_USERS_PREFIX: &str = "config/iam/users/";
const IAM_SERVICE_ACCOUNTS_PREFIX: &str = "config/iam/service-accounts/";
const IAM_STS_PREFIX: &str = "config/iam/sts/";
const IAM_GROUPS_PREFIX: &str = "config/iam/groups/";
const IAM_POLICIES_PREFIX: &str = "config/iam/policies/";
const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
const REPLICATION_META_DIR: &str = ".replication";
const RESYNC_META_FILE: &str = "resync.bin";
#[derive(Debug, Serialize, Deserialize)]
struct CompatIamFormat {
#[serde(default)]
version: i64,
}
#[derive(Debug, Serialize, Deserialize)]
struct CompatGroupInfo {
#[serde(default)]
version: i64,
#[serde(default = "default_group_status")]
status: String,
#[serde(default)]
members: Vec<String>,
#[serde(
rename = "updatedAt",
alias = "update_at",
default,
with = "rustfs_policy::serde_datetime::option"
)]
update_at: Option<OffsetDateTime>,
}
#[derive(Debug, Serialize, Deserialize)]
struct CompatMappedPolicy {
#[serde(default)]
version: i64,
#[serde(rename = "policy", alias = "policies", default)]
policy: String,
#[serde(
rename = "updatedAt",
alias = "update_at",
default,
with = "rustfs_policy::serde_datetime::option"
)]
update_at: Option<OffsetDateTime>,
}
fn default_group_status() -> String {
"enabled".to_string()
}
fn normalize_iam_config_blob(path: &str, data: &[u8]) -> std::result::Result<Option<Vec<u8>>, String> {
if path == IAM_FORMAT_FILE_PATH {
let mut format: CompatIamFormat =
serde_json::from_slice(data).map_err(|err| format!("parse IAM format failed: {err}"))?;
if format.version <= 0 {
format.version = 1;
}
return serde_json::to_vec(&format)
.map(Some)
.map_err(|err| format!("serialize IAM format failed: {err}"));
}
if is_identity_path(path) {
let mut identity: UserIdentity =
serde_json::from_slice(data).map_err(|err| format!("parse IAM identity failed: {err}"))?;
if identity.update_at.is_none() {
identity.update_at = Some(OffsetDateTime::now_utc());
}
return serde_json::to_vec(&identity)
.map(Some)
.map_err(|err| format!("serialize IAM identity failed: {err}"));
}
if is_group_path(path) {
let mut group: CompatGroupInfo = serde_json::from_slice(data).map_err(|err| format!("parse IAM group failed: {err}"))?;
if group.update_at.is_none() {
group.update_at = Some(OffsetDateTime::now_utc());
}
return serde_json::to_vec(&group)
.map(Some)
.map_err(|err| format!("serialize IAM group failed: {err}"));
}
if is_policy_doc_path(path) {
let mut doc = PolicyDoc::try_from(data.to_vec()).map_err(|err| format!("parse IAM policy doc failed: {err}"))?;
if doc.create_date.is_none() {
doc.create_date = doc.update_date;
}
if doc.update_date.is_none() {
doc.update_date = doc.create_date;
}
return serde_json::to_vec(&doc)
.map(Some)
.map_err(|err| format!("serialize IAM policy doc failed: {err}"));
}
if is_policy_mapping_path(path) {
let mut mapped: CompatMappedPolicy =
serde_json::from_slice(data).map_err(|err| format!("parse IAM policy mapping failed: {err}"))?;
if mapped.update_at.is_none() {
mapped.update_at = Some(OffsetDateTime::now_utc());
}
return serde_json::to_vec(&mapped)
.map(Some)
.map_err(|err| format!("serialize IAM policy mapping failed: {err}"));
}
Ok(None)
}
fn is_identity_path(path: &str) -> bool {
(path.starts_with(IAM_USERS_PREFIX) || path.starts_with(IAM_SERVICE_ACCOUNTS_PREFIX) || path.starts_with(IAM_STS_PREFIX))
&& path.ends_with("/identity.json")
}
fn is_group_path(path: &str) -> bool {
path.starts_with(IAM_GROUPS_PREFIX) && path.ends_with("/members.json")
}
fn is_policy_doc_path(path: &str) -> bool {
path.starts_with(IAM_POLICIES_PREFIX) && path.ends_with("/policy.json")
}
fn is_policy_mapping_path(path: &str) -> bool {
path.starts_with(IAM_POLICY_DB_PREFIX) && path.ends_with(".json")
}
fn is_resync_meta_path(path: &str) -> bool {
path.ends_with(&format!("{REPLICATION_META_DIR}/{RESYNC_META_FILE}"))
}
fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Option<Vec<u8>>, String> {
if !is_resync_meta_path(path) {
return Ok(None);
}
let status = decode_resync_file(data).map_err(|err| format!("decode resync meta failed: {err}"))?;
encode_resync_file(&status)
.map(Some)
.map_err(|err| format!("encode resync meta failed: {err}"))
}
/// Migrates bucket metadata from legacy format to RustFS.
/// Uses list_bucket (from disk volumes) to get bucket names, since list_objects_v2 on the legacy
/// meta bucket may not work (legacy format differs from object layer expectations).
/// Skips buckets that already exist in RustFS (idempotent).
pub async fn try_migrate_bucket_metadata<S: StorageAPI>(store: Arc<S>) {
let buckets_list = match store
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
{
Ok(b) => b,
Err(e) => {
warn!("list buckets failed (skip migration): {e}");
return;
}
};
let buckets: Vec<String> = buckets_list.into_iter().map(|b| b.name).collect();
if buckets.is_empty() {
debug!("No migrating bucket metadata found");
return;
}
debug!("Found {} migrating bucket metadata, migrating...", buckets.len());
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let h = HeaderMap::new();
for bucket in buckets {
let meta_path = format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{BUCKET_METADATA_FILE}");
migrate_one_if_missing(store.clone(), &opts, &h, &meta_path, &format!("bucket metadata: {bucket}")).await;
let resync_path = format!(
"{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{bucket}{SLASH_SEPARATOR}{REPLICATION_META_DIR}{SLASH_SEPARATOR}{RESYNC_META_FILE}"
);
migrate_one_if_missing(store.clone(), &opts, &h, &resync_path, &format!("bucket replication resync: {bucket}")).await;
}
}
async fn migrate_one_if_missing<S: StorageAPI>(
store: Arc<S>,
opts: &ObjectOptions,
headers: &HeaderMap,
path: &str,
label: &str,
) {
if store
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
.await
.is_ok()
{
debug!("{label} already exists in RustFS, skip");
return;
}
let mut rd = match store
.get_object_reader(MIGRATING_META_BUCKET, path, None, headers.clone(), opts)
.await
{
Ok(r) => r,
Err(e) => {
debug!("read migrating {label}: {e}");
return;
}
};
let data = match rd.read_all().await {
Ok(d) if !d.is_empty() => d,
Ok(_) => return,
Err(e) => {
debug!("read migrating {label} body: {e}");
return;
}
};
let data = match normalize_bucket_meta_blob(path, &data) {
Ok(Some(normalized)) => normalized,
Ok(None) => data,
Err(e) => {
warn!("skip {label} migration due to incompatible format: {e}");
return;
}
};
if let Err(e) = store
.put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), opts)
.await
{
warn!("write {label}: {e}");
} else {
info!("Migrated {label}");
}
}
/// Migrates IAM config from legacy meta bucket `config/iam/` to RustFS meta bucket.
/// Lists all objects under the IAM prefix in the source, copies each to the target if not present.
/// Skips objects that already exist in RustFS (idempotent).
/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
pub async fn try_migrate_iam_config<S: StorageAPI>(store: Arc<S>) {
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let h = HeaderMap::new();
let prefix = format!("{IAM_CONFIG_PREFIX}/");
let mut continuation: Option<String> = None;
let mut total_migrated = 0usize;
loop {
let list_result = match store
.clone()
.list_objects_v2(MIGRATING_META_BUCKET, &prefix, continuation, None, 500, false, None, false)
.await
{
Ok(r) => r,
Err(e) => {
debug!("list IAM config from legacy bucket failed (skip migration): {e}");
return;
}
};
for obj in list_result.objects {
let path = &obj.name;
if path.is_empty() || path.ends_with('/') {
continue;
}
if store
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
.await
.is_ok()
{
debug!("IAM config already exists in RustFS, skip: {path}");
continue;
}
let mut rd = match store
.get_object_reader(MIGRATING_META_BUCKET, path, None, h.clone(), &opts)
.await
{
Ok(r) => r,
Err(e) => {
debug!("read migrating IAM config {path}: {e}");
continue;
}
};
let data = match rd.read_all().await {
Ok(d) if !d.is_empty() => d,
Ok(_) => continue,
Err(e) => {
debug!("read migrating IAM config {path} body: {e}");
continue;
}
};
let data = match normalize_iam_config_blob(path, &data) {
Ok(Some(normalized)) => normalized,
Ok(None) => {
debug!("skip unsupported IAM config path during migration: {path}");
continue;
}
Err(e) => {
warn!("skip IAM config migration due to incompatible format, path: {path}, err: {e}");
continue;
}
};
if let Err(e) = store
.put_object(RUSTFS_META_BUCKET, path, &mut PutObjReader::from_vec(data), &opts)
.await
{
warn!("write IAM config {path}: {e}");
} else {
info!("Migrated IAM config: {path}");
total_migrated += 1;
}
}
continuation = list_result.next_continuation_token.or(list_result.continuation_token);
if !list_result.is_truncated || continuation.is_none() {
break;
}
}
if total_migrated > 0 {
info!("IAM migration complete: {} object(s) migrated", total_migrated);
}
}
#[cfg(test)]
mod tests {
use super::{normalize_bucket_meta_blob, normalize_iam_config_blob};
use crate::bucket::replication::{
BucketReplicationResyncStatus, ResyncStatusType, TargetReplicationResyncStatus, decode_resync_file, encode_resync_file,
};
use std::collections::HashMap;
#[test]
fn test_normalize_policy_mapping_legacy_timestamp_and_fields() {
let path = "config/iam/policydb/users/alice.json";
let input = r#"{"version":1,"policies":"readwrite","update_at":"2026-03-09 02:22:44.998954 +00:00:00"}"#;
let output = normalize_iam_config_blob(path, input.as_bytes())
.expect("normalize should succeed")
.expect("path should be supported");
let v: serde_json::Value = serde_json::from_slice(&output).expect("output should be valid JSON");
assert_eq!(v.get("policy").and_then(|x| x.as_str()), Some("readwrite"));
assert!(v.get("policies").is_none(), "legacy field should be normalized");
let updated_at = v
.get("updatedAt")
.and_then(|x| x.as_str())
.expect("updatedAt should exist as string");
assert!(updated_at.contains('T'), "updatedAt should be RFC3339-like");
}
#[test]
fn test_normalize_bucket_meta_blob_resync_reencode() {
let path = ".buckets/test/.replication/resync.bin";
let mut status = BucketReplicationResyncStatus::new();
status.id = 123;
status.targets_map = HashMap::from([(
"arn:replication::1:dest".to_string(),
TargetReplicationResyncStatus {
resync_id: "reset-1".to_string(),
resync_status: ResyncStatusType::ResyncStarted,
replicated_count: 1,
..Default::default()
},
)]);
let input = encode_resync_file(&status).expect("encode should succeed");
let output = normalize_bucket_meta_blob(path, &input)
.expect("normalize should succeed")
.expect("resync path should be normalized");
let decoded = decode_resync_file(&output).expect("decode should succeed");
assert_eq!(decoded.id, 123);
assert_eq!(decoded.targets_map["arn:replication::1:dest"].resync_id, "reset-1");
}
}
crates/ecstore/src/bucket/mod.rs
+4
View File
@@ -18,6 +18,10 @@ pub mod error;
pub mod lifecycle;
pub mod metadata;
pub mod metadata_sys;
#[cfg(test)]
mod metadata_test;
pub mod migration;
mod msgp_decode;
pub mod object_lock;
pub mod policy_sys;
pub mod quota;
crates/ecstore/src/bucket/msgp_decode.rs
+189
View File
@@ -0,0 +1,189 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! MessagePack decode helpers for bucket metadata, aligned with msgp format.
use crate::error::{Error, Result};
use byteorder::{BigEndian, ByteOrder};
use rmp::Marker;
use std::io::{Read, Write};
use time::OffsetDateTime;
/// Skip a single MessagePack value. Used for unknown map keys.
pub(crate) fn skip_msgp_value<R: Read>(rd: &mut R) -> Result<()> {
let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
let skip_len: usize = match marker {
Marker::Null | Marker::False | Marker::True => 0,
Marker::FixPos(_) | Marker::FixNeg(_) => 0,
Marker::U8 => 1,
Marker::U16 => 2,
Marker::U32 => 4,
Marker::U64 => 8,
Marker::I8 => 1,
Marker::I16 => 2,
Marker::I32 => 4,
Marker::I64 => 8,
Marker::F32 => 4,
Marker::F64 => 8,
Marker::FixStr(n) => n as usize,
Marker::Str8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::other)?;
b[0] as usize
}
Marker::Str16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
u16::from_be_bytes(b) as usize
}
Marker::Str32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
u32::from_be_bytes(b) as usize
}
Marker::Bin8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::other)?;
b[0] as usize
}
Marker::Bin16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
u16::from_be_bytes(b) as usize
}
Marker::Bin32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
u32::from_be_bytes(b) as usize
}
Marker::FixArray(n) => {
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Array16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
let n = u16::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Array32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
let n = u32::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::FixMap(n) => {
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Map16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
let n = u16::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Map32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
let n = u32::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::FixExt1 => 1,
Marker::FixExt2 => 2,
Marker::FixExt4 => 4,
Marker::FixExt8 => 8,
Marker::FixExt16 => 16,
Marker::Ext8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::other)?;
let len = b[0] as usize;
1 + len // type byte + data
}
Marker::Ext16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::other)?;
let len = u16::from_be_bytes(b) as usize;
2 + len
}
Marker::Ext32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::other)?;
let len = u32::from_be_bytes(b) as usize;
4 + len
}
Marker::Reserved => 0,
};
if skip_len > 0 {
let mut buf = vec![0u8; skip_len];
rd.read_exact(&mut buf).map_err(Error::other)?;
}
Ok(())
}
/// msgp time format: ext8 (0xc7), len 12, type 5, 8 bytes sec (BE) + 4 bytes nsec (BE).
pub(crate) const MSGP_TIME_EXT_TYPE: i8 = 5;
pub(crate) const MSGP_TIME_LEN: u8 = 12;
/// Read msgp ext8 time - caller must have already read the marker and verified it's ext8.
/// Ext8 format: 1 byte len, 1 byte type, then data bytes.
pub(crate) fn read_msgp_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
let mut len_buf = [0u8; 1];
rd.read_exact(&mut len_buf).map_err(Error::other)?;
let len = len_buf[0] as usize;
if len != MSGP_TIME_LEN as usize {
return Err(Error::other(format!("invalid msgp time len: {len}")));
}
let mut type_buf = [0u8; 1];
rd.read_exact(&mut type_buf).map_err(Error::other)?;
if type_buf[0] != MSGP_TIME_EXT_TYPE as u8 {
return Err(Error::other(format!("invalid msgp time type: {}", type_buf[0])));
}
let mut buf = [0u8; 12];
rd.read_exact(&mut buf).map_err(Error::other)?;
let sec = BigEndian::read_i64(&buf[0..8]);
let nsec = BigEndian::read_u32(&buf[8..12]);
OffsetDateTime::from_unix_timestamp(sec)
.map_err(|_| Error::other("invalid timestamp"))?
.replace_nanosecond(nsec)
.map_err(|_| Error::other("invalid nanosecond"))
}
/// Write msgp time as ext8 (0xc7), len 12, type 5. Always uses ext format (never nil).
pub(crate) fn write_msgp_time<W: Write>(wr: &mut W, t: OffsetDateTime) -> Result<()> {
wr.write_all(&[0xc7, MSGP_TIME_LEN, MSGP_TIME_EXT_TYPE as u8])
.map_err(Error::other)?;
let mut buf = [0u8; 12];
BigEndian::write_i64(&mut buf[0..8], t.unix_timestamp());
BigEndian::write_u32(&mut buf[8..12], t.nanosecond());
wr.write_all(&buf).map_err(Error::other)
}
crates/ecstore/src/bucket/quota/mod.rs
+68 -6
View File
@@ -15,7 +15,6 @@
pub mod checker;
use crate::error::Result;
use rmp_serde::Serializer as rmpSerializer;
use rustfs_config::{
QUOTA_API_PATH, QUOTA_EXCEEDED_ERROR_CODE, QUOTA_INTERNAL_ERROR_CODE, QUOTA_INVALID_CONFIG_ERROR_CODE,
QUOTA_NOT_FOUND_ERROR_CODE,
@@ -28,27 +27,35 @@ use time::OffsetDateTime;
pub enum QuotaType {
/// Hard quota: reject immediately when exceeded
#[default]
#[serde(alias = "HARD", alias = "hard")]
Hard,
}
/// Bucket quota configuration. quota_type defaults to Hard when omitted.
#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)]
pub struct BucketQuota {
#[serde(default)]
pub quota: Option<u64>,
/// Defaults to Hard when missing.
#[serde(default)]
pub quota_type: QuotaType,
/// Timestamp when this quota configuration was set (for audit purposes)
#[serde(default, with = "time::serde::rfc3339::option")]
pub created_at: Option<OffsetDateTime>,
/// Accept updated_at for compatibility; not used.
#[serde(default, with = "time::serde::rfc3339::option", skip_serializing_if = "Option::is_none")]
pub updated_at: Option<OffsetDateTime>,
}
impl BucketQuota {
/// Serialize to JSON bytes. Same format as parse_all_configs.
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
Ok(buf)
serde_json::to_vec(self).map_err(Into::into)
}
/// Deserialize from JSON bytes. Same format as parse_all_configs.
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: BucketQuota = rmp_serde::from_slice(buf)?;
Ok(t)
serde_json::from_slice(buf).map_err(Into::into)
}
pub fn new(quota: Option<u64>) -> Self {
@@ -57,6 +64,7 @@ impl BucketQuota {
quota,
quota_type: QuotaType::Hard,
created_at: Some(now),
updated_at: None,
}
}
@@ -156,3 +164,57 @@ impl QuotaErrorResponse {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Legacy format: quota, created_at, updated_at (no quota_type)
#[test]
fn deserialize_format_without_quota_type() {
let json = r#"{"quota":1073741824,"created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#;
let q: BucketQuota = serde_json::from_slice(json.as_bytes()).expect("should parse");
assert_eq!(q.quota, Some(1073741824));
assert_eq!(q.quota_type, QuotaType::Hard);
assert!(q.created_at.is_some());
assert!(q.updated_at.is_some());
}
/// RustFS format: quota, quota_type, created_at
#[test]
fn deserialize_rustfs_format() {
let json = r#"{"quota":1073741824,"quota_type":"Hard","created_at":"2024-01-01T00:00:00Z"}"#;
let q: BucketQuota = serde_json::from_slice(json.as_bytes()).expect("should parse");
assert_eq!(q.quota, Some(1073741824));
assert_eq!(q.quota_type, QuotaType::Hard);
assert!(q.created_at.is_some());
assert!(q.created_at.is_some_and(|t| t.unix_timestamp() == 1704067200));
}
/// E2E format uses "HARD" (uppercase)
#[test]
fn deserialize_quota_type_hard_uppercase() {
let json = r#"{"quota":2048,"quota_type":"HARD"}"#;
let q: BucketQuota = serde_json::from_slice(json.as_bytes()).expect("should parse");
assert_eq!(q.quota_type, QuotaType::Hard);
}
/// marshal_msg/unmarshal use JSON, same as parse_all_configs
#[test]
fn marshal_unmarshal_roundtrip() {
let q = BucketQuota::new(Some(1073741824));
let buf = q.marshal_msg().expect("marshal");
let restored = BucketQuota::unmarshal(&buf).expect("unmarshal");
assert_eq!(q.quota, restored.quota);
assert_eq!(q.quota_type, restored.quota_type);
}
/// unmarshal accepts format without quota_type
#[test]
fn unmarshal_format_without_quota_type() {
let json = r#"{"quota":1073741824,"created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#;
let q = BucketQuota::unmarshal(json.as_bytes()).expect("should parse");
assert_eq!(q.quota, Some(1073741824));
assert_eq!(q.quota_type, QuotaType::Hard);
}
}
crates/ecstore/src/bucket/replication/replication_pool.rs
+6 -37
View File
@@ -20,8 +20,8 @@ use crate::bucket::replication::ResyncStatusType;
use crate::bucket::replication::replicate_delete;
use crate::bucket::replication::replicate_object;
use crate::bucket::replication::replication_resyncer::{
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, ReplicationConfig, ReplicationResyncer,
get_heal_replicate_object_info,
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, REPLICATION_DIR, RESYNC_FILE_NAME, ReplicationConfig,
ReplicationResyncer, decode_resync_file, get_heal_replicate_object_info,
};
use crate::bucket::replication::replication_state::ReplicationStats;
use crate::config::com::read_config;
@@ -41,7 +41,7 @@ use rustfs_filemeta::VersionPurgeStatusType;
use rustfs_filemeta::replication_statuses_map;
use rustfs_filemeta::version_purge_statuses_map;
use rustfs_filemeta::{REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE};
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
use std::any::Any;
use std::sync::Arc;
use std::sync::atomic::AtomicI32;
@@ -861,17 +861,8 @@ async fn load_bucket_resync_metadata<S: StorageAPI>(
bucket: &str,
obj_api: Arc<S>,
) -> Result<BucketReplicationResyncStatus, EcstoreError> {
use std::convert::TryInto;
let mut brs = BucketReplicationResyncStatus::new();
// Constants that would be defined elsewhere
const REPLICATION_DIR: &str = "replication";
const RESYNC_FILE_NAME: &str = "resync.bin";
const RESYNC_META_FORMAT: u16 = 1;
const RESYNC_META_VERSION: u16 = 1;
const RESYNC_META_VERSION_V1: u16 = 1;
let resync_dir_path = format!("{BUCKET_META_PREFIX}/{bucket}/{REPLICATION_DIR}");
let resync_file_path = format!("{resync_dir_path}/{RESYNC_FILE_NAME}");
@@ -886,27 +877,7 @@ async fn load_bucket_resync_metadata<S: StorageAPI>(
return Ok(brs);
}
if data.len() <= 4 {
return Err(EcstoreError::CorruptedFormat);
}
// Read resync meta header
let format = u16::from_le_bytes(data[0..2].try_into().unwrap());
if format != RESYNC_META_FORMAT {
return Err(EcstoreError::CorruptedFormat);
}
let version = u16::from_le_bytes(data[2..4].try_into().unwrap());
if version != RESYNC_META_VERSION {
return Err(EcstoreError::CorruptedFormat);
}
// Parse data
brs = BucketReplicationResyncStatus::unmarshal_msg(&data[4..])?;
if brs.version != RESYNC_META_VERSION_V1 {
return Err(EcstoreError::CorruptedFormat);
}
brs = decode_resync_file(&data)?;
Ok(brs)
}
@@ -984,10 +955,8 @@ pub fn get_global_replication_pool() -> Option<Arc<DynReplicationPool>> {
pub async fn schedule_replication<S: StorageAPI>(oi: ObjectInfo, o: Arc<S>, dsc: ReplicateDecision, op_type: ReplicationType) {
let tgt_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
let purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
let tm = oi
.user_defined
.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"))
.map(|v| OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
let tm = get_str(&oi.user_defined, SUFFIX_REPLICATION_TIMESTAMP)
.map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
let mut rstate = oi.replication_state();
rstate.replicate_decision_str = dsc.to_string();
let asz = oi.get_actual_size().unwrap_or_default();
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+416 -68
View File
@@ -17,6 +17,7 @@ use crate::bucket::bucket_target_sys::{
AdvancedPutOptions, BucketTargetSys, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
};
use crate::bucket::metadata_sys;
use crate::bucket::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
use crate::bucket::replication::ResyncStatusType;
use crate::bucket::replication::replication_pool::GLOBAL_REPLICATION_STATS;
use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _};
@@ -50,7 +51,7 @@ use http_body::Frame;
use http_body_util::StreamBody;
use regex::Regex;
use rustfs_filemeta::{
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATION_RESET, ReplicateDecision, ReplicateObjectInfo,
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType,
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
@@ -58,8 +59,13 @@ use rustfs_filemeta::{
use rustfs_s3_common::EventName;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE,
RUSTFS_REPLICATION_RESET_STATUS, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, headers,
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
SUFFIX_REPLICATION_STATUS, SUFFIX_TAGGING_TIMESTAMP, headers,
};
use rustfs_utils::http::{
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_RESET_STATUS, SUFFIX_REPLICATION_SSEC_CRC, get_header_map, get_str,
has_internal_suffix, insert_header_map, insert_str, internal_key_strip_suffix_prefix, is_internal_key,
};
use rustfs_utils::path::path_join_buf;
use rustfs_utils::string::strings_has_prefix_fold;
@@ -69,7 +75,8 @@ use serde::Deserialize;
use serde::Serialize;
use std::any::Any;
use std::collections::HashMap;
use std::sync::Arc;
use std::io::{Cursor, Read};
use std::sync::{Arc, LazyLock};
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use tokio::io::AsyncRead;
@@ -80,14 +87,29 @@ use tokio_util::io::ReaderStream;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, instrument, warn};
const REPLICATION_DIR: &str = ".replication";
const RESYNC_FILE_NAME: &str = "resync.bin";
const RESYNC_META_FORMAT: u16 = 1;
const RESYNC_META_VERSION: u16 = 1;
pub(crate) const REPLICATION_DIR: &str = ".replication";
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
pub(crate) const RESYNC_META_FORMAT: u16 = 1;
pub(crate) const RESYNC_META_VERSION: u16 = 1;
const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60);
const WIRE_ZERO_TIME_UNIX: i64 = -62_135_596_800;
static WIRE_ZERO_TIME: LazyLock<OffsetDateTime> =
LazyLock::new(|| OffsetDateTime::from_unix_timestamp(WIRE_ZERO_TIME_UNIX).unwrap_or(OffsetDateTime::UNIX_EPOCH));
static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new();
fn wire_time_or_default(value: Option<OffsetDateTime>) -> OffsetDateTime {
value.unwrap_or(*WIRE_ZERO_TIME)
}
fn normalize_wire_time(value: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
match value {
Some(v) if v == *WIRE_ZERO_TIME || v == OffsetDateTime::UNIX_EPOCH => None,
other => other,
}
}
#[derive(Debug, Clone, Default)]
pub struct ResyncOpts {
pub bucket: String,
@@ -139,14 +161,199 @@ impl BucketReplicationResyncStatus {
}
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
Ok(rmp_serde::to_vec(&self)?)
let mut wr = Vec::new();
rmp::encode::write_map_len(&mut wr, 4)?;
rmp::encode::write_str(&mut wr, "v")?;
rmp::encode::write_i32(&mut wr, i32::from(self.version))?;
rmp::encode::write_str(&mut wr, "brs")?;
rmp::encode::write_map_len(&mut wr, self.targets_map.len() as u32)?;
for (arn, status) in &self.targets_map {
rmp::encode::write_str(&mut wr, arn)?;
status.marshal_wire_msg(&mut wr)?;
}
rmp::encode::write_str(&mut wr, "id")?;
rmp::encode::write_i32(&mut wr, self.id)?;
rmp::encode::write_str(&mut wr, "lu")?;
write_msgp_time(&mut wr, wire_time_or_default(self.last_update))?;
Ok(wr)
}
pub fn unmarshal_msg(data: &[u8]) -> Result<Self> {
let mut rd = Cursor::new(data);
let mut out = Self::new();
let mut fields = rmp::decode::read_map_len(&mut rd)?;
while fields > 0 {
fields -= 1;
let key = read_msgp_str(&mut rd)?;
match key.as_str() {
"v" => {
let v: i32 = rmp::decode::read_int(&mut rd)?;
out.version = u16::try_from(v).map_err(|_| Error::other("invalid resync version"))?;
}
"brs" => {
let map_len = rmp::decode::read_map_len(&mut rd)?;
let mut targets = HashMap::with_capacity(map_len as usize);
for _ in 0..map_len {
let arn = read_msgp_str(&mut rd)?;
let status = TargetReplicationResyncStatus::unmarshal_wire_msg(&mut rd)?;
targets.insert(arn, status);
}
out.targets_map = targets;
}
"id" => {
out.id = rmp::decode::read_int::<i32, _>(&mut rd)?;
}
"lu" => {
out.last_update = normalize_wire_time(read_msgp_time_or_nil(&mut rd)?);
}
_ => skip_msgp_value(&mut rd)?,
}
}
Ok(out)
}
pub fn unmarshal_legacy_msg(data: &[u8]) -> Result<Self> {
Ok(rmp_serde::from_slice(data)?)
}
}
pub(crate) fn encode_resync_file(status: &BucketReplicationResyncStatus) -> Result<Vec<u8>> {
let payload = status.marshal_msg()?;
let mut data = Vec::with_capacity(4 + payload.len());
let mut major = [0u8; 2];
byteorder::LittleEndian::write_u16(&mut major, RESYNC_META_FORMAT);
data.extend_from_slice(&major);
let mut minor = [0u8; 2];
byteorder::LittleEndian::write_u16(&mut minor, RESYNC_META_VERSION);
data.extend_from_slice(&minor);
data.extend_from_slice(&payload);
Ok(data)
}
pub(crate) fn decode_resync_file(data: &[u8]) -> Result<BucketReplicationResyncStatus> {
if data.len() <= 4 {
return Err(Error::CorruptedFormat);
}
let mut major = [0u8; 2];
major.copy_from_slice(&data[0..2]);
if byteorder::LittleEndian::read_u16(&major) != RESYNC_META_FORMAT {
return Err(Error::CorruptedFormat);
}
let mut minor = [0u8; 2];
minor.copy_from_slice(&data[2..4]);
if byteorder::LittleEndian::read_u16(&minor) != RESYNC_META_VERSION {
return Err(Error::CorruptedFormat);
}
let status = match BucketReplicationResyncStatus::unmarshal_msg(&data[4..]) {
Ok(v) => v,
Err(_) => BucketReplicationResyncStatus::unmarshal_legacy_msg(&data[4..])?,
};
if status.version != RESYNC_META_VERSION {
return Err(Error::CorruptedFormat);
}
Ok(status)
}
impl TargetReplicationResyncStatus {
fn marshal_wire_msg(&self, wr: &mut Vec<u8>) -> Result<()> {
rmp::encode::write_map_len(wr, 11)?;
rmp::encode::write_str(wr, "st")?;
write_msgp_time(wr, wire_time_or_default(self.start_time))?;
rmp::encode::write_str(wr, "lst")?;
write_msgp_time(wr, wire_time_or_default(self.last_update))?;
rmp::encode::write_str(wr, "id")?;
rmp::encode::write_str(wr, &self.resync_id)?;
rmp::encode::write_str(wr, "rdt")?;
write_msgp_time(wr, wire_time_or_default(self.resync_before_date))?;
rmp::encode::write_str(wr, "rst")?;
rmp::encode::write_i32(wr, resync_status_to_i32(self.resync_status))?;
rmp::encode::write_str(wr, "fs")?;
rmp::encode::write_i64(wr, self.failed_size)?;
rmp::encode::write_str(wr, "frc")?;
rmp::encode::write_i64(wr, self.failed_count)?;
rmp::encode::write_str(wr, "rs")?;
rmp::encode::write_i64(wr, self.replicated_size)?;
rmp::encode::write_str(wr, "rrc")?;
rmp::encode::write_i64(wr, self.replicated_count)?;
rmp::encode::write_str(wr, "bkt")?;
rmp::encode::write_str(wr, &self.bucket)?;
rmp::encode::write_str(wr, "obj")?;
rmp::encode::write_str(wr, &self.object)?;
Ok(())
}
fn unmarshal_wire_msg<R: Read>(rd: &mut R) -> Result<Self> {
let mut out = Self::new();
let mut fields = rmp::decode::read_map_len(rd)?;
while fields > 0 {
fields -= 1;
let key = read_msgp_str(rd)?;
match key.as_str() {
"st" => out.start_time = normalize_wire_time(read_msgp_time_or_nil(rd)?),
"lst" => out.last_update = normalize_wire_time(read_msgp_time_or_nil(rd)?),
"id" => out.resync_id = read_msgp_str(rd)?,
"rdt" => out.resync_before_date = normalize_wire_time(read_msgp_time_or_nil(rd)?),
"rst" => {
let v: i32 = rmp::decode::read_int(rd)?;
out.resync_status = resync_status_from_i32(v)?;
}
"fs" => out.failed_size = rmp::decode::read_int(rd)?,
"frc" => out.failed_count = rmp::decode::read_int(rd)?,
"rs" => out.replicated_size = rmp::decode::read_int(rd)?,
"rrc" => out.replicated_count = rmp::decode::read_int(rd)?,
"bkt" => out.bucket = read_msgp_str(rd)?,
"obj" => out.object = read_msgp_str(rd)?,
_ => skip_msgp_value(rd)?,
}
}
Ok(out)
}
}
fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
let len = rmp::decode::read_str_len(rd)? as usize;
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
Ok(String::from_utf8(buf)?)
}
fn read_msgp_time_or_nil<R: Read>(rd: &mut R) -> Result<Option<OffsetDateTime>> {
let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?;
match marker {
rmp::Marker::Null => Ok(None),
rmp::Marker::Ext8 => Ok(Some(read_msgp_ext8_time(rd)?)),
other => Err(Error::other(format!("expected time ext or nil, got marker: {other:?}"))),
}
}
fn resync_status_to_i32(status: ResyncStatusType) -> i32 {
match status {
ResyncStatusType::NoResync => 0,
ResyncStatusType::ResyncPending => 1,
ResyncStatusType::ResyncCanceled => 2,
ResyncStatusType::ResyncStarted => 3,
ResyncStatusType::ResyncCompleted => 4,
ResyncStatusType::ResyncFailed => 5,
}
}
fn resync_status_from_i32(code: i32) -> Result<ResyncStatusType> {
match code {
0 => Ok(ResyncStatusType::NoResync),
1 => Ok(ResyncStatusType::ResyncPending),
2 => Ok(ResyncStatusType::ResyncCanceled),
3 => Ok(ResyncStatusType::ResyncStarted),
4 => Ok(ResyncStatusType::ResyncCompleted),
5 => Ok(ResyncStatusType::ResyncFailed),
_ => Err(Error::other(format!("invalid resync status code: {code}"))),
}
}
static RESYNC_WORKER_COUNT: usize = 10;
#[derive(Debug)]
@@ -617,7 +824,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
let keys_to_update: Vec<_> = user_defined
.iter()
.filter(|(k, _)| k.eq_ignore_ascii_case(format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}").as_str()))
.filter(|(k, _)| has_internal_suffix(k, SUFFIX_REPLICATION_RESET))
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
@@ -695,19 +902,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
}
async fn save_resync_status<S: StorageAPI>(bucket: &str, status: &BucketReplicationResyncStatus, api: Arc<S>) -> Result<()> {
let buf = status.marshal_msg()?;
let mut data = Vec::new();
let mut major = [0u8; 2];
byteorder::LittleEndian::write_u16(&mut major, RESYNC_META_FORMAT);
data.extend_from_slice(&major);
let mut minor = [0u8; 2];
byteorder::LittleEndian::write_u16(&mut minor, RESYNC_META_VERSION);
data.extend_from_slice(&minor);
data.extend_from_slice(&buf);
let data = encode_resync_file(status)?;
let config_file = path_join_buf(&[BUCKET_META_PREFIX, bucket, REPLICATION_DIR, RESYNC_FILE_NAME]);
save_config(api, &config_file, data).await?;
@@ -900,8 +1095,8 @@ pub fn resync_target(
let rs = oi
.user_defined
.get(target_reset_header(arn).as_str())
.or(oi.user_defined.get(RUSTFS_REPLICATION_RESET_STATUS))
.map(|s| s.to_string());
.cloned()
.or_else(|| get_header_map(&oi.user_defined, SUFFIX_REPLICATION_RESET_STATUS));
let mut dec = ResyncTargetDecision::default();
@@ -1132,15 +1327,7 @@ impl ObjectInfoExt for ObjectInfo {
.user_defined
.iter()
.filter_map(|(k, v)| {
if k.starts_with(&format!("{RESERVED_METADATA_PREFIX_LOWER}-{REPLICATION_RESET}")) {
Some((
k.trim_start_matches(&format!("{RESERVED_METADATA_PREFIX_LOWER}-{REPLICATION_RESET}"))
.to_string(),
v.clone(),
))
} else {
None
}
internal_key_strip_suffix_prefix(k, SUFFIX_REPLICATION_RESET_ARN_PREFIX).map(|arn| (arn, v.clone()))
})
.collect(),
..Default::default()
@@ -1893,7 +2080,7 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() {
let mut eval_metadata = HashMap::new();
if let Some(ref s) = new_replication_internal {
eval_metadata.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}replication-status"), s.clone());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, s.clone());
}
let popts = ObjectOptions {
version_id: roi.version_id.map(|v| v.to_string()),
@@ -2549,8 +2736,6 @@ static VALID_SSE_REPLICATION_HEADERS: &[(&str, &str)] = &[
("X-Rustfs-Internal-Actual-Object-Size", "X-Rustfs-Replication-Actual-Object-Size"),
];
const REPLICATION_SSEC_CHECKSUM_HEADER: &str = "X-Rustfs-Replication-Ssec-Crc";
fn is_valid_sse_header(k: &str) -> Option<&str> {
VALID_SSE_REPLICATION_HEADERS
.iter()
@@ -2574,7 +2759,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
// In case of SSE-C objects copy the allowed internal headers as well
if !is_ssec || !has_valid_sse_header {
if strings_has_prefix_fold(k, RESERVED_METADATA_PREFIX) {
if is_internal_key(k) {
continue;
}
if is_standard_header(k) {
@@ -2598,7 +2783,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
// Add encrypted CRC to metadata for SSE-C objects
if is_ssec {
let encoded = BASE64_STANDARD.encode(checksum_data);
meta.insert(REPLICATION_SSEC_CHECKSUM_HEADER.to_string(), encoded);
insert_header_map(&mut meta, SUFFIX_REPLICATION_SSEC_CRC, encoded);
} else {
// Get checksum metadata for non-SSE-C objects
let (cs_meta, is_mp) = object_info.decrypt_checksums(0, &HeaderMap::new())?;
@@ -2658,11 +2843,8 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
if !tags.is_empty() {
put_op.user_tags = tags;
// set tag timestamp in opts
put_op.internal.tagging_timestamp = if let Some(ts) = object_info
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}tagging-timestamp"))
{
OffsetDateTime::parse(ts, &Rfc3339)
put_op.internal.tagging_timestamp = if let Some(ts) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
OffsetDateTime::parse(&ts, &Rfc3339)
.map_err(|e| Error::other(format!("Failed to parse tagging timestamp: {}", e)))?
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
@@ -2694,28 +2876,24 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
put_op.retain_until_date =
OffsetDateTime::parse(v, &Rfc3339).map_err(|e| Error::other(format!("Failed to parse retain until date: {}", e)))?;
// set retention timestamp in opts
put_op.internal.retention_timestamp = if let Some(v) = object_info
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}objectlock-retention-timestamp"))
{
OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
};
put_op.internal.retention_timestamp =
if let Some(v) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
};
}
if let Some(v) = lk_map.lookup(AMZ_OBJECT_LOCK_LEGAL_HOLD) {
let hold = v.to_uppercase();
put_op.legalhold = Some(ObjectLockLegalHoldStatus::from(hold.as_str()));
// set legalhold timestamp in opts
put_op.internal.legalhold_timestamp = if let Some(v) = object_info
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}objectlock-legalhold-timestamp"))
{
OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
};
put_op.internal.legalhold_timestamp =
if let Some(v) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP) {
OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
};
}
// Handle SSE-S3 encryption
@@ -2736,7 +2914,7 @@ fn put_replication_opts(sc: &str, object_info: &ObjectInfo) -> Result<(PutObject
// If KMS key ID replication is enabled (as by default)
// we include the object's KMS key ID. In any case, we
// always set the SSE-KMS header. If no KMS key ID is
// specified, MinIO is supposed to use whatever default
// specified, the server uses the default applicable
// config applies on the site or bucket.
// TODO: Implement SSE-KMS support with key ID replication
// let key_id = if kms::replicate_key_id() {
@@ -2859,13 +3037,10 @@ async fn replicate_object_with_multipart<S: StorageAPI>(ctx: MultipartReplicatio
let mut user_metadata = HashMap::new();
user_metadata.insert(
RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE.to_string(),
object_info
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX}actual-size"))
.map(|v| v.to_string())
.unwrap_or_default(),
insert_header_map(
&mut user_metadata,
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE,
rustfs_utils::http::get_str(&object_info.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE).unwrap_or_default(),
);
cli.complete_multipart_upload(
@@ -2994,6 +3169,9 @@ fn get_replication_action(oi1: &ObjectInfo, oi2: &HeadObjectOutput, op_type: Rep
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::msgp_decode::write_msgp_time;
use std::collections::HashMap;
use time::OffsetDateTime;
use uuid::Uuid;
#[test]
@@ -3010,6 +3188,176 @@ mod tests {
assert!(part_range_spec_from_actual_size(0, -1).is_err());
}
#[test]
fn test_unmarshal_resync_payload() {
let start = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid ts");
let last = OffsetDateTime::from_unix_timestamp(1_700_000_123).expect("valid ts");
let before = OffsetDateTime::from_unix_timestamp(1_699_000_000).expect("valid ts");
let bucket_last = OffsetDateTime::from_unix_timestamp(1_700_111_111).expect("valid ts");
let mut payload = Vec::new();
rmp::encode::write_map_len(&mut payload, 4).expect("write map");
rmp::encode::write_str(&mut payload, "v").expect("write key");
rmp::encode::write_i32(&mut payload, 1).expect("write version");
rmp::encode::write_str(&mut payload, "brs").expect("write key");
rmp::encode::write_map_len(&mut payload, 1).expect("write target map");
rmp::encode::write_str(&mut payload, "arn:replication::1:dest").expect("write arn");
rmp::encode::write_map_len(&mut payload, 11).expect("write target");
rmp::encode::write_str(&mut payload, "st").expect("write key");
write_msgp_time(&mut payload, start).expect("write time");
rmp::encode::write_str(&mut payload, "lst").expect("write key");
write_msgp_time(&mut payload, last).expect("write time");
rmp::encode::write_str(&mut payload, "id").expect("write key");
rmp::encode::write_str(&mut payload, "resync-1").expect("write id");
rmp::encode::write_str(&mut payload, "rdt").expect("write key");
write_msgp_time(&mut payload, before).expect("write time");
rmp::encode::write_str(&mut payload, "rst").expect("write key");
rmp::encode::write_i32(&mut payload, 3).expect("write status");
rmp::encode::write_str(&mut payload, "fs").expect("write key");
rmp::encode::write_i64(&mut payload, 11).expect("write fs");
rmp::encode::write_str(&mut payload, "frc").expect("write key");
rmp::encode::write_i64(&mut payload, 2).expect("write frc");
rmp::encode::write_str(&mut payload, "rs").expect("write key");
rmp::encode::write_i64(&mut payload, 101).expect("write rs");
rmp::encode::write_str(&mut payload, "rrc").expect("write key");
rmp::encode::write_i64(&mut payload, 9).expect("write rrc");
rmp::encode::write_str(&mut payload, "bkt").expect("write key");
rmp::encode::write_str(&mut payload, "bucket-a").expect("write bucket");
rmp::encode::write_str(&mut payload, "obj").expect("write key");
rmp::encode::write_str(&mut payload, "object-a").expect("write obj");
rmp::encode::write_str(&mut payload, "id").expect("write key");
rmp::encode::write_i32(&mut payload, 42).expect("write id");
rmp::encode::write_str(&mut payload, "lu").expect("write key");
write_msgp_time(&mut payload, bucket_last).expect("write lu");
let got = BucketReplicationResyncStatus::unmarshal_msg(&payload).expect("decode");
assert_eq!(got.version, 1);
assert_eq!(got.id, 42);
assert_eq!(got.last_update, Some(bucket_last));
let tgt = got.targets_map.get("arn:replication::1:dest").expect("target exists");
assert_eq!(tgt.resync_id, "resync-1");
assert_eq!(tgt.resync_status, ResyncStatusType::ResyncStarted);
assert_eq!(tgt.bucket, "bucket-a");
assert_eq!(tgt.object, "object-a");
assert_eq!(tgt.start_time, Some(start));
assert_eq!(tgt.last_update, Some(last));
assert_eq!(tgt.resync_before_date, Some(before));
}
#[test]
fn test_unmarshal_legacy_resync_payload() {
let mut status = BucketReplicationResyncStatus::new();
status.id = 7;
status.version = 1;
status.last_update = Some(OffsetDateTime::from_unix_timestamp(1_700_222_222).expect("valid ts"));
status.targets_map = HashMap::from([(
"legacy-arn".to_string(),
TargetReplicationResyncStatus {
resync_id: "legacy-1".to_string(),
resync_status: ResyncStatusType::ResyncCompleted,
..Default::default()
},
)]);
let old_payload = rmp_serde::to_vec(&status).expect("legacy encode");
let got = BucketReplicationResyncStatus::unmarshal_legacy_msg(&old_payload).expect("legacy decode");
assert_eq!(got.id, 7);
assert_eq!(got.version, 1);
assert_eq!(got.targets_map["legacy-arn"].resync_id, "legacy-1");
assert_eq!(got.targets_map["legacy-arn"].resync_status, ResyncStatusType::ResyncCompleted);
}
#[test]
fn test_resync_file_roundtrip_wire_format() {
let mut status = BucketReplicationResyncStatus::new();
status.id = 19;
status.last_update = Some(OffsetDateTime::from_unix_timestamp(1_700_333_333).expect("valid ts"));
status.targets_map = HashMap::from([(
"arn:replication::1:dest".to_string(),
TargetReplicationResyncStatus {
resync_id: "wire-1".to_string(),
resync_status: ResyncStatusType::ResyncStarted,
replicated_count: 5,
..Default::default()
},
)]);
let bytes = encode_resync_file(&status).expect("encode file");
assert_eq!(&bytes[0..2], &RESYNC_META_FORMAT.to_le_bytes());
assert_eq!(&bytes[2..4], &RESYNC_META_VERSION.to_le_bytes());
let got = decode_resync_file(&bytes).expect("decode file");
assert_eq!(got.version, RESYNC_META_VERSION);
assert_eq!(got.id, 19);
assert_eq!(got.targets_map["arn:replication::1:dest"].resync_id, "wire-1");
assert_eq!(got.targets_map["arn:replication::1:dest"].replicated_count, 5);
}
#[test]
fn test_resync_file_decodes_legacy_payload() {
let mut status = BucketReplicationResyncStatus::new();
status.id = 7;
status.version = RESYNC_META_VERSION;
status.targets_map = HashMap::from([(
"legacy-arn".to_string(),
TargetReplicationResyncStatus {
resync_id: "legacy-v1".to_string(),
resync_status: ResyncStatusType::ResyncCompleted,
..Default::default()
},
)]);
let legacy_payload = rmp_serde::to_vec(&status).expect("legacy encode");
let mut file_bytes = Vec::new();
file_bytes.extend_from_slice(&RESYNC_META_FORMAT.to_le_bytes());
file_bytes.extend_from_slice(&RESYNC_META_VERSION.to_le_bytes());
file_bytes.extend_from_slice(&legacy_payload);
let got = decode_resync_file(&file_bytes).expect("decode legacy");
assert_eq!(got.id, 7);
assert_eq!(got.targets_map["legacy-arn"].resync_id, "legacy-v1");
assert_eq!(got.targets_map["legacy-arn"].resync_status, ResyncStatusType::ResyncCompleted);
}
#[test]
fn test_resync_none_time_encodes_as_wire_zero_and_decodes_to_none() {
let wire_zero = OffsetDateTime::from_unix_timestamp(WIRE_ZERO_TIME_UNIX).expect("valid wire zero timestamp");
let mut with_none = BucketReplicationResyncStatus::new();
with_none.id = 77;
with_none.targets_map = HashMap::from([(
"arn:replication::1:dest".to_string(),
TargetReplicationResyncStatus {
resync_id: "wire-none".to_string(),
resync_status: ResyncStatusType::ResyncStarted,
replicated_count: 1,
..Default::default()
},
)]);
let mut with_zero = with_none.clone();
with_zero.last_update = Some(wire_zero);
if let Some(target) = with_zero.targets_map.get_mut("arn:replication::1:dest") {
target.start_time = Some(wire_zero);
target.last_update = Some(wire_zero);
target.resync_before_date = Some(wire_zero);
}
let encoded_none = encode_resync_file(&with_none).expect("encode with none");
let encoded_zero = encode_resync_file(&with_zero).expect("encode with zero");
assert_eq!(encoded_none, encoded_zero);
let decoded = decode_resync_file(&encoded_none).expect("decode");
let target = decoded
.targets_map
.get("arn:replication::1:dest")
.expect("target should exist");
assert_eq!(decoded.last_update, None);
assert_eq!(target.start_time, None);
assert_eq!(target.last_update, None);
assert_eq!(target.resync_before_date, None);
}
#[test]
fn test_heal_should_use_check_replicate_delete_failed_non_delete_marker() {
let oi = ObjectInfo {
crates/ecstore/src/bucket/utils.rs
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result, StorageError};
use regex::Regex;
use rustfs_utils::path::SLASH_SEPARATOR;
@@ -20,7 +20,7 @@ use s3s::xml;
use tracing::instrument;
pub fn is_meta_bucketname(name: &str) -> bool {
name.starts_with(RUSTFS_META_BUCKET)
name.starts_with(RUSTFS_META_BUCKET) || name.starts_with(MIGRATING_META_BUCKET)
}
lazy_static::lazy_static! {
crates/ecstore/src/config/com.rs
+331 -8
View File
@@ -13,16 +13,17 @@
// limitations under the License.
use crate::config::{Config, GLOBAL_STORAGE_CLASS, storageclass};
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::{DEFAULT_DELIMITER, RUSTFS_REGION};
use rustfs_utils::path::SLASH_SEPARATOR;
use std::collections::HashSet;
use serde_json::{Map, Value};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::sync::LazyLock;
use tracing::{error, instrument, warn};
use tracing::{debug, error, info, instrument, warn};
pub const CONFIG_PREFIX: &str = "config";
const CONFIG_FILE: &str = "config.json";
@@ -136,6 +137,226 @@ fn get_config_file() -> String {
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
}
fn storage_class_kvs_mut(cfg: &mut Config) -> &mut crate::config::KVS {
let sub_cfg = cfg.0.entry(STORAGE_CLASS_SUB_SYS.to_string()).or_insert_with(|| {
let mut section = HashMap::new();
section.insert(DEFAULT_DELIMITER.to_string(), storageclass::DEFAULT_KVS.clone());
section
});
sub_cfg
.entry(DEFAULT_DELIMITER.to_string())
.or_insert_with(|| storageclass::DEFAULT_KVS.clone())
}
fn parse_storage_class_value(value: &Value) -> Option<String> {
match value {
Value::String(v) => Some(v.trim().to_string()),
Value::Object(m) => m
.get("parity")
.and_then(Value::as_u64)
.map(|parity| if parity == 0 { String::new() } else { format!("EC:{parity}") }),
_ => None,
}
}
fn parse_inline_block_value(value: &Value) -> Option<String> {
match value {
Value::String(v) if !v.trim().is_empty() => Some(v.trim().to_string()),
Value::Number(v) => Some(v.to_string()),
_ => None,
}
}
fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
let sc = root.get("storageclass").or_else(|| root.get("storage_class"));
let Some(Value::Object(sc_obj)) = sc else {
return false;
};
let mut applied = false;
let kvs = storage_class_kvs_mut(cfg);
if let Some(v) = sc_obj.get("standard").and_then(parse_storage_class_value) {
kvs.insert(storageclass::CLASS_STANDARD.to_string(), v);
applied = true;
}
if let Some(v) = sc_obj.get("rrs").and_then(parse_storage_class_value) {
kvs.insert(storageclass::CLASS_RRS.to_string(), v);
applied = true;
}
if let Some(Value::String(v)) = sc_obj.get("optimize")
&& !v.trim().is_empty()
{
kvs.insert(storageclass::OPTIMIZE.to_string(), v.clone());
applied = true;
}
if let Some(v) = sc_obj.get("inline_block").and_then(parse_inline_block_value) {
kvs.insert(storageclass::INLINE_BLOCK.to_string(), v);
applied = true;
}
applied
}
fn decode_server_config_blob(data: &[u8]) -> Result<Config> {
if let Ok(cfg) = Config::unmarshal(data) {
return Ok(cfg);
}
let value: Value = serde_json::from_slice(data)?;
let Value::Object(root) = value else {
return Err(Error::other("unrecognized external server config shape"));
};
let mut cfg = Config::new();
let has_storage = apply_external_storage_class_map(&mut cfg, &root);
let has_header = root.contains_key("version") || root.contains_key("region") || root.contains_key("credential");
if !has_storage && !has_header {
return Err(Error::other("unrecognized external server config shape"));
}
Ok(cfg)
}
fn parse_object_seed(data: &[u8]) -> Option<Map<String, Value>> {
let value: Value = serde_json::from_slice(data).ok()?;
value.as_object().cloned()
}
fn build_storageclass_object(cfg: &Config) -> Map<String, Value> {
let kvs = cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_DELIMITER).unwrap_or_default();
let mut sc_obj = Map::new();
sc_obj.insert(
"standard".to_string(),
Value::String(kvs.lookup(storageclass::CLASS_STANDARD).unwrap_or_default()),
);
sc_obj.insert("rrs".to_string(), Value::String(kvs.lookup(storageclass::CLASS_RRS).unwrap_or_default()));
let optimize = kvs
.lookup(storageclass::OPTIMIZE)
.filter(|v| !v.trim().is_empty())
.unwrap_or_else(|| "availability".to_string());
sc_obj.insert("optimize".to_string(), Value::String(optimize));
if let Some(v) = kvs.lookup(storageclass::INLINE_BLOCK).filter(|v| !v.trim().is_empty()) {
sc_obj.insert("inline_block".to_string(), Value::String(v));
}
sc_obj
}
fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
let mut root = seed.and_then(parse_object_seed).unwrap_or_default();
if !matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty()) {
root.insert("version".to_string(), Value::String("33".to_string()));
}
if !matches!(root.get("region"), Some(Value::String(v)) if !v.trim().is_empty()) {
root.insert("region".to_string(), Value::String(RUSTFS_REGION.to_string()));
}
let mut sc_obj = match root.remove("storageclass") {
Some(Value::Object(v)) => v,
_ => Map::new(),
};
for (k, v) in build_storageclass_object(cfg) {
sc_obj.insert(k, v);
}
root.insert("storageclass".to_string(), Value::Object(sc_obj));
root.remove("storage_class");
Ok(serde_json::to_vec(&Value::Object(root))?)
}
fn is_standard_object_server_config(data: &[u8]) -> bool {
let Ok(value) = serde_json::from_slice::<Value>(data) else {
return false;
};
let Value::Object(root) = value else {
return false;
};
matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty())
&& matches!(root.get("storageclass"), Some(Value::Object(_)))
&& !root.contains_key("storage_class")
}
fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool {
build_storageclass_object(lhs) == build_storageclass_object(rhs)
}
fn is_object_not_found(err: &Error) -> bool {
*err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _) | Error::BucketNotFound(_))
}
pub async fn try_migrate_server_config<S: StorageAPI>(api: Arc<S>) {
let config_file = get_config_file();
match api
.get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default())
.await
{
Ok(_) => {
debug!("server config already exists in RustFS metadata bucket, skip migration");
return;
}
Err(err) if is_object_not_found(&err) => {}
Err(err) => {
warn!("check target server config failed, skip migration: {:?}", err);
return;
}
}
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let mut rd = match api
.get_object_reader(MIGRATING_META_BUCKET, &config_file, None, HeaderMap::new(), &opts)
.await
{
Ok(v) => v,
Err(err) => {
if !is_object_not_found(&err) {
warn!("read legacy server config failed: {:?}", err);
}
return;
}
};
let data = match rd.read_all().await {
Ok(v) if !v.is_empty() => v,
Ok(_) => {
debug!("legacy server config is empty, skip migration");
return;
}
Err(err) => {
warn!("read legacy server config body failed: {:?}", err);
return;
}
};
let cfg = match decode_server_config_blob(&data) {
Ok(v) => v,
Err(err) => {
warn!("legacy server config format is incompatible, skip migration: {:?}", err);
return;
}
};
let normalized = match encode_server_config_blob(&cfg, Some(&data)) {
Ok(v) => v,
Err(err) => {
warn!("serialize migrated server config failed, skip migration: {:?}", err);
return;
}
};
match save_config(api, &config_file, normalized).await {
Ok(()) => {
info!("Migrated compatible server config from legacy metadata bucket");
}
Err(err) => {
warn!("write migrated server config failed: {:?}", err);
}
}
}
/// Handle the situation where the configuration file does not exist, create and save a new configuration
async fn handle_missing_config<S: StorageAPI>(api: Arc<S>, context: &str) -> Result<Config> {
warn!("Configuration not found ({}): Start initializing new configuration", context);
@@ -171,7 +392,7 @@ async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<C
match read_config(api.clone(), &config_file).await {
Ok(cfg_data) => {
// TODO: decrypt
let cfg = Config::unmarshal(&cfg_data)?;
let cfg = decode_server_config_blob(&cfg_data)?;
return Ok(cfg.merge());
}
Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await,
@@ -180,14 +401,35 @@ async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<C
}
// Process non-empty configuration data
let cfg = Config::unmarshal(data)?;
let cfg = decode_server_config_blob(data)?;
Ok(cfg.merge())
}
pub async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &Config) -> Result<()> {
let data = cfg.marshal()?;
let config_file = get_config_file();
let existing = match read_config(api.clone(), &config_file).await {
Ok(v) => Some(v),
Err(Error::ConfigNotFound) => None,
Err(err) => {
warn!("read existing server config before save failed, continue with clean output: {:?}", err);
None
}
};
if let Some(current) = existing.as_deref()
&& is_standard_object_server_config(current)
&& let Ok(decoded_current) = decode_server_config_blob(current)
&& configs_semantically_equal(&decoded_current, cfg)
{
debug!("server config unchanged and already in standard object shape, skip write");
return Ok(());
}
let data = encode_server_config_blob(cfg, existing.as_deref())?;
if existing.as_deref().is_some_and(|current| current == data.as_slice()) {
debug!("server config bytes unchanged after encode, skip write");
return Ok(());
}
save_config(api, &config_file, data).await
}
@@ -232,3 +474,84 @@ async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api:
Ok(())
}
#[cfg(test)]
mod tests {
use super::{
configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config,
storage_class_kvs_mut,
};
use crate::config::Config;
use serde_json::Value;
#[test]
fn test_decode_server_config_accepts_legacy_hidden_if_empty_alias() {
let input = r#"{"storage_class":{"_":[{"key":"standard","value":"EC:2","hiddenIfEmpty":true}]}}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let kvs = cfg.get_value("storage_class", "_").expect("storage_class should exist");
assert!(kvs.0[0].hidden_if_empty);
}
#[test]
fn test_decode_server_config_accepts_missing_hidden_if_empty() {
let input = r#"{"storage_class":{"_":[{"key":"standard","value":"EC:2"}]}}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let kvs = cfg.get_value("storage_class", "_").expect("storage_class should exist");
assert!(!kvs.0[0].hidden_if_empty);
}
#[test]
fn test_decode_server_config_accepts_v33_object_shape() {
let input = r#"{
"version":"33",
"credential":{"accessKey":"test","secretKey":"testtesttest"},
"region":"us-east-1",
"worm":"off",
"storageclass":{"standard":"EC:2","rrs":"EC:1"},
"notify":{},
"logger":{},
"compress":{"enabled":false},
"openid":{},
"policy":{"opa":{}},
"ldapserverconfig":{}
}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let kvs = cfg.get_value("storage_class", "_").expect("storage_class should exist");
assert_eq!(kvs.get("standard"), "EC:2");
assert_eq!(kvs.get("rrs"), "EC:1");
assert_eq!(kvs.get("optimize"), "availability");
}
#[test]
fn test_encode_server_config_writes_external_object_shape() {
let mut cfg = Config::new();
let kvs = storage_class_kvs_mut(&mut cfg);
kvs.insert("standard".to_string(), "EC:2".to_string());
kvs.insert("rrs".to_string(), "EC:1".to_string());
let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
let v: Value = serde_json::from_slice(&out).expect("output should be json");
assert!(v.get("version").is_some(), "external object should have version");
assert!(v.get("storageclass").is_some(), "external object should have storageclass");
assert!(v.get("storage_class").is_none(), "should not write rustfs map shape");
}
#[test]
fn test_is_standard_object_server_config_detection() {
let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"}}"#;
assert!(is_standard_object_server_config(external));
let legacy = br#"{"storage_class":{"_":[{"key":"standard","value":"EC:2"}]}}"#;
assert!(!is_standard_object_server_config(legacy));
}
#[test]
fn test_configs_semantically_equal_for_equivalent_shapes() {
let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"}}"#;
let legacy = br#"{"storage_class":{"_":[{"key":"standard","value":"EC:2"},{"key":"rrs","value":"EC:1"},{"key":"optimize","value":"availability"}]}}"#;
let lhs = decode_server_config_blob(external).expect("decode external");
let rhs = decode_server_config_blob(legacy).expect("decode legacy");
assert!(configs_semantically_equal(&lhs, &rhs));
}
}
crates/ecstore/src/config/mod.rs
+5
View File
@@ -75,10 +75,15 @@ pub async fn init_global_config_sys(api: Arc<ECStore>) -> Result<()> {
GLOBAL_CONFIG_SYS.init(api).await
}
pub async fn try_migrate_server_config(api: Arc<ECStore>) {
com::try_migrate_server_config(api).await
}
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct KV {
pub key: String,
pub value: String,
#[serde(default, alias = "hiddenIfEmpty")]
pub hidden_if_empty: bool,
}
crates/ecstore/src/disk/local.rs
+7 -3
View File
@@ -697,7 +697,6 @@ impl LocalDisk {
match self.read_metadata_with_dmtime(meta_path).await {
Ok(res) => Ok(res),
Err(err) => {
warn!("read_raw: error: {:?}", err);
if err == Error::FileNotFound
&& !skip_access_checks(volume_dir.as_ref().to_string_lossy().to_string().as_str())
&& let Err(e) = access(volume_dir.as_ref()).await
@@ -1493,6 +1492,12 @@ impl DiskAPI for LocalDisk {
let erasure = &fi.erasure;
for (i, part) in fi.parts.iter().enumerate() {
let checksum_info = erasure.get_checksum_info(part.number);
let checksum_algo =
if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let part_path = self.get_object_path(
volume,
path_join_buf(&[
@@ -1506,7 +1511,7 @@ impl DiskAPI for LocalDisk {
.bitrot_verify(
&part_path,
erasure.shard_file_size(part.size as i64) as usize,
checksum_info.algorithm,
checksum_algo,
&checksum_info.hash,
erasure.shard_size(),
)
@@ -2058,7 +2063,6 @@ impl DiskAPI for LocalDisk {
let search_version_id = fi.version_id.or(Some(Uuid::nil()));
// Check if there's an existing version with the same version_id that has a data_dir to clean up
// Note: For non-versioned buckets, fi.version_id is None, but in xl.meta it's stored as Some(Uuid::nil())
let has_old_data_dir = {
xlmeta.find_version(search_version_id).ok().and_then(|(_, ver)| {
// shard_count == 0 means no other version shares this data_dir
crates/ecstore/src/disk/mod.rs
+1
View File
@@ -23,6 +23,7 @@ pub mod local;
pub mod os;
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
pub const MIGRATING_META_BUCKET: &str = ".minio.sys";
pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
crates/ecstore/src/erasure_coding/bitrot.rs
+1 -1
View File
@@ -173,7 +173,7 @@ where
}
pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize {
if algo != HashAlgorithm::HighwayHash256S {
if algo != HashAlgorithm::HighwayHash256S && algo != HashAlgorithm::HighwayHash256SLegacy {
return size;
}
size.div_ceil(shard_size) * algo.size() + size
crates/ecstore/src/erasure_coding/erasure.rs
+274 -117
View File
@@ -12,31 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Erasure coding implementation using Reed-Solomon SIMD backend.
//! Erasure coding implementation using reed-solomon-erasure (GF(2^8)).
//! Supports legacy (reed-solomon-simd) for reading/healing old-version files.
//!
//! This module provides erasure coding functionality with high-performance SIMD
//! Reed-Solomon implementation:
//!
//! ## Reed-Solomon Implementation
//!
//! ### SIMD Mode (Only)
//! - **Performance**: Uses SIMD optimization for high-performance encoding/decoding
//! - **Compatibility**: Works with any shard size through SIMD implementation
//! - **Reliability**: High-performance SIMD implementation for large data processing
//! - **Use case**: Optimized for maximum performance in large data processing scenarios
//!
//! ## Example
//!
//! ```ignore
//! use rustfs_ecstore::erasure_coding::Erasure;
//!
//! let erasure = Erasure::new(4, 2, 1024); // 4 data shards, 2 parity shards, 1KB block size
//! let data = b"hello world";
//! let shards = erasure.encode_data(data).unwrap();
//! // Simulate loss and recovery...
//! ```
use bytes::{Bytes, BytesMut};
use reed_solomon_erasure::galois_8::ReedSolomon;
use reed_solomon_simd;
use smallvec::SmallVec;
use std::io;
@@ -44,132 +25,88 @@ use tokio::io::AsyncRead;
use tracing::warn;
use uuid::Uuid;
/// Reed-Solomon encoder using SIMD implementation.
pub struct ReedSolomonEncoder {
/// Legacy calc_shard_size formula: (block_size.div_ceil(data_shards) + 1) & !1
/// Matches main branch and filemeta::ErasureInfo for old-version files.
pub fn calc_shard_size_legacy(block_size: usize, data_shards: usize) -> usize {
(block_size.div_ceil(data_shards) + 1) & !1
}
/// Reed-Solomon encoder for legacy (main branch) format using reed-solomon-simd.
/// Used when decoding/encoding files with uses_legacy_checksum == true.
struct LegacyReedSolomonEncoder {
data_shards: usize,
parity_shards: usize,
// Use RwLock to ensure thread safety, implementing Send + Sync
encoder_cache: std::sync::RwLock<Option<reed_solomon_simd::ReedSolomonEncoder>>,
decoder_cache: std::sync::RwLock<Option<reed_solomon_simd::ReedSolomonDecoder>>,
}
impl Clone for ReedSolomonEncoder {
impl Clone for LegacyReedSolomonEncoder {
fn clone(&self) -> Self {
Self {
data_shards: self.data_shards,
parity_shards: self.parity_shards,
// Create an empty cache for the new instance instead of sharing one
encoder_cache: std::sync::RwLock::new(None),
decoder_cache: std::sync::RwLock::new(None),
}
}
}
impl ReedSolomonEncoder {
/// Create a new Reed-Solomon encoder with specified data and parity shards.
pub fn new(data_shards: usize, parity_shards: usize) -> io::Result<Self> {
Ok(ReedSolomonEncoder {
data_shards,
parity_shards,
impl LegacyReedSolomonEncoder {
fn new(_data_shards: usize, _parity_shards: usize) -> io::Result<Self> {
Ok(Self {
data_shards: _data_shards,
parity_shards: _parity_shards,
encoder_cache: std::sync::RwLock::new(None),
decoder_cache: std::sync::RwLock::new(None),
})
}
/// Encode data shards with parity.
pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
if shards_vec.is_empty() {
return Ok(());
}
let simd_result = self.encode_with_simd(&mut shards_vec);
match simd_result {
Ok(()) => Ok(()),
Err(simd_error) => {
warn!("SIMD encoding failed: {}", simd_error);
Err(simd_error)
}
}
}
fn encode_with_simd(&self, shards_vec: &mut [&mut [u8]]) -> io::Result<()> {
let shard_len = shards_vec[0].len();
// Get or create encoder
let mut encoder = {
let mut cache_guard = self
.encoder_cache
.write()
.map_err(|_| io::Error::other("Failed to acquire encoder cache lock"))?;
match cache_guard.take() {
Some(mut cached_encoder) => {
// Use reset method to reset existing encoder to adapt to new parameters
if let Err(e) = cached_encoder.reset(self.data_shards, self.parity_shards, shard_len) {
warn!("Failed to reset SIMD encoder: {:?}, creating new one", e);
// If reset fails, create new encoder
Some(mut cached) => {
if cached.reset(self.data_shards, self.parity_shards, shard_len).is_err() {
reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?
} else {
cached_encoder
cached
}
}
None => {
// First use, create new encoder
reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?
}
None => reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?,
}
};
// Add original shards
for (i, shard) in shards_vec.iter().enumerate().take(self.data_shards) {
encoder
.add_original_shard(shard)
.map_err(|e| io::Error::other(format!("Failed to add shard {i}: {e:?}")))?;
}
// Encode and get recovery shards
let result = encoder
.encode()
.map_err(|e| io::Error::other(format!("SIMD encoding failed: {e:?}")))?;
// Copy recovery shards to output buffer
for (i, recovery_shard) in result.recovery_iter().enumerate() {
if i + self.data_shards < shards_vec.len() {
shards_vec[i + self.data_shards].copy_from_slice(recovery_shard);
}
}
// Return encoder to cache (encoder is automatically reset after result is dropped, can be reused)
drop(result); // Explicitly drop result to ensure encoder is reset
drop(result);
*self
.encoder_cache
.write()
.map_err(|_| io::Error::other("Failed to return encoder to cache"))? = Some(encoder);
Ok(())
}
/// Reconstruct missing shards.
pub fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
// Use SIMD for reconstruction
let simd_result = self.reconstruct_with_simd(shards);
match simd_result {
Ok(()) => Ok(()),
Err(simd_error) => {
warn!("SIMD reconstruction failed: {}", simd_error);
Err(simd_error)
}
}
}
fn reconstruct_with_simd(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
// Find a valid shard to determine length
fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
let shard_len = shards
.iter()
.find_map(|s| s.as_ref().map(|v| v.len()))
@@ -185,7 +122,6 @@ impl ReedSolomonEncoder {
Some(mut cached_decoder) => {
if let Err(e) = cached_decoder.reset(self.data_shards, self.parity_shards, shard_len) {
warn!("Failed to reset SIMD decoder: {:?}, creating new one", e);
reed_solomon_simd::ReedSolomonDecoder::new(self.data_shards, self.parity_shards, shard_len)
.map_err(|e| io::Error::other(format!("Failed to create SIMD decoder: {e:?}")))?
} else {
@@ -197,7 +133,6 @@ impl ReedSolomonEncoder {
}
};
// Add available shards (both data and parity)
for (i, shard_opt) in shards.iter().enumerate() {
if let Some(shard) = shard_opt {
if i < self.data_shards {
@@ -217,7 +152,6 @@ impl ReedSolomonEncoder {
.decode()
.map_err(|e| io::Error::other(format!("SIMD decode error: {e:?}")))?;
// Fill in missing data shards from reconstruction result
for (i, shard_opt) in shards.iter_mut().enumerate() {
if shard_opt.is_none() && i < self.data_shards {
for (restored_index, restored_data) in result.restored_original_iter() {
@@ -240,6 +174,67 @@ impl ReedSolomonEncoder {
}
}
/// Reed-Solomon encoder using reed-solomon-erasure
pub struct ReedSolomonEncoder {
data_shards: usize,
parity_shards: usize,
encoder: Option<ReedSolomon>,
}
impl Clone for ReedSolomonEncoder {
fn clone(&self) -> Self {
Self {
data_shards: self.data_shards,
parity_shards: self.parity_shards,
encoder: self.encoder.clone(),
}
}
}
impl ReedSolomonEncoder {
/// Create a new Reed-Solomon encoder with specified data and parity shards.
pub fn new(data_shards: usize, parity_shards: usize) -> io::Result<Self> {
let encoder = if parity_shards > 0 {
ReedSolomon::new(data_shards, parity_shards)
.map_err(|e| io::Error::other(format!("Failed to create Reed-Solomon encoder: {e:?}")))
.map(Some)?
} else {
None
};
Ok(ReedSolomonEncoder {
data_shards,
parity_shards,
encoder,
})
}
/// Encode data shards with parity.
pub fn encode(&self, shards: SmallVec<[&mut [u8]; 16]>) -> io::Result<()> {
let mut shards_vec: Vec<&mut [u8]> = shards.into_vec();
if shards_vec.is_empty() {
return Ok(());
}
if let Some(ref rs) = self.encoder {
rs.encode(&mut shards_vec)
.map_err(|e| io::Error::other(format!("Reed-Solomon encode failed: {e:?}")))
} else {
Ok(())
}
}
/// Reconstruct missing shards.
pub fn reconstruct(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if let Some(ref rs) = self.encoder {
rs.reconstruct_data(shards)
.map_err(|e| io::Error::other(format!("Reed-Solomon reconstruct failed: {e:?}")))
} else {
Ok(())
}
}
}
/// Erasure coding utility for data reliability using Reed-Solomon codes.
///
/// This struct provides encoding and decoding of data into data and parity shards.
@@ -262,24 +257,41 @@ impl ReedSolomonEncoder {
/// let shards = erasure.encode_data(data).unwrap();
/// // Simulate loss and recovery...
/// ```
#[derive(Default)]
pub struct Erasure {
pub data_shards: usize,
pub parity_shards: usize,
encoder: Option<ReedSolomonEncoder>,
legacy_encoder: Option<LegacyReedSolomonEncoder>,
pub block_size: usize,
uses_legacy: bool,
_id: Uuid,
_buf: Vec<u8>,
}
impl Default for Erasure {
fn default() -> Self {
Self {
data_shards: 0,
parity_shards: 0,
encoder: None,
legacy_encoder: None,
block_size: 0,
uses_legacy: false,
_id: Uuid::nil(),
_buf: vec![],
}
}
}
impl Clone for Erasure {
fn clone(&self) -> Self {
Self {
data_shards: self.data_shards,
parity_shards: self.parity_shards,
encoder: self.encoder.clone(),
legacy_encoder: self.legacy_encoder.clone(),
block_size: self.block_size,
uses_legacy: self.uses_legacy,
_id: Uuid::new_v4(), // Generate new ID for clone
_buf: vec![0u8; self.block_size],
}
@@ -287,28 +299,44 @@ impl Clone for Erasure {
}
pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize {
(block_size.div_ceil(data_shards) + 1) & !1
block_size.div_ceil(data_shards)
}
impl Erasure {
/// Create a new Erasure instance.
/// Create a new Erasure instance
///
/// # Arguments
/// * `data_shards` - Number of data shards.
/// * `parity_shards` - Number of parity shards.
/// * `block_size` - Block size for each shard.
pub fn new(data_shards: usize, parity_shards: usize, block_size: usize) -> Self {
let encoder = if parity_shards > 0 {
Self::new_with_options(data_shards, parity_shards, block_size, false)
}
/// Create a new Erasure instance with legacy format support.
///
/// When `uses_legacy` is true, uses main-branch shard_size formula and reed-solomon-simd
/// for decode/reconstruct (for reading and healing old-version files).
pub fn new_with_options(data_shards: usize, parity_shards: usize, block_size: usize, uses_legacy: bool) -> Self {
let encoder = if !uses_legacy && parity_shards > 0 {
Some(ReedSolomonEncoder::new(data_shards, parity_shards).unwrap())
} else {
None
};
let legacy_encoder = if uses_legacy && parity_shards > 0 {
Some(LegacyReedSolomonEncoder::new(data_shards, parity_shards).unwrap())
} else {
None
};
Erasure {
data_shards,
parity_shards,
block_size,
encoder,
legacy_encoder,
uses_legacy,
_id: Uuid::new_v4(),
_buf: vec![0u8; block_size],
}
@@ -323,28 +351,29 @@ impl Erasure {
/// A vector of encoded shards as `Bytes`.
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
// let shard_size = self.shard_size();
// let total_size = shard_size * self.total_shard_count();
// Data shard count
let per_shard_size = calc_shard_size(data.len(), self.data_shards);
// Total required size
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
} else {
calc_shard_size
};
let per_shard_size = shard_size_fn(data.len(), self.data_shards);
let need_total_size = per_shard_size * self.total_shard_count();
// Create a new buffer with the required total length for all shards
let mut data_buffer = BytesMut::with_capacity(need_total_size);
// Copy source data
data_buffer.extend_from_slice(data);
data_buffer.resize(need_total_size, 0u8);
{
// EC encode, the result will be written into data_buffer
let data_slices: SmallVec<[&mut [u8]; 16]> = data_buffer.chunks_exact_mut(per_shard_size).collect();
// Only do EC if parity_shards > 0
if self.parity_shards > 0 {
if let Some(encoder) = self.encoder.as_ref() {
if self.uses_legacy {
if let Some(encoder) = self.legacy_encoder.as_ref() {
encoder.encode(data_slices)?;
} else {
warn!("parity_shards > 0, uses_legacy but legacy_encoder is None");
}
} else if let Some(encoder) = self.encoder.as_ref() {
encoder.encode(data_slices)?;
} else {
warn!("parity_shards > 0, but encoder is None");
@@ -372,7 +401,13 @@ impl Erasure {
/// Ok if reconstruction succeeds, error otherwise.
pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if self.parity_shards > 0 {
if let Some(encoder) = self.encoder.as_ref() {
if self.uses_legacy {
if let Some(encoder) = self.legacy_encoder.as_ref() {
encoder.reconstruct(shards)?;
} else {
warn!("parity_shards > 0, uses_legacy but legacy_encoder is None");
}
} else if let Some(encoder) = self.encoder.as_ref() {
encoder.reconstruct(shards)?;
} else {
warn!("parity_shards > 0, but encoder is None");
@@ -395,7 +430,11 @@ impl Erasure {
/// Calculate the size of each shard.
pub fn shard_size(&self) -> usize {
calc_shard_size(self.block_size, self.data_shards)
if self.uses_legacy {
calc_shard_size_legacy(self.block_size, self.data_shards)
} else {
calc_shard_size(self.block_size, self.data_shards)
}
}
/// Calculate the total erasure file size for a given original size.
// Returns the final erasure size from the original size
@@ -408,10 +447,15 @@ impl Erasure {
}
let total_length = total_length as usize;
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
} else {
calc_shard_size
};
let num_shards = total_length / self.block_size;
let last_block_size = total_length % self.block_size;
let last_shard_size = calc_shard_size(last_block_size, self.data_shards);
let last_shard_size = shard_size_fn(last_block_size, self.data_shards);
(num_shards * self.shard_size() + last_shard_size) as i64
}
@@ -494,8 +538,7 @@ mod tests {
#[test]
fn test_shard_file_size_cases2() {
let erasure = Erasure::new(12, 4, 1024 * 1024);
assert_eq!(erasure.shard_file_size(1572864), 131074);
assert_eq!(erasure.shard_file_size(1572864), 131073);
}
#[test]
@@ -517,11 +560,14 @@ mod tests {
// Case 5: total_length > block_size, aligned
assert_eq!(erasure.shard_file_size(16), 4); // 16/8=2, last=0, 2*2+0=4
assert_eq!(erasure.shard_file_size(1248739), 312186); // 1248739/8=156092, last=3, 3 div_ceil 4=1, 156092*2+1=312185
// MinIO-compatible: 1248739/8=156092, last=3, ceil(3/4)=1, 156092*2+1=312185
assert_eq!(erasure.shard_file_size(1248739), 312185);
assert_eq!(erasure.shard_file_size(43), 12); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11
// MinIO-compatible: 43/8=5, last=3, ceil(3/4)=1, 5*2+1=11
assert_eq!(erasure.shard_file_size(43), 11);
assert_eq!(erasure.shard_file_size(1572864), 393216); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11
// 1572864 with block_size=8: 196608 full blocks, last=0, 196608*2+0=393216
assert_eq!(erasure.shard_file_size(1572864), 393216);
}
#[test]
@@ -601,10 +647,70 @@ mod tests {
#[test]
fn test_shard_size_and_file_size() {
let erasure = Erasure::new(4, 2, 8);
assert_eq!(erasure.shard_file_size(33), 9);
assert_eq!(erasure.shard_file_size(0), 0);
}
#[test]
fn test_legacy_shard_size_and_file_size() {
let erasure = Erasure::new_with_options(4, 2, 8, true);
assert_eq!(erasure.shard_size(), 2);
assert_eq!(calc_shard_size_legacy(8, 4), 2);
assert_eq!(calc_shard_size_legacy(1, 4), 2);
assert_eq!(erasure.shard_file_size(33), 10);
assert_eq!(erasure.shard_file_size(0), 0);
}
#[test]
fn test_legacy_encode_decode_roundtrip() {
let data_shards = 4;
let parity_shards = 2;
let block_size = 1024;
let erasure = Erasure::new_with_options(data_shards, parity_shards, block_size, true);
let data = b"Legacy encode/decode roundtrip test data with sufficient length.".repeat(20);
let encoded_shards = erasure.encode_data(&data).unwrap();
assert_eq!(encoded_shards.len(), data_shards + parity_shards);
let mut decode_input: Vec<Option<Vec<u8>>> = vec![None; data_shards + parity_shards];
for i in 0..data_shards {
decode_input[i] = Some(encoded_shards[i].to_vec());
}
erasure.decode_data(&mut decode_input).unwrap();
let mut recovered = Vec::new();
for shard in decode_input.iter().take(data_shards) {
recovered.extend_from_slice(shard.as_ref().unwrap());
}
recovered.truncate(data.len());
assert_eq!(&recovered, &data);
}
#[test]
fn test_legacy_decode_with_missing_shards() {
let data_shards = 4;
let parity_shards = 2;
let block_size = 256;
let erasure = Erasure::new_with_options(data_shards, parity_shards, block_size, true);
let data = b"Legacy decode with missing shards test.".repeat(10);
let encoded_shards = erasure.encode_data(&data).unwrap();
let mut shards_opt: Vec<Option<Vec<u8>>> = encoded_shards.iter().map(|s| Some(s.to_vec())).collect();
shards_opt[1] = None;
shards_opt[5] = None;
erasure.decode_data(&mut shards_opt).unwrap();
let mut recovered = Vec::new();
for shard in shards_opt.iter().take(data_shards) {
recovered.extend_from_slice(shard.as_ref().unwrap());
}
recovered.truncate(data.len());
assert_eq!(&recovered, &data);
}
#[test]
fn test_shard_file_offset() {
let erasure = Erasure::new(8, 8, 1024 * 1024);
@@ -887,6 +993,57 @@ mod tests {
assert_eq!(&recovered, &data);
}
/// Generates 7557 bytes identical to MinIO generateCompatTestData.
fn generate_compat_test_data(size: usize) -> Vec<u8> {
(0..size).map(|i| ((i * 7 + 13) % 256) as u8).collect()
}
/// Verifies reed-solomon-simd produces same shards.
/// Data shards (0-3) must match for MinIO to read RustFS part files.
/// Parity shards (4-5) differ: reed-solomon-simd vs klauspost use different RS encoding.
/// Run: cargo test -p rustfs-ecstore test_reed_solomon_compat
#[test]
fn test_reed_solomon_compat() {
let data = generate_compat_test_data(7557);
let erasure = Erasure::new(4, 2, 7557);
let shards = erasure.encode_data(&data).unwrap();
assert_eq!(shards.len(), 6, "expected 6 shards (4 data + 2 parity)");
// Per-shard HighwayHash
let expected_hashes: [&str; 6] = [
"fb3db9338e610cec541504ddae4b0bfd54445bcbd45318cf21f35f024240914d", // data 0
"a545269a3196e18e77ef9f5ec6e735a4f4ebe82d342db666b11a5256eb305720", // data 1
"2adbf0058f36c4cbcb5c9c16c38a6530c54198dfe504179a6f92d2349f245318", // data 2
"898e6d060b0cb4f0e830add7e1f936bc8b78442bf582283ee244a3a058602db8", // data 3
"4a20460bca044b3a777b26f2b0bcd371e3eab2f156f84778be3ccd8edd521ef2", // parity 4
"eb8ba4c0db15ca910d58d031f74e4601ba2fed62ad03ec29cadde3367ab0d415", // parity 5
];
let mut data_shards_match = true;
let mut parity_shards_match = true;
for (i, shard) in shards.iter().enumerate() {
let hash = rustfs_utils::HashAlgorithm::HighwayHash256S.hash_encode(shard);
let got = hex_simd::encode_to_string(hash.as_ref(), hex_simd::AsciiCase::Lower);
let matches = got == expected_hashes[i];
if i < 4 {
data_shards_match &= matches;
} else {
parity_shards_match &= matches;
}
if !matches {
eprintln!(
"Shard {} ({}): got {} want {}",
i,
if i < 4 { "data" } else { "parity" },
got,
expected_hashes[i]
);
}
}
assert!(data_shards_match, "Data shards (0-3) must match");
assert!(parity_shards_match, "Parity shards (4-5): reed-solomon-simd differs");
}
#[test]
fn test_simd_small_data_handling() {
let data_shards = 4;
crates/ecstore/src/erasure_coding/mod.rs
+1 -1
View File
@@ -19,4 +19,4 @@ pub mod erasure;
pub mod heal;
pub use bitrot::*;
pub use erasure::{Erasure, ReedSolomonEncoder, calc_shard_size};
pub use erasure::{Erasure, ReedSolomonEncoder, calc_shard_size, calc_shard_size_legacy};
crates/ecstore/src/pools.rs
+698 -62
View File
@@ -13,6 +13,17 @@
// limitations under the License.
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::bucket::{
lifecycle::{
bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{
LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule, eval_action_from_lifecycle,
},
lifecycle::IlmAction,
},
metadata_sys,
object_lock::objectlock_sys::BucketObjectLockSys,
};
use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
use crate::config::com::{CONFIG_PREFIX, read_config, save_config};
use crate::data_usage::DATA_USAGE_CACHE_NAME;
@@ -30,25 +41,32 @@ use crate::store_api::{
BucketOperations, BucketOptions, CompletePart, GetObjectReader, HealOperations, MakeBucketOptions, MultipartOperations,
ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, StorageAPI,
};
use crate::{sets::Sets, store::ECStore};
use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore};
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use bytes::Bytes;
use futures::future::BoxFuture;
use http::HeaderMap;
use rmp_serde::{Deserializer, Serializer};
use rustfs_common::defer;
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_rio::{HashReader, WarpReader};
use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, Reader, TryGetIndex, WarpReader};
use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join};
use rustfs_workers::workers::Workers;
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
use std::io::{Cursor, Write};
use std::path::PathBuf;
use std::sync::Arc;
use std::pin::Pin;
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use std::task::{Context, Poll};
use time::{Duration, OffsetDateTime};
use tokio::io::{AsyncReadExt, BufReader};
use tokio::io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
@@ -75,6 +93,147 @@ pub struct PoolMeta {
pub dont_save: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistedPoolMeta {
pub version: u16,
pub pools: Vec<PersistedPoolStatus>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistedPoolStatus {
#[serde(rename = "id")]
pub id: usize,
#[serde(rename = "cmdline")]
pub cmd_line: String,
#[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
pub last_update: OffsetDateTime,
#[serde(rename = "decommissionInfo")]
pub decommission: Option<PersistedPoolDecommissionInfo>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct PersistedPoolDecommissionInfo {
#[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
pub start_time: Option<OffsetDateTime>,
#[serde(rename = "startSize")]
pub start_size: usize,
#[serde(rename = "totalSize")]
pub total_size: usize,
#[serde(rename = "currentSize")]
pub current_size: usize,
#[serde(rename = "complete")]
pub complete: bool,
#[serde(rename = "failed")]
pub failed: bool,
#[serde(rename = "canceled")]
pub canceled: bool,
#[serde(rename = "queuedBuckets", default)]
pub queued_buckets: Vec<String>,
#[serde(rename = "decommissionedBuckets", default)]
pub decommissioned_buckets: Vec<String>,
#[serde(rename = "bucket", default)]
pub bucket: String,
#[serde(rename = "prefix", default)]
pub prefix: String,
#[serde(rename = "object", default)]
pub object: String,
#[serde(rename = "objectsDecommissioned")]
pub items_decommissioned: usize,
#[serde(rename = "objectsDecommissionedFailed")]
pub items_decommission_failed: usize,
#[serde(rename = "bytesDecommissioned")]
pub bytes_done: usize,
#[serde(rename = "bytesDecommissionedFailed")]
pub bytes_failed: usize,
}
impl From<PersistedPoolMeta> for PoolMeta {
fn from(value: PersistedPoolMeta) -> Self {
Self {
version: value.version,
pools: value.pools.into_iter().map(Into::into).collect(),
dont_save: false,
}
}
}
impl From<PersistedPoolStatus> for PoolStatus {
fn from(value: PersistedPoolStatus) -> Self {
Self {
id: value.id,
cmd_line: value.cmd_line,
last_update: value.last_update,
decommission: value.decommission.map(Into::into),
}
}
}
impl From<PersistedPoolDecommissionInfo> for PoolDecommissionInfo {
fn from(value: PersistedPoolDecommissionInfo) -> Self {
Self {
start_time: value.start_time,
start_size: value.start_size,
total_size: value.total_size,
current_size: value.current_size,
complete: value.complete,
failed: value.failed,
canceled: value.canceled,
queued_buckets: value.queued_buckets,
decommissioned_buckets: value.decommissioned_buckets,
bucket: value.bucket,
prefix: value.prefix,
object: value.object,
items_decommissioned: value.items_decommissioned,
items_decommission_failed: value.items_decommission_failed,
bytes_done: value.bytes_done,
bytes_failed: value.bytes_failed,
}
}
}
impl From<&PoolMeta> for PersistedPoolMeta {
fn from(value: &PoolMeta) -> Self {
Self {
version: value.version,
pools: value.pools.iter().map(Into::into).collect(),
}
}
}
impl From<&PoolStatus> for PersistedPoolStatus {
fn from(value: &PoolStatus) -> Self {
Self {
id: value.id,
cmd_line: value.cmd_line.clone(),
last_update: value.last_update,
decommission: value.decommission.as_ref().map(Into::into),
}
}
}
impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo {
fn from(value: &PoolDecommissionInfo) -> Self {
Self {
start_time: value.start_time,
start_size: value.start_size,
total_size: value.total_size,
current_size: value.current_size,
complete: value.complete,
failed: value.failed,
canceled: value.canceled,
queued_buckets: value.queued_buckets.clone(),
decommissioned_buckets: value.decommissioned_buckets.clone(),
bucket: value.bucket.clone(),
prefix: value.prefix.clone(),
object: value.object.clone(),
items_decommissioned: value.items_decommissioned,
items_decommission_failed: value.items_decommission_failed,
bytes_done: value.bytes_done,
bytes_failed: value.bytes_failed,
}
}
}
impl PoolMeta {
pub fn new(pools: &[Arc<Sets>], prev_meta: &PoolMeta) -> Self {
let mut new_meta = Self {
@@ -144,8 +303,8 @@ impl PoolMeta {
}
let mut buf = Deserializer::new(Cursor::new(&data[4..]));
let meta: PoolMeta = Deserialize::deserialize(&mut buf)?;
*self = meta;
let meta: PersistedPoolMeta = Deserialize::deserialize(&mut buf)?;
*self = meta.into();
if self.version != POOL_META_VERSION {
return Err(Error::other(format!("unexpected PoolMeta version: {}", self.version)));
@@ -161,7 +320,7 @@ impl PoolMeta {
data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
data.write_u16::<LittleEndian>(POOL_META_VERSION)?;
let mut buf = Vec::new();
self.serialize(&mut Serializer::new(&mut buf))?;
PersistedPoolMeta::from(self).serialize(&mut Serializer::new(&mut buf))?;
data.write_all(&buf)?;
for pool in pools {
@@ -178,7 +337,6 @@ impl PoolMeta {
stats.last_update = OffsetDateTime::now_utc();
let mut pd = d.clone();
pd.start_time = None;
pd.canceled = true;
pd.failed = false;
pd.complete = false;
@@ -202,7 +360,6 @@ impl PoolMeta {
stats.last_update = OffsetDateTime::now_utc();
let mut pd = d.clone();
pd.start_time = None;
pd.canceled = false;
pd.failed = true;
pd.complete = false;
@@ -226,7 +383,6 @@ impl PoolMeta {
stats.last_update = OffsetDateTime::now_utc();
let mut pd = d.clone();
pd.start_time = None;
pd.canceled = false;
pd.failed = false;
pd.complete = true;
@@ -576,6 +732,132 @@ impl Display for DecomBucketInfo {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DecommissionFinalState {
Complete,
Failed,
}
fn determine_decommission_final_state(items_failed: usize, was_cancelled: bool) -> DecommissionFinalState {
if items_failed > 0 || was_cancelled {
DecommissionFinalState::Failed
} else {
DecommissionFinalState::Complete
}
}
fn remaining_versions_after_decommission(fivs: &FileInfoVersions) -> usize {
fivs.versions.iter().filter(|version| !version.deleted).count()
}
fn decommission_delete_marker_opts(
version: &rustfs_filemeta::FileInfo,
version_id: Option<String>,
src_pool_idx: usize,
) -> ObjectOptions {
ObjectOptions {
versioned: true,
version_id,
mod_time: version.mod_time,
src_pool_idx,
data_movement: true,
delete_marker: true,
skip_decommissioned: true,
delete_replication: version.replication_state_internal.clone(),
..Default::default()
}
}
async fn should_skip_lifecycle_for_decommission(
store: Arc<ECStore>,
bucket: &str,
version: &rustfs_filemeta::FileInfo,
lifecycle_config: Option<&BucketLifecycleConfiguration>,
lock_retention: Option<DefaultRetention>,
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
apply_actions: bool,
) -> bool {
let Some(lifecycle_config) = lifecycle_config else {
return false;
};
let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await;
let object_info = crate::store_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
let event = eval_action_from_lifecycle(lifecycle_config, lock_retention, replication_config, &object_info).await;
match event.action {
IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => {
if apply_actions && object_info.is_remote() {
let _ = apply_expiry_on_transitioned_object(store, &object_info, &event, &LcEventSrc::Decom).await;
}
false
}
IlmAction::DeleteAction
| IlmAction::DeleteVersionAction
| IlmAction::DeleteAllVersionsAction
| IlmAction::DelMarkerDeleteAllVersionsAction => {
if apply_actions {
let _ = apply_expiry_rule(&event, &LcEventSrc::Decom, &object_info).await;
}
true
}
_ => false,
}
}
struct IndexedDecommissionReader<R> {
inner: R,
index: Option<Index>,
}
impl<R> IndexedDecommissionReader<R> {
fn new(inner: R, index: Option<Index>) -> Self {
Self { inner, index }
}
}
impl<R: AsyncRead + Unpin + Send + Sync> AsyncRead for IndexedDecommissionReader<R> {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
Pin::new(&mut self.inner).poll_read(cx, buf)
}
}
impl<R: AsyncRead + Unpin + Send + Sync> EtagResolvable for IndexedDecommissionReader<R> {}
impl<R: AsyncRead + Unpin + Send + Sync> HashReaderDetector for IndexedDecommissionReader<R> {}
impl<R: AsyncRead + Unpin + Send + Sync> TryGetIndex for IndexedDecommissionReader<R> {
fn try_get_index(&self) -> Option<&Index> {
self.index.as_ref()
}
}
impl<R: AsyncRead + Unpin + Send + Sync> Reader for IndexedDecommissionReader<R> {}
fn decode_part_index(index: Option<&Bytes>) -> Option<Index> {
let bytes = index?;
let mut decoded = Index::new();
if decoded.load(bytes.as_ref()).is_ok() {
Some(decoded)
} else {
None
}
}
fn put_obj_reader_from_chunk(chunk: Vec<u8>, size: i64, actual_size: i64, index: Option<Index>) -> Result<PutObjReader> {
use sha2::{Digest, Sha256};
let sha256hex = if !chunk.is_empty() {
Some(hex_simd::encode_to_string(Sha256::digest(&chunk), hex_simd::AsciiCase::Lower))
} else {
None
};
let reader = IndexedDecommissionReader::new(WarpReader::new(Cursor::new(chunk)), index);
let hash_reader = HashReader::new(Box::new(reader), size, actual_size, None, sha256hex, false)?;
Ok(PutObjReader::new(hash_reader))
}
impl ECStore {
pub async fn status(&self, idx: usize) -> Result<PoolStatus> {
let space_info = self.get_decommission_pool_space_info(idx).await?;
@@ -621,13 +903,18 @@ impl ECStore {
return Err(Error::other("InvalidArgument"));
}
let Some(has_canceler) = self.decommission_cancelers.get(idx) else {
return Err(Error::other("InvalidArgument"));
};
let canceler = {
let mut cancelers = self.decommission_cancelers.write().await;
let Some(slot) = cancelers.get_mut(idx) else {
return Err(Error::other("InvalidArgument"));
};
if has_canceler.is_none() {
return Err(StorageError::DecommissionNotStarted);
}
let Some(canceler) = slot.take() else {
return Err(StorageError::DecommissionNotStarted);
};
canceler
};
let mut lock = self.pool_meta.write().await;
if lock.decommission_cancel(idx) {
@@ -640,6 +927,8 @@ impl ECStore {
}
}
canceler.cancel();
Ok(())
}
pub async fn is_decommission_running(&self) -> bool {
@@ -684,8 +973,8 @@ impl ECStore {
Ok(())
}
#[allow(unused_assignments)]
#[tracing::instrument(skip(self, set, wk, rcfg))]
#[allow(unused_assignments, clippy::too_many_arguments)]
#[tracing::instrument(skip(self, set, wk, lifecycle_config, lock_retention, replication_config))]
async fn decommission_entry(
self: &Arc<Self>,
idx: usize,
@@ -693,7 +982,9 @@ impl ECStore {
bucket: String,
set: Arc<SetDisks>,
wk: Arc<Workers>,
rcfg: Option<String>,
lifecycle_config: Option<BucketLifecycleConfiguration>,
lock_retention: Option<DefaultRetention>,
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
) {
warn!("decommission_entry: {} {}", &bucket, &entry.name);
wk.give().await;
@@ -713,12 +1004,27 @@ impl ECStore {
fivs.versions.sort_by(|a, b| b.mod_time.cmp(&a.mod_time));
let mut decommissioned: usize = 0;
let expired: usize = 0;
let mut expired: usize = 0;
for version in fivs.versions.iter() {
// TODO: filterLifecycle
if should_skip_lifecycle_for_decommission(
self.clone(),
&bucket,
version,
lifecycle_config.as_ref(),
lock_retention.clone(),
replication_config.clone(),
true,
)
.await
{
expired += 1;
decommissioned += 1;
continue;
}
let remaining_versions = fivs.versions.len() - expired;
if version.deleted && remaining_versions == 1 && rcfg.is_none() {
if version.deleted && remaining_versions == 1 && replication_config.is_none() {
//
decommissioned += 1;
info!("decommission_pool: DELETE marked object with no other non-current versions will be skipped");
@@ -731,25 +1037,19 @@ impl ECStore {
let mut failure = false;
let mut error = None;
if version.deleted {
// TODO: other params
if let Err(err) = self
.delete_object(
bucket.as_str(),
&version.name,
ObjectOptions {
versioned: true,
version_id: version_id.clone(),
mod_time: version.mod_time,
src_pool_idx: idx,
data_movement: true,
delete_marker: true,
skip_decommissioned: true,
..Default::default()
},
decommission_delete_marker_opts(version, version_id.clone(), idx),
)
.await
{
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err) {
warn!(
"decommission_pool: ignore delete-marker copy for {}/{} version {:?}: {:?}",
&bucket, &version.name, &version_id, &err
);
ignore = true;
continue;
}
@@ -776,7 +1076,33 @@ impl ECStore {
for _i in 0..3 {
if version.is_remote() {
// TODO: DecomTieredObject
if let Err(err) = self
.decommission_tiered_object(
bucket.as_str(),
&version.name,
version,
&ObjectOptions {
version_id: version_id.clone(),
mod_time: version.mod_time,
user_defined: version.metadata.clone(),
src_pool_idx: idx,
data_movement: true,
..Default::default()
},
)
.await
{
if is_err_object_not_found(&err) || is_err_version_not_found(&err) || is_err_data_movement_overwrite(&err)
{
ignore = true;
break;
}
failure = true;
error!("decommission_pool: decommission_tiered_object err {:?}", &err);
error = Some(err);
}
break;
}
let bucket = bucket.clone();
@@ -847,7 +1173,8 @@ impl ECStore {
}
{
self.pool_meta.write().await.count_item(idx, decommissioned, failure);
let size = usize::try_from(version.size).unwrap_or_default();
self.pool_meta.write().await.count_item(idx, size, failure);
}
if failure {
@@ -872,6 +1199,14 @@ impl ECStore {
.await
{
error!("decommission_pool: delete_object err {:?}", &err);
} else if decommissioned != fivs.versions.len() {
warn!(
"decommission_pool: source object retained for {}/{} because only {}/{} versions were decommissioned",
&bucket,
&entry.name,
decommissioned,
fivs.versions.len()
);
}
{
@@ -903,16 +1238,19 @@ impl ECStore {
) -> Result<()> {
let wk = Workers::new(pool.disk_set.len() * 2).map_err(Error::other)?;
// let mut vc = None;
// replication
let rcfg: Option<String> = None;
let mut lifecycle_config = None;
let mut lock_retention = None;
let mut replication_config = None;
if bi.name != RUSTFS_META_BUCKET {
let _versioning = BucketVersioningSys::get(&bi.name).await?;
// vc = Some(versioning);
// TODO: LifecycleSys
// TODO: BucketObjectLockSys
// TODO: ReplicationConfig
let _ = BucketVersioningSys::get(&bi.name).await?;
lifecycle_config = GLOBAL_LifecycleSys.get(&bi.name).await;
lock_retention = BucketObjectLockSys::get(&bi.name).await;
replication_config = match metadata_sys::get_replication_config(&bi.name).await {
Ok(config) => Some(config),
Err(Error::ConfigNotFound) => None,
Err(err) => return Err(err),
};
}
for (set_idx, set) in pool.disk_set.iter().enumerate() {
@@ -925,17 +1263,22 @@ impl ECStore {
let bucket = bi.name.clone();
let wk = wk.clone();
let set = set.clone();
let rcfg = rcfg.clone();
let lifecycle_config = lifecycle_config.clone();
let lock_retention = lock_retention.clone();
let replication_config = replication_config.clone();
move |entry: MetaCacheEntry| {
let this = this.clone();
let bucket = bucket.clone();
let wk = wk.clone();
let set = set.clone();
let rcfg = rcfg.clone();
let lifecycle_config = lifecycle_config.clone();
let lock_retention = lock_retention.clone();
let replication_config = replication_config.clone();
Box::pin(async move {
wk.take().await;
this.decommission_entry(idx, entry, bucket, set, wk, rcfg).await
this.decommission_entry(idx, entry, bucket, set, wk, lifecycle_config, lock_retention, replication_config)
.await
})
}
});
@@ -988,7 +1331,15 @@ impl ECStore {
#[tracing::instrument(skip(self, rx))]
pub async fn do_decommission_in_routine(self: &Arc<Self>, rx: CancellationToken, idx: usize) {
if let Err(err) = self.decommission_in_background(rx, idx).await {
let decommission_token = rx.child_token();
{
let mut cancelers = self.decommission_cancelers.write().await;
if let Some(slot) = cancelers.get_mut(idx) {
*slot = Some(decommission_token.clone());
}
}
if let Err(err) = self.decommission_in_background(decommission_token.clone(), idx).await {
error!("decom err {:?}", &err);
if let Err(er) = self.decommission_failed(idx).await {
error!("decom failed err {:?}", &er);
@@ -1001,29 +1352,49 @@ impl ECStore {
warn!("decommission: decommission_in_background complete {}", idx);
let (failed, cmd_line) = {
let (final_state, cmd_line) = {
let pool_meta = self.pool_meta.read().await;
let failed = {
let final_state = {
if let Some(info) = &pool_meta.pools[idx].decommission {
info.items_decommission_failed > 0
determine_decommission_final_state(info.items_decommission_failed, info.canceled)
} else {
false
DecommissionFinalState::Failed
}
};
let cmd_line = pool_meta.pools[idx].cmd_line.clone();
(failed, cmd_line)
(final_state, cmd_line)
};
if !failed {
let mut completed_successfully = false;
if final_state == DecommissionFinalState::Complete {
warn!("Decommissioning complete for pool {}, verifying for any pending objects", cmd_line);
if let Err(er) = self.decommission_failed(idx).await {
error!("decom failed err {:?}", &er);
if let Err(err) = self.check_after_decommission(idx).await {
error!("decom post-check err {:?}", &err);
if let Err(er) = self.decommission_failed(idx).await {
error!("decom failed err {:?}", &er);
}
} else if let Err(er) = self.complete_decommission(idx).await {
error!("decom complete err {:?}", &er);
} else {
completed_successfully = true;
}
} else if let Err(er) = self.complete_decommission(idx).await {
error!("decom complete err {:?}", &er);
} else if let Err(er) = self.decommission_failed(idx).await {
error!("decom failed err {:?}", &er);
}
warn!("Decommissioning complete for pool {}", cmd_line);
{
let mut cancelers = self.decommission_cancelers.write().await;
if let Some(slot) = cancelers.get_mut(idx) {
*slot = None;
}
}
if completed_successfully {
warn!("Decommissioning complete for pool {}", cmd_line);
} else {
warn!("Decommissioning finished in failed state for pool {}", cmd_line);
}
}
#[tracing::instrument(skip(self))]
@@ -1043,6 +1414,14 @@ impl ECStore {
}
}
let canceler = {
let mut cancelers = self.decommission_cancelers.write().await;
cancelers.get_mut(idx).and_then(Option::take)
};
if let Some(canceler) = canceler {
canceler.cancel();
}
Ok(())
}
@@ -1061,6 +1440,14 @@ impl ECStore {
}
}
let canceler = {
let mut cancelers = self.decommission_cancelers.write().await;
cancelers.get_mut(idx).and_then(Option::take)
};
if let Some(canceler) = canceler {
canceler.cancel();
}
Ok(())
}
@@ -1190,6 +1577,94 @@ impl ECStore {
Ok(ret)
}
async fn check_after_decommission(self: &Arc<Self>, idx: usize) -> Result<()> {
let buckets = self.get_buckets_to_decommission().await?;
let pool = self.pools[idx].clone();
for set in &pool.disk_set {
for bucket_info in &buckets {
let mut lifecycle_config = None;
let mut lock_retention = None;
let mut replication_config = None;
if bucket_info.name != RUSTFS_META_BUCKET {
lifecycle_config = GLOBAL_LifecycleSys.get(&bucket_info.name).await;
lock_retention = BucketObjectLockSys::get(&bucket_info.name).await;
replication_config = match metadata_sys::get_replication_config(&bucket_info.name).await {
Ok(config) => Some(config),
Err(Error::ConfigNotFound) => None,
Err(err) => return Err(err),
};
}
let versions_found = Arc::new(AtomicUsize::new(0));
let versions_found_cb = versions_found.clone();
let bucket_name = bucket_info.name.clone();
let lifecycle_config_cb = lifecycle_config.clone();
let lock_retention_cb = lock_retention.clone();
let replication_config_cb = replication_config.clone();
let store = Arc::clone(self);
let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| {
let versions_found = versions_found_cb.clone();
let bucket_name = bucket_name.clone();
let lifecycle_config = lifecycle_config_cb.clone();
let lock_retention = lock_retention_cb.clone();
let replication_config = replication_config_cb.clone();
let store = Arc::clone(&store);
Box::pin(async move {
if !entry.is_object() {
return;
}
if bucket_name == RUSTFS_META_BUCKET && entry.name.contains(DATA_USAGE_CACHE_NAME) {
return;
}
let Ok(fivs) = entry.file_info_versions(&bucket_name) else {
return;
};
let mut remaining = 0;
for version in &fivs.versions {
if version.deleted {
continue;
}
if should_skip_lifecycle_for_decommission(
Arc::clone(&store),
&bucket_name,
version,
lifecycle_config.as_ref(),
lock_retention.clone(),
replication_config.clone(),
false,
)
.await
{
continue;
}
remaining += 1;
}
versions_found.fetch_add(remaining, Ordering::Relaxed);
})
});
set.list_objects_to_decommission(CancellationToken::new(), bucket_info.clone(), callback)
.await?;
let versions_found = versions_found.load(Ordering::Relaxed);
if versions_found > 0 {
return Err(Error::other(format!(
"at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning",
bucket_info.name
)));
}
}
}
Ok(())
}
#[tracing::instrument(skip(self, rd))]
async fn decommission_object(self: Arc<Self>, pool_idx: usize, bucket: String, rd: GetObjectReader) -> Result<()> {
warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
@@ -1238,7 +1713,10 @@ impl ECStore {
reader.read_exact(&mut chunk).await?;
let mut data = PutObjReader::from_vec(chunk);
let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?;
let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size };
let index = decode_part_index(part.index.as_ref());
let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?;
let pi = match self
.put_object_part(
@@ -1294,8 +1772,13 @@ impl ECStore {
return Ok(());
}
let reader = BufReader::new(rd.stream);
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?;
let actual_size = object_info.get_actual_size()?;
let index = object_info
.parts
.first()
.and_then(|part| decode_part_index(part.index.as_ref()));
let reader = IndexedDecommissionReader::new(WarpReader::new(BufReader::new(rd.stream)), index);
let hrd = HashReader::new(Box::new(reader), object_info.size, actual_size, object_info.etag.clone(), None, false)?;
let mut data = PutObjReader::new(hrd);
if let Err(err) = self
@@ -1325,6 +1808,159 @@ impl ECStore {
}
}
#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use super::*;
#[test]
fn determine_decommission_final_state_marks_failures_and_cancellations() {
assert_eq!(determine_decommission_final_state(0, false), DecommissionFinalState::Complete);
assert_eq!(determine_decommission_final_state(1, false), DecommissionFinalState::Failed);
assert_eq!(determine_decommission_final_state(0, true), DecommissionFinalState::Failed);
}
#[test]
fn remaining_versions_after_decommission_ignores_delete_markers() {
let fivs = FileInfoVersions {
versions: vec![
rustfs_filemeta::FileInfo {
deleted: false,
size: 128,
..Default::default()
},
rustfs_filemeta::FileInfo {
deleted: true,
size: 0,
..Default::default()
},
],
..Default::default()
};
assert_eq!(remaining_versions_after_decommission(&fivs), 1);
}
#[test]
fn decommission_delete_marker_opts_preserves_replication_state() {
let mod_time = OffsetDateTime::now_utc();
let version = rustfs_filemeta::FileInfo {
mod_time: Some(mod_time),
replication_state_internal: Some(rustfs_filemeta::ReplicationState {
replica_status: rustfs_filemeta::ReplicationStatusType::Replica,
delete_marker: true,
replicate_decision_str: "existing".to_string(),
..Default::default()
}),
..Default::default()
};
let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7);
let replication = opts.delete_replication.expect("replication state should be preserved");
assert!(opts.versioned);
assert!(opts.data_movement);
assert!(opts.delete_marker);
assert!(opts.skip_decommissioned);
assert_eq!(opts.src_pool_idx, 7);
assert_eq!(opts.version_id.as_deref(), Some("version-id"));
assert_eq!(opts.mod_time, Some(mod_time));
assert_eq!(replication.replica_status, rustfs_filemeta::ReplicationStatusType::Replica);
assert!(replication.delete_marker);
assert_eq!(replication.replicate_decision_str, "existing");
}
#[test]
fn decommission_state_transitions_preserve_start_time() {
let start_time = OffsetDateTime::now_utc();
let mut pool_meta = PoolMeta {
version: POOL_META_VERSION,
pools: vec![PoolStatus {
id: 0,
cmd_line: "/tmp/pool".to_string(),
last_update: start_time,
decommission: Some(PoolDecommissionInfo {
start_time: Some(start_time),
..Default::default()
}),
}],
dont_save: true,
};
assert!(pool_meta.decommission_failed(0));
assert_eq!(
pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
Some(start_time)
);
assert!(pool_meta.decommission_complete(0));
assert_eq!(
pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
Some(start_time)
);
assert!(pool_meta.decommission_cancel(0));
assert_eq!(
pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
Some(start_time)
);
}
#[test]
fn pool_meta_persists_decommission_resume_queues() {
let start_time = OffsetDateTime::now_utc();
let pool_meta = PoolMeta {
version: POOL_META_VERSION,
pools: vec![PoolStatus {
id: 1,
cmd_line: "/data/pool1/disk{1...4}".to_string(),
last_update: start_time,
decommission: Some(PoolDecommissionInfo {
start_time: Some(start_time),
queued_buckets: vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()],
decommissioned_buckets: vec!["bucket-done".to_string()],
bucket: "bucket-b".to_string(),
prefix: "prefix".to_string(),
object: "object.txt".to_string(),
items_decommissioned: 7,
items_decommission_failed: 1,
bytes_done: 1024,
bytes_failed: 128,
..Default::default()
}),
}],
dont_save: false,
};
let mut buf = Vec::new();
PersistedPoolMeta::from(&pool_meta)
.serialize(&mut Serializer::new(&mut buf))
.expect("pool meta should serialize");
let mut deserializer = Deserializer::new(Cursor::new(&buf));
let restored: PoolMeta = PersistedPoolMeta::deserialize(&mut deserializer)
.expect("pool meta should deserialize")
.into();
let restored_decommission = restored.pools[0]
.decommission
.as_ref()
.expect("decommission info should survive round-trip");
assert_eq!(
restored_decommission.queued_buckets,
vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()]
);
assert_eq!(restored_decommission.decommissioned_buckets, vec!["bucket-done".to_string()]);
assert_eq!(restored_decommission.bucket, "bucket-b");
assert_eq!(restored_decommission.prefix, "prefix");
assert_eq!(restored_decommission.object, "object.txt");
assert_eq!(restored_decommission.items_decommissioned, 7);
assert_eq!(restored_decommission.items_decommission_failed, 1);
assert_eq!(restored_decommission.bytes_done, 1024);
assert_eq!(restored_decommission.bytes_failed, 128);
}
}
// impl Fn(MetaCacheEntry) -> impl Future<Output = Result<(), Error>>
pub type ListCallback = Arc<dyn Fn(MetaCacheEntry) -> BoxFuture<'static, ()> + Send + Sync + 'static>;
crates/ecstore/src/set_disk.rs
+147 -25
View File
@@ -80,9 +80,12 @@ use rustfs_lock::{FastLockGuard, NamespaceLock, NamespaceLockGuard, NamespaceLoc
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _, WarpReader};
use rustfs_s3_common::EventName;
use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER};
use rustfs_utils::http::{
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
contains_key_str, get_header_map, get_str, insert_str, remove_header_map,
};
use rustfs_utils::{
HashAlgorithm,
crypto::hex,
@@ -136,6 +139,30 @@ pub fn get_lock_acquire_timeout() -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
}
fn build_tiered_decommission_file_info(
bucket: &str,
object: &str,
fi: &FileInfo,
disk_count: usize,
default_parity_count: usize,
storage_class: Option<&str>,
) -> (FileInfo, usize) {
let parity_drives = GLOBAL_STORAGE_CLASS
.get()
.and_then(|sc| sc.get_parity_for_sc(storage_class.unwrap_or_default()))
.unwrap_or(default_parity_count);
let data_drives = disk_count - parity_drives;
let mut write_quorum = data_drives;
if data_drives == parity_drives {
write_quorum += 1;
}
let mut updated = fi.clone();
updated.erasure = FileInfo::new([bucket, object].join("/").as_str(), data_drives, parity_drives).erasure;
(updated, write_quorum)
}
#[derive(Clone, Debug)]
pub struct SetDisks {
pub locker_owner: String,
@@ -620,7 +647,7 @@ impl ObjectIO for SetDisks {
&tmp_object,
erasure.shard_file_size(data.size()),
erasure.shard_size(),
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await
{
@@ -678,8 +705,8 @@ impl ObjectIO for SetDisks {
)));
}
if user_defined.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")) {
user_defined.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression-size"), w_size.to_string());
if contains_key_str(&user_defined, SUFFIX_COMPRESSION) {
insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string());
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
@@ -765,7 +792,7 @@ impl ObjectIO for SetDisks {
.await?;
if let Some(old_dir) = op_old_dir {
self.commit_rename_data_dir(&shuffle_disks, bucket, object, &old_dir.to_string(), write_quorum)
self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum)
.await?;
}
@@ -1336,6 +1363,12 @@ impl ObjectOperations for SetDisks {
let mut delete_marker = opts.versioned;
if opts.version_id.is_some() {
// Decommission/rebalance may recreate a delete marker on a new pool before that
// exact version exists there, so we must still treat it as a mark-delete write.
if opts.data_movement && opts.delete_marker && !version_found {
mark_delete = true;
}
if version_found && opts.delete_marker_replication_status() == ReplicationStatusType::Replica {
mark_delete = false;
}
@@ -1889,6 +1922,68 @@ impl ObjectOperations for SetDisks {
}
}
impl SetDisks {
#[tracing::instrument(skip(self, fi, opts))]
pub(crate) async fn decommission_tiered_object(
&self,
bucket: &str,
object: &str,
fi: &FileInfo,
opts: &ObjectOptions,
) -> Result<()> {
let _lock_guard = if !opts.no_lock {
Some(
self.new_ns_lock(bucket, object)
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
Error::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?,
)
} else {
None
};
let disks = self.disks.read().await.clone();
let storage_class = opts.user_defined.get(AMZ_STORAGE_CLASS).map(String::as_str);
let (fi, write_quorum) =
build_tiered_decommission_file_info(bucket, object, fi, disks.len(), self.default_parity_count, storage_class);
let parts_metadata = vec![fi.clone(); disks.len()];
let (shuffle_disks, parts_metadata) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
let mut errs = Vec::with_capacity(shuffle_disks.len());
let mut futures = Vec::with_capacity(shuffle_disks.len());
for (index, disk) in shuffle_disks.iter().enumerate() {
let mut file_info = parts_metadata[index].clone();
file_info.erasure.index = index + 1;
futures.push(async move {
if let Some(disk) = disk {
disk.write_metadata("", bucket, object, file_info).await
} else {
Err(DiskError::DiskNotFound)
}
});
}
for result in join_all(futures).await {
match result {
Ok(_) => errs.push(None),
Err(err) => errs.push(Some(err)),
}
}
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(to_object_err(err.into(), vec![bucket, object]));
}
Ok(())
}
}
#[async_trait::async_trait]
impl ListOperations for SetDisks {
#[tracing::instrument(skip(self))]
@@ -2007,7 +2102,7 @@ impl MultipartOperations for SetDisks {
&tmp_part_path,
erasure.shard_file_size(data.size()),
erasure.shard_size(),
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await
{
@@ -2454,11 +2549,11 @@ impl MultipartOperations for SetDisks {
fi.data_dir = Some(Uuid::new_v4());
if let Some(cssum) = user_defined.get(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM)
if let Some(cssum) = get_header_map(&user_defined, SUFFIX_REPLICATION_SSEC_CRC)
&& !cssum.is_empty()
{
fi.checksum = base64_simd::STANDARD.decode_to_vec(cssum).ok().map(Bytes::from);
user_defined.remove(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM);
fi.checksum = base64_simd::STANDARD.decode_to_vec(&cssum).ok().map(Bytes::from);
remove_header_map(&mut user_defined, SUFFIX_REPLICATION_SSEC_CRC);
}
let parts_metadata = vec![fi.clone(); disks.len()];
@@ -2809,8 +2904,8 @@ impl MultipartOperations for SetDisks {
}
}
if let Some(rc_crc) = opts.user_defined.get(RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM) {
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(rc_crc) {
if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
fi.checksum = Some(Bytes::from(rc_crc_bytes));
} else {
error!("complete_multipart_upload decode rc_crc failed rc_crc={}", rc_crc);
@@ -2849,25 +2944,19 @@ impl MultipartOperations for SetDisks {
fi.metadata.insert("etag".to_owned(), etag);
if opts.replication_request {
if let Some(actual_size) = opts
.user_defined
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}Actual-Object-Size").as_str())
{
if let Some(actual_size) = get_str(&opts.user_defined, SUFFIX_ACTUAL_OBJECT_SIZE_CAP) {
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, actual_size.clone());
fi.metadata
.insert(format!("{RESERVED_METADATA_PREFIX}actual-size"), actual_size.clone());
fi.metadata
.insert("x-rustfs-encryption-original-size".to_string(), actual_size.to_string());
.insert("x-rustfs-encryption-original-size".to_string(), actual_size);
}
} else {
fi.metadata
.insert(format!("{RESERVED_METADATA_PREFIX}actual-size"), object_actual_size.to_string());
insert_str(&mut fi.metadata, SUFFIX_ACTUAL_SIZE, object_actual_size.to_string());
fi.metadata
.insert("x-rustfs-encryption-original-size".to_string(), object_actual_size.to_string());
}
if fi.is_compressed() {
fi.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression-size"), object_size.to_string());
insert_str(&mut fi.metadata, SUFFIX_COMPRESSION_SIZE, object_size.to_string());
}
if opts.data_movement {
@@ -2927,7 +3016,7 @@ impl MultipartOperations for SetDisks {
.await?;
if let Some(old_dir) = op_old_dir {
self.commit_rename_data_dir(&shuffle_disks, bucket, object, &old_dir.to_string(), write_quorum)
self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum)
.await?;
}
@@ -3356,12 +3445,18 @@ async fn disks_with_all_parts(
if (meta.data.is_some() || meta.size == 0) && !meta.parts.is_empty() {
if let Some(data) = &meta.data {
let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number);
let checksum_algo =
if meta.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let data_len = data.len();
let verify_err = bitrot_verify(
Box::new(Cursor::new(data.clone())),
data_len,
meta.erasure.shard_file_size(meta.size) as usize,
checksum_info.algorithm,
checksum_algo,
checksum_info.hash,
meta.erasure.shard_size(),
)
@@ -4161,6 +4256,33 @@ mod tests {
assert!(e_tag_matches("\"abc\"", "*"));
}
#[test]
fn test_build_tiered_decommission_file_info_preserves_transition_metadata() {
let version_id = Uuid::new_v4();
let transition_version_id = Uuid::new_v4();
let original = FileInfo {
version_id: Some(version_id),
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_tier: "WARM-TIER".to_string(),
transition_version_id: Some(transition_version_id),
erasure: FileInfo::new("old-bucket/old-object", 8, 8).erasure,
..Default::default()
};
let (updated, write_quorum) = build_tiered_decommission_file_info("bucket", "object", &original, 16, 4, None);
assert_eq!(updated.version_id, original.version_id);
assert_eq!(updated.transition_status, original.transition_status);
assert_eq!(updated.transitioned_objname, original.transitioned_objname);
assert_eq!(updated.transition_tier, original.transition_tier);
assert_eq!(updated.transition_version_id, original.transition_version_id);
assert_eq!(updated.erasure.data_blocks, 12);
assert_eq!(updated.erasure.parity_blocks, 4);
assert_eq!(write_quorum, 12);
assert_ne!(updated.erasure.distribution, original.erasure.distribution);
}
#[test]
fn test_should_prevent_write() {
let oi = ObjectInfo {
crates/ecstore/src/set_disk/heal.rs
+12 -4
View File
@@ -124,11 +124,12 @@ impl SetDisks {
);
let erasure = if !latest_meta.deleted && !latest_meta.is_remote() {
// Initialize erasure coding
erasure_coding::Erasure::new(
// Initialize erasure coding; use legacy mode for old-version files
erasure_coding::Erasure::new_with_options(
latest_meta.erasure.data_blocks,
latest_meta.erasure.parity_blocks,
latest_meta.erasure.block_size,
latest_meta.uses_legacy_checksum,
)
} else {
erasure_coding::Erasure::default()
@@ -347,7 +348,14 @@ impl SetDisks {
for (part_index, part) in latest_meta.parts.iter().enumerate() {
let till_offset = erasure.shard_file_offset(0, part.size, part.size);
let checksum_algo = erasure_info.get_checksum_info(part.number).algorithm;
let checksum_info = erasure_info.get_checksum_info(part.number);
let checksum_algo = if latest_meta.uses_legacy_checksum
&& checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S
{
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let mut readers = Vec::with_capacity(latest_disks.len());
let mut writers = Vec::with_capacity(out_dated_disks.len());
// let mut errors = Vec::with_capacity(out_dated_disks.len());
@@ -420,7 +428,7 @@ impl SetDisks {
]),
erasure.shard_file_size(part.size as i64),
erasure.shard_size(),
HashAlgorithm::HighwayHash256,
HashAlgorithm::HighwayHash256S,
)
.await
{
crates/ecstore/src/set_disk/read.rs
+15 -2
View File
@@ -603,7 +603,12 @@ impl SetDisks {
object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds"
);
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let erasure = erasure_coding::Erasure::new_with_options(
fi.erasure.data_blocks,
fi.erasure.parity_blocks,
fi.erasure.block_size,
fi.uses_legacy_checksum,
);
let part_indices: Vec<usize> = (part_index..=last_part_index).collect();
debug!(bucket, object, ?part_indices, "Multipart part indices to stream");
@@ -648,6 +653,14 @@ impl SetDisks {
"Streaming multipart part"
);
let checksum_info = fi.erasure.get_checksum_info(part_number);
let checksum_algo =
if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
rustfs_utils::HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
let mut readers = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
for (idx, disk_op) in disks.iter().enumerate() {
@@ -659,7 +672,7 @@ impl SetDisks {
read_offset,
till_offset,
erasure.shard_size(),
HashAlgorithm::HighwayHash256,
checksum_algo.clone(),
skip_verify_bitrot,
)
.await
crates/ecstore/src/store.rs
+1 -1
View File
@@ -156,7 +156,7 @@ pub struct ECStore {
// pub local_disks: Vec<DiskStore>,
pub pool_meta: RwLock<PoolMeta>,
pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
pub decommission_cancelers: Vec<Option<usize>>,
pub decommission_cancelers: RwLock<Vec<Option<CancellationToken>>>,
}
// impl Clone for ECStore {
crates/ecstore/src/store/init.rs
+2 -1
View File
@@ -149,7 +149,7 @@ impl ECStore {
let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
pool_meta.dont_save = true;
let decommission_cancelers = vec![None; pools.len()];
let decommission_cancelers = RwLock::new(vec![None; pools.len()]);
let ec = Arc::new(ECStore {
id: deployment_id.unwrap(),
disk_map,
@@ -265,6 +265,7 @@ impl ECStore {
init_background_expiry(self.clone()).await;
TransitionState::init(self.clone()).await;
crate::tier::tier::try_migrate_tiering_config(self.clone()).await;
if let Err(err) = GLOBAL_TierConfigMgr.write().await.init(self.clone()).await {
info!("TierConfigMgr init error: {}", err);
crates/ecstore/src/store/object.rs
+105 -6
View File
@@ -14,7 +14,64 @@
use super::*;
fn select_data_movement_target_pool(
existing_pool_idx: Result<usize>,
src_pool_idx: usize,
delete_marker: bool,
) -> Result<Option<usize>> {
match existing_pool_idx {
Ok(pool_idx) => {
if delete_marker && pool_idx == src_pool_idx {
Ok(None)
} else {
Ok(Some(pool_idx))
}
}
Err(err) => {
if is_err_read_quorum(&err) {
return Err(StorageError::ErasureWriteQuorum);
}
if delete_marker && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
Ok(None)
} else {
Err(err)
}
}
}
}
impl ECStore {
#[instrument(skip(self, fi, opts))]
pub(crate) async fn decommission_tiered_object(
&self,
bucket: &str,
object: &str,
fi: &rustfs_filemeta::FileInfo,
opts: &ObjectOptions,
) -> Result<()> {
check_put_object_args(bucket, object)?;
let object = encode_dir_object(object);
if self.single_pool() {
return Err(Error::other(format!("error decommissioning {bucket}/{object}")));
}
let idx = self.get_pool_idx_no_lock(bucket, &object, fi.size).await?;
if opts.data_movement && idx == opts.src_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.clone().unwrap_or_default(),
));
}
self.pools[idx]
.get_disks_by_key(&object)
.decommission_tiered_object(bucket, &object, fi, opts)
.await
}
#[instrument(level = "debug", skip(self))]
pub(super) async fn handle_get_object_reader(
&self,
@@ -179,6 +236,30 @@ impl ECStore {
let mut gopts = opts.clone();
gopts.no_lock = true;
if opts.data_movement {
let existing_pool_idx = self
.get_pool_info_existing_with_opts(bucket, object, &gopts)
.await
.map(|(pinfo, _)| pinfo.index);
let target_pool_idx =
match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
Some(pool_idx) => pool_idx,
None => self.get_pool_idx_no_lock(bucket, object, 0).await?,
};
if opts.src_pool_idx == target_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.unwrap_or_default(),
));
}
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
// Determine which pool contains it
let (mut pinfo, errs) = self
.get_pool_info_existing_with_opts(bucket, object, &gopts)
@@ -204,12 +285,6 @@ impl ECStore {
));
}
if opts.data_movement {
let mut obj = self.pools[pinfo.index].delete_object(bucket, object, opts).await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
if !errs.is_empty() && !opts.versioned && !opts.version_suspended {
return self.delete_object_from_all_pools(bucket, object, &opts, errs).await;
}
@@ -565,3 +640,27 @@ impl ECStore {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() {
let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap();
assert_eq!(target, None);
}
#[test]
fn delete_marker_data_movement_falls_back_when_version_does_not_exist_yet() {
let err = StorageError::ObjectNotFound("bucket".to_string(), "object".to_string());
let target = select_data_movement_target_pool(Err(err), 1, true).unwrap();
assert_eq!(target, None);
}
#[test]
fn non_delete_marker_data_movement_keeps_existing_pool() {
let target = select_data_movement_target_pool(Ok(0), 1, false).unwrap();
assert_eq!(target, Some(0));
}
}
crates/ecstore/src/store_api.rs
+3 -3
View File
@@ -27,8 +27,8 @@ use bytes::Bytes;
use http::{HeaderMap, HeaderValue};
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision, ReplicationState,
ReplicationStatusType, RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicateDecision, ReplicationState, ReplicationStatusType,
RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
version_purge_statuses_map,
};
use rustfs_lock::NamespaceLockWrapper;
@@ -36,7 +36,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader};
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
use rustfs_utils::path::decode_dir_object;
use serde::{Deserialize, Serialize};
crates/ecstore/src/store_api/types.rs
+14 -25
View File
@@ -118,11 +118,8 @@ impl ObjectOptions {
}
pub fn put_replication_state(&self) -> ReplicationState {
let rs = match self
.user_defined
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_STATUS}").as_str())
{
Some(v) => v.to_string(),
let rs = match rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS) {
Some(v) => v,
None => return ReplicationState::default(),
};
@@ -341,15 +338,11 @@ impl Clone for ObjectInfo {
impl ObjectInfo {
pub fn is_compressed(&self) -> bool {
self.user_defined
.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression"))
rustfs_utils::http::contains_key_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION)
}
pub fn is_compressed_ok(&self) -> Result<(CompressionAlgorithm, bool)> {
let scheme = self
.user_defined
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression"))
.cloned();
let scheme = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
if let Some(scheme) = scheme {
let algorithm = CompressionAlgorithm::from_str(&scheme)?;
@@ -369,7 +362,7 @@ impl ObjectInfo {
}
if self.is_compressed() {
if let Some(size_str) = self.user_defined.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"))
if let Some(size_str) = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
&& !size_str.is_empty()
{
// Todo: deal with error
@@ -745,15 +738,11 @@ impl ObjectInfo {
.user_defined
.iter()
.filter_map(|(k, v)| {
if k.starts_with(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}")) {
Some((
k.trim_start_matches(&format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-"))
.to_string(),
v.clone(),
))
} else {
None
}
rustfs_utils::http::internal_key_strip_suffix_prefix(
k,
rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
)
.map(|arn| (arn, v.clone()))
})
.collect(),
..Default::default()
@@ -1035,8 +1024,8 @@ mod tests {
fn get_actual_size_uses_compressed_metadata_size() {
let user_defined = {
let mut map = HashMap::new();
map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string());
map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"), "42".to_string());
rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "42".to_string());
map
};
@@ -1072,7 +1061,7 @@ mod tests {
fn get_actual_size_uses_compressed_parts_actual_size_when_metadata_missing() {
let user_defined = {
let mut map = HashMap::new();
map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string());
rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
map
};
@@ -1100,7 +1089,7 @@ mod tests {
fn get_actual_size_returns_error_when_compressed_parts_missing_and_size_mismatch() {
let user_defined = {
let mut map = HashMap::new();
map.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}compression"), "zstd".to_string());
rustfs_utils::http::insert_str(&mut map, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
map
};
crates/ecstore/src/store_init.rs
+60 -4
View File
@@ -18,7 +18,7 @@ use crate::disk::{self, DiskAPI};
use crate::error::{Error, Result};
use crate::{
disk::{
DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET,
DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET,
error::DiskError,
format::{FormatErasureVersion, FormatMetaVersion, FormatV3},
new_disk,
@@ -71,11 +71,13 @@ pub async fn connect_load_init_formats(
check_format_erasure_values(&formats, set_drive_count)?;
if first_disk && should_init_erasure_disks(&errs) {
// UnformattedDisk, not format file create
// UnformattedDisk, try migrate from MinIO format first, else create new format
info!("first_disk && should_init_erasure_disks");
// new format and save
if let Ok(fm) = try_migrate_format(disks, set_count, set_drive_count).await {
info!("Migrated format from MinIO config");
return Ok(fm);
}
let fm = init_format_erasure(disks, set_count, set_drive_count, deployment_id).await?;
return Ok(fm);
}
@@ -149,6 +151,60 @@ async fn init_format_erasure(
get_format_erasure_in_quorum(&fms)
}
/// Tries to migrate format
/// Returns Ok(FormatV3) if migration succeeds, Err otherwise.
async fn try_migrate_format(disks: &[Option<DiskStore>], set_count: usize, set_drive_count: usize) -> Result<FormatV3> {
for disk in disks.iter().flatten() {
let data = match disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await {
Ok(d) if !d.is_empty() => d,
_ => continue,
};
let fm = FormatV3::try_from(data.as_ref()).map_err(|e| Error::other(format!("parse MinIO format: {e}")))?;
let first_set = fm
.erasure
.sets
.first()
.ok_or_else(|| Error::other("MinIO format: erasure.sets is empty"))?;
if fm.erasure.sets.len() != set_count || first_set.len() != set_drive_count {
debug!(
"MinIO format set count mismatch: got {}x{}, expected {}x{}",
fm.erasure.sets.len(),
first_set.len(),
set_count,
set_drive_count
);
continue;
}
if fm.erasure.version != FormatErasureVersion::V3 {
debug!("MinIO format erasure version not V3: {:?}", fm.erasure.version);
continue;
}
let mut fms = vec![None; disks.len()];
for (idx, disk_opt) in disks.iter().enumerate() {
if disk_opt.is_none() {
continue;
}
let set_idx = idx / set_drive_count;
let disk_idx = idx % set_drive_count;
if set_idx >= fm.erasure.sets.len() || disk_idx >= fm.erasure.sets[set_idx].len() {
continue;
}
let mut newfm = fm.clone();
newfm.erasure.this = fm.erasure.sets[set_idx][disk_idx];
fms[idx] = Some(newfm);
}
save_format_file_all(disks, &fms).await?;
return get_format_erasure_in_quorum(&fms);
}
Err(Error::other("no MinIO format to migrate"))
}
pub fn get_format_erasure_in_quorum(formats: &[Option<FormatV3>]) -> Result<FormatV3> {
let mut countmap = HashMap::new();
crates/ecstore/src/store_list_objects.rs
+1 -1
View File
@@ -492,7 +492,7 @@ impl ECStore {
}
pub async fn list_path(self: Arc<Self>, o: &ListPathOptions) -> Result<MetaCacheEntriesSortedResult> {
// warn!("list_path opt {:?}", &o);
// tracing::warn!("list_path opt {:?}", &o);
check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?;
// if opts.prefix.ends_with(SLASH_SEPARATOR) {
crates/ecstore/src/store_utils.rs
+4 -2
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::config::storageclass::STANDARD;
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use regex::Regex;
use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS};
use std::collections::HashMap;
@@ -45,7 +45,7 @@ pub fn clean_metadata_keys(metadata: &mut HashMap<String, String>, key_names: &[
// Check whether the bucket is the metadata bucket
fn is_meta_bucket(bucket_name: &str) -> bool {
bucket_name == RUSTFS_META_BUCKET
bucket_name == RUSTFS_META_BUCKET || bucket_name == MIGRATING_META_BUCKET
}
// Check whether the bucket is reserved
@@ -164,6 +164,8 @@ mod tests {
fn test_meta_bucket_is_invalid() {
assert!(is_reserved_or_invalid_bucket(RUSTFS_META_BUCKET, false));
assert!(is_reserved_or_invalid_bucket(RUSTFS_META_BUCKET, true));
assert!(is_reserved_or_invalid_bucket(MIGRATING_META_BUCKET, false));
assert!(is_reserved_or_invalid_bucket(MIGRATING_META_BUCKET, true));
}
#[test]
crates/ecstore/src/tier/tier.rs
+839 -37
View File
@@ -18,14 +18,16 @@
#![allow(unused_must_use)]
#![allow(clippy::all)]
use byteorder::{ByteOrder, LittleEndian};
use bytes::Bytes;
use http::HeaderMap;
use http::status::StatusCode;
use lazy_static::lazy_static;
use rand::{Rng, RngExt};
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, hash_map::Entry},
io::Cursor,
io::{self, Cursor},
sync::Arc,
time::Duration,
};
@@ -46,9 +48,9 @@ use crate::tier::{
use crate::{
StorageAPI,
config::com::{CONFIG_PREFIX, read_config},
disk::RUSTFS_META_BUCKET,
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
store::ECStore,
store_api::{ObjectOptions, PutObjReader},
store_api::{ObjectIO as _, ObjectOptions, PutObjReader},
};
use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
@@ -61,10 +63,17 @@ use super::{
const TIER_CFG_REFRESH: Duration = Duration::from_secs(15 * 60);
pub const TIER_CONFIG_FILE: &str = "tier-config.json";
const TIER_CONFIG_LEGACY_FILE: &str = "tier-config.json";
pub const TIER_CONFIG_FILE: &str = "tier-config.bin";
pub const TIER_CONFIG_FORMAT: u16 = 1;
pub const TIER_CONFIG_V1: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 1;
pub const TIER_CONFIG_VERSION: u16 = 2;
const EXTERNAL_TIER_TYPE_UNSUPPORTED: i32 = 0;
const EXTERNAL_TIER_TYPE_S3: i32 = 1;
const EXTERNAL_TIER_TYPE_AZURE: i32 = 2;
const EXTERNAL_TIER_TYPE_GCS: i32 = 3;
const EXTERNAL_TIER_TYPE_MINIO: i32 = 4;
const _TIER_CFG_REFRESH_AT_HDR: &str = "X-RustFS-TierCfg-RefreshedAt";
@@ -104,6 +113,609 @@ pub struct TierConfigMgr {
pub last_refreshed_at: OffsetDateTime,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfigMgr {
#[serde(rename = "Tiers")]
tiers: HashMap<String, ExternalTierConfig>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierConfig {
#[serde(rename = "Version")]
version: String,
#[serde(rename = "Type")]
tier_type: i32,
#[serde(rename = "Name")]
name: String,
#[serde(rename = "S3")]
s3: Option<ExternalTierS3>,
#[serde(rename = "Azure")]
azure: Option<ExternalTierAzure>,
#[serde(rename = "GCS")]
gcs: Option<ExternalTierGcs>,
#[serde(rename = "MinIO", alias = "Compatible")]
compatible_backend: Option<ExternalTierCompatible>,
#[serde(rename = "XTierType", skip_serializing_if = "Option::is_none")]
tier_type_hint: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierS3 {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccessKey")]
access_key: String,
#[serde(rename = "SecretKey")]
secret_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
#[serde(rename = "AWSRole")]
aws_role: bool,
#[serde(rename = "AWSRoleWebIdentityTokenFile")]
aws_role_web_identity_token_file: String,
#[serde(rename = "AWSRoleARN")]
aws_role_arn: String,
#[serde(rename = "AWSRoleSessionName")]
aws_role_session_name: String,
#[serde(rename = "AWSRoleDurationSeconds")]
aws_role_duration_seconds: i32,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalServicePrincipalAuth {
#[serde(rename = "TenantID")]
tenant_id: String,
#[serde(rename = "ClientID")]
client_id: String,
#[serde(rename = "ClientSecret")]
client_secret: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierAzure {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccountName")]
account_name: String,
#[serde(rename = "AccountKey")]
account_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
#[serde(rename = "SPAuth")]
sp_auth: ExternalServicePrincipalAuth,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierGcs {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "Creds")]
creds: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
#[serde(rename = "StorageClass")]
storage_class: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(default)]
struct ExternalTierCompatible {
#[serde(rename = "Endpoint")]
endpoint: String,
#[serde(rename = "AccessKey")]
access_key: String,
#[serde(rename = "SecretKey")]
secret_key: String,
#[serde(rename = "Bucket")]
bucket: String,
#[serde(rename = "Prefix")]
prefix: String,
#[serde(rename = "Region")]
region: String,
}
fn tier_config_path(file: &str) -> String {
format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, file)
}
fn tier_hint_for_type(tier_type: TierType) -> Option<&'static str> {
match tier_type {
TierType::RustFS => Some("rustfs"),
TierType::Aliyun => Some("aliyun"),
TierType::Tencent => Some("tencent"),
TierType::Huaweicloud => Some("huaweicloud"),
TierType::R2 => Some("r2"),
_ => None,
}
}
fn tier_type_from_hint(hint: Option<&str>) -> Option<TierType> {
match hint {
Some("rustfs") => Some(TierType::RustFS),
Some("aliyun") => Some(TierType::Aliyun),
Some("tencent") => Some(TierType::Tencent),
Some("huaweicloud") => Some(TierType::Huaweicloud),
Some("r2") => Some(TierType::R2),
_ => None,
}
}
fn external_tier_s3_from_internal(s3: &crate::tier::tier_config::TierS3) -> ExternalTierS3 {
ExternalTierS3 {
endpoint: s3.endpoint.clone(),
access_key: s3.access_key.clone(),
secret_key: s3.secret_key.clone(),
bucket: s3.bucket.clone(),
prefix: s3.prefix.clone(),
region: s3.region.clone(),
storage_class: s3.storage_class.clone(),
aws_role: s3.aws_role,
aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
aws_role_arn: s3.aws_role_arn.clone(),
aws_role_session_name: s3.aws_role_session_name.clone(),
aws_role_duration_seconds: s3.aws_role_duration_seconds,
}
}
fn external_tier_s3_from_compatible_payload(
endpoint: String,
access_key: String,
secret_key: String,
bucket: String,
prefix: String,
region: String,
) -> ExternalTierS3 {
ExternalTierS3 {
endpoint,
access_key,
secret_key,
bucket,
prefix,
region,
storage_class: String::new(),
aws_role: false,
aws_role_web_identity_token_file: String::new(),
aws_role_arn: String::new(),
aws_role_session_name: String::new(),
aws_role_duration_seconds: 0,
}
}
fn external_tier_alias_from_compatible_payload(
endpoint: String,
access_key: String,
secret_key: String,
bucket: String,
prefix: String,
region: String,
) -> ExternalTierCompatible {
ExternalTierCompatible {
endpoint,
access_key,
secret_key,
bucket,
prefix,
region,
}
}
fn to_external_tier_config(name: &str, tier: &TierConfig) -> io::Result<ExternalTierConfig> {
let mut out = ExternalTierConfig {
version: if tier.version.is_empty() {
"v1".to_string()
} else {
tier.version.clone()
},
name: if tier.name.is_empty() {
name.to_string()
} else {
tier.name.clone()
},
..Default::default()
};
match tier.tier_type {
TierType::S3 => {
let s3 = tier
.s3
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing s3 backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.s3 = Some(external_tier_s3_from_internal(s3));
}
TierType::Azure => {
let az = tier
.azure
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing azure backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_AZURE;
out.azure = Some(ExternalTierAzure {
endpoint: az.endpoint.clone(),
account_name: az.access_key.clone(),
account_key: az.secret_key.clone(),
bucket: az.bucket.clone(),
prefix: az.prefix.clone(),
region: az.region.clone(),
storage_class: az.storage_class.clone(),
sp_auth: ExternalServicePrincipalAuth {
tenant_id: az.sp_auth.tenant_id.clone(),
client_id: az.sp_auth.client_id.clone(),
client_secret: az.sp_auth.client_secret.clone(),
},
});
}
TierType::GCS => {
let gcs = tier
.gcs
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing gcs backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_GCS;
out.gcs = Some(ExternalTierGcs {
endpoint: gcs.endpoint.clone(),
creds: gcs.creds.clone(),
bucket: gcs.bucket.clone(),
prefix: gcs.prefix.clone(),
region: gcs.region.clone(),
storage_class: gcs.storage_class.clone(),
});
}
TierType::MinIO => {
let backend = tier
.minio
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_MINIO;
out.compatible_backend = Some(external_tier_alias_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::RustFS => {
let backend = tier
.rustfs
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Aliyun => {
let backend = tier
.aliyun
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Tencent => {
let backend = tier
.tencent
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Huaweicloud => {
let backend = tier
.huaweicloud
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::R2 => {
let backend = tier
.r2
.as_ref()
.ok_or_else(|| io::Error::other("tier config missing compatible backend payload"))?;
out.tier_type = EXTERNAL_TIER_TYPE_S3;
out.tier_type_hint = tier_hint_for_type(tier.tier_type.clone()).map(ToString::to_string);
out.s3 = Some(external_tier_s3_from_compatible_payload(
backend.endpoint.clone(),
backend.access_key.clone(),
backend.secret_key.clone(),
backend.bucket.clone(),
backend.prefix.clone(),
backend.region.clone(),
));
}
TierType::Unsupported => {
out.tier_type = EXTERNAL_TIER_TYPE_UNSUPPORTED;
}
}
Ok(out)
}
fn decode_legacy_s3_like(name: &str, ext: &ExternalTierConfig) -> io::Result<ExternalTierS3> {
if let Some(s3) = ext.s3.as_ref() {
return Ok(s3.clone());
}
if let Some(m) = ext.compatible_backend.as_ref() {
return Ok(external_tier_s3_from_compatible_payload(
m.endpoint.clone(),
m.access_key.clone(),
m.secret_key.clone(),
m.bucket.clone(),
m.prefix.clone(),
m.region.clone(),
));
}
Err(io::Error::other(format!("tier config '{name}' missing compatible backend payload")))
}
fn from_external_tier_config(name: String, ext: ExternalTierConfig) -> io::Result<TierConfig> {
let mut cfg = TierConfig {
version: if ext.version.is_empty() {
"v1".to_string()
} else {
ext.version.clone()
},
name: if ext.name.is_empty() { name.clone() } else { ext.name.clone() },
..Default::default()
};
let hinted = tier_type_from_hint(ext.tier_type_hint.as_deref());
let tier_type = if let Some(h) = hinted {
h
} else {
match ext.tier_type {
EXTERNAL_TIER_TYPE_S3 => TierType::S3,
EXTERNAL_TIER_TYPE_AZURE => TierType::Azure,
EXTERNAL_TIER_TYPE_GCS => TierType::GCS,
EXTERNAL_TIER_TYPE_MINIO => TierType::MinIO,
_ => TierType::Unsupported,
}
};
cfg.tier_type = tier_type.clone();
match tier_type {
TierType::S3 => {
let s3 = ext
.s3
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing s3 backend payload", cfg.name)))?;
cfg.s3 = Some(crate::tier::tier_config::TierS3 {
name: cfg.name.clone(),
endpoint: s3.endpoint.clone(),
access_key: s3.access_key.clone(),
secret_key: s3.secret_key.clone(),
bucket: s3.bucket.clone(),
prefix: s3.prefix.clone(),
region: s3.region.clone(),
storage_class: s3.storage_class.clone(),
aws_role: s3.aws_role,
aws_role_web_identity_token_file: s3.aws_role_web_identity_token_file.clone(),
aws_role_arn: s3.aws_role_arn.clone(),
aws_role_session_name: s3.aws_role_session_name.clone(),
aws_role_duration_seconds: s3.aws_role_duration_seconds,
});
}
TierType::Azure => {
let az = ext
.azure
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing azure backend payload", cfg.name)))?;
cfg.azure = Some(crate::tier::tier_config::TierAzure {
name: cfg.name.clone(),
endpoint: az.endpoint.clone(),
access_key: az.account_name.clone(),
secret_key: az.account_key.clone(),
bucket: az.bucket.clone(),
prefix: az.prefix.clone(),
region: az.region.clone(),
storage_class: az.storage_class.clone(),
sp_auth: crate::tier::tier_config::ServicePrincipalAuth {
tenant_id: az.sp_auth.tenant_id.clone(),
client_id: az.sp_auth.client_id.clone(),
client_secret: az.sp_auth.client_secret.clone(),
},
});
}
TierType::GCS => {
let gcs = ext
.gcs
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing gcs backend payload", cfg.name)))?;
cfg.gcs = Some(crate::tier::tier_config::TierGCS {
name: cfg.name.clone(),
endpoint: gcs.endpoint.clone(),
creds: gcs.creds.clone(),
bucket: gcs.bucket.clone(),
prefix: gcs.prefix.clone(),
region: gcs.region.clone(),
storage_class: gcs.storage_class.clone(),
});
}
TierType::MinIO => {
let m = ext
.compatible_backend
.as_ref()
.ok_or_else(|| io::Error::other(format!("tier config '{}' missing compatible backend payload", cfg.name)))?;
cfg.minio = Some(crate::tier::tier_config::TierMinIO {
name: cfg.name.clone(),
endpoint: m.endpoint.clone(),
access_key: m.access_key.clone(),
secret_key: m.secret_key.clone(),
bucket: m.bucket.clone(),
prefix: m.prefix.clone(),
region: m.region.clone(),
});
}
TierType::RustFS => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.rustfs = Some(crate::tier::tier_config::TierRustFS {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
storage_class: m.storage_class,
});
}
TierType::Aliyun => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.aliyun = Some(crate::tier::tier_config::TierAliyun {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Tencent => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.tencent = Some(crate::tier::tier_config::TierTencent {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Huaweicloud => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.huaweicloud = Some(crate::tier::tier_config::TierHuaweicloud {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::R2 => {
let m = decode_legacy_s3_like(&cfg.name, &ext)?;
cfg.r2 = Some(crate::tier::tier_config::TierR2 {
name: cfg.name.clone(),
endpoint: m.endpoint,
access_key: m.access_key,
secret_key: m.secret_key,
bucket: m.bucket,
prefix: m.prefix,
region: m.region,
});
}
TierType::Unsupported => {}
}
Ok(cfg)
}
fn encode_external_tiering_config_blob(cfg: &TierConfigMgr) -> io::Result<Bytes> {
let mut tiers = HashMap::with_capacity(cfg.tiers.len());
for (name, tier_cfg) in &cfg.tiers {
tiers.insert(name.clone(), to_external_tier_config(name, tier_cfg)?);
}
let payload = rmp_serde::to_vec(&ExternalTierConfigMgr { tiers })
.map_err(|err| io::Error::other(format!("serialize tier config payload failed: {err}")))?;
let mut data = Vec::with_capacity(4 + payload.len());
let mut format = [0u8; 2];
LittleEndian::write_u16(&mut format, TIER_CONFIG_FORMAT);
data.extend_from_slice(&format);
let mut version = [0u8; 2];
LittleEndian::write_u16(&mut version, TIER_CONFIG_VERSION);
data.extend_from_slice(&version);
data.extend_from_slice(&payload);
Ok(Bytes::from(data))
}
fn decode_external_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
if data.len() <= 4 {
return Err(io::Error::other("tierConfigInit: no data"));
}
let format = LittleEndian::read_u16(&data[0..2]);
if format != TIER_CONFIG_FORMAT {
return Err(io::Error::other(format!("tierConfigInit: unknown format: {format}")));
}
let version = LittleEndian::read_u16(&data[2..4]);
if version != TIER_CONFIG_V1 && version != TIER_CONFIG_VERSION {
return Err(io::Error::other(format!("tierConfigInit: unknown version: {version}")));
}
let external: ExternalTierConfigMgr =
rmp_serde::from_slice(&data[4..]).map_err(|err| io::Error::other(format!("decode tier config payload failed: {err}")))?;
let mut tiers = HashMap::with_capacity(external.tiers.len());
for (name, ext_cfg) in external.tiers {
tiers.insert(name.clone(), from_external_tier_config(name, ext_cfg)?);
}
Ok(TierConfigMgr {
driver_cache: HashMap::new(),
tiers,
last_refreshed_at: OffsetDateTime::now_utc(),
})
}
fn decode_tiering_config_blob(data: &[u8]) -> io::Result<TierConfigMgr> {
if let Ok(cfg) = TierConfigMgr::unmarshal(data) {
return Ok(cfg);
}
decode_external_tiering_config_blob(data)
}
impl TierConfigMgr {
pub fn new() -> Arc<RwLock<Self>> {
Arc::new(RwLock::new(Self {
@@ -285,12 +897,12 @@ impl TierConfigMgr {
rustfs.secret_key = creds.secret_key;
}
TierType::MinIO => {
let mut minio = tier_config.minio.as_mut().expect("err");
let compatible_backend = tier_config.minio.as_mut().expect("err");
if creds.access_key == "" || creds.secret_key == "" {
return Err(ERR_TIER_MISSING_CREDENTIALS.clone());
}
minio.access_key = creds.access_key;
minio.secret_key = creds.secret_key;
compatible_backend.access_key = creds.access_key;
compatible_backend.secret_key = creds.secret_key;
}
TierType::Aliyun => {
let mut aliyun = tier_config.aliyun.as_mut().expect("err");
@@ -401,9 +1013,8 @@ impl TierConfigMgr {
}
pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
let data = self.marshal()?;
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
let data = encode_external_tiering_config_blob(self)?;
let config_file = tier_config_path(TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
@@ -483,38 +1094,229 @@ async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierC
#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
let data = read_config(api.clone(), config_file.as_str()).await;
if let Err(err) = data {
if is_err_config_not_found(&err) {
warn!("config not found, start to init");
let cfg = new_and_save_tiering_config(api).await?;
return Ok(cfg);
} else {
error!("read config err {:?}", &err);
return Err(std::io::Error::other(err));
}
}
let cfg;
let version = 1; //LittleEndian::read_u16(&data[2..4]);
match version {
TIER_CONFIG_V1/* | TIER_CONFIG_VERSION */ => {
cfg = match TierConfigMgr::unmarshal(&data.unwrap()) {
Ok(cfg) => cfg,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
let config_file = tier_config_path(TIER_CONFIG_FILE);
match read_config(api.clone(), config_file.as_str()).await {
Ok(data) => decode_tiering_config_blob(&data),
Err(err) if is_err_config_not_found(&err) => {
let legacy_file = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_config(api.clone(), legacy_file.as_str()).await {
Ok(data) => {
let cfg = TierConfigMgr::unmarshal(&data)?;
let normalized = encode_external_tiering_config_blob(&cfg)?;
let _ = api
.put_object(
RUSTFS_META_BUCKET,
&config_file,
&mut PutObjReader::from_vec(normalized.to_vec()),
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await;
Ok(cfg)
}
};
Err(legacy_err) if is_err_config_not_found(&legacy_err) => {
warn!("config not found, start to init");
new_and_save_tiering_config(api).await.map_err(io::Error::other)
}
Err(legacy_err) => Err(io::Error::other(legacy_err)),
}
}
_ => {
return Err(std::io::Error::other(format!("tierConfigInit: unknown version: {}", version)));
Err(err) => {
error!("read config err {:?}", &err);
Err(io::Error::other(err))
}
}
}
Ok(cfg)
async fn read_tier_config_from_bucket<S: StorageAPI>(
api: Arc<S>,
bucket: &str,
path: &str,
opts: &ObjectOptions,
) -> io::Result<Option<Vec<u8>>> {
let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await {
Ok(v) => v,
Err(err) if is_err_config_not_found(&err) => return Ok(None),
Err(err) => return Err(io::Error::other(err)),
};
let data = rd.read_all().await.map_err(io::Error::other)?;
if data.is_empty() {
return Ok(None);
}
Ok(Some(data))
}
async fn write_tier_config_to_rustfs<S: StorageAPI>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()> {
api.put_object(
RUSTFS_META_BUCKET,
path,
&mut PutObjReader::from_vec(data.to_vec()),
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await
.map_err(io::Error::other)?;
Ok(())
}
pub async fn try_migrate_tiering_config<S: StorageAPI>(api: Arc<S>) {
let target_path = tier_config_path(TIER_CONFIG_FILE);
if api
.get_object_info(RUSTFS_META_BUCKET, &target_path, &ObjectOptions::default())
.await
.is_ok()
{
debug!("tier config already exists in RustFS metadata bucket, skip migration");
return;
}
let opts = ObjectOptions {
max_parity: true,
no_lock: true,
..Default::default()
};
let legacy_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
match read_tier_config_from_bucket(api.clone(), RUSTFS_META_BUCKET, &legacy_path, &opts).await {
Ok(Some(data)) => match TierConfigMgr::unmarshal(&data)
.and_then(|cfg| encode_external_tiering_config_blob(&cfg).map_err(io::Error::other))
{
Ok(out) => {
if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
info!("Migrated tier config from legacy RustFS metadata format");
return;
}
}
Err(err) => warn!("legacy tier config is incompatible, skip local migration: {}", err),
},
Ok(None) => {}
Err(err) => warn!("read legacy local tier config failed: {}", err),
}
match read_tier_config_from_bucket(api.clone(), MIGRATING_META_BUCKET, &target_path, &opts).await {
Ok(Some(data)) => match decode_tiering_config_blob(&data).and_then(|cfg| encode_external_tiering_config_blob(&cfg)) {
Ok(out) => {
if write_tier_config_to_rustfs(api.clone(), &target_path, out).await.is_ok() {
info!("Migrated compatible tier config from migrating metadata bucket");
}
}
Err(err) => warn!("migrating tier config is incompatible, skip migration: {}", err),
},
Ok(None) => {}
Err(err) => warn!("read migrating tier config failed: {}", err),
}
}
pub fn is_err_config_not_found(err: &StorageError) -> bool {
matches!(err, StorageError::ObjectNotFound(_, _)) || err == &StorageError::ConfigNotFound
matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::BucketNotFound(_)) || err == &StorageError::ConfigNotFound
}
#[cfg(test)]
mod tests {
use super::*;
fn build_s3_tier(name: &str) -> TierConfig {
TierConfig {
version: "v1".to_string(),
tier_type: TierType::S3,
name: name.to_string(),
s3: Some(crate::tier::tier_config::TierS3 {
name: name.to_string(),
endpoint: "https://example-s3.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-a".to_string(),
prefix: "prefix-a".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
aws_role: false,
aws_role_web_identity_token_file: String::new(),
aws_role_arn: String::new(),
aws_role_session_name: String::new(),
aws_role_duration_seconds: 0,
}),
..Default::default()
}
}
#[test]
fn test_tiering_external_blob_roundtrip_for_standard_type() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
assert_eq!(&bytes[0..2], &TIER_CONFIG_FORMAT.to_le_bytes());
assert_eq!(&bytes[2..4], &TIER_CONFIG_VERSION.to_le_bytes());
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
let tier = decoded.tiers.get("COLD-A").expect("tier should exist");
assert_eq!(tier.tier_type.as_lowercase(), "s3");
assert_eq!(tier.s3.as_ref().expect("s3 should exist").endpoint, "https://example-s3.invalid");
}
#[test]
fn test_tiering_external_blob_roundtrip_for_extended_type_hint() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert(
"COLD-B".to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::RustFS,
name: "COLD-B".to_string(),
rustfs: Some(crate::tier::tier_config::TierRustFS {
name: "COLD-B".to_string(),
endpoint: "https://example-compat.invalid".to_string(),
access_key: "ak".to_string(),
secret_key: "sk".to_string(),
bucket: "bucket-b".to_string(),
prefix: "prefix-b".to_string(),
region: "us-east-1".to_string(),
storage_class: "STANDARD".to_string(),
}),
..Default::default()
},
);
let bytes = encode_external_tiering_config_blob(&cfg).expect("encode should succeed");
let decoded = decode_external_tiering_config_blob(&bytes).expect("decode should succeed");
let tier = decoded.tiers.get("COLD-B").expect("tier should exist");
assert_eq!(tier.tier_type.as_lowercase(), "rustfs");
assert_eq!(
tier.rustfs.as_ref().expect("backend should exist").endpoint,
"https://example-compat.invalid"
);
}
#[test]
fn test_decode_tiering_config_blob_accepts_legacy_json() {
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
last_refreshed_at: OffsetDateTime::now_utc(),
};
cfg.tiers.insert("COLD-A".to_string(), build_s3_tier("COLD-A"));
let data = serde_json::to_vec(&cfg).expect("legacy json should encode");
let decoded = decode_tiering_config_blob(&data).expect("legacy json should decode");
assert_eq!(
decoded
.tiers
.get("COLD-A")
.and_then(|tier| tier.s3.as_ref())
.map(|s3| s3.bucket.as_str()),
Some("bucket-a")
);
}
}
crates/ecstore/src/tier/tier_config.rs
+8 -8
View File
@@ -146,7 +146,7 @@ impl Clone for TierConfig {
fn clone(&self) -> TierConfig {
let mut s3 = None;
let mut r = None;
let mut m = None;
let mut compatible_backend = None;
let mut aliyun = None;
let mut tencent = None;
let mut huaweicloud = None;
@@ -165,9 +165,9 @@ impl Clone for TierConfig {
r = Some(r_);
}
TierType::MinIO => {
let mut m_ = self.minio.as_ref().expect("err").clone();
m_.secret_key = "REDACTED".to_string();
m = Some(m_);
let mut compatible_backend_ = self.minio.as_ref().expect("err").clone();
compatible_backend_.secret_key = "REDACTED".to_string();
compatible_backend = Some(compatible_backend_);
}
TierType::Aliyun => {
let mut aliyun_ = self.aliyun.as_ref().expect("err").clone();
@@ -207,7 +207,7 @@ impl Clone for TierConfig {
name: self.name.clone(),
s3,
rustfs: r,
minio: m,
minio: compatible_backend,
aliyun,
tencent,
huaweicloud,
@@ -408,7 +408,7 @@ impl TierMinIO {
if name.is_empty() {
return Err(std::io::Error::other(ERR_TIER_NAME_EMPTY));
}
let m = TierMinIO {
let backend = TierMinIO {
access_key: access_key.to_string(),
secret_key: secret_key.to_string(),
bucket: bucket.to_string(),
@@ -417,7 +417,7 @@ impl TierMinIO {
};
for option in options {
let option = option(m.clone());
let option = option(backend.clone());
let option = *option;
option?;
}
@@ -426,7 +426,7 @@ impl TierMinIO {
version: C_TIER_CONFIG_VER.to_string(),
tier_type: TierType::MinIO,
name: name.to_string(),
minio: Some(m),
minio: Some(backend),
..Default::default()
})
}
crates/ecstore/tests/legacy_bitrot_read_test.rs
+238
View File
@@ -0,0 +1,238 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Integration test: verify create_bitrot_reader with HighwayHash256SLegacy reads main-version objects.
//!
//! Supports both EC (part files) and inline objects. Reads xl.meta to compute params.
//! Uses create_bitrot_reader only (no ECStore/get_object_reader) to isolate bitrot compatibility.
//!
//! Run from workspace root:
//! cargo test -p rustfs-ecstore test_legacy_bitrot_read -- --nocapture
//!
//! Environment:
//! RUSTFS_LEGACY_TEST_ROOT - Test data root (default: workspace root)
//! RUSTFS_LEGACY_TEST_DISK - Disk name under root (default: "test1" for rustfs, "test" for minio)
//! RUSTFS_SKIP_LEGACY_TEST - Set to 1 to skip
//!
//! For MinIO data: RUSTFS_LEGACY_TEST_ROOT=/path/to/minio (disk "test" and .minio.sys auto-detected).
use rustfs_ecstore::bitrot::create_bitrot_reader;
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::disk::{DiskOption, STORAGE_FORMAT_FILE, new_disk};
use rustfs_filemeta::{FileInfoOpts, get_file_info};
use rustfs_utils::HashAlgorithm;
use std::path::PathBuf;
use tokio::fs;
fn workspace_root() -> PathBuf {
let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".into());
PathBuf::from(&manifest)
.ancestors()
.nth(2)
.unwrap_or(std::path::Path::new("."))
.to_path_buf()
}
fn legacy_test_data_exists() -> bool {
if std::env::var("RUSTFS_SKIP_LEGACY_TEST").unwrap_or_default() == "1" {
return false;
}
let root = workspace_root();
let format_path = root.join("test1/.rustfs.sys/format.json");
let ktvzip_meta = root.join("test1/vvvv/ktvzip.tar.gz").join(STORAGE_FORMAT_FILE);
let path_traversal_meta = root.join("test1/vvvv/path_traversal.md").join(STORAGE_FORMAT_FILE);
format_path.exists() && (ktvzip_meta.exists() || path_traversal_meta.exists())
}
async fn run_legacy_bitrot_test_for_object(root: &std::path::Path, disk_name: &str, bucket: &str, object: &str) -> bool {
let xl_meta_path = root.join(disk_name).join(bucket).join(object).join(STORAGE_FORMAT_FILE);
if !xl_meta_path.exists() {
eprintln!("xl_meta_path not found: {:?}", xl_meta_path);
return false;
}
let buf = match fs::read(&xl_meta_path).await {
Ok(b) => b,
Err(_) => {
eprintln!("Failed to read xl_meta_path: {:?}", xl_meta_path);
return false;
}
};
let fi = match get_file_info(
&buf,
bucket,
object,
"",
FileInfoOpts {
data: true, // need inline data for inline objects
include_free_versions: false,
},
) {
Ok(f) => f,
Err(_) => {
eprintln!("Failed to get file info: {:?}", xl_meta_path);
return false;
}
};
if fi.deleted || fi.parts.is_empty() {
eprintln!("File is deleted or has no parts: {:?}", xl_meta_path);
return false;
}
let shard_size = fi.erasure.shard_size();
let part_number = 1;
let checksum_info = fi.erasure.get_checksum_info(part_number);
let checksum_algo = if fi.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm
};
eprintln!("fi.inline_data(): {:?}", fi.inline_data());
eprintln!("fi.is_remote(): {:?}", fi.metadata);
// Inline path: use fi.data
if fi.inline_data() {
let inline_bytes = match &fi.data {
Some(b) if !b.is_empty() => b.as_ref(),
_ => {
eprintln!("Inline data is empty: {:?}", xl_meta_path);
return false;
}
};
let read_length = fi.size as usize;
if read_length == 0 {
eprintln!("Read length is 0: {:?}", xl_meta_path);
return false;
}
let mut reader = match create_bitrot_reader(
Some(inline_bytes),
None,
bucket,
"",
0,
read_length,
shard_size,
checksum_algo.clone(),
false,
)
.await
{
Ok(Some(r)) => r,
_ => {
eprintln!("Failed to create bitrot reader for inline data: {:?}", xl_meta_path);
return false;
}
};
let mut buf = vec![0u8; shard_size];
match reader.read(&mut buf).await {
Ok(n) if n > 0 => {
eprintln!("Successfully read {} bytes (inline) via create_bitrot_reader with {:?}", n, checksum_algo);
return true;
}
_ => {
eprintln!("Failed to read inline data: {:?}", xl_meta_path);
return false;
}
}
}
// EC path: use disk + part file
let data_dir = match fi.data_dir {
Some(d) => d,
None => {
eprintln!("Data dir is empty: {:?}", xl_meta_path);
return false;
}
};
let path = format!("{object}/{data_dir}/part.{}", part_number);
let part_path = root.join(disk_name).join(bucket).join(&path);
if !part_path.exists() {
eprintln!("Part file not found: {:?}", part_path);
return false;
}
let disk_path = root.join(disk_name);
let path_str = disk_path.to_str().expect("path");
let mut ep = Endpoint::try_from(path_str).expect("endpoint");
ep.set_pool_index(0);
ep.set_set_index(0);
ep.set_disk_index(0);
let opt = DiskOption {
cleanup: false,
health_check: false,
};
let disk = match new_disk(&ep, &opt).await {
Ok(d) => d,
Err(_) => {
eprintln!("Failed to create disk: {:?}", disk_path);
return false;
}
};
let read_length = shard_size;
let mut reader =
match create_bitrot_reader(None, Some(&disk), bucket, &path, 0, read_length, shard_size, checksum_algo.clone(), false)
.await
{
Ok(Some(r)) => r,
_ => {
eprintln!("Failed to create bitrot reader for EC part: {:?}", part_path);
return false;
}
};
let mut buf = vec![0u8; shard_size];
match reader.read(&mut buf).await {
Ok(n) if n > 0 => {
eprintln!(
"Successfully read {} bytes (EC part) via create_bitrot_reader with {:?}",
n, checksum_algo
);
true
}
_ => {
eprintln!("Failed to read EC part: {:?}", part_path);
false
}
}
}
#[tokio::test]
async fn test_legacy_bitrot_read() {
if !legacy_test_data_exists() {
eprintln!("Skipping legacy bitrot test: test1/vvvv xl.meta or RUSTFS_SKIP_LEGACY_TEST");
return;
}
// let root = workspace_root();
let root = PathBuf::from("/Users/weisd/project/minio");
let disk_name = "test";
let bucket = "vvvv";
// Try both EC (part files) and inline objects
let ok = run_legacy_bitrot_test_for_object(&root, disk_name, bucket, "ktvzip.tar.gz").await;
eprintln!("ok: {:?}", ok);
assert!(ok, "create_bitrot_reader failed for both ktvzip.tar.gz and path_traversal.md");
let ok = run_legacy_bitrot_test_for_object(&root, disk_name, bucket, "path_traversal.md").await;
assert!(ok, "create_bitrot_reader failed for path_traversal.md");
}
crates/filemeta/examples/dump_versions.rs
+24
View File
@@ -0,0 +1,24 @@
use rustfs_filemeta::FileMeta;
use std::{env, fs, path::PathBuf};
fn main() {
let path = env::args()
.nth(1)
.map(PathBuf::from)
.expect("usage: dump_versions <xl.meta path>");
let data = fs::read(&path).expect("read xl.meta");
let meta = FileMeta::load(&data).expect("load xl.meta");
let versions = meta
.into_file_info_versions("debug-bucket", "debug-object", true)
.expect("decode versions");
println!("path: {}", path.display());
println!("versions: {}", versions.versions.len());
for (idx, version) in versions.versions.iter().enumerate() {
println!(
"#{idx}: version_id={:?} deleted={} mark_deleted={} is_latest={} size={} mod_time={:?}",
version.version_id, version.deleted, version.mark_deleted, version.is_latest, version.size, version.mod_time
);
}
}
crates/filemeta/src/fileinfo.rs
+17 -22
View File
@@ -16,7 +16,10 @@ use crate::{Error, ReplicationState, ReplicationStatusType, Result, TRANSITION_C
use bytes::Bytes;
use rmp_serde::Serializer;
use rustfs_utils::HashAlgorithm;
use rustfs_utils::http::headers::{RESERVED_METADATA_PREFIX_LOWER, RUSTFS_HEALING};
use rustfs_utils::http::{
SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_HEALING, SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER,
SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, insert_str,
};
use s3s::dto::{RestoreStatus, Timestamp};
use s3s::header::X_AMZ_RESTORE;
use serde::{Deserialize, Serialize};
@@ -236,6 +239,8 @@ pub struct FileInfo {
// Combined checksum when object was uploaded
pub checksum: Option<Bytes>,
pub versioned: bool,
/// True when version meta was parsed via rmp_serde fallback (legacy format).
pub uses_legacy_checksum: bool,
}
impl FileInfo {
@@ -367,58 +372,48 @@ impl FileInfo {
}
pub fn set_healing(&mut self) {
self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string());
insert_str(&mut self.metadata, SUFFIX_HEALING, "true".to_string());
}
pub fn set_tier_free_version_id(&mut self, version_id: &str) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_ID}"), version_id.to_string());
insert_str(&mut self.metadata, SUFFIX_TIER_FV_ID, version_id.to_string());
}
pub fn tier_free_version_id(&self) -> String {
self.metadata[&format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_ID}")].clone()
get_str(&self.metadata, SUFFIX_TIER_FV_ID).unwrap_or_default()
}
pub fn set_tier_free_version(&mut self) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_MARKER}"), "".to_string());
insert_str(&mut self.metadata, SUFFIX_TIER_FV_MARKER, "".to_string());
}
pub fn set_skip_tier_free_version(&mut self) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_SKIP_FV_ID}"), "".to_string());
insert_str(&mut self.metadata, SUFFIX_TIER_SKIP_FV_ID, "".to_string());
}
pub fn skip_tier_free_version(&self) -> bool {
self.metadata
.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_SKIP_FV_ID}"))
contains_key_str(&self.metadata, SUFFIX_TIER_SKIP_FV_ID)
}
pub fn tier_free_version(&self) -> bool {
self.metadata
.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_MARKER}"))
contains_key_str(&self.metadata, SUFFIX_TIER_FV_MARKER)
}
pub fn set_inline_data(&mut self) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}inline-data").to_owned(), "true".to_owned());
insert_str(&mut self.metadata, SUFFIX_INLINE_DATA, "true".to_string());
}
pub fn set_data_moved(&mut self) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}data-moved").to_owned(), "true".to_owned());
insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, "true".to_string());
}
pub fn inline_data(&self) -> bool {
self.metadata
.contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}inline-data").as_str())
&& !self.is_remote()
contains_key_str(&self.metadata, SUFFIX_INLINE_DATA) && !self.is_remote()
}
/// Check if the object is compressed
pub fn is_compressed(&self) -> bool {
self.metadata
.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression"))
contains_key_str(&self.metadata, SUFFIX_COMPRESSION)
}
/// Check if the object is remote (transitioned to another tier)
crates/filemeta/src/filemeta.rs
+185 -95
View File
@@ -13,17 +13,20 @@
// limitations under the License.
use crate::{
ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, ObjectPartInfo, RawFileInfo, ReplicationState,
ReplicationStatusType, Result, TIER_FV_ID, TIER_FV_MARKER, VersionPurgeStatusType, is_restored_object_on_disk,
ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, NULL_VERSION_ID, ObjectPartInfo, RawFileInfo,
ReplicationState, ReplicationStatusType, Result, VersionPurgeStatusType, is_restored_object_on_disk,
replication_statuses_map, version_purge_statuses_map,
};
use byteorder::ByteOrder;
use bytes::Bytes;
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
use rustfs_utils::http::headers::{
self, AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_RESTORE_EXPIRY_DAYS,
AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER,
VERSION_PURGE_STATUS_KEY,
AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE,
AMZ_STORAGE_CLASS,
};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS,
SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, has_internal_suffix, insert_bytes,
is_internal_key,
};
use s3s::header::X_AMZ_RESTORE;
use serde::{Deserialize, Serialize};
@@ -48,7 +51,8 @@ pub static XL_FILE_HEADER: [u8; 4] = [b'X', b'L', b'2', b' '];
static XL_FILE_VERSION_MAJOR: u16 = 1;
static XL_FILE_VERSION_MINOR: u16 = 3;
static XL_HEADER_VERSION: u8 = 3;
pub static XL_META_VERSION: u8 = 2;
pub static XL_META_VERSION: u8 = 3;
/// Legacy format (main branch): meta_ver=2 with file/header versions 1.3.3.
static XXHASH_SEED: u64 = 0;
const XL_FLAG_FREE_VERSION: u8 = 1 << 0;
@@ -58,6 +62,18 @@ const _XL_FLAG_INLINE_DATA: u8 = 1 << 2;
const META_DATA_READ_DEFAULT: usize = 4 << 10;
const MSGP_UINT32_SIZE: usize = 5;
/// Max object versions per object, default is 10000
const DEFAULT_OBJECT_MAX_VERSIONS: usize = 10000;
/// Returns the inline data map key for a version_id. "null" for null version.
pub(crate) fn data_key_for_version(version_id: Option<Uuid>) -> String {
if version_id.is_none() || version_id == Some(Uuid::nil()) {
NULL_VERSION_ID.to_string()
} else {
version_id.unwrap_or_default().to_string()
}
}
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
@@ -68,8 +84,14 @@ pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
pub const TRANSITION_TIER: &str = "transition-tier";
/// Returns true if the key is a transient internal flag that should not be persisted to meta_sys.
pub fn is_skip_meta_key(key: &str) -> bool {
has_internal_suffix(key, SUFFIX_HEALING) || has_internal_suffix(key, SUFFIX_DATA_MOV)
}
mod codec;
mod inline_data;
mod msgp_decode;
mod validation;
mod version;
@@ -171,11 +193,10 @@ impl FileMeta {
for (k, v) in fi.metadata.iter() {
// Split metadata into meta_user and meta_sys based on prefix
// This logic must match From<FileInfo> for MetaObject
if k.len() > RESERVED_METADATA_PREFIX.len()
&& (k.starts_with(RESERVED_METADATA_PREFIX) || k.starts_with(RESERVED_METADATA_PREFIX_LOWER))
{
let is_system = is_internal_key(k);
if is_system {
// Skip internal flags that shouldn't be persisted
if k == headers::X_RUSTFS_HEALING || k == headers::X_RUSTFS_DATA_MOV {
if is_skip_meta_key(k) {
continue;
}
// Insert into meta_sys
@@ -221,18 +242,26 @@ impl FileMeta {
}
pub fn add_version(&mut self, mut fi: FileInfo) -> Result<()> {
// empty version_id means "null" (versioning disabled/suspended)
if fi.version_id.is_none() {
fi.version_id = Some(Uuid::nil());
}
let version_key = data_key_for_version(fi.version_id);
let mut next_data = self.data.clone();
if let Some(ref data) = fi.data {
let key = fi.version_id.unwrap_or_default().to_string();
self.data.replace(&key, data.to_vec())?;
next_data.replace(&version_key, data.to_vec())?;
} else {
let _ = next_data.remove_key(&version_key)?;
}
let version = FileMetaVersion::from(fi);
self.add_version_filemata(version)
self.add_version_filemata(version)?;
self.data = next_data;
Ok(())
}
pub fn add_version_filemata(&mut self, version: FileMetaVersion) -> Result<()> {
@@ -240,13 +269,12 @@ impl FileMeta {
return Err(Error::other("file meta version invalid"));
}
// TODO: make it configurable
// 1000 is the limit of versions
// if self.versions.len() + 1 > 1000 {
// return Err(Error::other(
// "You've exceeded the limit on the number of versions you can create on this object",
// ));
// }
// check max versions limit
if self.versions.len() + 1 > DEFAULT_OBJECT_MAX_VERSIONS {
return Err(Error::other(
"You've exceeded the limit on the number of versions you can create on this object",
));
}
if self.versions.is_empty() {
self.versions.push(FileMetaShallowVersion::try_from(version)?);
@@ -255,21 +283,44 @@ impl FileMeta {
let vid = version.get_version_id();
if let Some(fidx) = self.versions.iter().position(|v| v.header.version_id == vid) {
// Match existing version for replace; null version: None and Some(nil) are equivalent
let matches = |h: &Option<Uuid>| {
let v_null = vid.is_none() || vid == Some(Uuid::nil());
let h_null = h.is_none() || *h == Some(Uuid::nil());
(v_null && h_null) || (vid == *h)
};
if let Some(fidx) = self.versions.iter().position(|v| matches(&v.header.version_id)) {
return self.set_idx(fidx, version);
}
// append placeholder to find insert position
let placeholder = FileMetaShallowVersion {
header: FileMetaVersionHeader {
mod_time: None, // None sorts before any real mod_time
..Default::default()
},
meta: Vec::new(),
};
self.versions.push(placeholder);
let mod_time = version.get_mod_time();
let new_shallow = FileMetaShallowVersion::try_from(version)?;
for (idx, exist) in self.versions.iter().enumerate() {
if let Some(ref ex_mt) = exist.header.mod_time
&& let Some(ref in_md) = mod_time
&& ex_mt <= in_md
{
self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?);
let ex_mt = exist.header.mod_time;
let insert_here = match (ex_mt, mod_time) {
(None, _) => true, // placeholder: always insert before
(Some(em), Some(nm)) => em <= nm,
(Some(_), None) => false,
};
if insert_here {
self.versions.insert(idx, new_shallow);
self.versions.pop(); // remove placeholder
return Ok(());
}
}
self.versions.pop(); // remove placeholder on fallback
Err(Error::other("add_version failed"))
// if !ver.valid() {
@@ -343,8 +394,9 @@ impl FileMeta {
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
{
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_status.clone())
@@ -353,8 +405,9 @@ impl FileMeta {
.as_bytes()
.to_vec(),
);
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-timestamp"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
@@ -363,8 +416,9 @@ impl FileMeta {
.to_vec(),
);
} else {
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_status_internal.clone().unwrap_or_default())
@@ -372,8 +426,9 @@ impl FileMeta {
.as_bytes()
.to_vec(),
);
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
@@ -387,15 +442,14 @@ impl FileMeta {
if !fi.version_purge_status().is_empty()
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
{
delete_marker.meta_sys.insert(
VERSION_PURGE_STATUS_KEY.to_string(),
fi.replication_state_internal
.as_ref()
.map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
let value = fi
.replication_state_internal
.as_ref()
.map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec();
insert_bytes(&mut delete_marker.meta_sys, SUFFIX_PURGESTATUS, value);
}
if let Some(delete_marker) = ventry.delete_marker.as_mut() {
@@ -433,8 +487,9 @@ impl FileMeta {
if let Some(delete_marker) = v.delete_marker.as_mut() {
if !fi.delete_marker_replication_status().is_empty() {
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_status.clone())
@@ -443,8 +498,9 @@ impl FileMeta {
.as_bytes()
.to_vec(),
);
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-timestamp"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
@@ -453,8 +509,9 @@ impl FileMeta {
.to_vec(),
);
} else {
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_status_internal.clone().unwrap_or_default())
@@ -462,8 +519,9 @@ impl FileMeta {
.as_bytes()
.to_vec(),
);
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
@@ -502,15 +560,14 @@ impl FileMeta {
let mut v = self.get_idx(i)?;
if let Some(obj) = v.object.as_mut() {
obj.meta_sys.insert(
VERSION_PURGE_STATUS_KEY.to_string(),
fi.replication_state_internal
.as_ref()
.map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
let value = fi
.replication_state_internal
.as_ref()
.map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec();
insert_bytes(&mut obj.meta_sys, SUFFIX_PURGESTATUS, value);
for (k, v) in fi
.replication_state_internal
.as_ref()
@@ -617,13 +674,14 @@ impl FileMeta {
// TODO: freeVersion
if header.free_version() {
non_free_versions -= 1;
if include_free_versions && found_free_version.is_none() {
let mut found_free_fi = FileMetaVersion::default();
if found_free_fi.unmarshal_msg(&ver.meta).is_ok() && found_free_fi.version_type != VersionType::Invalid {
let mut free_fi = found_free_fi.into_fileinfo(volume, path, all_parts);
free_fi.is_latest = true;
found_free_version = Some(free_fi);
}
if include_free_versions
&& found_free_version.is_none()
&& let Ok(found_free_fi) = ver.parse_version_meta()
&& found_free_fi.version_type != VersionType::Invalid
{
let mut free_fi = found_free_fi.into_fileinfo(volume, path, all_parts);
free_fi.is_latest = true;
found_free_version = Some(free_fi);
}
if header.version_id != Some(vid) {
@@ -650,10 +708,10 @@ impl FileMeta {
fi.successor_mod_time = succ_mod_time;
}
if read_data {
if read_data && fi.inline_data() {
fi.data = self
.data
.find(fi.version_id.unwrap_or_default().to_string().as_str())?
.find(data_key_for_version(fi.version_id).as_str())?
.map(bytes::Bytes::from);
}
@@ -725,9 +783,7 @@ impl FileMeta {
pub fn into_file_info_versions(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfoVersions> {
let mut versions = Vec::new();
for version in self.versions.iter() {
let mut file_version = FileMetaVersion::default();
file_version.unmarshal_msg(&version.meta)?;
let fi = file_version.into_fileinfo(volume, path, all_parts);
let fi = version.into_fileinfo(volume, path, all_parts)?;
versions.push(fi);
}
@@ -770,29 +826,9 @@ impl FileMeta {
self.versions.first().unwrap().header.mod_time
}
/// Load or convert from buffer
/// Load or convert from buffer. Handles both current (meta_ver=3) and legacy (meta_ver=2) formats.
pub fn load_or_convert(buf: &[u8]) -> Result<Self> {
// Try to load as current format first
match Self::load(buf) {
Ok(meta) => Ok(meta),
Err(_) => {
// Try to convert from legacy format
Self::load_legacy(buf)
}
}
}
/// Load legacy format
pub fn load_legacy(_buf: &[u8]) -> Result<Self> {
// Implementation for loading legacy xl.meta formats
// This would handle conversion from older formats
Err(Error::other("Legacy format not yet implemented"))
}
/// Add legacy version
pub fn add_legacy(&mut self, _legacy_obj: &str) -> Result<()> {
// Implementation for adding legacy xl.meta v1 objects
Err(Error::other("Legacy version addition not yet implemented"))
Self::load(buf)
}
/// List all versions as FileInfo
@@ -1061,6 +1097,7 @@ mod test {
object: None,
delete_marker: None,
write_version: 1,
uses_legacy_checksum: false,
};
assert!(legacy_version.is_legacy(), "Should be recognized as a Legacy version");
@@ -1178,6 +1215,7 @@ mod test {
}),
delete_marker: None,
write_version: 1,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(version).expect("Conversion failed");
@@ -1391,6 +1429,7 @@ mod test {
object: None,
delete_marker: Some(delete_marker),
write_version: (i + 100) as u64,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(delete_version).unwrap();
@@ -1458,6 +1497,57 @@ mod test {
assert!(fm.validate_integrity().is_ok());
}
#[test]
fn test_add_version_clears_stale_inline_data_for_null_version() {
let mut fm = FileMeta::new();
let mut inline_fi = crate::fileinfo::FileInfo::new("test", 2, 1);
inline_fi.mod_time = Some(OffsetDateTime::now_utc());
inline_fi.data = Some(Bytes::new());
inline_fi.set_inline_data();
fm.add_version(inline_fi).unwrap();
let inline_version = fm.into_fileinfo("bucket", "test", "", true, false, true).unwrap();
assert!(inline_version.inline_data());
assert_eq!(inline_version.data, Some(Bytes::new()));
let mut disk_fi = crate::fileinfo::FileInfo::new("test", 2, 1);
disk_fi.mod_time = Some(OffsetDateTime::now_utc() + time::Duration::seconds(1));
disk_fi.size = 1024;
fm.add_version(disk_fi).unwrap();
let latest = fm.into_fileinfo("bucket", "test", "", true, false, true).unwrap();
assert!(!latest.inline_data());
assert!(latest.data.is_none());
assert!(
fm.data
.find(data_key_for_version(latest.version_id).as_str())
.unwrap()
.is_none()
);
}
#[test]
fn test_add_version_keeps_inline_data_when_version_insert_fails() {
let mut fm = FileMeta::new();
let mut inline_fi = crate::fileinfo::FileInfo::new("test", 2, 1);
inline_fi.mod_time = Some(OffsetDateTime::now_utc());
inline_fi.data = Some(Bytes::from_static(b"inline"));
inline_fi.set_inline_data();
fm.add_version(inline_fi).unwrap();
let before = fm.data.find(data_key_for_version(Some(Uuid::nil())).as_str()).unwrap();
assert_eq!(before, Some(Bytes::from_static(b"inline").to_vec()));
let invalid_disk_fi = crate::fileinfo::FileInfo::new("test", 2, 1);
let err = fm.add_version(invalid_disk_fi).unwrap_err();
assert!(err.to_string().contains("file meta version invalid"));
let after = fm.data.find(data_key_for_version(Some(Uuid::nil())).as_str()).unwrap();
assert_eq!(after, Some(Bytes::from_static(b"inline").to_vec()));
}
#[test]
fn test_version_merge_scenarios() {
// Test various version merge scenarios
crates/filemeta/src/filemeta/codec.rs
+19 -5
View File
@@ -22,10 +22,26 @@ impl FileMeta {
pub fn load(buf: &[u8]) -> Result<FileMeta> {
let mut xl = FileMeta::default();
xl.unmarshal_msg(buf)?;
Ok(xl)
}
/// Read (major, minor, header_ver, meta_ver) from xl.meta without full parse.
/// Returns Err if format is not feat (e.g. legacy uses rmp_serde in meta block).
pub fn read_format_versions(buf: &[u8]) -> Result<(u16, u16, u8, u8)> {
let (buf, major, minor) = Self::check_xl2_v1(buf)?;
if buf.len() < 5 {
return Err(Error::other("insufficient data for metadata length prefix"));
}
let (mut size_buf, buf) = buf.split_at(5);
let bin_len = rmp::decode::read_bin_len(&mut size_buf)?;
if buf.len() < bin_len as usize {
return Err(Error::other("insufficient data for metadata"));
}
let (meta, _) = buf.split_at(bin_len as usize);
let (_, header_ver, meta_ver, _) = Self::decode_xl_headers(meta)?;
Ok((major, minor, header_ver, meta_ver))
}
pub fn check_xl2_v1(buf: &[u8]) -> Result<(&[u8], u16, u16)> {
if buf.len() < 8 {
return Err(Error::other("xl file header not exists"));
@@ -294,11 +310,9 @@ impl FileMeta {
let offset = wr.len();
// xl header
rmp::encode::write_uint8(&mut wr, XL_HEADER_VERSION)?;
rmp::encode::write_uint8(&mut wr, XL_META_VERSION)?;
rmp::encode::write_uint(&mut wr, XL_HEADER_VERSION as u64)?;
rmp::encode::write_uint(&mut wr, XL_META_VERSION as u64)?;
// versions
rmp::encode::write_sint(&mut wr, self.versions.len() as i64)?;
for ver in self.versions.iter() {
crates/filemeta/src/filemeta/inline_data.rs
+7 -5
View File
@@ -40,10 +40,14 @@ impl FileMeta {
/// Count shared data directories
pub fn shared_data_dir_count(&self, version_id: Option<Uuid>, data_dir: Option<Uuid>) -> usize {
let version_id = version_id.unwrap_or_default();
let vid = version_id.unwrap_or_default();
if self.data.entries().unwrap_or_default() > 0
&& self.data.find(version_id.to_string().as_str()).unwrap_or_default().is_some()
&& self
.data
.find(super::data_key_for_version(version_id).as_str())
.unwrap_or_default()
.is_some()
{
return 0;
}
@@ -51,9 +55,7 @@ impl FileMeta {
self.versions
.iter()
.filter(|v| {
v.header.version_type == VersionType::Object
&& v.header.version_id != Some(version_id)
&& v.header.uses_data_dir()
v.header.version_type == VersionType::Object && v.header.version_id != Some(vid) && v.header.uses_data_dir()
})
.filter_map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).ok())
.filter(|&dir| dir.is_some() && dir == data_dir)
crates/filemeta/src/filemeta/msgp_decode.rs
+212
View File
@@ -0,0 +1,212 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{Error, Result};
use rmp::Marker;
use std::io::Read;
/// Reader that prepends a single byte to the stream. Used when we've read the marker
/// and need to pass it to a decoder that expects to read the marker itself.
pub(crate) struct PrependByteReader<'a, R> {
pub(crate) byte: Option<u8>,
pub(crate) inner: &'a mut R,
}
impl<R: Read> Read for PrependByteReader<'_, R> {
#[allow(clippy::collapsible_if)]
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
if let Some(b) = self.byte.take() {
if !buf.is_empty() {
buf[0] = b;
return Ok(1);
}
self.byte = Some(b);
}
self.inner.read(buf)
}
}
/// Read MessagePack nil or array length. Returns None for nil, Some(len) for array.
pub(crate) fn read_nil_or_array_len<R: Read>(rd: &mut R) -> Result<Option<usize>> {
let mut buf = [0u8; 1];
rd.read_exact(&mut buf).map_err(Error::from)?;
let marker = buf[0];
match marker {
0xc0 => Ok(None), // nil
0x90..=0x9f => Ok(Some((marker & 0x0f) as usize)),
0xdc => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
Ok(Some(u16::from_be_bytes(b) as usize))
}
0xdd => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
Ok(Some(u32::from_be_bytes(b) as usize))
}
_ => Err(Error::other(format!("expected nil or array, got marker 0x{marker:02x}"))),
}
}
/// Read MessagePack nil or map length. Returns None for nil, Some(len) for map.
pub(crate) fn read_nil_or_map_len<R: Read>(rd: &mut R) -> Result<Option<usize>> {
let mut buf = [0u8; 1];
rd.read_exact(&mut buf).map_err(Error::from)?;
let marker = buf[0];
match marker {
0xc0 => Ok(None), // nil
0x80..=0x8f => Ok(Some((marker & 0x0f) as usize)),
0xde => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
Ok(Some(u16::from_be_bytes(b) as usize))
}
0xdf => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
Ok(Some(u32::from_be_bytes(b) as usize))
}
_ => Err(Error::other(format!("expected nil or map, got marker 0x{marker:02x}"))),
}
}
/// Skip a single MessagePack value. Used for unknown map keys.
pub(crate) fn skip_msgp_value<R: Read>(rd: &mut R) -> Result<()> {
let marker = rmp::decode::read_marker(rd).map_err(Error::from)?;
let skip_len: usize = match marker {
Marker::Null | Marker::False | Marker::True => 0,
Marker::FixPos(_) | Marker::FixNeg(_) => 0,
Marker::U8 => 1,
Marker::U16 => 2,
Marker::U32 => 4,
Marker::U64 => 8,
Marker::I8 => 1,
Marker::I16 => 2,
Marker::I32 => 4,
Marker::I64 => 8,
Marker::F32 => 4,
Marker::F64 => 8,
Marker::FixStr(n) => n as usize,
Marker::Str8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::from)?;
b[0] as usize
}
Marker::Str16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
u16::from_be_bytes(b) as usize
}
Marker::Str32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
u32::from_be_bytes(b) as usize
}
Marker::Bin8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::from)?;
b[0] as usize
}
Marker::Bin16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
u16::from_be_bytes(b) as usize
}
Marker::Bin32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
u32::from_be_bytes(b) as usize
}
Marker::FixArray(n) => {
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Array16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
let n = u16::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Array32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
let n = u32::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::FixMap(n) => {
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Map16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
let n = u16::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Map32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
let n = u32::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::FixExt1 => 1,
Marker::FixExt2 => 2,
Marker::FixExt4 => 4,
Marker::FixExt8 => 8,
Marker::FixExt16 => 16,
Marker::Ext8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::from)?;
let len = b[0] as usize;
1 + len // type byte + data
}
Marker::Ext16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
let len = u16::from_be_bytes(b) as usize;
2 + len // type bytes + data
}
Marker::Ext32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
let len = u32::from_be_bytes(b) as usize;
4 + len // type bytes + data
}
Marker::Reserved => 0,
};
if skip_len > 0 {
let mut buf = vec![0u8; skip_len];
rd.read_exact(&mut buf).map_err(Error::from)?;
}
Ok(())
}
crates/filemeta/src/filemeta/version.rs
+768 -240
View File
File diff suppressed because it is too large Load Diff
crates/filemeta/src/filemeta_inline.rs
+52
View File
@@ -181,6 +181,58 @@ impl InlineData {
self.serialize(keys, values)
}
pub fn remove_key(&mut self, key: &str) -> Result<bool> {
let buf = self.after_version();
if buf.is_empty() {
return Ok(false);
}
let mut cur = Cursor::new(buf);
let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize;
let mut keys = Vec::with_capacity(fields_len);
let mut values = Vec::with_capacity(fields_len);
let mut found = false;
while fields_len > 0 {
fields_len -= 1;
let str_len = rmp::decode::read_str_len(&mut cur)?;
let mut field_buff = vec![0u8; str_len as usize];
cur.read_exact(&mut field_buff)?;
let find_key = String::from_utf8(field_buff)?;
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
let start = cur.position() as usize;
let end = start + bin_len;
cur.set_position(end as u64);
if find_key == key {
found = true;
continue;
}
keys.push(find_key);
values.push(buf[start..end].to_vec());
}
if !found {
return Ok(false);
}
if keys.is_empty() {
self.0 = Vec::new();
return Ok(true);
}
self.serialize(keys, values)?;
Ok(true)
}
pub fn remove(&mut self, remove_keys: Vec<Uuid>) -> Result<bool> {
let buf = self.after_version();
if buf.is_empty() {
crates/filemeta/src/replication.rs
+2 -2
View File
@@ -15,7 +15,7 @@
use bytes::Bytes;
use core::fmt;
use regex::Regex;
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_utils::http::internal_key_rustfs;
use serde::{Deserialize, Serialize};
use std::any::Any;
use std::collections::HashMap;
@@ -859,7 +859,7 @@ pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationS
}
pub fn target_reset_header(arn: &str) -> String {
format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-{arn}")
internal_key_rustfs(&format!("{REPLICATION_RESET}-{arn}"))
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
crates/filemeta/src/test_data.rs
+6
View File
@@ -56,6 +56,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
object: Some(object_version),
delete_marker: None,
write_version: 1,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
@@ -74,6 +75,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
object: None,
delete_marker: Some(delete_marker),
write_version: 2,
uses_legacy_checksum: false,
};
let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?;
@@ -86,6 +88,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
object: None,
delete_marker: None,
write_version: 3,
uses_legacy_checksum: false,
};
let mut legacy_shallow = FileMetaShallowVersion::try_from(legacy_version)?;
@@ -139,6 +142,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
object: Some(object_version),
delete_marker: None,
write_version: (i + 1) as u64,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
@@ -158,6 +162,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
object: None,
delete_marker: Some(delete_marker),
write_version: (i + 100) as u64,
uses_legacy_checksum: false,
};
let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?;
@@ -245,6 +250,7 @@ pub fn create_xlmeta_with_inline_data() -> Result<Vec<u8>> {
object: Some(object_version),
delete_marker: None,
write_version: 1,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
crates/heal/tests/heal_integration_test.rs
+21 -49
View File
@@ -38,6 +38,11 @@ use walkdir::WalkDir;
const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
const NON_INLINE_TEST_DATA_SIZE: usize = 256 * 1024 + 137;
fn non_inline_test_data() -> Vec<u8> {
(0..NON_INLINE_TEST_DATA_SIZE).map(|idx| (idx % 251) as u8).collect()
}
async fn wait_for_path_exists(path: &Path, timeout: Duration, interval: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
@@ -177,10 +182,10 @@ mod serial_tests {
// Create test bucket and object
let bucket_name = "test-heal-object-basic";
let object_name = "test-object.txt";
let test_data = b"Hello, this is test data for healing!";
let test_data = non_inline_test_data();
create_test_bucket(&ecstore, bucket_name).await;
upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
upload_test_object(&ecstore, bucket_name, object_name, &test_data).await;
let _obj_dir = disk_paths[0].join(bucket_name).join(object_name);
// ─── 1️⃣ delete single data shard file ─────────────────────────────────────
let obj_dir = disk_paths[0].join(bucket_name).join(object_name);
@@ -198,55 +203,22 @@ mod serial_tests {
assert!(!target_part.exists());
println!("✅ Deleted shard part file: {target_part:?}");
// Create heal manager with faster interval
let cfg = HealConfig {
heal_interval: Duration::from_millis(1),
let heal_opts = HealOpts {
recreate: true,
remove: false,
update_parity: true,
..Default::default()
};
let heal_manager = HealManager::new(heal_storage.clone(), Some(cfg));
heal_manager.start().await.unwrap();
// Submit heal request for the object
let heal_request = HealRequest::new(
HealType::Object {
bucket: bucket_name.to_string(),
object: object_name.to_string(),
version_id: None,
},
HealOptions {
dry_run: false,
recursive: false,
remove_corrupted: false,
recreate_missing: true,
scan_mode: HealScanMode::Normal,
update_parity: true,
timeout: Some(Duration::from_secs(300)),
pool_index: None,
set_index: None,
},
HealPriority::Normal,
);
let task_id = heal_manager
.submit_heal_request(heal_request)
let (object_result, object_error) = heal_storage
.heal_object(bucket_name, object_name, None, &heal_opts)
.await
.expect("Failed to submit heal request");
.expect("failed to heal object");
info!("heal_object result: {:?}, error: {:?}", object_result, object_error);
assert!(object_error.is_none(), "heal_object returned error: {object_error:?}");
info!("Submitted heal request with task ID: {}", task_id);
// Wait for task completion
tokio::time::sleep(tokio::time::Duration::from_secs(8)).await;
// Attempt to fetch task status (might be removed if finished)
match heal_manager.get_task_status(&task_id).await {
Ok(status) => info!("Task status: {:?}", status),
Err(e) => info!("Task status not found (likely completed): {}", e),
}
// ─── 2️⃣ verify each part file is restored ───────
assert!(target_part.exists());
// ─── 3️⃣ verify object data integrity by actually reading it ───────
// `test_heal_format_with_data` covers on-disk shard restoration. Here we
// focus on the object-level healing contract: the object must remain
// readable with intact contents after healing.
let mut reader = ecstore
.get_object_reader(bucket_name, object_name, None, HeaderMap::new(), &ObjectOptions::default())
.await
@@ -364,10 +336,10 @@ mod serial_tests {
// Create test bucket and object
let bucket_name = "test-heal-format-with-data";
let object_name = "test-object.txt";
let test_data = b"Hello, this is test data for healing!";
let test_data = non_inline_test_data();
create_test_bucket(&ecstore, bucket_name).await;
upload_test_object(&ecstore, bucket_name, object_name, test_data).await;
upload_test_object(&ecstore, bucket_name, object_name, &test_data).await;
let obj_dir = disk_paths[0].join(bucket_name).join(object_name);
let target_part = WalkDir::new(&obj_dir)
.min_depth(2)
crates/iam/src/manager.rs
+58 -13
View File
@@ -32,7 +32,7 @@ use rustfs_policy::{
format::Format,
policy::{Policy, PolicyDoc, default::DEFAULT_POLICIES, iam_policy_claim_name_sa},
};
use rustfs_utils::path::path_join_buf;
use rustfs_utils::{get_env_opt_str, path::path_join_buf};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::sync::atomic::AtomicU8;
@@ -147,7 +147,7 @@ where
// Background ticker for synchronization
// Check if environment variable is set
let skip_background_task = std::env::var("RUSTFS_SKIP_BACKGROUND_TASK").is_ok();
let skip_background_task = get_env_opt_str("RUSTFS_SKIP_BACKGROUND_TASK").is_some();
if !skip_background_task {
// Background thread starts periodic updates or receives signal updates
@@ -595,7 +595,11 @@ where
Ok(users
.values()
.filter_map(|x| {
if !access_key.is_empty() && x.credentials.parent_user.as_str() == access_key && x.credentials.is_temp() {
if !access_key.is_empty()
&& x.credentials.parent_user.as_str() == access_key
&& x.credentials.is_temp()
&& !x.credentials.is_service_account()
{
let mut c = x.credentials.clone();
c.secret_key = String::new();
c.session_token = String::new();
@@ -1419,9 +1423,6 @@ where
}
pub async fn get_group_description(&self, name: &str) -> Result<GroupDesc> {
let (ps, updated_at) = self.policy_db_get_internal(name, true, false).await?;
let policy = ps.join(",");
let gi = self
.cache
.groups
@@ -1430,13 +1431,25 @@ where
.cloned()
.ok_or(Error::NoSuchGroup(name.to_string()))?;
Ok(GroupDesc {
name: name.to_string(),
policy,
members: gi.members,
updated_at: Some(updated_at),
status: gi.status,
})
let mapped_policy = if let Some(policy) = self.cache.group_policies.load().get(name).cloned() {
Some(policy)
} else {
let mut policies = HashMap::new();
if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, true, &mut policies).await
&& !is_err_no_such_policy(&err)
{
return Err(err);
}
if let Some(policy) = policies.get(name).cloned() {
Cache::add_or_update(&self.cache.group_policies, name, &policy, OffsetDateTime::now_utc());
Some(policy)
} else {
None
}
};
Ok(build_group_desc(name, gi, mapped_policy))
}
pub async fn list_groups(&self) -> Result<Vec<String>> {
@@ -1882,6 +1895,20 @@ fn filter_policies(cache: &Cache, policy_name: &str, bucket_name: &str) -> (Stri
(policies.join(","), Policy::merge_policies(to_merge))
}
fn build_group_desc(name: &str, group_info: GroupInfo, mapped_policy: Option<MappedPolicy>) -> GroupDesc {
let (policy, updated_at) = mapped_policy
.map(|policy| (policy.policies, Some(policy.update_at)))
.unwrap_or_else(|| (String::new(), Some(OffsetDateTime::now_utc())));
GroupDesc {
name: name.to_string(),
policy,
members: group_info.members,
updated_at,
status: group_info.status,
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -2226,4 +2253,22 @@ mod tests {
assert!(merged.statements.is_empty());
assert!(merged.is_empty());
}
#[test]
fn test_build_group_desc_preserves_policy_for_disabled_group() {
let group_info = GroupInfo {
status: STATUS_DISABLED.to_string(),
members: vec!["alice".to_string()],
..Default::default()
};
let mapped_policy = MappedPolicy::new("readonly");
let desc = build_group_desc("ops", group_info, Some(mapped_policy));
assert_eq!(desc.name, "ops");
assert_eq!(desc.status, STATUS_DISABLED);
assert_eq!(desc.policy, "readonly");
assert_eq!(desc.members, vec!["alice".to_string()]);
assert!(desc.updated_at.is_some());
}
}
crates/iam/src/store.rs
+54
View File
@@ -113,7 +113,11 @@ impl UserType {
#[derive(Serialize, Deserialize, Clone)]
pub struct MappedPolicy {
pub version: i64,
/// policy, legacy: policies. Serialize as policy.
#[serde(rename = "policy", alias = "policies")]
pub policies: String,
/// updatedAt (RFC3339), legacy: update_at. Serialize as updatedAt.
#[serde(rename = "updatedAt", alias = "update_at", with = "rustfs_policy::serde_datetime")]
pub update_at: OffsetDateTime,
}
@@ -158,6 +162,13 @@ pub struct GroupInfo {
pub version: i64,
pub status: String,
pub members: Vec<String>,
/// updatedAt (RFC3339), legacy: update_at. Serialize as updatedAt.
#[serde(
rename = "updatedAt",
alias = "update_at",
default,
with = "rustfs_policy::serde_datetime::option"
)]
pub update_at: Option<OffsetDateTime>,
}
@@ -171,3 +182,46 @@ impl GroupInfo {
}
}
}
#[cfg(test)]
mod tests {
use super::{GroupInfo, MappedPolicy};
/// uses RFC3339 for updatedAt. MappedPolicy must serialize as RFC3339.
#[test]
fn test_mapped_policy_timestamps_serialize_as_rfc3339() {
let mp = MappedPolicy::new("readwrite");
let json = serde_json::to_string(&mp).expect("serialize");
assert!(json.contains('T'), "MappedPolicy updatedAt should be RFC3339; got: {}", json);
assert!(
json.contains('Z') || json.contains("+00:00"),
"MappedPolicy updatedAt should be RFC3339; got: {}",
json
);
}
/// Deserialize MappedPolicy from JSON (RFC3339 updatedAt).
#[test]
fn test_mapped_policy_deserialize_minio_style_rfc3339() {
let minio_style = r#"{"version":1,"policy":"readwrite","updatedAt":"2025-03-07T12:00:00Z"}"#;
let mp: MappedPolicy = serde_json::from_str(minio_style).expect("deserialize");
assert_eq!(mp.policies, "readwrite");
}
/// GroupInfo updatedAt: uses RFC3339.
#[test]
fn test_group_info_timestamps_serialize_as_rfc3339() {
let g = GroupInfo::new(vec!["u1".to_string()]);
let json = serde_json::to_string(&g).expect("serialize");
assert!(json.contains('T'), "GroupInfo updatedAt should be RFC3339; got: {}", json);
}
/// Deserialize GroupInfo from JSON (RFC3339 updatedAt).
#[test]
fn test_group_info_deserialize_minio_style_rfc3339() {
let minio_style = r#"{"version":1,"status":"enabled","members":["u1"],"updatedAt":"2025-03-07T12:00:00Z"}"#;
let g: GroupInfo = serde_json::from_str(minio_style).expect("deserialize");
assert_eq!(g.members, ["u1"]);
assert!(g.update_at.is_some());
}
}
crates/iam/src/store/object.rs
+123 -3
View File
@@ -129,15 +129,55 @@ impl ObjectStore {
}
fn decrypt_data(data: &[u8]) -> Result<Vec<u8>> {
let de = rustfs_crypto::decrypt_data(get_global_action_cred().unwrap_or_default().secret_key.as_bytes(), data)?;
Ok(de)
if Self::is_plaintext_json(data) {
return Ok(data.to_vec());
}
let cred = get_global_action_cred().unwrap_or_default();
let secret_key = cred.secret_key;
let mut keys: Vec<(Vec<u8>, bool)> = vec![(secret_key.clone().into_bytes(), false)];
if !cred.access_key.is_empty() && !secret_key.is_empty() {
keys.push((format!("{}:{secret_key}", cred.access_key).into_bytes(), true));
}
const STREAM_IO_HEADER_LEN: usize = 41;
let mut last_err = None;
for (key, is_access_secret) in keys {
if is_access_secret
&& data.len() >= STREAM_IO_HEADER_LEN
&& let Ok(plain) = rustfs_crypto::decrypt_stream_io(&key, data)
{
return Ok(plain);
}
match rustfs_crypto::decrypt_data(&key, data) {
Ok(plain) => return Ok(plain),
Err(err) => last_err = Some(err),
}
}
Err(last_err.unwrap_or(rustfs_crypto::Error::ErrUnexpectedHeader).into())
}
fn encrypt_data(data: &[u8]) -> Result<Vec<u8>> {
let en = rustfs_crypto::encrypt_data(get_global_action_cred().unwrap_or_default().secret_key.as_bytes(), data)?;
let cred = get_global_action_cred().unwrap_or_default();
let password = if !cred.access_key.is_empty() && !cred.secret_key.is_empty() {
format!("{}:{}", cred.access_key, cred.secret_key).into_bytes()
} else {
cred.secret_key.clone().into_bytes()
};
let en = rustfs_crypto::encrypt_stream_io(&password, data)?;
Ok(en)
}
fn is_plaintext_json(data: &[u8]) -> bool {
std::str::from_utf8(data).is_ok() && serde_json::from_slice::<serde_json::Value>(data).is_ok()
}
#[cfg(test)]
fn encrypt_data_for_test(data: &[u8]) -> Result<Vec<u8>> {
Self::encrypt_data(data)
}
async fn load_iamconfig_bytes_with_metadata(&self, path: impl AsRef<str> + Send) -> Result<(Vec<u8>, ObjectInfo)> {
let (data, obj) = read_config_with_metadata(self.object_api.clone(), path.as_ref(), &ObjectOptions::default()).await?;
@@ -1164,3 +1204,83 @@ impl Store for ObjectStore {
// Ok(())
// }
}
#[cfg(test)]
mod tests {
use super::ObjectStore;
use rustfs_credentials::{Credentials, get_global_action_cred, init_global_action_credentials};
fn test_cred() -> Credentials {
if let Some(cred) = get_global_action_cred() {
return cred;
}
let _ = init_global_action_credentials(Some("COMPATTESTAK".to_string()), Some("COMPATTESTSK1234567890".to_string()));
get_global_action_cred().unwrap_or_default()
}
#[test]
fn test_decrypt_data_accepts_plaintext_json() {
let raw = br#"{"Version":1,"policy":"readonly"}"#;
let out = ObjectStore::decrypt_data(raw).expect("plaintext json should pass through");
assert_eq!(out, raw);
}
#[test]
fn test_decrypt_data_accepts_rustfs_legacy_secret_encryption() {
let cred = test_cred();
let plain = br#"{"accessKey":"ak","secretKey":"sk"}"#;
let encrypted = rustfs_crypto::encrypt_data(cred.secret_key.as_bytes(), plain).expect("encrypt with rustfs secret");
let out = ObjectStore::decrypt_data(&encrypted).expect("decrypt rustfs legacy encryption");
assert_eq!(out, plain);
}
#[test]
fn test_decrypt_data_accepts_access_secret_encryption() {
let cred = test_cred();
let plain = br#"{"Version":1,"updatedAt":"2025-03-07T12:00:00Z"}"#;
let root_cred = format!("{}:{}", cred.access_key, cred.secret_key);
let encrypted = rustfs_crypto::encrypt_stream_io(root_cred.as_bytes(), plain).expect("encrypt with stream_io");
let out = ObjectStore::decrypt_data(&encrypted).expect("decrypt stream_io");
assert_eq!(out, plain);
}
#[test]
fn test_decrypt_data_corrupt_stream_io_fails() {
let cred = test_cred();
let plain = br#"{"Version":1}"#;
let root_cred = format!("{}:{}", cred.access_key, cred.secret_key);
let mut encrypted = rustfs_crypto::encrypt_stream_io(root_cred.as_bytes(), plain).expect("encrypt with stream_io");
if encrypted.len() > 50 {
encrypted[50] ^= 0xFF; // corrupt one byte
}
let result = ObjectStore::decrypt_data(&encrypted);
assert!(result.is_err(), "corrupt stream_io data should fail decrypt");
}
#[test]
fn test_decrypt_data_short_data_fails() {
let short = &[0x00u8; 40]; // less than 41-byte stream_io header, not valid JSON
let result = ObjectStore::decrypt_data(short);
assert!(result.is_err(), "short non-JSON data should fail decrypt");
}
#[test]
fn test_encrypt_data_produces_stream_io_format() {
let _ = test_cred();
let plain = br#"{"Version":1,"policy":"readonly"}"#;
let encrypted = ObjectStore::encrypt_data_for_test(plain).expect("encrypt should succeed");
// stream_io header: salt(32) + alg_id(1) + nonce_prefix(8) = 41 bytes
const STREAM_IO_HEADER_LEN: usize = 41;
assert!(
encrypted.len() >= STREAM_IO_HEADER_LEN,
"encrypted should have at least 41-byte stream_io header"
);
assert!(
encrypted[32] == 0x00 || encrypted[32] == 0x01 || encrypted[32] == 0x02,
"alg_id should be 0x00, 0x01, or 0x02"
);
// Round-trip: encrypt then decrypt
let decrypted = ObjectStore::decrypt_data(&encrypted).expect("decrypt should succeed");
assert_eq!(plain, decrypted.as_slice());
}
}
crates/iam/src/sys.rs
+1 -4
View File
@@ -361,10 +361,6 @@ impl<T: Store> IamSys<T> {
return Err(IamError::IAMActionNotAllowed);
}
if opts.expiration.is_none() {
return Err(IamError::InvalidExpiration);
}
// TODO: check allow_site_replicator_account
let policy_buf = if let Some(policy) = opts.session_policy {
@@ -619,6 +615,7 @@ impl<T: Store> IamSys<T> {
}
let updated_at = self.store.add_user(access_key, args).await?;
self.load_user(access_key, UserType::Reg).await?;
self.notify_for_user(access_key, false).await;
crates/keystone/Cargo.toml
+2
View File
@@ -36,6 +36,7 @@ time = { workspace = true }
moka = { workspace = true }
rustfs-credentials = { workspace = true }
rustfs-policy = { workspace = true }
rustfs-utils = { workspace = true }
# Middleware dependencies
tower = { workspace = true }
http = { workspace = true }
@@ -50,6 +51,7 @@ tokio = { workspace = true, features = ["test-util"] }
tower = { workspace = true, features = ["util"] }
hyper = { workspace = true, features = ["server"] }
serde_json = { workspace = true }
temp-env = { workspace = true }
[[test]]
name = "integration"
crates/keystone/src/config.rs
+57 -39
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use crate::{KeystoneError, KeystoneVersion, Result};
use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_str, get_env_u64};
use serde::{Deserialize, Serialize};
use std::time::Duration;
@@ -80,59 +81,35 @@ pub struct RoleMapping {
impl KeystoneConfig {
/// Load configuration from environment variables
pub fn from_env() -> Result<Self> {
let enable = std::env::var("RUSTFS_KEYSTONE_ENABLE")
.unwrap_or_else(|_| "false".to_string())
.parse()
.unwrap_or(false);
let enable = get_env_bool("RUSTFS_KEYSTONE_ENABLE", false);
if !enable {
return Ok(Self::default());
}
let auth_url = std::env::var("RUSTFS_KEYSTONE_AUTH_URL")
.map_err(|_| KeystoneError::ConfigError("RUSTFS_KEYSTONE_AUTH_URL not set".to_string()))?;
let auth_url = get_env_opt_str("RUSTFS_KEYSTONE_AUTH_URL")
.ok_or_else(|| KeystoneError::ConfigError("RUSTFS_KEYSTONE_AUTH_URL not set".to_string()))?;
let version = std::env::var("RUSTFS_KEYSTONE_VERSION").unwrap_or_else(|_| "v3".to_string());
let version = get_env_str("RUSTFS_KEYSTONE_VERSION", "v3");
let admin_user = std::env::var("RUSTFS_KEYSTONE_ADMIN_USER").ok();
let admin_password = std::env::var("RUSTFS_KEYSTONE_ADMIN_PASSWORD").ok();
let admin_project = std::env::var("RUSTFS_KEYSTONE_ADMIN_PROJECT").ok();
let admin_domain = std::env::var("RUSTFS_KEYSTONE_ADMIN_DOMAIN").ok();
let admin_user = get_env_opt_str("RUSTFS_KEYSTONE_ADMIN_USER");
let admin_password = get_env_opt_str("RUSTFS_KEYSTONE_ADMIN_PASSWORD");
let admin_project = get_env_opt_str("RUSTFS_KEYSTONE_ADMIN_PROJECT");
let admin_domain = get_env_opt_str("RUSTFS_KEYSTONE_ADMIN_DOMAIN");
let verify_ssl = std::env::var("RUSTFS_KEYSTONE_VERIFY_SSL")
.unwrap_or_else(|_| "true".to_string())
.parse()
.unwrap_or(true);
let verify_ssl = get_env_bool("RUSTFS_KEYSTONE_VERIFY_SSL", true);
let enable_cache = std::env::var("RUSTFS_KEYSTONE_ENABLE_CACHE")
.unwrap_or_else(|_| "true".to_string())
.parse()
.unwrap_or(true);
let enable_cache = get_env_bool("RUSTFS_KEYSTONE_ENABLE_CACHE", true);
let cache_size = std::env::var("RUSTFS_KEYSTONE_CACHE_SIZE")
.unwrap_or_else(|_| "10000".to_string())
.parse()
.unwrap_or(10000);
let cache_size = get_env_u64("RUSTFS_KEYSTONE_CACHE_SIZE", 10000);
let cache_ttl_seconds = std::env::var("RUSTFS_KEYSTONE_CACHE_TTL")
.unwrap_or_else(|_| "300".to_string())
.parse()
.unwrap_or(300);
let cache_ttl_seconds = get_env_u64("RUSTFS_KEYSTONE_CACHE_TTL", 300);
let enable_tenant_prefix = std::env::var("RUSTFS_KEYSTONE_TENANT_PREFIX")
.unwrap_or_else(|_| "true".to_string())
.parse()
.unwrap_or(true);
let enable_tenant_prefix = get_env_bool("RUSTFS_KEYSTONE_TENANT_PREFIX", true);
let implicit_tenants = std::env::var("RUSTFS_KEYSTONE_IMPLICIT_TENANTS")
.unwrap_or_else(|_| "true".to_string())
.parse()
.unwrap_or(true);
let implicit_tenants = get_env_bool("RUSTFS_KEYSTONE_IMPLICIT_TENANTS", true);
let timeout_seconds = std::env::var("RUSTFS_KEYSTONE_TIMEOUT")
.unwrap_or_else(|_| "30".to_string())
.parse()
.unwrap_or(30);
let timeout_seconds = get_env_u64("RUSTFS_KEYSTONE_TIMEOUT", 30);
Ok(Self {
enable,
@@ -224,6 +201,7 @@ impl Default for KeystoneConfig {
#[cfg(test)]
mod tests {
use super::*;
use temp_env::with_vars;
#[test]
fn test_default_config() {
@@ -248,4 +226,44 @@ mod tests {
config.version = "invalid".to_string();
assert!(config.get_version().is_err());
}
#[test]
fn test_from_env_reads_configuration() {
with_vars(
vec![
("RUSTFS_KEYSTONE_ENABLE", Some("true")),
("RUSTFS_KEYSTONE_AUTH_URL", Some("https://keystone.example.com")),
("RUSTFS_KEYSTONE_VERSION", Some("v2.0")),
("RUSTFS_KEYSTONE_ADMIN_USER", Some("admin")),
("RUSTFS_KEYSTONE_ADMIN_PASSWORD", Some("secret")),
("RUSTFS_KEYSTONE_ADMIN_PROJECT", Some("service")),
("RUSTFS_KEYSTONE_ADMIN_DOMAIN", Some("Default")),
("RUSTFS_KEYSTONE_VERIFY_SSL", Some("false")),
("RUSTFS_KEYSTONE_ENABLE_CACHE", Some("false")),
("RUSTFS_KEYSTONE_CACHE_SIZE", Some("2048")),
("RUSTFS_KEYSTONE_CACHE_TTL", Some("900")),
("RUSTFS_KEYSTONE_TENANT_PREFIX", Some("false")),
("RUSTFS_KEYSTONE_IMPLICIT_TENANTS", Some("false")),
("RUSTFS_KEYSTONE_TIMEOUT", Some("99")),
],
|| {
let config = KeystoneConfig::from_env().expect("keystone config should load from env");
assert!(config.enable);
assert_eq!(config.auth_url, "https://keystone.example.com");
assert_eq!(config.version, "v2.0");
assert_eq!(config.admin_user.as_deref(), Some("admin"));
assert_eq!(config.admin_password.as_deref(), Some("secret"));
assert_eq!(config.admin_project.as_deref(), Some("service"));
assert_eq!(config.admin_domain.as_deref(), Some("Default"));
assert!(!config.verify_ssl);
assert!(!config.enable_cache);
assert_eq!(config.cache_size, 2048);
assert_eq!(config.cache_ttl_seconds, 900);
assert!(!config.enable_tenant_prefix);
assert!(!config.implicit_tenants);
assert_eq!(config.timeout_seconds, 99);
},
);
}
}
crates/kms/Cargo.toml
+2
View File
@@ -56,6 +56,7 @@ moka = { workspace = true, features = ["future"] }
# Additional dependencies
md5 = { workspace = true }
arc-swap = { workspace = true }
rustfs-utils = { workspace = true }
# HTTP client for Vault
reqwest = { workspace = true }
@@ -63,6 +64,7 @@ vaultrs = { workspace = true }
[dev-dependencies]
tempfile = { workspace = true }
temp-env = { workspace = true }
[features]
default = []
crates/kms/src/config.rs
+50 -16
View File
@@ -15,6 +15,7 @@
//! KMS configuration management
use crate::error::{KmsError, Result};
use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_str};
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use std::time::Duration;
@@ -304,7 +305,7 @@ impl KmsConfig {
let mut config = Self::default();
// Backend type
if let Ok(backend_type) = std::env::var("RUSTFS_KMS_BACKEND") {
if let Some(backend_type) = get_env_opt_str("RUSTFS_KMS_BACKEND") {
config.backend = match backend_type.to_lowercase().as_str() {
"local" => KmsBackend::Local,
"vault" => KmsBackend::Vault,
@@ -313,12 +314,12 @@ impl KmsConfig {
}
// Default key ID
if let Ok(key_id) = std::env::var("RUSTFS_KMS_DEFAULT_KEY_ID") {
if let Some(key_id) = get_env_opt_str("RUSTFS_KMS_DEFAULT_KEY_ID") {
config.default_key_id = Some(key_id);
}
// Timeout
if let Ok(timeout_str) = std::env::var("RUSTFS_KMS_TIMEOUT_SECS") {
if let Some(timeout_str) = get_env_opt_str("RUSTFS_KMS_TIMEOUT_SECS") {
let timeout_secs = timeout_str
.parse::<u64>()
.map_err(|_| KmsError::configuration_error("Invalid timeout value"))?;
@@ -326,22 +327,20 @@ impl KmsConfig {
}
// Retry attempts
if let Ok(retries_str) = std::env::var("RUSTFS_KMS_RETRY_ATTEMPTS") {
if let Some(retries_str) = get_env_opt_str("RUSTFS_KMS_RETRY_ATTEMPTS") {
config.retry_attempts = retries_str
.parse()
.map_err(|_| KmsError::configuration_error("Invalid retry attempts value"))?;
}
// Enable cache
if let Ok(cache_str) = std::env::var("RUSTFS_KMS_ENABLE_CACHE") {
config.enable_cache = cache_str.parse().unwrap_or(true);
}
config.enable_cache = get_env_bool("RUSTFS_KMS_ENABLE_CACHE", config.enable_cache);
// Backend-specific configuration
match config.backend {
KmsBackend::Local => {
let key_dir = std::env::var("RUSTFS_KMS_LOCAL_KEY_DIR").unwrap_or_else(|_| "./kms_keys".to_string());
let master_key = std::env::var("RUSTFS_KMS_LOCAL_MASTER_KEY").ok();
let key_dir = get_env_str("RUSTFS_KMS_LOCAL_KEY_DIR", "./kms_keys");
let master_key = get_env_opt_str("RUSTFS_KMS_LOCAL_MASTER_KEY");
config.backend_config = BackendConfig::Local(LocalConfig {
key_dir: PathBuf::from(key_dir),
@@ -350,17 +349,16 @@ impl KmsConfig {
});
}
KmsBackend::Vault => {
let address = std::env::var("RUSTFS_KMS_VAULT_ADDRESS").unwrap_or_else(|_| "http://localhost:8200".to_string());
let token = std::env::var("RUSTFS_KMS_VAULT_TOKEN").unwrap_or_else(|_| "dev-token".to_string());
let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
let token = get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token");
config.backend_config = BackendConfig::Vault(Box::new(VaultConfig {
address,
auth_method: VaultAuthMethod::Token { token },
namespace: std::env::var("RUSTFS_KMS_VAULT_NAMESPACE").ok(),
mount_path: std::env::var("RUSTFS_KMS_VAULT_MOUNT_PATH").unwrap_or_else(|_| "transit".to_string()),
kv_mount: std::env::var("RUSTFS_KMS_VAULT_KV_MOUNT").unwrap_or_else(|_| "secret".to_string()),
key_path_prefix: std::env::var("RUSTFS_KMS_VAULT_KEY_PREFIX")
.unwrap_or_else(|_| "rustfs/kms/keys".to_string()),
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
tls: None,
}));
}
@@ -374,6 +372,7 @@ impl KmsConfig {
#[cfg(test)]
mod tests {
use super::*;
use temp_env::with_vars;
use tempfile::TempDir;
#[test]
@@ -423,4 +422,39 @@ mod tests {
config.retry_attempts = 0;
assert!(config.validate().is_err());
}
#[test]
fn test_from_env_reads_vault_settings() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault")),
("RUSTFS_KMS_DEFAULT_KEY_ID", Some("tenant-key")),
("RUSTFS_KMS_TIMEOUT_SECS", Some("42")),
("RUSTFS_KMS_RETRY_ATTEMPTS", Some("7")),
("RUSTFS_KMS_ENABLE_CACHE", Some("false")),
("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")),
("RUSTFS_KMS_VAULT_TOKEN", Some("vault-token")),
("RUSTFS_KMS_VAULT_NAMESPACE", Some("tenant-a")),
("RUSTFS_KMS_VAULT_MOUNT_PATH", Some("transit-alt")),
("RUSTFS_KMS_VAULT_KV_MOUNT", Some("secret-alt")),
("RUSTFS_KMS_VAULT_KEY_PREFIX", Some("tenant/keys")),
],
|| {
let config = KmsConfig::from_env().expect("kms config should load from env");
assert_eq!(config.backend, KmsBackend::Vault);
assert_eq!(config.default_key_id.as_deref(), Some("tenant-key"));
assert_eq!(config.timeout, Duration::from_secs(42));
assert_eq!(config.retry_attempts, 7);
assert!(!config.enable_cache);
let vault = config.vault_config().expect("vault backend config");
assert_eq!(vault.address, "https://vault.example.com");
assert_eq!(vault.namespace.as_deref(), Some("tenant-a"));
assert_eq!(vault.mount_path, "transit-alt");
assert_eq!(vault.kv_mount, "secret-alt");
assert_eq!(vault.key_path_prefix, "tenant/keys");
},
);
}
}
crates/madmin/src/group.rs
+51 -2
View File
@@ -13,10 +13,11 @@
// limitations under the License.
use serde::Deserialize;
use serde::Deserializer;
use serde::Serialize;
use time::OffsetDateTime;
#[derive(Debug, Serialize, Deserialize, Default)]
#[derive(Debug, Serialize, Default, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum GroupStatus {
#[default]
@@ -24,6 +25,20 @@ pub enum GroupStatus {
Disabled,
}
impl<'de> Deserialize<'de> for GroupStatus {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = String::deserialize(deserializer)?;
match value.as_str() {
"" | "enabled" => Ok(Self::Enabled),
"disabled" => Ok(Self::Disabled),
_ => Err(serde::de::Error::unknown_variant(&value, &["enabled", "disabled"])),
}
}
}
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct GroupAddRemove {
pub group: String,
@@ -40,6 +55,40 @@ pub struct GroupDesc {
pub status: String,
pub members: Vec<String>,
pub policy: String,
#[serde(rename = "updatedAt", skip_serializing_if = "Option::is_none")]
#[serde(
rename = "updatedAt",
skip_serializing_if = "Option::is_none",
with = "time::serde::rfc3339::option"
)]
pub updated_at: Option<OffsetDateTime>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn group_desc_updated_at_serializes_as_rfc3339() {
let now = OffsetDateTime::now_utc().replace_nanosecond(0).unwrap();
let group = GroupDesc {
name: "group-a".to_string(),
status: "enabled".to_string(),
members: vec!["user-a".to_string()],
policy: "readwrite".to_string(),
updated_at: Some(now),
};
let json = serde_json::to_string(&group).unwrap();
let decoded: GroupDesc = serde_json::from_str(&json).unwrap();
assert!(json.contains("\"updatedAt\":\""));
assert!(json.contains('T'));
assert_eq!(decoded.updated_at, Some(now));
}
#[test]
fn group_status_accepts_empty_string_as_enabled() {
let status: GroupStatus = serde_json::from_str(r#""""#).unwrap();
assert_eq!(status, GroupStatus::Enabled);
}
}
crates/madmin/src/user.rs
+325 -71
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
use serde_json::value::RawValue;
use std::collections::HashMap;
use time::OffsetDateTime;
@@ -88,7 +89,7 @@ pub struct UserInfo {
#[serde(rename = "memberOf", skip_serializing_if = "Option::is_none")]
pub member_of: Option<Vec<String>>,
#[serde(rename = "updatedAt")]
#[serde(rename = "updatedAt", with = "time::serde::rfc3339::option")]
pub updated_at: Option<OffsetDateTime>,
}
@@ -134,47 +135,61 @@ pub struct ListServiceAccountsResp {
pub accounts: Vec<ServiceAccountInfo>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct ListAccessKeysResp {
#[serde(rename = "serviceAccounts", default)]
pub service_accounts: Vec<ServiceAccountInfo>,
#[serde(rename = "stsKeys", default)]
pub sts_keys: Vec<ServiceAccountInfo>,
}
pub const ACCESS_KEY_LIST_USERS_ONLY: &str = "users-only";
pub const ACCESS_KEY_LIST_STS_ONLY: &str = "sts-only";
pub const ACCESS_KEY_LIST_SVCACC_ONLY: &str = "svcacc-only";
pub const ACCESS_KEY_LIST_ALL: &str = "all";
#[derive(Debug, Serialize, Deserialize)]
pub struct AddServiceAccountReq {
#[serde(rename = "policy", skip_serializing_if = "Option::is_none")]
pub policy: Option<String>,
#[serde(
rename = "policy",
skip_serializing_if = "Option::is_none",
default,
deserialize_with = "deserialize_optional_policy_value"
)]
pub policy: Option<Value>,
#[serde(rename = "targetUser", skip_serializing_if = "Option::is_none")]
pub target_user: Option<String>,
#[serde(rename = "accessKey")]
#[serde(rename = "accessKey", default)]
pub access_key: String,
#[serde(rename = "secretKey")]
#[serde(rename = "secretKey", default)]
pub secret_key: String,
#[serde(rename = "name")]
#[serde(rename = "name", skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(rename = "description", skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(rename = "expiration", with = "time::serde::rfc3339::option")]
#[serde(
rename = "expiration",
skip_serializing_if = "Option::is_none",
default,
with = "time::serde::rfc3339::option"
)]
pub expiration: Option<OffsetDateTime>,
#[serde(rename = "comment", skip_serializing_if = "Option::is_none")]
pub comment: Option<String>,
}
impl AddServiceAccountReq {
pub fn validate(&self) -> Result<(), String> {
if self.access_key.is_empty() {
return Err("accessKey is empty".to_string());
}
if self.secret_key.is_empty() {
return Err("secretKey is empty".to_string());
}
if self.name.is_none() {
return Err("name is empty".to_string());
}
// TODO: validate
Ok(())
validate_service_account_name(self.name.as_deref())?;
validate_service_account_description(self.description.as_deref().or(self.comment.as_deref()))?;
validate_service_account_expiration(self.expiration)
}
}
@@ -195,7 +210,7 @@ pub struct AddServiceAccountResp<'a> {
pub credentials: Credentials<'a>,
}
#[derive(Serialize)]
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct InfoServiceAccountResp {
pub parent_user: String,
@@ -213,10 +228,70 @@ pub struct InfoServiceAccountResp {
pub expiration: Option<OffsetDateTime>,
}
pub type TemporaryAccountInfoResp = InfoServiceAccountResp;
#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq)]
pub struct LDAPSpecificAccessKeyInfo {
#[serde(rename = "username", skip_serializing_if = "Option::is_none")]
pub username: Option<String>,
}
impl LDAPSpecificAccessKeyInfo {
pub fn is_empty(&self) -> bool {
self.username.is_none()
}
}
#[derive(Debug, Serialize, Deserialize, Default, PartialEq, Eq)]
pub struct OpenIDSpecificAccessKeyInfo {
#[serde(rename = "configName", skip_serializing_if = "Option::is_none")]
pub config_name: Option<String>,
#[serde(rename = "userID", skip_serializing_if = "Option::is_none")]
pub user_id: Option<String>,
#[serde(rename = "userIDClaim", skip_serializing_if = "Option::is_none")]
pub user_id_claim: Option<String>,
#[serde(rename = "displayName", skip_serializing_if = "Option::is_none")]
pub display_name: Option<String>,
#[serde(rename = "displayNameClaim", skip_serializing_if = "Option::is_none")]
pub display_name_claim: Option<String>,
}
impl OpenIDSpecificAccessKeyInfo {
pub fn is_empty(&self) -> bool {
self.config_name.is_none()
&& self.user_id.is_none()
&& self.user_id_claim.is_none()
&& self.display_name.is_none()
&& self.display_name_claim.is_none()
}
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct InfoAccessKeyResp {
pub access_key: String,
#[serde(flatten)]
pub info: InfoServiceAccountResp,
pub user_type: String,
pub user_provider: String,
#[serde(rename = "ldapSpecificInfo", skip_serializing_if = "LDAPSpecificAccessKeyInfo::is_empty")]
pub ldap_specific_info: LDAPSpecificAccessKeyInfo,
#[serde(
rename = "openIDSpecificInfo",
skip_serializing_if = "OpenIDSpecificAccessKeyInfo::is_empty"
)]
pub open_id_specific_info: OpenIDSpecificAccessKeyInfo,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct UpdateServiceAccountReq {
#[serde(rename = "newPolicy", skip_serializing_if = "Option::is_none")]
pub new_policy: Option<String>,
#[serde(
rename = "newPolicy",
skip_serializing_if = "Option::is_none",
default,
deserialize_with = "deserialize_optional_policy_value"
)]
pub new_policy: Option<Value>,
#[serde(rename = "newSecretKey", skip_serializing_if = "Option::is_none")]
pub new_secret_key: Option<String>,
@@ -230,18 +305,95 @@ pub struct UpdateServiceAccountReq {
#[serde(rename = "newDescription", skip_serializing_if = "Option::is_none")]
pub new_description: Option<String>,
#[serde(rename = "newExpiration", skip_serializing_if = "Option::is_none")]
#[serde(rename = "newExpiration", skip_serializing_if = "Option::is_none", default)]
#[serde(with = "time::serde::rfc3339::option")]
pub new_expiration: Option<OffsetDateTime>,
}
impl UpdateServiceAccountReq {
pub fn validate(&self) -> Result<(), String> {
// TODO: validate
Ok(())
validate_service_account_name(self.new_name.as_deref())?;
validate_service_account_description(self.new_description.as_deref())?;
validate_service_account_expiration(self.new_expiration)
}
}
fn deserialize_optional_policy_value<'de, D>(deserializer: D) -> Result<Option<Value>, D::Error>
where
D: Deserializer<'de>,
{
let value = Option::<Value>::deserialize(deserializer)?;
Ok(value.map(normalize_policy_value))
}
fn normalize_policy_value(value: Value) -> Value {
match value {
Value::String(policy) => serde_json::from_str(&policy).unwrap_or(Value::String(policy)),
other => other,
}
}
fn validate_service_account_name(name: Option<&str>) -> Result<(), String> {
let Some(name) = name else {
return Ok(());
};
if name.is_empty() {
return Ok(());
}
if name.len() > 32 {
return Err("name must not be longer than 32 characters".to_string());
}
let mut chars = name.chars();
let Some(first) = chars.next() else {
return Ok(());
};
if !first.is_ascii_alphabetic() {
return Err(
"name must contain only ASCII letters, digits, underscores and hyphens and must start with a letter".to_string(),
);
}
if chars.any(|c| !c.is_ascii_alphanumeric() && c != '_' && c != '-') {
return Err(
"name must contain only ASCII letters, digits, underscores and hyphens and must start with a letter".to_string(),
);
}
Ok(())
}
fn validate_service_account_description(description: Option<&str>) -> Result<(), String> {
let Some(description) = description else {
return Ok(());
};
if description.len() > 256 {
return Err("description must be at most 256 bytes long".to_string());
}
Ok(())
}
fn validate_service_account_expiration(expiration: Option<OffsetDateTime>) -> Result<(), String> {
let Some(expiration) = expiration else {
return Ok(());
};
if expiration.unix_timestamp() == 0 {
return Ok(());
}
if expiration < OffsetDateTime::now_utc() {
return Err("the expiration time should be in the future".to_string());
}
Ok(())
}
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct AccountInfo {
pub account_name: String,
@@ -423,6 +575,54 @@ pub struct IAMEntities {
pub sts_policies: Vec<HashMap<String, Vec<String>>>,
}
/// PolicyEntitiesResult - contains response to a policy entities query.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PolicyEntitiesResult {
#[serde(rename = "timestamp", with = "time::serde::rfc3339")]
pub timestamp: time::OffsetDateTime,
#[serde(rename = "userMappings", skip_serializing_if = "Vec::is_empty")]
pub user_mappings: Vec<UserPolicyEntities>,
#[serde(rename = "groupMappings", skip_serializing_if = "Vec::is_empty")]
pub group_mappings: Vec<GroupPolicyEntities>,
#[serde(rename = "policyMappings", skip_serializing_if = "Vec::is_empty")]
pub policy_mappings: Vec<PolicyEntities>,
}
impl Default for PolicyEntitiesResult {
fn default() -> Self {
Self {
timestamp: time::OffsetDateTime::UNIX_EPOCH,
user_mappings: Vec::new(),
group_mappings: Vec::new(),
policy_mappings: Vec::new(),
}
}
}
/// UserPolicyEntities - user -> policies mapping
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
pub struct UserPolicyEntities {
pub user: String,
pub policies: Vec<String>,
#[serde(rename = "memberOfMappings", skip_serializing_if = "Vec::is_empty")]
pub member_of_mappings: Vec<GroupPolicyEntities>,
}
/// GroupPolicyEntities - group -> policies mapping
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
pub struct GroupPolicyEntities {
pub group: String,
pub policies: Vec<String>,
}
/// PolicyEntities - policy -> user+group mapping
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
pub struct PolicyEntities {
pub policy: String,
pub users: Vec<String>,
pub groups: Vec<String>,
}
/// IAMErrEntities - represents errored out IAM entries while import with error
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct IAMErrEntities {
@@ -659,13 +859,14 @@ mod tests {
#[test]
fn test_add_service_account_req_validate_success() {
let req = AddServiceAccountReq {
policy: Some("ReadOnlyAccess".to_string()),
policy: Some(serde_json::json!({"Version": "2012-10-17"})),
target_user: Some("testuser".to_string()),
access_key: "AKIAIOSFODNN7EXAMPLE".to_string(),
secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
name: Some("test-service".to_string()),
description: Some("Test service account".to_string()),
expiration: None,
comment: None,
};
let result = req.validate();
@@ -673,54 +874,82 @@ mod tests {
}
#[test]
fn test_add_service_account_req_validate_empty_access_key() {
fn test_add_service_account_req_validate_allows_generated_credentials() {
let req = AddServiceAccountReq {
policy: None,
target_user: None,
access_key: "".to_string(),
secret_key: "secret".to_string(),
name: Some("test".to_string()),
description: None,
expiration: None,
};
let result = req.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("accessKey is empty"));
}
#[test]
fn test_add_service_account_req_validate_empty_secret_key() {
let req = AddServiceAccountReq {
policy: None,
target_user: None,
access_key: "AKIAIOSFODNN7EXAMPLE".to_string(),
secret_key: "".to_string(),
name: Some("test".to_string()),
description: None,
expiration: None,
};
let result = req.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("secretKey is empty"));
}
#[test]
fn test_add_service_account_req_validate_empty_name() {
let req = AddServiceAccountReq {
policy: None,
target_user: None,
access_key: "AKIAIOSFODNN7EXAMPLE".to_string(),
secret_key: "secret".to_string(),
name: None,
description: None,
expiration: None,
comment: None,
};
assert!(req.validate().is_ok());
}
#[test]
fn test_add_service_account_req_deserializes_stringified_policy_json() {
let req: AddServiceAccountReq = serde_json::from_str(
r#"{
"policy":"{\"Version\":\"2012-10-17\",\"Statement\":[]}",
"accessKey":"AKIAIOSFODNN7EXAMPLE",
"secretKey":"secret"
}"#,
)
.unwrap();
assert_eq!(req.policy, Some(serde_json::json!({"Version":"2012-10-17","Statement":[]})));
}
#[test]
fn test_add_service_account_req_allows_missing_policy_field() {
let req: AddServiceAccountReq = serde_json::from_str(
r#"{
"accessKey":"AKIAIOSFODNN7EXAMPLE",
"secretKey":"secret"
}"#,
)
.unwrap();
assert_eq!(req.policy, None);
}
#[test]
fn test_add_service_account_req_validate_invalid_name() {
let req = AddServiceAccountReq {
policy: None,
target_user: None,
access_key: "AKIAIOSFODNN7EXAMPLE".to_string(),
secret_key: "secret".to_string(),
name: Some("1invalid".to_string()),
description: None,
expiration: None,
comment: None,
};
let result = req.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("name is empty"));
assert!(result.unwrap_err().contains("must start with a letter"));
}
#[test]
fn test_add_service_account_req_validate_rejects_long_description() {
let req = AddServiceAccountReq {
policy: None,
target_user: None,
access_key: "AKIAIOSFODNN7EXAMPLE".to_string(),
secret_key: "secret".to_string(),
name: Some("test".to_string()),
description: Some("a".repeat(257)),
expiration: None,
comment: None,
};
let result = req.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("at most 256 bytes"));
}
#[test]
@@ -793,7 +1022,7 @@ mod tests {
#[test]
fn test_update_service_account_req_validate() {
let req = UpdateServiceAccountReq {
new_policy: Some("FullAccess".to_string()),
new_policy: Some(serde_json::json!({"Version": "2012-10-17"})),
new_secret_key: Some("newsecret".to_string()),
new_status: Some("enabled".to_string()),
new_name: Some("updated-service".to_string()),
@@ -805,6 +1034,25 @@ mod tests {
assert!(result.is_ok());
}
#[test]
fn test_update_service_account_req_deserializes_stringified_policy_json() {
let req: UpdateServiceAccountReq = serde_json::from_str(
r#"{
"newPolicy":"{\"Version\":\"2012-10-17\",\"Statement\":[]}"
}"#,
)
.unwrap();
assert_eq!(req.new_policy, Some(serde_json::json!({"Version":"2012-10-17","Statement":[]})));
}
#[test]
fn test_update_service_account_req_allows_missing_policy_field() {
let req: UpdateServiceAccountReq = serde_json::from_str(r#"{}"#).unwrap();
assert_eq!(req.new_policy, None);
}
#[test]
fn test_account_info_creation() {
use crate::BackendInfo;
@@ -904,6 +1152,7 @@ mod tests {
#[test]
fn test_serialization_deserialization_roundtrip() {
let now = OffsetDateTime::now_utc().replace_nanosecond(0).unwrap();
let user_info = UserInfo {
auth_info: Some(UserAuthInfo {
auth_type: UserAuthType::Ldap,
@@ -914,16 +1163,19 @@ mod tests {
policy_name: Some("ReadOnlyAccess".to_string()),
status: AccountStatus::Enabled,
member_of: Some(vec!["group1".to_string()]),
updated_at: None,
updated_at: Some(now),
};
let json = serde_json::to_string(&user_info).unwrap();
let deserialized: UserInfo = serde_json::from_str(&json).unwrap();
assert!(json.contains("\"updatedAt\":\""));
assert!(json.contains('T'));
assert_eq!(deserialized.secret_key.unwrap(), "secret123");
assert_eq!(deserialized.policy_name.unwrap(), "ReadOnlyAccess");
assert_eq!(deserialized.status, AccountStatus::Enabled);
assert_eq!(deserialized.member_of.unwrap().len(), 1);
assert_eq!(deserialized.updated_at, Some(now));
}
#[test]
@@ -962,13 +1214,14 @@ mod tests {
fn test_edge_cases() {
// Test empty strings and edge cases
let req = AddServiceAccountReq {
policy: Some("".to_string()),
policy: Some(serde_json::Value::Null),
target_user: Some("".to_string()),
access_key: "valid_key".to_string(),
secret_key: "valid_secret".to_string(),
name: Some("valid_name".to_string()),
description: Some("".to_string()),
expiration: None,
comment: None,
};
// Should still validate successfully with empty optional strings
@@ -977,13 +1230,14 @@ mod tests {
// Test very long strings
let long_string = "a".repeat(1000);
let long_req = AddServiceAccountReq {
policy: Some(long_string.clone()),
policy: Some(serde_json::json!({"Statement": [long_string.clone()]})),
target_user: Some(long_string.clone()),
access_key: long_string.clone(),
secret_key: long_string.clone(),
name: Some(long_string.clone()),
description: Some(long_string),
name: Some("valid_name".to_string()),
description: Some("valid description".to_string()),
expiration: None,
comment: None,
};
assert!(long_req.validate().is_ok());
crates/notify/src/stream.rs
+2 -4
View File
@@ -18,6 +18,7 @@ use rustfs_targets::{
store::{Key, Store},
target::EntityTarget,
};
use rustfs_utils::get_env_usize;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{Semaphore, mpsc};
@@ -179,10 +180,7 @@ pub async fn stream_events_with_batching(
// Configuration parameters
const DEFAULT_BATCH_SIZE: usize = 1;
let batch_size = std::env::var("RUSTFS_EVENT_BATCH_SIZE")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(DEFAULT_BATCH_SIZE);
let batch_size = get_env_usize("RUSTFS_EVENT_BATCH_SIZE", DEFAULT_BATCH_SIZE);
const BATCH_TIMEOUT: Duration = Duration::from_secs(5);
const MAX_RETRIES: usize = 5;
const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
crates/obs/src/telemetry/local.rs
+39 -1
View File
@@ -96,7 +96,14 @@ pub(super) fn init_local_logging(
let log_dir_str = config.log_directory.as_deref().filter(|s| !s.is_empty());
if let Some(log_directory) = log_dir_str {
init_file_logging_internal(config, log_directory, logger_level, is_production)
match init_file_logging_internal(config, log_directory, logger_level, is_production) {
Ok(guard) => Ok(guard),
Err(error) if should_fallback_to_stdout(&error) => {
emit_file_logging_fallback_warning(log_directory, &error);
Ok(init_stdout_only(config, logger_level, is_production))
}
Err(error) => Err(error),
}
} else {
Ok(init_stdout_only(config, logger_level, is_production))
}
@@ -331,6 +338,24 @@ pub fn ensure_dir_permissions(log_directory: &str) -> Result<(), TelemetryError>
}
}
pub(super) fn should_fallback_to_stdout(error: &TelemetryError) -> bool {
match error {
TelemetryError::SetPermissions(_) => true,
TelemetryError::Io(message) => {
let message = message.to_ascii_lowercase();
message.contains("permission denied") || message.contains("os error 13")
}
_ => false,
}
}
pub(super) fn emit_file_logging_fallback_warning(log_directory: &str, error: &TelemetryError) {
eprintln!(
"[WARN] Failed to initialize file observability logging at '{}': {}. Falling back to stdout logging.",
log_directory, error
);
}
// ─── Cleanup task ─────────────────────────────────────────────────────────────
/// Spawn a background task that periodically cleans up old log files.
@@ -496,4 +521,17 @@ mod tests {
assert!(result.is_err(), "invalid filename must return Err, not panic");
});
}
#[test]
fn test_permission_denied_errors_fall_back_to_stdout() {
assert!(should_fallback_to_stdout(&TelemetryError::Io(
"Permission denied (os error 13)".to_string()
)));
assert!(should_fallback_to_stdout(&TelemetryError::SetPermissions(
"dir='/logs', want=0o755, have=0o777, err=Permission denied (os error 13)".to_string()
)));
assert!(!should_fallback_to_stdout(&TelemetryError::Io(
"No such file or directory (os error 2)".to_string()
)));
}
}
crates/obs/src/telemetry/otel.rs
+48 -43
View File
@@ -175,6 +175,7 @@ pub(super) fn init_observability_http(
let mut cleanup_handle = None;
let mut tracing_guard = None; // Guard for file writer
let mut stdout_guard = None; // Guard for stdout writer (File mode)
let mut force_stdout_logging = false;
// ── Case 1: OTLP Logging
if !log_ep.is_empty() {
@@ -201,40 +202,34 @@ pub(super) fn init_observability_http(
{
let log_filename = config.log_filename.as_deref().unwrap_or(&service_name);
let keep_files = config.log_keep_files.unwrap_or(DEFAULT_LOG_KEEP_FILES);
let file_logging_result = (|| -> Result<_, TelemetryError> {
fs::create_dir_all(log_directory).map_err(|e| TelemetryError::Io(e.to_string()))?;
// 1. Ensure dir exists
if let Err(e) = fs::create_dir_all(log_directory) {
return Err(TelemetryError::Io(e.to_string()));
}
// 2. Permissions
#[cfg(unix)]
crate::telemetry::local::ensure_dir_permissions(log_directory)?;
#[cfg(unix)]
crate::telemetry::local::ensure_dir_permissions(log_directory)?;
// 3. Rotation
let rotation_str = config
.log_rotation_time
.as_deref()
.unwrap_or(DEFAULT_LOG_ROTATION_TIME)
.to_lowercase();
let match_mode = FileMatchMode::from_config_str(config.log_match_mode.as_deref().unwrap_or(DEFAULT_OBS_LOG_MATCH_MODE));
let rotation = match rotation_str.as_str() {
"minutely" => Rotation::Minutely,
"hourly" => Rotation::Hourly,
"daily" => Rotation::Daily,
_ => Rotation::Daily,
};
let max_single_file_size = config
.log_max_single_file_size_bytes
.unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES);
let rotation_str = config
.log_rotation_time
.as_deref()
.unwrap_or(DEFAULT_LOG_ROTATION_TIME)
.to_lowercase();
let match_mode =
FileMatchMode::from_config_str(config.log_match_mode.as_deref().unwrap_or(DEFAULT_OBS_LOG_MATCH_MODE));
let rotation = match rotation_str.as_str() {
"minutely" => Rotation::Minutely,
"hourly" => Rotation::Hourly,
"daily" => Rotation::Daily,
_ => Rotation::Daily,
};
let max_single_file_size = config
.log_max_single_file_size_bytes
.unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES);
let file_appender =
RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?;
let file_appender =
RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?;
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
tracing_guard = Some(guard);
file_layer_opt = Some(
tracing_subscriber::fmt::layer()
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
let file_layer = tracing_subscriber::fmt::layer()
.with_timer(LocalTime::rfc_3339())
.with_target(true)
.with_ansi(false)
@@ -247,17 +242,28 @@ pub(super) fn init_observability_http(
.with_current_span(true)
.with_span_list(true)
.with_span_events(span_events.clone())
.with_filter(build_env_filter(logger_level, None)),
);
.with_filter(build_env_filter(logger_level, None));
let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files);
Ok((file_layer, guard, cleanup_handle, rotation_str))
})();
// The cleanup task keeps rotated files bounded while the OTLP trace and
// metric exporters continue to operate independently.
cleanup_handle = Some(spawn_cleanup_task(config, log_directory, log_filename, keep_files));
match file_logging_result {
Ok((file_layer, guard, new_cleanup_handle, rotation_str)) => {
tracing_guard = Some(guard);
file_layer_opt = Some(file_layer);
cleanup_handle = Some(new_cleanup_handle);
info!(
"Init file logging at '{}', rotation: {}, keep {} files",
log_directory, rotation_str, keep_files
);
info!(
"Init file logging at '{}', rotation: {}, keep {} files",
log_directory, rotation_str, keep_files
);
}
Err(error) if crate::telemetry::local::should_fallback_to_stdout(&error) => {
crate::telemetry::local::emit_file_logging_fallback_warning(log_directory, &error);
force_stdout_logging = true;
}
Err(error) => return Err(error),
}
}
// ── Tracing subscriber registry ───────────────────────────────────────────
@@ -266,10 +272,9 @@ pub(super) fn init_observability_http(
.map(|p| OpenTelemetryLayer::new(p.tracer(service_name.to_string())));
let metrics_layer = meter_provider.as_ref().map(|p| MetricsLayer::new(p.clone()));
// Optional stdout mirror (matching `init_file_logging_internal` logic).
// This is separate from OTLP stdout exporting; it only affects local human
// readable output for the tracing subscriber.
if config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) || !is_production {
// Optional stdout mirror (matching init_file_logging_internal logic)
// This is separate from OTLP stdout logic. If file logging is enabled, we honor its stdout rules.
if force_stdout_logging || config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) || !is_production {
let (stdout_nb, stdout_g) = tracing_appender::non_blocking(std::io::stdout());
stdout_guard = Some(stdout_g);
stdout_layer_opt = Some(
crates/policy/Cargo.toml
+1 -1
View File
@@ -32,7 +32,7 @@ workspace = true
rustfs-credentials = { workspace = true }
rustfs-config = { workspace = true, features = ["constants", "opa"] }
tokio = { workspace = true, features = ["full"] }
time = { workspace = true, features = ["serde-human-readable"] }
time = { workspace = true, features = ["serde", "parsing", "formatting", "macros"] }
serde = { workspace = true, features = ["derive", "rc"] }
serde_json.workspace = true
thiserror.workspace = true
crates/policy/src/auth/mod.rs
+18
View File
@@ -26,6 +26,8 @@ use time::OffsetDateTime;
pub struct UserIdentity {
pub version: i64,
pub credentials: Credentials,
/// updatedAt (RFC3339), legacy RustFS: update_at. Serialize as updatedAt
#[serde(rename = "updatedAt", alias = "update_at", default, with = "crate::serde_datetime::option")]
pub update_at: Option<OffsetDateTime>,
}
@@ -74,3 +76,19 @@ impl From<Credentials> for UserIdentity {
}
}
}
#[cfg(test)]
mod tests {
use super::UserIdentity;
/// Deserialize UserIdentity from MinIO-style JSON (RFC3339 updatedAt).
#[test]
fn test_user_identity_deserialize_minio_style_rfc3339() {
let minio_style =
r#"{"version":1,"credentials":{"accessKey":"ak","secretKey":"sk12345678"},"updatedAt":"2025-03-07T12:00:00Z"}"#;
let u: UserIdentity = serde_json::from_str(minio_style).expect("deserialize MinIO-style identity");
assert_eq!(u.version, 1);
assert_eq!(u.credentials.access_key, "ak");
assert!(u.update_at.is_some());
}
}
crates/policy/src/lib.rs
+1
View File
@@ -17,5 +17,6 @@ pub mod auth;
pub mod error;
pub mod format;
pub mod policy;
pub mod serde_datetime;
pub mod service_type;
pub mod utils;
crates/policy/src/policy/doc.rs
+58
View File
@@ -19,9 +19,27 @@ use super::Policy;
#[derive(Serialize, Deserialize, Default, Clone)]
pub struct PolicyDoc {
/// Version (omitempty), legacy: version.
#[serde(rename = "Version", alias = "version", default)]
pub version: i64,
/// Policy, legacy: policy. Serialize as Policy.
#[serde(rename = "Policy", alias = "policy")]
pub policy: Policy,
/// CreateDate (RFC3339), legacy: create_date. Serialize as CreateDate.
#[serde(
rename = "CreateDate",
alias = "create_date",
default,
with = "crate::serde_datetime::option"
)]
pub create_date: Option<OffsetDateTime>,
/// UpdateDate (RFC3339), legacy: update_date. Serialize as UpdateDate.
#[serde(
rename = "UpdateDate",
alias = "update_date",
default,
with = "crate::serde_datetime::option"
)]
pub update_date: Option<OffsetDateTime>,
}
@@ -73,3 +91,43 @@ impl TryFrom<Vec<u8>> for PolicyDoc {
.map_err(|_| serde_json::Error::custom("Failed to parse as PolicyDoc or Policy".to_string()))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::policy::Policy;
#[test]
fn test_policy_doc_timestamps_serialize_as_rfc3339() {
let policy = Policy::default();
let doc = PolicyDoc::new(policy);
let json = serde_json::to_string(&doc).expect("serialize");
// RFC3339 uses 'T' between date and time and 'Z' or offset for UTC
assert!(json.contains('T'), "PolicyDoc timestamps should be RFC3339 (contain 'T'); got: {}", json);
assert!(
json.contains('Z') || json.contains("+00:00"),
"PolicyDoc timestamps should be RFC3339 (contain 'Z' or +00:00); got: {}",
json
);
}
#[test]
fn test_policy_doc_deserialize_minio_style_rfc3339_timestamps() {
let minio_style = r#"{"Version":1,"Policy":{"Version":"2012-10-17","Statement":[]},"CreateDate":"2025-03-07T12:00:00Z","UpdateDate":"2025-03-07T12:00:00Z"}"#;
let doc: PolicyDoc = serde_json::from_str(minio_style).expect("deserialize MinIO-style JSON");
assert_eq!(doc.version, 1);
assert!(doc.create_date.is_some());
assert!(doc.update_date.is_some());
}
/// Round-trip: serialize then deserialize PolicyDoc; timestamps must match.
#[test]
fn test_policy_doc_timestamp_roundtrip() {
let policy = Policy::default();
let doc = PolicyDoc::new(policy);
let json = serde_json::to_string(&doc).expect("serialize");
let restored: PolicyDoc = serde_json::from_str(&json).expect("deserialize");
assert_eq!(doc.create_date, restored.create_date);
assert_eq!(doc.update_date, restored.update_date);
}
}
crates/policy/src/policy/policy.rs
+10 -2
View File
@@ -55,9 +55,9 @@ impl Args<'_> {
pub struct Policy {
#[serde(default, rename = "ID")]
pub id: ID,
#[serde(rename = "Version")]
#[serde(default, rename = "Version")]
pub version: String,
#[serde(rename = "Statement")]
#[serde(default, rename = "Statement")]
pub statements: Vec<Statement>,
}
@@ -1084,6 +1084,14 @@ mod test {
);
}
#[test]
fn test_parse_empty_policy_object_as_implied_policy() {
let policy = Policy::parse_config(b"{}").expect("empty JSON object should parse");
assert!(policy.version.is_empty());
assert!(policy.statements.is_empty());
}
#[test]
fn test_statement_with_both_action_and_notaction_is_invalid() {
// Test: A statement with both Action and NotAction returns BothActionAndNotAction error
crates/policy/src/serde_datetime.rs
+121
View File
@@ -0,0 +1,121 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Serde helpers for IAM timestamps: serialize as RFC3339 (MinIO-compatible),
//! deserialize from RFC3339 or legacy RustFS human-readable format.
use serde::{Deserialize, Deserializer, Serializer};
use time::OffsetDateTime;
use time::format_description;
use time::format_description::well_known::Rfc3339;
/// Legacy RustFS format: `YYYY-MM-DD HH:MM:SS.ffffff +00:00:00` (time crate serde-human-readable style).
static LEGACY_FORMAT: std::sync::OnceLock<time::format_description::OwnedFormatItem> = std::sync::OnceLock::new();
fn legacy_format() -> &'static time::format_description::OwnedFormatItem {
LEGACY_FORMAT.get_or_init(|| {
format_description::parse_owned::<2>(
"[year]-[month]-[day] [hour]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]",
)
.expect("legacy format description is valid")
});
LEGACY_FORMAT.get().expect("initialized above")
}
fn parse_rfc3339_or_legacy(s: &str) -> Result<OffsetDateTime, time::Error> {
OffsetDateTime::parse(s, &Rfc3339).or_else(|_| OffsetDateTime::parse(s, legacy_format()).map_err(Into::into))
}
/// Serialize as RFC3339; deserialize from RFC3339 or legacy RustFS format.
pub fn serialize<S>(dt: &OffsetDateTime, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
time::serde::rfc3339::serialize(dt, serializer)
}
/// Deserialize from RFC3339 or legacy RustFS human-readable format.
pub fn deserialize<'de, D>(deserializer: D) -> Result<OffsetDateTime, D::Error>
where
D: Deserializer<'de>,
{
let s = <&str>::deserialize(deserializer)?;
parse_rfc3339_or_legacy(s).map_err(serde::de::Error::custom)
}
/// Option version: serialize as RFC3339; deserialize from RFC3339 or legacy.
pub mod option {
use serde::{Deserialize, Deserializer, Serializer};
use time::OffsetDateTime;
use super::{Rfc3339, parse_rfc3339_or_legacy};
pub fn serialize<S>(opt: &Option<OffsetDateTime>, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match opt {
Some(dt) => {
let s = dt.format(&Rfc3339).map_err(serde::ser::Error::custom)?;
serializer.serialize_some(&s)
}
None => serializer.serialize_none(),
}
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<OffsetDateTime>, D::Error>
where
D: Deserializer<'de>,
{
let opt: Option<&str> = Option::deserialize(deserializer)?;
match opt {
None => Ok(None),
Some(s) => parse_rfc3339_or_legacy(s).map(Some).map_err(serde::de::Error::custom),
}
}
}
#[cfg(test)]
mod tests {
use serde::{Deserialize, Serialize};
use super::*;
#[derive(Serialize, Deserialize)]
#[serde(transparent)]
struct Dt(#[serde(with = "crate::serde_datetime")] OffsetDateTime);
#[test]
fn test_deserialize_legacy_rustfs_timestamp() {
// Legacy RustFS human-readable format (time serde-human-readable).
let json = r#""2026-03-09 02:22:44.998954 +00:00:00""#;
let Dt(dt) = serde_json::from_str(json).expect("deserialize legacy timestamp");
assert_eq!(dt.year(), 2026);
assert_eq!(dt.month(), time::Month::March);
assert_eq!(dt.day(), 9);
assert_eq!(dt.hour(), 2);
assert_eq!(dt.minute(), 22);
assert_eq!(dt.second(), 44);
}
#[test]
fn test_deserialize_rfc3339_timestamp() {
let json = r#""2025-03-07T12:00:00Z""#;
let Dt(dt) = serde_json::from_str(json).expect("deserialize RFC3339");
assert_eq!(dt.year(), 2025);
assert_eq!(dt.month(), time::Month::March);
assert_eq!(dt.day(), 7);
assert_eq!(dt.hour(), 12);
}
}
crates/rio/src/compress_index.rs
+4
View File
@@ -71,6 +71,10 @@ impl Index {
self.info.len()
}
pub fn is_empty(&self) -> bool {
self.info.is_empty()
}
fn alloc_infos(&mut self, n: usize) {
if n > MAX_INDEX_ENTRIES {
panic!("n > MAX_INDEX_ENTRIES");
crates/rio/src/http_reader.rs
+3 -5
View File
@@ -18,6 +18,7 @@ use futures::{Stream, TryStreamExt as _};
use http::HeaderMap;
use pin_project_lite::pin_project;
use reqwest::{Certificate, Client, Identity, Method, RequestBuilder};
use rustfs_utils::get_env_opt_str;
use std::error::Error as _;
use std::io::{self, Error};
use std::ops::Not as _;
@@ -32,11 +33,8 @@ use tracing::error;
/// Get the TLS path from the RUSTFS_TLS_PATH environment variable.
/// If the variable is not set, return None.
fn tls_path() -> Option<&'static std::path::PathBuf> {
static TLS_PATH: LazyLock<Option<std::path::PathBuf>> = LazyLock::new(|| {
std::env::var("RUSTFS_TLS_PATH")
.ok()
.and_then(|s| if s.is_empty() { None } else { Some(s.into()) })
});
static TLS_PATH: LazyLock<Option<std::path::PathBuf>> =
LazyLock::new(|| get_env_opt_str("RUSTFS_TLS_PATH").and_then(|s| if s.is_empty() { None } else { Some(s.into()) }));
TLS_PATH.as_ref()
}
crates/rio/src/lib.rs
+1 -1
View File
@@ -49,7 +49,7 @@ pub use writer::*;
mod http_reader;
pub use http_reader::*;
pub use compress_index::TryGetIndex;
pub use compress_index::{Index, TryGetIndex};
mod etag;
crates/scanner/src/data_usage_define.rs
-11
View File
@@ -310,17 +310,6 @@ impl DataUsageHash {
let hash = self.calculate_hash();
warn!(
"mod_alt: key: {} hash: {} cycle: {} cycles: {} mod: {} hash >> 32: {} result: {}",
self.0,
hash,
cycle,
cycles,
(hash >> 32) as u32,
(hash >> 32) as u32 % cycles,
cycle % cycles,
);
(hash >> 32) as u32 % cycles == cycle % cycles
}
crates/scanner/src/scanner_io.rs
+10 -6
View File
@@ -162,7 +162,8 @@ impl ScannerIO for ECStore {
let all_buckets_clone = all_buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>();
tokio::spawn(async move {
let mut last_update = SystemTime::now();
let mut last_update = SystemTime::UNIX_EPOCH;
let mut has_sent_once = false;
let mut ticker = tokio::time::interval(Duration::from_secs(30));
loop {
@@ -184,7 +185,8 @@ impl ScannerIO for ECStore {
all_merged.merge(result);
}
if all_merged.root().is_some() && all_merged.info.last_update.unwrap() > last_update {
let merged_last_update = all_merged.info.last_update.unwrap_or(SystemTime::UNIX_EPOCH);
if all_merged.root().is_some() && (!has_sent_once || merged_last_update > last_update) {
let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone);
if let Err(e) = updates.send(dui).await {
error!("Failed to send data usage info: {}", e);
@@ -202,12 +204,14 @@ impl ScannerIO for ECStore {
all_merged.merge(result);
}
if all_merged.root().is_some() && all_merged.info.last_update.unwrap() > last_update {
let merged_last_update = all_merged.info.last_update.unwrap_or(SystemTime::UNIX_EPOCH);
if all_merged.root().is_some() && (!has_sent_once || merged_last_update > last_update) {
let dui = all_merged.dui(&all_merged.info.name, &all_buckets_clone);
if let Err(e) = updates.send(dui).await {
error!("Failed to send data usage info: {}", e);
}
last_update = all_merged.info.last_update.unwrap();
has_sent_once = true;
last_update = merged_last_update;
}
}
}
@@ -511,8 +515,8 @@ impl ScannerIODisk for Disk {
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
Ok(versions) => versions,
Err(e) => {
error!("Failed to get file info versions: {}", e);
return Err(StorageError::other("failed to get file info".to_string()));
error!("Failed to get file info versions: {}/{}, err: {e}", item.bucket, item.object_path());
return Err(StorageError::other("skip file".to_string()));
}
};
crates/scanner/tests/lifecycle_integration_test.rs
+2 -2
View File
@@ -267,12 +267,12 @@ async fn create_test_tier(server: u32) {
..Default::default()
})
} else if server == 2 {
let test_minio_server = std::env::var("TEST_MINIO_SERVER").unwrap_or_else(|_| "localhost:9000".to_string());
let test_compatible_server = std::env::var("TEST_MINIO_SERVER").unwrap_or_else(|_| "localhost:9000".to_string());
Some(TierMinIO {
access_key: "minioadmin".to_string(),
secret_key: "minioadmin".to_string(),
bucket: "mblock2".to_string(),
endpoint: format!("http://{}", test_minio_server),
endpoint: format!("http://{}", test_compatible_server),
prefix: format!("mypre{}/", uuid::Uuid::new_v4()),
region: "".to_string(),
..Default::default()
crates/trusted-proxies/src/cloud/detector.rs
+46 -13
View File
@@ -15,6 +15,7 @@
//! Cloud provider detection and metadata fetching.
use async_trait::async_trait;
use rustfs_utils::get_env_opt_str;
use std::str::FromStr;
use std::time::Duration;
use tracing::{debug, info, warn};
@@ -56,37 +57,33 @@ impl FromStr for CloudProvider {
impl CloudProvider {
/// Detects the cloud provider based on environment variables.
pub fn detect_from_env() -> Option<Self> {
let has_env = |key| get_env_opt_str(key).is_some();
// Check for AWS environment variables.
if std::env::var("RUSTFS_AWS_EXECUTION_ENV").is_ok()
|| std::env::var("RUSTFS_AWS_REGION").is_ok()
|| std::env::var("RUSTFS_EC2_INSTANCE_ID").is_ok()
{
if has_env("RUSTFS_AWS_EXECUTION_ENV") || has_env("RUSTFS_AWS_REGION") || has_env("RUSTFS_EC2_INSTANCE_ID") {
return Some(Self::Aws);
}
// Check for Azure environment variables.
if std::env::var("RUSTFS_WEBSITE_SITE_NAME").is_ok()
|| std::env::var("RUSTFS_WEBSITE_INSTANCE_ID").is_ok()
|| std::env::var("RUSTFS_APPSETTING_WEBSITE_SITE_NAME").is_ok()
if has_env("RUSTFS_WEBSITE_SITE_NAME")
|| has_env("RUSTFS_WEBSITE_INSTANCE_ID")
|| has_env("RUSTFS_APPSETTING_WEBSITE_SITE_NAME")
{
return Some(Self::Azure);
}
// Check for GCP environment variables.
if std::env::var("RUSTFS_GCP_PROJECT").is_ok()
|| std::env::var("RUSTFS_GOOGLE_CLOUD_PROJECT").is_ok()
|| std::env::var("RUSTFS_GAE_INSTANCE").is_ok()
{
if has_env("RUSTFS_GCP_PROJECT") || has_env("RUSTFS_GOOGLE_CLOUD_PROJECT") || has_env("RUSTFS_GAE_INSTANCE") {
return Some(Self::Gcp);
}
// Check for DigitalOcean environment variables.
if std::env::var("RUSTFS_DIGITALOCEAN_REGION").is_ok() {
if has_env("RUSTFS_DIGITALOCEAN_REGION") {
return Some(Self::DigitalOcean);
}
// Check for Cloudflare environment variables.
if std::env::var("RUSTFS_CF_PAGES").is_ok() || std::env::var("RUSTFS_CF_WORKERS").is_ok() {
if has_env("RUSTFS_CF_PAGES") || has_env("RUSTFS_CF_WORKERS") {
return Some(Self::Cloudflare);
}
@@ -106,6 +103,42 @@ impl CloudProvider {
}
}
#[cfg(test)]
mod tests {
use super::*;
use temp_env::{with_var, with_vars_unset};
#[test]
fn test_detect_from_env_prefers_known_provider_markers() {
with_var("RUSTFS_AWS_REGION", Some("us-east-1"), || {
assert_eq!(CloudProvider::detect_from_env(), Some(CloudProvider::Aws));
});
}
#[test]
fn test_detect_from_env_returns_none_without_markers() {
with_vars_unset(
vec![
"RUSTFS_AWS_EXECUTION_ENV",
"RUSTFS_AWS_REGION",
"RUSTFS_EC2_INSTANCE_ID",
"RUSTFS_WEBSITE_SITE_NAME",
"RUSTFS_WEBSITE_INSTANCE_ID",
"RUSTFS_APPSETTING_WEBSITE_SITE_NAME",
"RUSTFS_GCP_PROJECT",
"RUSTFS_GOOGLE_CLOUD_PROJECT",
"RUSTFS_GAE_INSTANCE",
"RUSTFS_DIGITALOCEAN_REGION",
"RUSTFS_CF_PAGES",
"RUSTFS_CF_WORKERS",
],
|| {
assert_eq!(CloudProvider::detect_from_env(), None);
},
);
}
}
/// Trait for fetching metadata from a specific cloud provider.
#[async_trait]
pub trait CloudMetadataFetcher: Send + Sync {
crates/utils/Cargo.toml
+3 -1
View File
@@ -26,6 +26,7 @@ categories = ["web-programming", "development-tools", "cryptography"]
[dependencies]
base64-simd = { workspace = true, optional = true }
blake2 = { workspace = true, optional = true }
blake3 = { workspace = true, optional = true }
brotli = { workspace = true, optional = true }
bytes = { workspace = true, optional = true }
@@ -69,6 +70,7 @@ zstd = { workspace = true, optional = true }
tempfile = { workspace = true }
rand = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
temp-env = { workspace = true }
[target.'cfg(windows)'.dependencies]
windows = { workspace = true, optional = true, features = ["Win32_Storage_FileSystem", "Win32_Foundation"] }
@@ -87,7 +89,7 @@ notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde",
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
string = ["dep:regex"]
crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1"]
hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:crc-fast"]
hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:crc-fast"]
os = ["dep:rustix", "dep:tempfile", "dep:windows"] # operating system utilities
integration = [] # integration test features
sys = ["dep:sysinfo"] # system information features
crates/utils/src/envs.rs
+293 -27
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use std::{
collections::BTreeSet,
collections::HashSet,
env,
sync::{Mutex, OnceLock},
@@ -30,7 +31,7 @@ use tracing::warn;
/// - `i8`: The parsed value as i8 if successful, otherwise the default value.
///
pub fn get_env_i8(key: &str, default: i8) -> i8 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
@@ -43,7 +44,7 @@ pub fn get_env_i8(key: &str, default: i8) -> i8 {
/// - `Option<i8>`: The parsed value as i8 if successful, otherwise None
///
pub fn get_env_opt_i8(key: &str) -> Option<i8> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -57,7 +58,7 @@ pub fn get_env_opt_i8(key: &str) -> Option<i8> {
/// - `u8`: The parsed value as u8 if successful, otherwise the default value.
///
pub fn get_env_u8(key: &str, default: u8) -> u8 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
@@ -70,7 +71,7 @@ pub fn get_env_u8(key: &str, default: u8) -> u8 {
/// - `Option<u8>`: The parsed value as u8 if successful, otherwise None
///
pub fn get_env_opt_u8(key: &str) -> Option<u8> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
static WARNED_ENV_MESSAGES: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();
@@ -86,19 +87,182 @@ fn log_once(key: &str, message: impl FnOnce() -> String) {
}
}
fn external_alias_for_key(key: &str) -> Option<String> {
let suffix = key.strip_prefix("RUSTFS_")?;
if is_external_compatible_suffix(suffix) {
Some(format!("{}{}", external_env_prefix(), suffix))
} else {
None
}
}
fn resolve_env_with_aliases(key: &str, deprecated: &[&str]) -> Option<(String, String)> {
if let Ok(value) = env::var(key) {
return Some((key.to_string(), value));
}
let (alias, value) = deprecated
if let Some((alias, value)) = deprecated
.iter()
.find_map(|alias| env::var(alias).ok().map(|value| (*alias, value)))?;
.find_map(|alias| env::var(alias).ok().map(|value| (*alias, value)))
{
let deprecated_key = format!("env_alias:{alias}->{key}");
log_once(&deprecated_key, || {
format!("Environment variable {alias} is deprecated, use {key} instead")
});
return Some((alias.to_string(), value));
}
let alias = external_alias_for_key(key)?;
let value = env::var(&alias).ok()?;
let deprecated_key = format!("env_alias:{alias}->{key}");
log_once(&deprecated_key, || {
format!("Environment variable {alias} is deprecated, use {key} instead")
});
Some((alias.to_string(), value))
Some((alias, value))
}
const EXTERNAL_ENV_PREFIX_BYTES: [u8; 6] = [77, 73, 78, 73, 79, 95];
const EXTERNAL_COMPATIBLE_SUFFIXES: &[&str] = &[
"ACCESS_KEY",
"ACCESS_KEY_FILE",
"ADDRESS",
"API_XFF_HEADER",
"AUDIT_WEBHOOK_AUTH_TOKEN",
"AUDIT_WEBHOOK_CLIENT_CERT",
"AUDIT_WEBHOOK_CLIENT_KEY",
"AUDIT_WEBHOOK_ENABLE",
"AUDIT_WEBHOOK_ENDPOINT",
"AUDIT_WEBHOOK_QUEUE_DIR",
"COMPRESS_ENABLE",
"COMPRESS_EXTENSIONS",
"COMPRESS_MIME_TYPES",
"CONSOLE_ADDRESS",
"DRIVE_ACTIVE_MONITORING",
"ERASURE_SET_DRIVE_COUNT",
"IDENTITY_OPENID_CLAIM_NAME",
"IDENTITY_OPENID_CLAIM_PREFIX",
"IDENTITY_OPENID_CLIENT_ID",
"IDENTITY_OPENID_CLIENT_SECRET",
"IDENTITY_OPENID_CONFIG_URL",
"IDENTITY_OPENID_DISPLAY_NAME",
"IDENTITY_OPENID_REDIRECT_URI",
"IDENTITY_OPENID_SCOPES",
"ILM_EXPIRATION_WORKERS",
"LICENSE",
"NOTIFY_MQTT_BROKER",
"NOTIFY_MQTT_ENABLE",
"NOTIFY_MQTT_KEEP_ALIVE_INTERVAL",
"NOTIFY_MQTT_PASSWORD",
"NOTIFY_MQTT_QOS",
"NOTIFY_MQTT_QUEUE_DIR",
"NOTIFY_MQTT_QUEUE_LIMIT",
"NOTIFY_MQTT_RECONNECT_INTERVAL",
"NOTIFY_MQTT_TOPIC",
"NOTIFY_MQTT_USERNAME",
"NOTIFY_WEBHOOK_AUTH_TOKEN",
"NOTIFY_WEBHOOK_CLIENT_CERT",
"NOTIFY_WEBHOOK_CLIENT_KEY",
"NOTIFY_WEBHOOK_ENABLE",
"NOTIFY_WEBHOOK_ENDPOINT",
"NOTIFY_WEBHOOK_QUEUE_DIR",
"NOTIFY_WEBHOOK_QUEUE_LIMIT",
"POLICY_PLUGIN_AUTH_TOKEN",
"POLICY_PLUGIN_URL",
"PORT",
"REGION",
"ROOT_PASSWORD",
"ROOT_USER",
"SECRET_KEY",
"SECRET_KEY_FILE",
"STORAGE_CLASS_INLINE_BLOCK",
"STORAGE_CLASS_OPTIMIZE",
"STORAGE_CLASS_RRS",
"STORAGE_CLASS_STANDARD",
"VERSION",
"VOLUMES",
];
const EXTERNAL_DYNAMIC_COMPATIBLE_PREFIXES: &[&str] = &["AUDIT_MQTT_", "AUDIT_WEBHOOK_", "NOTIFY_MQTT_", "NOTIFY_WEBHOOK_"];
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct ExternalEnvCompatReport {
pub mapped_pairs: Vec<(String, String)>,
pub conflict_keys: Vec<String>,
}
impl ExternalEnvCompatReport {
pub fn mapped_count(&self) -> usize {
self.mapped_pairs.len()
}
pub fn conflict_count(&self) -> usize {
self.conflict_keys.len()
}
}
fn external_env_prefix() -> &'static str {
static PREFIX: OnceLock<String> = OnceLock::new();
PREFIX
.get_or_init(|| EXTERNAL_ENV_PREFIX_BYTES.iter().map(|&byte| char::from(byte)).collect())
.as_str()
}
fn is_external_compatible_suffix(suffix: &str) -> bool {
EXTERNAL_COMPATIBLE_SUFFIXES.contains(&suffix)
|| EXTERNAL_DYNAMIC_COMPATIBLE_PREFIXES
.iter()
.any(|prefix| suffix.starts_with(prefix))
}
fn build_external_env_compat_report_from_entries<I>(entries: I) -> ExternalEnvCompatReport
where
I: IntoIterator<Item = (String, String)>,
{
let env_map: std::collections::BTreeMap<String, String> = entries.into_iter().collect();
let mut mapped_pairs = BTreeSet::new();
let mut conflict_keys = BTreeSet::new();
let source_prefix = external_env_prefix();
for (source_key, source_value) in env_map.iter() {
let Some(suffix) = source_key.strip_prefix(source_prefix) else {
continue;
};
if !is_external_compatible_suffix(suffix) {
continue;
}
let rustfs_key = format!("RUSTFS_{suffix}");
match env_map.get(&rustfs_key) {
None => {
mapped_pairs.insert((source_key.clone(), rustfs_key));
}
Some(rustfs_value) if rustfs_value != source_value => {
conflict_keys.insert(rustfs_key);
}
Some(_) => {}
}
}
ExternalEnvCompatReport {
mapped_pairs: mapped_pairs.into_iter().collect(),
conflict_keys: conflict_keys.into_iter().collect(),
}
}
/// Build compatibility plan between source-prefixed variables and `RUSTFS_*`.
///
/// Precedence rule:
/// - If both `RUSTFS_*` and source-prefixed variables exist, keep `RUSTFS_*` and record a conflict.
/// - If only source-prefixed variables exist, mark them as mappable to `RUSTFS_*`.
pub fn build_external_env_compat_report() -> ExternalEnvCompatReport {
build_external_env_compat_report_from_entries(env::vars())
}
fn parse_env_value<T>(key: &str) -> Option<T>
where
T: std::str::FromStr,
{
resolve_env_with_aliases(key, &[]).and_then(|(_, value)| value.parse().ok())
}
pub fn get_env_str_with_aliases(key: &str, deprecated: &[&str], default: &str) -> String {
@@ -134,7 +298,7 @@ pub fn get_env_bool_with_aliases(key: &str, deprecated: &[&str], default: bool)
/// - `i16`: The parsed value as i16 if successful, otherwise the default value.
///
pub fn get_env_i16(key: &str, default: i16) -> i16 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 16-bit type: signed i16
@@ -146,7 +310,7 @@ pub fn get_env_i16(key: &str, default: i16) -> i16 {
/// - `Option<i16>`: The parsed value as i16 if successful, otherwise None
///
pub fn get_env_opt_i16(key: &str) -> Option<i16> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -160,7 +324,7 @@ pub fn get_env_opt_i16(key: &str) -> Option<i16> {
/// - `u16`: The parsed value as u16 if successful, otherwise the default value.
///
pub fn get_env_u16(key: &str, default: u16) -> u16 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 16-bit type: unsigned u16
@@ -172,7 +336,7 @@ pub fn get_env_u16(key: &str, default: u16) -> u16 {
/// - `Option<u16>`: The parsed value as u16 if successful, otherwise None
///
pub fn get_env_u16_opt(key: &str) -> Option<u16> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 16-bit type: unsigned u16
@@ -197,7 +361,7 @@ pub fn get_env_opt_u16(key: &str) -> Option<u16> {
/// - `i32`: The parsed value as i32 if successful, otherwise the default value.
///
pub fn get_env_i32(key: &str, default: i32) -> i32 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 32-bit type: signed i32
@@ -209,7 +373,7 @@ pub fn get_env_i32(key: &str, default: i32) -> i32 {
/// - `Option<i32>`: The parsed value as i32 if successful, otherwise None
///
pub fn get_env_opt_i32(key: &str) -> Option<i32> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -223,7 +387,7 @@ pub fn get_env_opt_i32(key: &str) -> Option<i32> {
/// - `u32`: The parsed value as u32 if successful, otherwise the default value.
///
pub fn get_env_u32(key: &str, default: u32) -> u32 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 32-bit type: unsigned u32
@@ -235,7 +399,7 @@ pub fn get_env_u32(key: &str, default: u32) -> u32 {
/// - `Option<u32>`: The parsed value as u32 if successful, otherwise None
///
pub fn get_env_opt_u32(key: &str) -> Option<u32> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
///
@@ -247,7 +411,7 @@ pub fn get_env_opt_u32(key: &str) -> Option<u32> {
/// - `f32`: The parsed value as f32 if successful, otherwise the default value
///
pub fn get_env_f32(key: &str, default: f32) -> f32 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
///
@@ -258,7 +422,7 @@ pub fn get_env_f32(key: &str, default: f32) -> f32 {
/// - `Option<f32>`: The parsed value as f32 if successful, otherwise None
///
pub fn get_env_opt_f32(key: &str) -> Option<f32> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -271,7 +435,7 @@ pub fn get_env_opt_f32(key: &str) -> Option<f32> {
/// - `i64`: The parsed value as i64 if successful, otherwise the default value
///
pub fn get_env_i64(key: &str, default: i64) -> i64 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
///
@@ -282,7 +446,7 @@ pub fn get_env_i64(key: &str, default: i64) -> i64 {
/// - `Option<i64>`: The parsed value as i64 if successful, otherwise None
///
pub fn get_env_opt_i64(key: &str) -> Option<i64> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, returning Option<Option<i64>> if not set or parsing fails.
@@ -294,7 +458,7 @@ pub fn get_env_opt_i64(key: &str) -> Option<i64> {
/// - `Option<Option<i64>>`: The parsed value as i64 if successful, otherwise None
///
pub fn get_env_opt_opt_i64(key: &str) -> Option<Option<i64>> {
env::var(key).ok().map(|v| v.parse().ok())
resolve_env_with_aliases(key, &[]).map(|(_, value)| value.parse().ok())
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -307,7 +471,7 @@ pub fn get_env_opt_opt_i64(key: &str) -> Option<Option<i64>> {
/// - `u64`: The parsed value as u64 if successful, otherwise the default value.
///
pub fn get_env_u64(key: &str, default: u64) -> u64 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as an unsigned 64-bit integer, returning `None` if not set or parsing fails.
@@ -341,7 +505,7 @@ pub fn get_env_opt_u64_with_aliases(key: &str, deprecated: &[&str]) -> Option<u6
/// - `Option<u64>`: The parsed value as u64 if successful, otherwise None
///
pub fn get_env_opt_u64(key: &str) -> Option<u64> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -354,7 +518,7 @@ pub fn get_env_opt_u64(key: &str) -> Option<u64> {
/// - `f64`: The parsed value as f64 if successful, otherwise the default value.
///
pub fn get_env_f64(key: &str, default: f64) -> f64 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
@@ -366,7 +530,7 @@ pub fn get_env_f64(key: &str, default: f64) -> f64 {
/// - `Option<f64>`: The parsed value as f64 if successful, otherwise None
///
pub fn get_env_opt_f64(key: &str) -> Option<f64> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -379,7 +543,7 @@ pub fn get_env_opt_f64(key: &str) -> Option<f64> {
/// - `usize`: The parsed value as usize if successful, otherwise the default value.
///
pub fn get_env_usize(key: &str, default: usize) -> usize {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
///
@@ -390,7 +554,7 @@ pub fn get_env_usize(key: &str, default: usize) -> usize {
/// - `Option<usize>`: The parsed value as usize if successful, otherwise None
///
pub fn get_env_usize_opt(key: &str) -> Option<usize> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
@@ -427,7 +591,7 @@ pub fn get_env_str(key: &str, default: &str) -> String {
/// - `Option<String>`: The environment variable value if set, otherwise None.
///
pub fn get_env_opt_str(key: &str) -> Option<String> {
env::var(key).ok()
resolve_env_with_aliases(key, &[]).map(|(_, value)| value)
}
/// Retrieve an environment variable as a boolean, with a default value if not set or parsing fails.
@@ -476,3 +640,105 @@ pub fn get_env_opt_bool(key: &str) -> Option<bool> {
None
})
}
/// Copy supported external-prefix variables such as `MINIO_*` into their
/// canonical `RUSTFS_*` names in the current process when the canonical key is
/// missing.
#[allow(unsafe_code)]
pub fn apply_external_env_compat() -> ExternalEnvCompatReport {
let report = build_external_env_compat_report();
for (source_key, rustfs_key) in &report.mapped_pairs {
if let Ok(value) = env::var(source_key) {
// Safety: this helper is intended for early startup bootstrap
// before any background threads are created.
unsafe {
env::set_var(rustfs_key, value);
}
}
}
report
}
#[cfg(test)]
mod tests {
use super::{apply_external_env_compat, build_external_env_compat_report_from_entries, get_env_str};
fn source_key(suffix: &str) -> String {
let mut key = super::external_env_prefix().to_string();
key.push_str(suffix);
key
}
#[test]
fn source_value_is_mapped_when_rustfs_missing() {
let report =
build_external_env_compat_report_from_entries(vec![(source_key("STORAGE_CLASS_STANDARD"), "EC:2".to_string())]);
assert_eq!(report.mapped_count(), 1);
assert!(
report
.mapped_pairs
.iter()
.any(|(input_key, rustfs_key)| input_key == &source_key("STORAGE_CLASS_STANDARD")
&& rustfs_key == "RUSTFS_STORAGE_CLASS_STANDARD")
);
assert_eq!(report.conflict_count(), 0);
}
#[test]
fn rustfs_value_takes_precedence_on_conflict() {
let report = build_external_env_compat_report_from_entries(vec![
("RUSTFS_ERASURE_SET_DRIVE_COUNT".to_string(), "8".to_string()),
(source_key("ERASURE_SET_DRIVE_COUNT"), "16".to_string()),
]);
assert_eq!(report.mapped_count(), 0);
assert_eq!(report.conflict_count(), 1);
assert!(report.conflict_keys.iter().any(|key| key == "RUSTFS_ERASURE_SET_DRIVE_COUNT"));
}
#[test]
fn dynamic_notify_suffix_is_mapped() {
let report =
build_external_env_compat_report_from_entries(vec![(source_key("NOTIFY_WEBHOOK_ENABLE_PRIMARY"), "on".to_string())]);
assert_eq!(report.mapped_count(), 1);
assert!(
report
.mapped_pairs
.iter()
.any(|(input_key, rustfs_key)| input_key == &source_key("NOTIFY_WEBHOOK_ENABLE_PRIMARY")
&& rustfs_key == "RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY")
);
assert_eq!(report.conflict_count(), 0);
}
#[test]
fn unrelated_source_key_is_ignored() {
let report = build_external_env_compat_report_from_entries(vec![(source_key("UNKNOWN_COMPAT_TEST"), "1".to_string())]);
assert_eq!(report.mapped_count(), 0);
assert_eq!(report.conflict_count(), 0);
}
#[test]
fn minio_alias_is_used_for_rustfs_reads() {
temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || {
temp_env::with_var_unset("RUSTFS_ROOT_USER", || {
assert_eq!(get_env_str("RUSTFS_ROOT_USER", "default-user"), "compat-admin");
});
});
}
#[test]
fn apply_external_env_compat_copies_missing_rustfs_keys() {
temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || {
temp_env::with_var_unset("RUSTFS_ROOT_USER", || {
let report = apply_external_env_compat();
assert!(
report
.mapped_pairs
.iter()
.any(|(source_key, rustfs_key)| source_key == "MINIO_ROOT_USER" && rustfs_key == "RUSTFS_ROOT_USER")
);
assert_eq!(std::env::var("RUSTFS_ROOT_USER").as_deref(), Ok("compat-admin"));
});
});
}
}
crates/utils/src/hash.rs
+101 -11
View File
@@ -12,13 +12,30 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use blake2::{Blake2b512, Digest as Blake2Digest};
use highway::{HighwayHash, HighwayHasher, Key};
use md5::{Digest, Md5};
use md5::Md5;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
/// The fixed key for HighwayHash256. DO NOT change for compatibility.
const HIGHWAY_HASH256_KEY: [u64; 4] = [3, 4, 2, 1];
/// Magic HH-256 key: HH-256 hash of first 100 decimals of π as utf-8 with zero key.
const MAGIC_HIGHWAY_HASH256_KEY: [u8; 32] = [
0x4b, 0xe7, 0x34, 0xfa, 0x8e, 0x23, 0x8a, 0xcd, 0x26, 0x3e, 0x83, 0xe6, 0xbb, 0x96, 0x85, 0x52, 0x04, 0x0f, 0x93, 0x5d, 0xa3,
0x9f, 0x44, 0x14, 0x97, 0xe0, 0x9d, 0x13, 0x22, 0xde, 0x36, 0xa0,
];
/// Legacy HH-256 key (main branch): fixed [3,4,2,1] as u64 LE.
const LEGACY_HIGHWAY_HASH256_KEY: [u8; 32] = [
3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
];
fn highway_key_from_bytes(bytes: &[u8; 32]) -> [u64; 4] {
let mut key = [0u64; 4];
for (i, chunk) in bytes.chunks_exact(8).enumerate() {
key[i] = u64::from_le_bytes(chunk.try_into().unwrap());
}
key
}
#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)]
/// Supported hash algorithms for bitrot protection.
@@ -30,6 +47,8 @@ pub enum HashAlgorithm {
// HighwayHash256S represents the Streaming HighwayHash-256 hash function
#[default]
HighwayHash256S,
/// Legacy HighwayHash256S (main branch) with fixed key [3,4,2,1]
HighwayHash256SLegacy,
// BLAKE2b512 represents the BLAKE2b-512 hash function
BLAKE2b512,
/// MD5 (128-bit)
@@ -43,7 +62,8 @@ enum HashEncoded {
Sha256([u8; 32]),
HighwayHash256([u8; 32]),
HighwayHash256S([u8; 32]),
Blake2b512(blake3::Hash),
HighwayHash256SLegacy([u8; 32]),
Blake2b512([u8; 64]),
None,
}
@@ -55,7 +75,8 @@ impl AsRef<[u8]> for HashEncoded {
HashEncoded::Sha256(hash) => hash.as_ref(),
HashEncoded::HighwayHash256(hash) => hash.as_ref(),
HashEncoded::HighwayHash256S(hash) => hash.as_ref(),
HashEncoded::Blake2b512(hash) => hash.as_bytes(),
HashEncoded::HighwayHash256SLegacy(hash) => hash.as_ref(),
HashEncoded::Blake2b512(hash) => hash.as_ref(),
HashEncoded::None => &[],
}
}
@@ -83,17 +104,30 @@ impl HashAlgorithm {
match self {
HashAlgorithm::Md5 => HashEncoded::Md5(Md5::digest(data).into()),
HashAlgorithm::HighwayHash256 => {
let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY));
let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
let mut hasher = HighwayHasher::new(key);
hasher.append(data);
HashEncoded::HighwayHash256(u8x32_from_u64x4(hasher.finalize256()))
}
HashAlgorithm::SHA256 => HashEncoded::Sha256(Sha256::digest(data).into()),
HashAlgorithm::HighwayHash256S => {
let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY));
let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
let mut hasher = HighwayHasher::new(key);
hasher.append(data);
HashEncoded::HighwayHash256S(u8x32_from_u64x4(hasher.finalize256()))
}
HashAlgorithm::BLAKE2b512 => HashEncoded::Blake2b512(blake3::hash(data)),
HashAlgorithm::HighwayHash256SLegacy => {
let key = Key(highway_key_from_bytes(&LEGACY_HIGHWAY_HASH256_KEY));
let mut hasher = HighwayHasher::new(key);
hasher.append(data);
HashEncoded::HighwayHash256SLegacy(u8x32_from_u64x4(hasher.finalize256()))
}
HashAlgorithm::BLAKE2b512 => {
let hash = Blake2b512::digest(data);
let mut out = [0u8; 64];
out.copy_from_slice(hash.as_ref());
HashEncoded::Blake2b512(out)
}
HashAlgorithm::None => HashEncoded::None,
}
}
@@ -108,7 +142,8 @@ impl HashAlgorithm {
HashAlgorithm::SHA256 => 32,
HashAlgorithm::HighwayHash256 => 32,
HashAlgorithm::HighwayHash256S => 32,
HashAlgorithm::BLAKE2b512 => 32, // blake3 outputs 32 bytes by default
HashAlgorithm::HighwayHash256SLegacy => 32,
HashAlgorithm::BLAKE2b512 => 64,
HashAlgorithm::Md5 => 16,
HashAlgorithm::None => 0,
}
@@ -167,7 +202,7 @@ mod tests {
assert_eq!(HashAlgorithm::HighwayHash256.size(), 32);
assert_eq!(HashAlgorithm::HighwayHash256S.size(), 32);
assert_eq!(HashAlgorithm::SHA256.size(), 32);
assert_eq!(HashAlgorithm::BLAKE2b512.size(), 32);
assert_eq!(HashAlgorithm::BLAKE2b512.size(), 64);
assert_eq!(HashAlgorithm::None.size(), 0);
}
@@ -220,13 +255,68 @@ mod tests {
let data = b"test data";
let hash = HashAlgorithm::BLAKE2b512.hash_encode(data);
let hash = hash.as_ref();
assert_eq!(hash.len(), 32); // blake3 outputs 32 bytes by default
assert_eq!(hash.len(), 64);
// BLAKE2b512 should be deterministic
let hash2 = HashAlgorithm::BLAKE2b512.hash_encode(data);
let hash2 = hash2.as_ref();
assert_eq!(hash, hash2);
}
#[test]
fn test_bitrot_selftest() {
let checksums: [(HashAlgorithm, &str); 5] = [
(HashAlgorithm::SHA256, "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004"),
(
HashAlgorithm::BLAKE2b512,
"e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669",
),
(
HashAlgorithm::HighwayHash256,
"39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313",
),
(
HashAlgorithm::HighwayHash256S,
"39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313",
),
(
HashAlgorithm::HighwayHash256SLegacy,
"a5592a831588836b0f61bff43da4bd957c376d9b6412a9ecbbd144a3ecf34649",
),
];
for (algo, expected_hex) in checksums {
let block_size = match algo {
HashAlgorithm::SHA256 => 64,
HashAlgorithm::BLAKE2b512 => 128,
HashAlgorithm::HighwayHash256 | HashAlgorithm::HighwayHash256S | HashAlgorithm::HighwayHash256SLegacy => 32,
_ => continue,
};
let mut msg = Vec::new();
let mut sum = Vec::new();
for _ in 0..block_size {
sum = algo.hash_encode(&msg).as_ref().to_vec();
msg.extend_from_slice(&sum);
}
let got = hex_simd::encode_to_string(&sum, hex_simd::AsciiCase::Lower);
assert_eq!(got, expected_hex, "{:?} selftest mismatch: got {} want {}", algo, got, expected_hex);
}
}
/// Generates 7557 bytes
/// Pattern: (i*7+13)%256 for each byte.
fn generate_compat_test_data(size: usize) -> Vec<u8> {
(0..size).map(|i| ((i * 7 + 13) % 256) as u8).collect()
}
/// Run: cargo test -p rustfs-utils test_highwayhash_compat
#[test]
fn test_highwayhash_compat() {
let data = generate_compat_test_data(7557);
let hash = HashAlgorithm::HighwayHash256S.hash_encode(&data);
let got = hex_simd::encode_to_string(hash.as_ref(), hex_simd::AsciiCase::Lower);
let expected = "06543bf1c637e67386922a43b71cca08e5faa0f9131105a2bf96dec880529551";
assert_eq!(got, expected, "HighwayHash256S must match: got {} want {}", got, expected);
}
#[test]
fn test_different_data_different_hashes() {
let data1 = b"test data 1";
crates/utils/src/http/header_compat.rs
+123
View File
@@ -0,0 +1,123 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! HTTP header compatibility: read both x-rustfs-* and x-minio-* headers for MinIO
//! interoperability. Write both when sending replication requests.
//!
//! Use suffix-based API: `get_header(headers, SUFFIX_FORCE_DELETE)` queries both
//! x-rustfs-force-delete and x-minio-force-delete.
use http::{HeaderMap, HeaderValue};
use std::borrow::Cow;
const RUSTFS_PREFIX: &str = "x-rustfs-";
const MINIO_PREFIX: &str = "x-minio-";
const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
// Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header.
pub const SUFFIX_FORCE_DELETE: &str = "force-delete";
pub const SUFFIX_INCLUDE_DELETED: &str = "include-deleted";
pub const SUFFIX_REPLICATION_RESET_STATUS: &str = "replication-reset-status";
pub const SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE: &str = "replication-actual-object-size";
pub const SUFFIX_SOURCE_VERSION_ID: &str = "source-version-id";
pub const SUFFIX_SOURCE_MTIME: &str = "source-mtime";
pub const SUFFIX_SOURCE_ETAG: &str = "source-etag";
pub const SUFFIX_SOURCE_DELETEMARKER: &str = "source-deletemarker";
pub const SUFFIX_SOURCE_PROXY_REQUEST: &str = "source-proxy-request";
pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request";
pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check";
pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc";
/// Returns true if the key is an internal encryption metadata key (x-rustfs-encryption-* or
/// x-minio-encryption-*). Case-insensitive for metadata filtering.
pub fn is_encryption_metadata_key(key: &str) -> bool {
let lower = key.to_lowercase();
lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) || lower.starts_with(MINIO_ENCRYPTION_PREFIX)
}
fn rustfs_key(suffix: &str) -> String {
format!("{RUSTFS_PREFIX}{suffix}")
}
fn minio_key(suffix: &str) -> String {
format!("{MINIO_PREFIX}{suffix}")
}
/// Get header value: tries x-rustfs-{suffix} first, then x-minio-{suffix}. Case-insensitive.
pub fn get_header<'a>(headers: &'a HeaderMap, suffix: &str) -> Option<Cow<'a, str>> {
let rk = rustfs_key(suffix);
let mk = minio_key(suffix);
headers
.get(&rk)
.or_else(|| headers.get(&mk))
.and_then(|v| v.to_str().ok().map(Cow::Borrowed))
}
/// Insert header with both x-rustfs-{suffix} and x-minio-{suffix}.
pub fn insert_header(headers: &mut HeaderMap, suffix: &str, value: impl AsRef<[u8]>) {
if let Ok(v) = HeaderValue::from_bytes(value.as_ref()) {
if let Ok(k1) = rustfs_key(suffix).parse::<http::HeaderName>() {
headers.insert(k1, v.clone());
}
if let Ok(k2) = minio_key(suffix).parse::<http::HeaderName>() {
headers.insert(k2, v);
}
}
}
/// Get from HashMap: tries x-rustfs-{suffix} first, then x-minio-{suffix}.
pub fn get_header_map(map: &std::collections::HashMap<String, String>, suffix: &str) -> Option<String> {
let rk = rustfs_key(suffix);
let mk = minio_key(suffix);
map.get(&rk).cloned().or_else(|| map.get(&mk).cloned())
}
/// Insert into HashMap with both x-rustfs-{suffix} and x-minio-{suffix}.
pub fn insert_header_map(map: &mut std::collections::HashMap<String, String>, suffix: &str, value: impl Into<String>) {
let v = value.into();
map.insert(rustfs_key(suffix), v.clone());
map.insert(minio_key(suffix), v);
}
/// Remove from HashMap both x-rustfs-{suffix} and x-minio-{suffix}.
pub fn remove_header_map(map: &mut std::collections::HashMap<String, String>, suffix: &str) {
map.remove(&rustfs_key(suffix));
map.remove(&minio_key(suffix));
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_is_encryption_metadata_key() {
assert!(is_encryption_metadata_key("x-rustfs-encryption-iv"));
assert!(is_encryption_metadata_key("X-Rustfs-Encryption-Key"));
assert!(is_encryption_metadata_key("x-minio-encryption-iv"));
assert!(!is_encryption_metadata_key("x-amz-meta-custom"));
assert!(!is_encryption_metadata_key("x-rustfs-internal-healing"));
}
#[test]
fn test_get_header() {
let mut headers = HeaderMap::new();
headers.insert("x-minio-force-delete", HeaderValue::from_static("true"));
assert_eq!(get_header(&headers, SUFFIX_FORCE_DELETE).as_deref(), Some("true"));
let mut headers2 = HeaderMap::new();
headers2.insert("X-Rustfs-Force-Delete", HeaderValue::from_static("true"));
assert_eq!(get_header(&headers2, SUFFIX_FORCE_DELETE).as_deref(), Some("true"));
}
}
crates/utils/src/http/headers.rs
-32
View File
@@ -148,41 +148,9 @@ pub const AMZ_META_NAME: &str = "X-Amz-Meta-Name";
pub const AMZ_META_UNENCRYPTED_CONTENT_LENGTH: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length";
pub const AMZ_META_UNENCRYPTED_CONTENT_MD5: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5";
pub const RUSTFS_ENCRYPTION: &str = "X-Rustfs-Encryption-";
pub const RUSTFS_ENCRYPTION_LOWER: &str = "x-rustfs-encryption-";
pub const RESERVED_METADATA_PREFIX: &str = "X-RustFS-Internal-";
pub const RESERVED_METADATA_PREFIX_LOWER: &str = "x-rustfs-internal-";
pub const RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing";
// pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
// pub const X_RUSTFS_INLINE_DATA: &str = "x-rustfs-inline-data";
pub const VERSION_PURGE_STATUS_KEY: &str = "X-Rustfs-Internal-purgestatus";
pub const X_RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing";
pub const X_RUSTFS_DATA_MOV: &str = "X-Rustfs-Internal-data-mov";
pub const AMZ_TAGGING_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
pub const RUSTFS_FORCE_DELETE: &str = "X-Rustfs-Force-Delete";
pub const RUSTFS_INCLUDE_DELETED: &str = "X-Rustfs-Include-Deleted";
pub const RUSTFS_REPLICATION_RESET_STATUS: &str = "X-Rustfs-Replication-Reset-Status";
pub const RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE: &str = "X-Rustfs-Replication-Actual-Object-Size";
pub const RUSTFS_BUCKET_SOURCE_VERSION_ID: &str = "X-Rustfs-Source-Version-Id";
pub const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-RustFS-Source-Mtime";
pub const RUSTFS_BUCKET_SOURCE_ETAG: &str = "X-Rustfs-Source-Etag";
pub const RUSTFS_BUCKET_REPLICATION_DELETE_MARKER: &str = "X-Rustfs-Source-DeleteMarker";
pub const RUSTFS_BUCKET_REPLICATION_PROXY_REQUEST: &str = "X-Rustfs-Source-Proxy-Request";
pub const RUSTFS_BUCKET_REPLICATION_REQUEST: &str = "X-Rustfs-Source-Replication-Request";
pub const RUSTFS_BUCKET_REPLICATION_CHECK: &str = "X-Rustfs-Source-Replication-Check";
pub const RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM: &str = "X-Rustfs-Source-Replication-Ssec-Crc";
// SSEC encryption header constants
pub const SSEC_ALGORITHM_HEADER: &str = "x-amz-server-side-encryption-customer-algorithm";
pub const SSEC_KEY_HEADER: &str = "x-amz-server-side-encryption-customer-key";
crates/utils/src/http/metadata_compat.rs
+181
View File
@@ -0,0 +1,181 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! System metadata compatibility: write both x-rustfs-internal-* and x-minio-internal-*
//! for MinIO interoperability. Read prefers RustFS, fallback to MinIO.
use std::collections::HashMap;
pub const RUSTFS_INTERNAL_PREFIX: &str = "x-rustfs-internal-";
pub const MINIO_INTERNAL_PREFIX: &str = "x-minio-internal-";
// Key suffixes (lowercase, no prefix)
pub const SUFFIX_INLINE_DATA: &str = "inline-data";
pub const SUFFIX_DATA_MOVED: &str = "data-moved";
/// Transient flag for data movement
pub const SUFFIX_DATA_MOV: &str = "data-mov";
/// Transient flag for healing
pub const SUFFIX_HEALING: &str = "healing";
pub const SUFFIX_COMPRESSION: &str = "compression";
pub const SUFFIX_COMPRESSION_SIZE: &str = "compression-size";
pub const SUFFIX_ACTUAL_SIZE: &str = "actual-size";
pub const SUFFIX_ACTUAL_OBJECT_SIZE: &str = "actual-object-size";
/// Used by replication; key stored with capital A
pub const SUFFIX_ACTUAL_OBJECT_SIZE_CAP: &str = "Actual-Object-Size";
pub const SUFFIX_CRC: &str = "crc";
pub const SUFFIX_TRANSITION_STATUS: &str = "transition-status";
pub const SUFFIX_TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier";
pub const SUFFIX_FREE_VERSION: &str = "free-version";
pub const SUFFIX_PURGESTATUS: &str = "purgestatus";
pub const SUFFIX_REPLICA_STATUS: &str = "replica-status";
pub const SUFFIX_REPLICA_TIMESTAMP: &str = "replica-timestamp";
pub const SUFFIX_REPLICATION_STATUS: &str = "replication-status";
pub const SUFFIX_REPLICATION_TIMESTAMP: &str = "replication-timestamp";
pub const SUFFIX_TAGGING_TIMESTAMP: &str = "tagging-timestamp";
pub const SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP: &str = "objectlock-retention-timestamp";
pub const SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP: &str = "objectlock-legalhold-timestamp";
pub const SUFFIX_REPLICATION_RESET: &str = "replication-reset";
/// Prefix for replication-reset-{arn} keys; use with internal_key_strip_suffix_prefix to extract arn.
pub const SUFFIX_REPLICATION_RESET_ARN_PREFIX: &str = "replication-reset-";
pub const SUFFIX_TIER_FV_ID: &str = "tier-free-versionID";
pub const SUFFIX_TIER_FV_MARKER: &str = "tier-free-marker";
pub const SUFFIX_TIER_SKIP_FV_ID: &str = "tier-skip-fvid";
/// Returns true if the key is an internal metadata key (x-rustfs-internal-* or x-minio-internal-*)
/// for xl.meta compatibility. Case-insensitive.
pub fn is_internal_key(key: &str) -> bool {
let lower = key.to_lowercase();
lower.starts_with(RUSTFS_INTERNAL_PREFIX) || lower.starts_with(MINIO_INTERNAL_PREFIX)
}
/// Returns true if the key matches the given suffix for either x-rustfs-internal-* or x-minio-internal-*.
pub fn has_internal_suffix(key: &str, suffix: &str) -> bool {
let lower = key.to_lowercase();
let rustfs_key = format!("{RUSTFS_INTERNAL_PREFIX}{suffix}");
let minio_key = format!("{MINIO_INTERNAL_PREFIX}{suffix}");
lower == rustfs_key || lower == minio_key
}
/// Strips x-rustfs-internal- or x-minio-internal- prefix from key. Returns the suffix part.
/// Case-insensitive. Returns None if key is not an internal key.
pub fn strip_internal_prefix(key: &str) -> Option<String> {
let lower = key.to_lowercase();
lower
.strip_prefix(RUSTFS_INTERNAL_PREFIX)
.or_else(|| lower.strip_prefix(MINIO_INTERNAL_PREFIX))
.map(|s| s.to_string())
}
/// Returns true if key is internal and its suffix part starts with the given suffix_prefix.
/// E.g. internal_key_starts_with("x-rustfs-internal-replication-reset-arn1", "replication-reset") == true.
pub fn internal_key_starts_with(key: &str, suffix_prefix: &str) -> bool {
strip_internal_prefix(key).is_some_and(|s| s.starts_with(suffix_prefix))
}
/// For keys like x-rustfs-internal-replication-reset-{arn}, strips the internal prefix and suffix_prefix,
/// returning the remainder (e.g. "arn1"). Returns None if key does not match.
pub fn internal_key_strip_suffix_prefix(key: &str, suffix_prefix: &str) -> Option<String> {
let rest = strip_internal_prefix(key)?;
rest.strip_prefix(suffix_prefix).map(|s| s.to_string())
}
fn both_keys(suffix: &str) -> (String, String) {
(format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"), format!("{MINIO_INTERNAL_PREFIX}{suffix}"))
}
/// Builds the RustFS internal key for the given suffix. Use when a single key is needed (e.g. for
/// backward compat). Prefer insert_str/get_str when both keys should be written/read.
pub fn internal_key_rustfs(suffix: &str) -> String {
format!("{RUSTFS_INTERNAL_PREFIX}{suffix}")
}
// === String type (FileInfo.metadata, user_defined) ===
pub fn insert_str(map: &mut HashMap<String, String>, suffix: &str, value: String) {
let (k1, k2) = both_keys(suffix);
map.insert(k1, value.clone());
map.insert(k2, value);
}
pub fn get_str(map: &HashMap<String, String>, suffix: &str) -> Option<String> {
let (k1, k2) = both_keys(suffix);
map.get(&k1).cloned().or_else(|| map.get(&k2).cloned())
}
pub fn contains_key_str(map: &HashMap<String, String>, suffix: &str) -> bool {
let (k1, k2) = both_keys(suffix);
map.contains_key(&k1) || map.contains_key(&k2)
}
pub fn remove_str(map: &mut HashMap<String, String>, suffix: &str) {
let (k1, k2) = both_keys(suffix);
map.remove(&k1);
map.remove(&k2);
}
// === Vec<u8> type (meta_sys) ===
pub fn insert_bytes(map: &mut HashMap<String, Vec<u8>>, suffix: &str, value: Vec<u8>) {
let (k1, k2) = both_keys(suffix);
let v = value.clone();
map.insert(k1, value);
map.insert(k2, v);
}
pub fn get_bytes(map: &HashMap<String, Vec<u8>>, suffix: &str) -> Option<Vec<u8>> {
let (k1, k2) = both_keys(suffix);
map.get(&k1).cloned().or_else(|| map.get(&k2).cloned())
}
pub fn contains_key_bytes(map: &HashMap<String, Vec<u8>>, suffix: &str) -> bool {
let (k1, k2) = both_keys(suffix);
map.contains_key(&k1) || map.contains_key(&k2)
}
pub fn remove_bytes(map: &mut HashMap<String, Vec<u8>>, suffix: &str) {
let (k1, k2) = both_keys(suffix);
map.remove(&k1);
map.remove(&k2);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_is_internal_key() {
assert!(is_internal_key("x-rustfs-internal-healing"));
assert!(is_internal_key("x-rustfs-internal-purgestatus"));
assert!(is_internal_key("X-RustFS-Internal-purgestatus"));
assert!(is_internal_key("x-minio-internal-compression"));
assert!(is_internal_key("x-minio-internal-replication-status"));
assert!(is_internal_key("X-Minio-Internal-Compression"));
assert!(!is_internal_key("x-amz-meta-custom"));
assert!(!is_internal_key("content-type"));
assert!(!is_internal_key("x-rustfs-meta-custom"));
}
#[test]
fn test_has_internal_suffix() {
assert!(has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_PURGESTATUS));
assert!(has_internal_suffix("X-Minio-Internal-purgestatus", SUFFIX_PURGESTATUS));
assert!(has_internal_suffix("x-minio-internal-compression", SUFFIX_COMPRESSION));
assert!(has_internal_suffix("x-rustfs-internal-healing", SUFFIX_HEALING));
assert!(has_internal_suffix("x-minio-internal-data-mov", SUFFIX_DATA_MOV));
assert!(!has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_HEALING));
assert!(!has_internal_suffix("x-amz-meta-custom", SUFFIX_PURGESTATUS));
}
}
crates/utils/src/http/mod.rs
+4
View File
@@ -12,7 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod header_compat;
pub mod headers;
pub mod ip;
pub mod metadata_compat;
pub use header_compat::*;
pub use headers::*;
pub use ip::*;
pub use metadata_compat::*;
crates/utils/src/obj/metadata.rs
+2 -3
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::http::{RESERVED_METADATA_PREFIX_LOWER, is_minio_header, is_rustfs_header};
use crate::http::{is_internal_key, is_minio_header, is_rustfs_header};
use std::collections::HashMap;
/// Extract user-defined metadata keys from object metadata.
@@ -80,7 +80,7 @@ pub fn extract_user_defined_metadata(metadata: &HashMap<String, String>) -> Hash
for (key, value) in metadata {
let lower_key = key.to_ascii_lowercase();
if lower_key.starts_with(RESERVED_METADATA_PREFIX_LOWER) {
if is_internal_key(key) {
continue;
}
@@ -188,7 +188,6 @@ mod tests {
let mut metadata: HashMap<String, String> = HashMap::new();
metadata.insert("x-rustfs-internal-healing".to_string(), "true".to_string());
metadata.insert("x-rustfs-internal-data-mov".to_string(), "value".to_string());
metadata.insert("X-RustFS-Internal-purgestatus".to_string(), "status".to_string());
metadata.insert("x-rustfs-meta-custom".to_string(), "custom-value".to_string());
metadata.insert("my-key".to_string(), "my-value".to_string());
docker-compose.decommission.yml
+49
View File
@@ -0,0 +1,49 @@
services:
rustfs-decommission-latest:
image: ${RUSTFS_DOCKER_IMAGE:-rustfs-local:decommission-latest}
entrypoint:
- /bin/sh
- -lc
- |
set -eu
mkdir -p \
/data/pool0/disk1 /data/pool0/disk2 /data/pool0/disk3 /data/pool0/disk4 \
/data/pool1/disk1 /data/pool1/disk2 /data/pool1/disk3 /data/pool1/disk4 \
/logs
exec /usr/bin/rustfs '/data/pool0/disk{1...4}' '/data/pool1/disk{1...4}'
ports:
- "9100:9000"
- "9101:9001"
environment:
RUSTFS_VOLUMES: "/data/pool0/disk{1...4} /data/pool1/disk{1...4}"
RUSTFS_ADDRESS: "0.0.0.0:9000"
RUSTFS_CONSOLE_ADDRESS: "0.0.0.0:9001"
RUSTFS_CONSOLE_ENABLE: "true"
RUSTFS_CORS_ALLOWED_ORIGINS: "*"
RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS: "*"
RUSTFS_ACCESS_KEY: "rustfsadmin"
RUSTFS_SECRET_KEY: "rustfsadmin"
RUSTFS_OBS_LOGGER_LEVEL: "info"
RUSTFS_OBS_LOG_DIRECTORY: "/logs"
RUST_LOG: "info,rustfs=debug,rustfs_ecstore=debug"
volumes:
- ./deploy/data/decommission:/data
- ./deploy/logs/decommission:/logs
healthcheck:
test:
[
"CMD",
"sh",
"-c",
"curl -f http://127.0.0.1:9000/health && curl -f http://127.0.0.1:9001/rustfs/console/health"
]
interval: 20s
timeout: 10s
retries: 6
start_period: 300s
restart: unless-stopped
networks:
- rustfs-decommission-network
networks:
rustfs-decommission-network:
entrypoint.sh
+2 -1
View File
@@ -31,11 +31,12 @@ process_data_volumes() {
VOLUME_LIST=""
for vol in $VOLUME_LIST_RAW; do
# Helper to manually expand {N..M} since sh doesn't support it on variables
if echo "$vol" | grep -E -q "\{[0-9]+\.\.[0-9]+\}"; then
if echo "$vol" | grep -E -q "\{[0-9]+\.\.\.?[0-9]+\}"; then
PREFIX=${vol%%\{*}
SUFFIX=${vol##*\}}
RANGE=${vol#*\{}
RANGE=${RANGE%\}}
RANGE=$(echo "$RANGE" | sed 's/\.\.\./../')
START=${RANGE%%..*}
END=${RANGE##*..}
rustfs/Cargo.toml
+2
View File
@@ -49,6 +49,7 @@ rustfs-appauth = { workspace = true }
rustfs-audit = { workspace = true }
rustfs-common = { workspace = true }
rustfs-config = { workspace = true, features = ["constants", "notify"] }
rustfs-crypto = { workspace = true }
rustfs-credentials = { workspace = true }
rustfs-ecstore = { workspace = true }
rustfs-filemeta.workspace = true
@@ -168,6 +169,7 @@ aws-sdk-s3 = { workspace = true }
aws-config = { workspace = true }
anyhow = { workspace = true }
tokio = { workspace = true, features = ["test-util"] }
temp-env = { workspace = true }
[build-dependencies]
http.workspace = true

Some files were not shown because too many files have changed in this diff Show More