gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): support MinIO DARE fixture compatibility (#3590)

Browse Source
This commit is contained in:
houseme
2026-06-19 10:13:39 +08:00
committed by GitHub
parent 80b1fca02a
commit 8cf3c0bfbd
13 changed files with 563 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+159 -132
View File
@@ -81,17 +81,6 @@ dependencies = [
"zeroize",
]
[[package]]
name = "ahash"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
dependencies = [
"getrandom 0.2.17",
"once_cell",
"version_check",
]
[[package]]
name = "ahash"
version = "0.8.12"
@@ -132,9 +121,9 @@ checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
[[package]]
name = "alloc-stdlib"
version = "0.2.2"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"
dependencies = [
"alloc-no-stdlib",
]
@@ -388,7 +377,7 @@ version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
dependencies = [
"ahash 0.8.12",
"ahash",
"arrow-buffer",
"arrow-data",
"arrow-schema",
@@ -546,7 +535,7 @@ version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
dependencies = [
"ahash 0.8.12",
"ahash",
"arrow-array",
"arrow-buffer",
"arrow-data",
@@ -1637,9 +1626,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
version = "5.0.1"
version = "5.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924"
checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -2606,7 +2595,7 @@ dependencies = [
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core 0.9.12",
"parking_lot_core",
]
[[package]]
@@ -2658,7 +2647,7 @@ dependencies = [
"liblzma",
"log",
"object_store",
"parking_lot 0.12.5",
"parking_lot",
"parquet",
"sqlparser",
"tempfile",
@@ -2689,7 +2678,7 @@ dependencies = [
"itertools 0.14.0",
"log",
"object_store",
"parking_lot 0.12.5",
"parking_lot",
"tokio",
]
@@ -2781,7 +2770,7 @@ dependencies = [
"liblzma",
"log",
"object_store",
"parking_lot 0.12.5",
"parking_lot",
"rand 0.9.4",
"tokio",
"tokio-util",
@@ -2885,7 +2874,7 @@ dependencies = [
"itertools 0.14.0",
"log",
"object_store",
"parking_lot 0.12.5",
"parking_lot",
"parquet",
"tokio",
]
@@ -2912,7 +2901,7 @@ dependencies = [
"futures",
"log",
"object_store",
"parking_lot 0.12.5",
"parking_lot",
"rand 0.9.4",
"tempfile",
"url",
@@ -3056,7 +3045,7 @@ dependencies = [
"datafusion-expr",
"datafusion-physical-expr",
"datafusion-physical-plan",
"parking_lot 0.12.5",
"parking_lot",
]
[[package]]
@@ -3133,7 +3122,7 @@ dependencies = [
"hashbrown 0.17.1",
"indexmap 2.14.0",
"itertools 0.14.0",
"parking_lot 0.12.5",
"parking_lot",
"petgraph 0.8.3",
"recursive",
"tokio",
@@ -3167,7 +3156,7 @@ dependencies = [
"hashbrown 0.17.1",
"indexmap 2.14.0",
"itertools 0.14.0",
"parking_lot 0.12.5",
"parking_lot",
"pin-project",
]
@@ -3218,7 +3207,7 @@ dependencies = [
"itertools 0.14.0",
"log",
"num-traits",
"parking_lot 0.12.5",
"parking_lot",
"pin-project-lite",
"tokio",
]
@@ -3250,7 +3239,7 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-plan",
"parking_lot 0.12.5",
"parking_lot",
]
[[package]]
@@ -3293,7 +3282,7 @@ dependencies = [
"log",
"lru 0.16.4",
"mime_guess",
"parking_lot 0.12.5",
"parking_lot",
"percent-encoding",
"pin-project-lite",
"reflink-copy",
@@ -4671,9 +4660,6 @@ name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
dependencies = [
"ahash 0.7.8",
]
[[package]]
name = "hashbrown"
@@ -4715,6 +4701,15 @@ dependencies = [
"serde_core",
]
[[package]]
name = "hashlink"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5081f264ed7adee96ea4b4778b6bb9da0a7228b084587aa3bd3ff05da7c5a3b"
dependencies = [
"hashbrown 0.17.1",
]
[[package]]
name = "headers"
version = "0.4.1"
@@ -4843,7 +4838,7 @@ dependencies = [
"moka",
"ndk-context",
"once_cell",
"parking_lot 0.12.5",
"parking_lot",
"rand 0.10.1",
"resolv-conf",
"smallvec",
@@ -5252,7 +5247,7 @@ version = "0.11.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
dependencies = [
"ahash 0.8.12",
"ahash",
"indexmap 2.14.0",
"is-terminal",
"itoa",
@@ -5270,7 +5265,7 @@ version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90807d610575744524d9bdc69f3885d96f0e6c3354565b0828354a7ff2a262b8"
dependencies = [
"ahash 0.8.12",
"ahash",
"clap",
"crossbeam-channel",
"crossbeam-utils",
@@ -5306,15 +5301,6 @@ dependencies = [
"hybrid-array",
]
[[package]]
name = "instant"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
dependencies = [
"cfg-if",
]
[[package]]
name = "integer-encoding"
version = "3.0.4"
@@ -5934,15 +5920,6 @@ version = "0.4.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a"
[[package]]
name = "lru"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a"
dependencies = [
"hashbrown 0.12.3",
]
[[package]]
name = "lru"
version = "0.16.4"
@@ -6226,7 +6203,7 @@ version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae9e23184d81616fcdfcdc5768e14c2bca765096094d08e7895259fe658378e4"
dependencies = [
"ahash 0.8.12",
"ahash",
"crossbeam-queue",
"crossbeam-utils",
"metrics",
@@ -6366,7 +6343,7 @@ dependencies = [
"equivalent",
"event-listener",
"futures-util",
"parking_lot 0.12.5",
"parking_lot",
"portable-atomic",
"smallvec",
"tagptr",
@@ -6490,7 +6467,7 @@ dependencies = [
"libc",
"log",
"neli-proc-macros",
"parking_lot 0.12.5",
"parking_lot",
]
[[package]]
@@ -6917,7 +6894,7 @@ dependencies = [
"http 1.4.2",
"humantime",
"itertools 0.14.0",
"parking_lot 0.12.5",
"parking_lot",
"percent-encoding",
"thiserror 2.0.18",
"tokio",
@@ -7253,17 +7230,6 @@ version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]]
name = "parking_lot"
version = "0.12.5"
@@ -7271,21 +7237,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
dependencies = [
"lock_api",
"parking_lot_core 0.9.12",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
"parking_lot_core",
]
[[package]]
@@ -7296,7 +7248,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall 0.5.18",
"redox_syscall",
"smallvec",
"windows-link",
]
@@ -7307,7 +7259,7 @@ version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
dependencies = [
"ahash 0.8.12",
"ahash",
"arrow-array",
"arrow-buffer",
"arrow-data",
@@ -7928,7 +7880,7 @@ dependencies = [
"fnv",
"lazy_static",
"memchr",
"parking_lot 0.12.5",
"parking_lot",
"thiserror 2.0.18",
]
@@ -8308,7 +8260,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -8453,7 +8405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dc94ed8e3de45f6d8d052869d48c0dbeebcaa7a6c345ec7f0f917e10347428e"
dependencies = [
"clocksource",
"parking_lot 0.12.5",
"parking_lot",
"thiserror 2.0.18",
]
@@ -8566,15 +8518,6 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -8595,21 +8538,6 @@ dependencies = [
"thiserror 2.0.18",
]
[[package]]
name = "reed-solomon-erasure"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f"
dependencies = [
"cc",
"libc",
"libm",
"lru 0.7.8",
"parking_lot 0.11.2",
"smallvec",
"spin 0.9.8",
]
[[package]]
name = "reed-solomon-simd"
version = "3.1.0"
@@ -9090,6 +9018,7 @@ dependencies = [
"base64 0.22.1",
"base64-simd",
"bytes",
"chacha20poly1305",
"clap",
"const-str",
"datafusion",
@@ -9333,6 +9262,7 @@ dependencies = [
"byteorder",
"bytes",
"bytesize",
"chacha20poly1305",
"chrono",
"criterion",
"enumset",
@@ -9359,14 +9289,13 @@ dependencies = [
"num_cpus",
"opentelemetry",
"opentelemetry_sdk",
"parking_lot 0.12.5",
"parking_lot",
"path-absolutize",
"pin-project-lite",
"proptest",
"quick-xml 0.40.1",
"rand 0.10.1",
"ratelimit",
"reed-solomon-erasure",
"reed-solomon-simd",
"regex",
"reqwest",
@@ -9378,6 +9307,7 @@ dependencies = [
"rustfs-config",
"rustfs-credentials",
"rustfs-data-usage",
"rustfs-erasure-codec",
"rustfs-filemeta",
"rustfs-io-metrics",
"rustfs-kms",
@@ -9422,6 +9352,22 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "rustfs-erasure-codec"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcda6886ece4bfc6917640007a6b9693dc8424a9412d18c3dda5be2431500f31"
dependencies = [
"cc",
"hashlink",
"libc",
"libm",
"parking_lot",
"rayon",
"smallvec",
"spin 0.12.0",
]
[[package]]
name = "rustfs-extension-schema"
version = "1.0.0-beta.8"
@@ -9656,7 +9602,7 @@ dependencies = [
"async-trait",
"crossbeam-queue",
"futures",
"parking_lot 0.12.5",
"parking_lot",
"rustfs-io-metrics",
"rustfs-utils",
"serde",
@@ -9932,6 +9878,7 @@ version = "1.0.0-beta.8"
dependencies = [
"aes-gcm",
"bytes",
"chacha20poly1305",
"hex",
"hmac 0.13.0",
"minlz",
@@ -9975,7 +9922,7 @@ dependencies = [
"http 1.4.2",
"metrics",
"object_store",
"parking_lot 0.12.5",
"parking_lot",
"pin-project-lite",
"rustfs-common",
"rustfs-ecstore",
@@ -9999,7 +9946,7 @@ dependencies = [
"datafusion",
"derive_builder",
"futures",
"parking_lot 0.12.5",
"parking_lot",
"rustfs-s3select-api",
"s3s",
"snafu 0.9.1",
@@ -10092,7 +10039,7 @@ dependencies = [
"libc",
"metrics",
"mysql_async",
"parking_lot 0.12.5",
"parking_lot",
"pulsar",
"redis",
"regex",
@@ -10785,7 +10732,7 @@ dependencies = [
"futures-util",
"log",
"once_cell",
"parking_lot 0.12.5",
"parking_lot",
"serial_test_derive",
]
@@ -11121,6 +11068,12 @@ dependencies = [
"lock_api",
]
[[package]]
name = "spin"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1527984ca054dfca79333baec451042863f485fbee01b7bf6d911de915cac865"
[[package]]
name = "spki"
version = "0.7.3"
@@ -11518,7 +11471,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050"
dependencies = [
"futures",
"parking_lot 0.12.5",
"parking_lot",
]
[[package]]
@@ -11775,7 +11728,7 @@ dependencies = [
"io-uring",
"libc",
"mio",
"parking_lot 0.12.5",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"slab",
@@ -11808,7 +11761,7 @@ dependencies = [
"futures-channel",
"futures-util",
"log",
"parking_lot 0.12.5",
"parking_lot",
"percent-encoding",
"phf 0.13.1",
"pin-project-lite",
@@ -12821,7 +12774,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
@@ -12830,7 +12783,16 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
]
[[package]]
@@ -12848,14 +12810,31 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
]
[[package]]
@@ -12873,48 +12852,96 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "1.0.3"
Cargo.toml
+3 -2
View File
@@ -137,7 +137,7 @@ futures-core = "0.3.32"
futures-lite = "2.6.1"
futures-util = "0.3.32"
pollster = "0.4.0"
pulsar = { version = "6.8.0", default-features = false, features = ["tokio-rustls-runtime","telemetry"] }
pulsar = { version = "6.8.0", default-features = false, features = ["tokio-rustls-runtime", "telemetry"] }
lapin = { version = "4.10.0", default-features = false, features = ["tokio", "rustls", "rustls--aws_lc_rs"] }
hyper = { version = "1.10.1", features = ["http2", "http1", "server"] }
hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
@@ -263,7 +263,8 @@ pretty_assertions = "1.4.1"
rand = { version = "0.10.1", features = ["serde"] }
ratelimit = "0.10.1"
rayon = "1.12.0"
reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
reed-solomon-erasure = { package = "rustfs-erasure-codec", version = "7.0.0", features = ["simd-accel"] }
#reed-solomon-erasure = { version = "6.0", features = ["simd-accel"], git = "https://github.com/houseme/reed-solomon-erasure",rev = "main" }
reed-solomon-simd = "3.1.0"
regex = { version = "1.12.4" }
rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] }
crates/ecstore/Cargo.toml
+1
View File
@@ -113,6 +113,7 @@ rustfs-madmin.workspace = true
rustfs-concurrency.workspace = true
reqwest = { workspace = true }
aes-gcm.workspace = true
chacha20poly1305.workspace = true
aws-sdk-s3 = { workspace = true }
urlencoding = { workspace = true }
smallvec = { workspace = true }
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+5 -6
View File
@@ -2887,7 +2887,9 @@ mod tests {
use uuid::Uuid;
#[tokio::test]
#[serial]
async fn expiry_enqueue_reports_missed_without_worker_channel() {
let before = global_metrics().report().await.lifecycle_expiry;
let state = ExpiryState::new();
let mut state = state.write().await;
let object = ObjectInfo {
@@ -2904,12 +2906,9 @@ mod tests {
assert!(!queued);
assert_eq!(state.stats.missed_tasks(), 1);
let expiry = global_metrics().report().await.lifecycle_expiry;
assert_eq!(expiry.current_queue_capacity, 0);
assert_eq!(expiry.current_queued, 0);
assert_eq!(expiry.current_active, 0);
assert_eq!(expiry.current_workers, 0);
assert_eq!(expiry.queue_missed, 1);
let after = global_metrics().report().await.lifecycle_expiry;
assert!(after.queue_missed >= before.queue_missed.saturating_add(1));
assert!(after.scanner_missed >= before.scanner_missed.saturating_add(1));
}
#[tokio::test]
crates/ecstore/src/global.rs
+7
View File
@@ -154,6 +154,13 @@ pub fn get_global_endpoints_opt() -> Option<EndpointServerPools> {
GLOBAL_Endpoints.get().cloned()
}
#[cfg(test)]
pub async fn reset_local_disk_test_state() {
GLOBAL_LOCAL_DISK_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
}
pub async fn is_first_cluster_node_local() -> bool {
get_global_endpoints().first_local()
}
crates/ecstore/src/rpc/peer_s3_client.rs
+9 -15
View File
@@ -1082,7 +1082,7 @@ mod tests {
use crate::disk::endpoint::Endpoint;
use crate::disk::local::LocalDisk;
use crate::endpoints::{Endpoints, PoolEndpoints};
use crate::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
use crate::global::reset_local_disk_test_state;
use crate::store::init_local_disks;
use rustfs_filemeta::FileInfo;
use serial_test::serial;
@@ -1110,12 +1110,6 @@ mod tests {
}
}
async fn reset_local_disk_globals() {
GLOBAL_LOCAL_DISK_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
}
#[derive(Debug)]
struct TestPeerS3Client {
pools: Option<Vec<usize>>,
@@ -1292,7 +1286,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn local_get_bucket_info_survives_prior_walk_timeout() {
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for local peer listing regression");
let disks = init_test_local_disks(&temp_dir, 1, "local-get-bucket-info-survives-prior-walk-timeout").await;
@@ -1337,13 +1331,13 @@ mod tests {
})
.await;
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn local_get_bucket_info_requires_local_write_quorum() {
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for partial bucket regression");
let disks = init_test_local_disks(&temp_dir, 2, "local-get-bucket-info-requires-local-write-quorum").await;
@@ -1360,13 +1354,13 @@ mod tests {
assert_eq!(err, Error::ErasureWriteQuorum);
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn local_peer_filters_disks_by_pool() {
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for pool filtered local peer regression");
let disks = init_test_local_disks_for_pools(&temp_dir, &[(0, 2), (1, 2)], "local-peer-filters-disks-by-pool").await;
@@ -1399,13 +1393,13 @@ mod tests {
.expect("pool 1 local listing should succeed against its own disks");
assert!(pool1_buckets.is_empty());
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn heal_bucket_local_recreates_missing_bucket_volumes() {
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for bucket heal regression");
let disks = init_test_local_disks(&temp_dir, 2, "heal-bucket-local-recreates-missing-bucket-volumes").await;
@@ -1434,7 +1428,7 @@ mod tests {
disk.stat_volume(bucket).await.expect("bucket should exist after heal");
}
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
}
#[test]
crates/ecstore/src/store.rs
+3 -9
View File
@@ -841,17 +841,11 @@ impl ServerPoolsAvailableSpace {
mod tests {
use super::*;
use crate::endpoints::{Endpoints, PoolEndpoints};
use crate::global::{GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
use crate::global::{GLOBAL_LOCAL_DISK_ID_MAP, reset_local_disk_test_state};
use crate::store_init::{connect_load_init_formats, init_disks};
use serial_test::serial;
use tempfile::TempDir;
async fn reset_local_disk_globals() {
GLOBAL_LOCAL_DISK_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_ID_MAP.write().await.clear();
GLOBAL_LOCAL_DISK_SET_DRIVES.write().await.clear();
}
#[tokio::test]
async fn test_get_disk_infos() {
let disks = vec![None, None]; // Empty disks for testing
@@ -906,7 +900,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn test_find_local_disk_by_ref_backfills_uuid_map() {
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for local disk ref test");
let disk_paths = (0..4)
@@ -967,7 +961,7 @@ mod tests {
Some(first_disk.endpoint().to_string())
);
reset_local_disk_globals().await;
reset_local_disk_test_state().await;
}
#[tokio::test]
crates/ecstore/src/store_api/readers.rs
+128 -26
View File
@@ -1,10 +1,14 @@
use super::*;
#[cfg(feature = "rio-v2")]
use aes_gcm::aead::Payload;
use aes_gcm::{
Aes256Gcm, Key, Nonce,
aead::{Aead, KeyInit},
};
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
#[cfg(feature = "rio-v2")]
use chacha20poly1305::ChaCha20Poly1305;
#[cfg(feature = "rio-v2")]
use hmac::{Hmac, Mac};
use md5::{Digest, Md5};
use rustfs_kms::{service_manager::get_global_encryption_service, types::ObjectEncryptionContext};
@@ -53,6 +57,8 @@ const DARE_VERSION_20: u8 = 0x20;
#[cfg(feature = "rio-v2")]
const DARE_CIPHER_AES_256_GCM: u8 = 0x00;
#[cfg(feature = "rio-v2")]
const DARE_CIPHER_CHACHA20_POLY1305: u8 = 0x01;
#[cfg(feature = "rio-v2")]
const DARE_HEADER_SIZE: usize = 16;
#[cfg(feature = "rio-v2")]
const DARE_TAG_SIZE: usize = 16;
@@ -1025,6 +1031,35 @@ fn metadata_get<'a>(metadata: &'a HashMap<String, String>, key: &str) -> Option<
})
}
#[cfg(feature = "rio-v2")]
fn is_supported_sealed_object_key_cipher(cipher: u8) -> bool {
matches!(cipher, DARE_CIPHER_AES_256_GCM | DARE_CIPHER_CHACHA20_POLY1305)
}
#[cfg(feature = "rio-v2")]
fn decrypt_sealed_object_key_payload(sealing_key: [u8; 32], header: &[u8], sealed_key: &[u8]) -> Result<Vec<u8>> {
let nonce = &header[4..16];
let ciphertext = &sealed_key[DARE_HEADER_SIZE..];
let aad = &header[..4];
match header[1] {
DARE_CIPHER_AES_256_GCM => {
let cipher = Aes256Gcm::new_from_slice(&sealing_key)
.map_err(|err| Error::other(format!("invalid AES-GCM sealing key: {err}")))?;
let nonce = Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?;
cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
}
DARE_CIPHER_CHACHA20_POLY1305 => {
let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key)
.map_err(|err| Error::other(format!("invalid ChaCha20-Poly1305 sealing key: {err}")))?;
let nonce =
chacha20poly1305::Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?;
cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
}
_ => return Err(Error::other("unsupported sealed object-key DARE header")),
}
.map_err(|err| Error::other(format!("failed to unseal object key: {err}")))
}
async fn resolve_encryption_material(oi: &ObjectInfo, headers: &HeaderMap<HeaderValue>) -> Result<EncryptionMaterial> {
if metadata_get(&oi.user_defined, SSEC_ALGORITHM_HEADER).is_some() {
return resolve_ssec_material(oi, headers);
@@ -1154,7 +1189,7 @@ fn try_unseal_minio_object_key(
return Ok(None);
};
let header = &sealed_key[..DARE_HEADER_SIZE];
if header[0] != DARE_VERSION_20 || header[1] != DARE_CIPHER_AES_256_GCM {
if header[0] != DARE_VERSION_20 || !is_supported_sealed_object_key_cipher(header[1]) {
return Err(Error::other("unsupported sealed object-key DARE header"));
}
if u16::from_le_bytes([header[2], header[3]]) != 31 || header[4] & 0x80 == 0 {
@@ -1162,17 +1197,7 @@ fn try_unseal_minio_object_key(
}
let sealing_key = derive_sealing_key(external_key, iv, managed_sse_domain(metadata), bucket, object);
let cipher = Aes256Gcm::new_from_slice(&sealing_key).map_err(|err| Error::other(format!("invalid sealing key: {err}")))?;
let nonce = Nonce::try_from(&header[4..16]).map_err(|_| Error::other("invalid sealed object-key package nonce"))?;
let plaintext = cipher
.decrypt(
&nonce,
aes_gcm::aead::Payload {
msg: &sealed_key[DARE_HEADER_SIZE..],
aad: &header[..4],
},
)
.map_err(|err| Error::other(format!("failed to unseal object key: {err}")))?;
let plaintext = decrypt_sealed_object_key_payload(sealing_key, header, &sealed_key)?;
let object_key: [u8; 32] = plaintext
.as_slice()
.try_into()
@@ -1888,34 +1913,70 @@ mod tests {
object: &str,
data_key: [u8; 32],
object_key: [u8; 32],
) -> ([u8; 32], Vec<u8>) {
seal_managed_s3_object_key_for_test_with_cipher(bucket, object, data_key, object_key, DARE_CIPHER_AES_256_GCM)
}
#[cfg(feature = "rio-v2")]
fn seal_managed_s3_object_key_for_test_with_cipher(
bucket: &str,
object: &str,
data_key: [u8; 32],
object_key: [u8; 32],
cipher_id: u8,
) -> ([u8; 32], Vec<u8>) {
let iv = [0x24u8; SEALED_KEY_IV_SIZE];
let sealing_key = derive_sealing_key(data_key, iv, "SSE-S3", bucket, object);
let cipher = Aes256Gcm::new_from_slice(&sealing_key).expect("valid sealing key");
let mut header = [0u8; DARE_HEADER_SIZE];
header[0] = DARE_VERSION_20;
header[1] = DARE_CIPHER_AES_256_GCM;
header[1] = cipher_id;
header[2..4].copy_from_slice(&31u16.to_le_bytes());
header[4] = 0x80;
header[5..16].copy_from_slice(&[0x46u8; 11]);
let nonce = Nonce::try_from(&header[4..16]).expect("valid nonce");
let ciphertext = match cipher_id {
DARE_CIPHER_AES_256_GCM => {
let cipher = Aes256Gcm::new_from_slice(&sealing_key).expect("valid sealing key");
let nonce = Nonce::try_from(&header[4..16]).expect("valid nonce");
cipher
.encrypt(
&nonce,
Payload {
msg: &object_key,
aad: &header[..4],
},
)
.expect("seal managed object key")
}
DARE_CIPHER_CHACHA20_POLY1305 => {
let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key).expect("valid sealing key");
let nonce = chacha20poly1305::Nonce::try_from(&header[4..16]).expect("valid nonce");
cipher
.encrypt(
&nonce,
Payload {
msg: &object_key,
aad: &header[..4],
},
)
.expect("seal managed object key")
}
_ => panic!("unsupported test cipher"),
};
let mut sealed = header.to_vec();
sealed.extend_from_slice(
&cipher
.encrypt(
&nonce,
aes_gcm::aead::Payload {
msg: &object_key,
aad: &header[..4],
},
)
.expect("seal managed object key"),
);
sealed.extend_from_slice(&ciphertext);
(iv, sealed)
}
#[cfg(feature = "rio-v2")]
#[test]
fn test_supported_sealed_object_key_cipher_accepts_current_minio_fixture_value() {
assert!(is_supported_sealed_object_key_cipher(DARE_CIPHER_AES_256_GCM));
assert!(is_supported_sealed_object_key_cipher(DARE_CIPHER_CHACHA20_POLY1305));
assert!(!is_supported_sealed_object_key_cipher(0x02));
}
#[tokio::test]
async fn resolve_managed_material_accepts_case_insensitive_metadata_keys() {
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
@@ -1937,6 +1998,47 @@ mod tests {
.await;
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn resolve_managed_material_accepts_chacha20_poly1305_header_variant() {
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
let data_key = [0x24; 32];
let object_key = [0x33; 32];
let (iv, sealed_key) = seal_managed_s3_object_key_for_test_with_cipher(
"bucket",
"object",
data_key,
object_key,
DARE_CIPHER_CHACHA20_POLY1305,
);
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
let metadata = HashMap::from([
(
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(sealed_key),
),
(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(iv)),
(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
),
(
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(encrypted_dek.as_bytes()),
),
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
]);
let material = resolve_managed_material("bucket", "object", &metadata)
.await
.expect("managed material should accept current MinIO header variant");
assert_eq!(material.key_kind, EncryptionKeyKind::Object);
assert_eq!(material.key_bytes, object_key);
})
.await;
}
#[tokio::test]
async fn resolve_encryption_material_accepts_case_insensitive_metadata_keys() {
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
crates/ecstore/tests/minio_generated_read_test.rs
+86 -47
View File
@@ -7,12 +7,13 @@ use std::path::{Path, PathBuf};
use rustfs_ecstore::bitrot::create_bitrot_reader;
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::disk::{DiskAPI as _, DiskOption, new_disk};
use rustfs_ecstore::erasure_coding::Erasure;
use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use temp_env::async_with_vars;
use tokio::io::AsyncReadExt;
use tokio::io::{AsyncReadExt, AsyncWrite};
#[derive(Debug, Deserialize)]
struct ManifestRecord {
@@ -21,6 +22,30 @@ struct ManifestRecord {
backend_files: Vec<String>,
}
#[derive(Default)]
struct VecAsyncWriter {
bytes: Vec<u8>,
}
impl AsyncWrite for VecAsyncWriter {
fn poll_write(
mut self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<std::io::Result<usize>> {
self.bytes.extend_from_slice(buf);
std::task::Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
}
fn fixture_root() -> PathBuf {
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
.map(PathBuf::from)
@@ -93,23 +118,32 @@ fn sha256_hex(bytes: &[u8]) -> String {
}
async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
let disk_root = case_dir.join("backend").join("disk1");
let disk_root_str = disk_root
.to_str()
.unwrap_or_else(|| panic!("non-utf8 disk root {}", disk_root.display()));
let mut endpoint = Endpoint::try_from(disk_root_str).expect("fixture disk endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("open fixture disk");
let mut disks = Vec::with_capacity(file_info.erasure.distribution.len());
for disk_number in 1..=file_info.erasure.distribution.len() {
let disk_root = case_dir.join("backend").join(format!("disk{disk_number}"));
let disk_root_str = disk_root
.to_str()
.unwrap_or_else(|| panic!("non-utf8 disk root {}", disk_root.display()));
let mut endpoint = Endpoint::try_from(disk_root_str).expect("fixture disk endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_number - 1);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.unwrap_or_else(|err| panic!("open fixture disk {disk_number}: {err}"));
disks.push(disk);
}
let mut disk_order = vec![None; disks.len()];
for (idx, disk) in disks.iter().enumerate() {
let block_index = file_info.erasure.distribution[idx];
disk_order[block_index - 1] = Some(disk);
}
let data_dir = file_info
.data_dir
.as_ref()
@@ -119,37 +153,42 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
for part in &file_info.parts {
let checksum_info = file_info.erasure.get_checksum_info(part.number);
let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number);
let mut reader = create_bitrot_reader(
None,
Some(&disk),
&manifest.bucket,
&path,
0,
part.size,
file_info.erasure.shard_size(),
checksum_info.algorithm.clone(),
false,
false,
)
.await
.unwrap_or_else(|err| panic!("create bitrot reader for {path}: {err:?}"))
.unwrap_or_else(|| panic!("missing bitrot reader for {path}"));
let mut block = vec![0u8; file_info.erasure.shard_size()];
loop {
let n = reader
.read(&mut block)
.await
.unwrap_or_else(|err| panic!("read decoded encrypted bytes from {path}: {err}"));
if n == 0 {
break;
}
encrypted.extend_from_slice(&block[..n]);
if n < block.len() {
break;
}
let shard_read_len = file_info.erasure.shard_file_size(part.size as i64);
let mut readers = Vec::with_capacity(disks.len());
for (idx, disk) in disk_order.iter().enumerate() {
let reader = create_bitrot_reader(
None,
*disk,
&manifest.bucket,
&path,
0,
shard_read_len as usize,
file_info.erasure.shard_size(),
checksum_info.algorithm.clone(),
false,
false,
)
.await
.unwrap_or_else(|err| panic!("create bitrot reader for disk{} {path}: {err:?}", idx + 1));
readers.push(reader);
}
let erasure = Erasure::new(
file_info.erasure.data_blocks,
file_info.erasure.parity_blocks,
file_info.erasure.block_size,
);
let mut writer = VecAsyncWriter::default();
let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
if let Some(err) = err {
panic!("decode erasure shards for {path}: {err}");
}
assert_eq!(written, part.size, "decoded part size should match xl.meta part size");
encrypted.extend_from_slice(&writer.bytes);
}
for disk in disks {
disk.close().await.expect("close fixture disk");
}
disk.close().await.expect("close fixture disk");
encrypted
}
crates/rio-v2/Cargo.toml
+1
View File
@@ -31,6 +31,7 @@ workspace = true
[dependencies]
aes-gcm.workspace = true
bytes.workspace = true
chacha20poly1305.workspace = true
hex.workspace = true
hmac.workspace = true
minlz = "1.1.0"
crates/rio-v2/src/encrypt_reader.rs
+114 -19
View File
@@ -12,8 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use aes_gcm::aead::Aead;
use aes_gcm::aead::{Aead, Payload};
use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
use chacha20poly1305::ChaCha20Poly1305;
use hmac::{Hmac, Mac};
use pin_project_lite::pin_project;
use rand::RngExt;
@@ -27,12 +28,49 @@ use tokio::io::{AsyncRead, ReadBuf};
const DARE_VERSION_20: u8 = 0x20;
const DARE_CIPHER_AES_256_GCM: u8 = 0x00;
const DARE_CIPHER_CHACHA20_POLY1305: u8 = 0x01;
const DARE_HEADER_SIZE: usize = 16;
const DARE_TAG_SIZE: usize = 16;
const DARE_PAYLOAD_SIZE: usize = 64 * 1024;
type HmacSha256 = Hmac<Sha256>;
enum DareDecryptCipher {
Aes256Gcm(Box<Aes256Gcm>),
ChaCha20Poly1305(ChaCha20Poly1305),
}
impl DareDecryptCipher {
fn new(cipher_id: u8, key: [u8; 32]) -> io::Result<Self> {
match cipher_id {
DARE_CIPHER_AES_256_GCM => Aes256Gcm::new_from_slice(&key)
.map(Box::new)
.map(Self::Aes256Gcm)
.map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, format!("invalid AES-GCM key: {err}"))),
DARE_CIPHER_CHACHA20_POLY1305 => ChaCha20Poly1305::new_from_slice(&key)
.map(Self::ChaCha20Poly1305)
.map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, format!("invalid ChaCha20-Poly1305 key: {err}"))),
_ => Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE cipher suite")),
}
}
fn cipher_id(&self) -> u8 {
match self {
Self::Aes256Gcm(_) => DARE_CIPHER_AES_256_GCM,
Self::ChaCha20Poly1305(_) => DARE_CIPHER_CHACHA20_POLY1305,
}
}
fn decrypt(&self, nonce: &[u8; 12], ciphertext: &[u8], aad: &[u8]) -> Result<Vec<u8>, aes_gcm::aead::Error> {
match self {
Self::Aes256Gcm(cipher) => cipher.decrypt(&Nonce::from(*nonce), Payload { msg: ciphertext, aad }),
Self::ChaCha20Poly1305(cipher) => {
cipher.decrypt(&chacha20poly1305::Nonce::from(*nonce), Payload { msg: ciphertext, aad })
}
}
}
}
#[derive(Clone, Copy)]
enum MultipartKeySource {
LegacyNonce { base_nonce: [u8; 12] },
@@ -215,7 +253,8 @@ pin_project! {
pub struct DecryptReader<R> {
#[pin]
inner: R,
cipher: Aes256Gcm,
key: [u8; 32],
cipher: Option<DareDecryptCipher>,
expected_base_nonce: Option<[u8; 12]>,
sequence_number: u32,
multipart_parts: Vec<usize>,
@@ -250,7 +289,8 @@ where
pub fn new_with_object_key_and_sequence(inner: R, object_key: [u8; 32], sequence_number: u32) -> Self {
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&object_key).expect("valid AES-256-GCM key"),
key: object_key,
cipher: None,
expected_base_nonce: None,
sequence_number,
multipart_parts: Vec::new(),
@@ -273,7 +313,8 @@ where
pub fn new_with_sequence(inner: R, key: [u8; 32], nonce: [u8; 12], sequence_number: u32) -> Self {
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"),
key,
cipher: None,
expected_base_nonce: Some(nonce),
sequence_number,
multipart_parts: Vec::new(),
@@ -311,7 +352,8 @@ where
let first_part = multipart_parts.first().copied().unwrap_or(1);
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"),
key,
cipher: None,
expected_base_nonce: Some(multipart_part_nonce(base_nonce, first_part)),
sequence_number,
multipart_parts,
@@ -341,7 +383,8 @@ where
let first_key = derive_part_key(object_key, first_part as u32);
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&first_key).expect("valid AES-256-GCM key"),
key: first_key,
cipher: None,
expected_base_nonce: None,
sequence_number,
multipart_parts,
@@ -395,7 +438,8 @@ where
}
Some(MultipartKeySource::ObjectKey { object_key }) => {
let part_key = derive_part_key(object_key, next_part as u32);
*this.cipher = Aes256Gcm::new_from_slice(&part_key).expect("valid AES-256-GCM key");
*this.key = part_key;
*this.cipher = None;
*this.expected_base_nonce = None;
}
None => {}
@@ -433,6 +477,12 @@ where
}
let header = this.header_buf;
if header[0] != DARE_VERSION_20 {
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE version")));
}
if !matches!(header[1], DARE_CIPHER_AES_256_GCM | DARE_CIPHER_CHACHA20_POLY1305) {
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE cipher suite")));
}
let payload_len = usize::from(u16::from_le_bytes([header[2], header[3]])) + 1;
let package_len = payload_len + DARE_TAG_SIZE;
if payload_len == 0 || payload_len > DARE_PAYLOAD_SIZE {
@@ -441,7 +491,10 @@ where
if !is_final_header(*header) && payload_len != DARE_PAYLOAD_SIZE {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::InvalidData,
"non-final DARE package must carry a full 64KiB payload",
format!(
"non-final DARE package must carry a full 64KiB payload: cipher={}, payload_len={}, sequence_number={}, header={:02x?}",
header[1], payload_len, *this.sequence_number, header
),
)));
}
if this.ciphertext_buf.len() < package_len {
@@ -469,6 +522,7 @@ where
}
match open_dare_package(
*this.key,
this.cipher,
*this.sequence_number,
*this.expected_base_nonce,
@@ -573,7 +627,8 @@ fn build_dare_package(
}
fn open_dare_package(
cipher: &Aes256Gcm,
key: [u8; 32],
cipher: &mut Option<DareDecryptCipher>,
sequence_number: u32,
expected_base_nonce: Option<[u8; 12]>,
header: [u8; DARE_HEADER_SIZE],
@@ -583,9 +638,19 @@ fn open_dare_package(
if header[0] != DARE_VERSION_20 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE version"));
}
if header[1] != DARE_CIPHER_AES_256_GCM {
if !matches!(header[1], DARE_CIPHER_AES_256_GCM | DARE_CIPHER_CHACHA20_POLY1305) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE cipher suite"));
}
if let Some(cipher) = cipher.as_ref() {
if cipher.cipher_id() != header[1] {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"DARE package cipher suite does not match the stream reference cipher",
));
}
} else {
*cipher = Some(DareDecryptCipher::new(header[1], key)?);
}
let header_nonce: [u8; 12] = header[4..16].try_into().expect("nonce slice");
if let Some(expected_base_nonce) = expected_base_nonce {
let masked_expected = apply_final_flag(expected_base_nonce, is_final_header(header));
@@ -608,16 +673,10 @@ fn open_dare_package(
let mut package_nonce = header_nonce;
xor_sequence_into_nonce(&mut package_nonce, sequence_number);
let nonce = Nonce::try_from(package_nonce.as_slice())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid DARE nonce length"))?;
let plaintext = cipher
.decrypt(
&nonce,
aes_gcm::aead::Payload {
msg: ciphertext,
aad: &header[..4],
},
)
.as_ref()
.expect("DARE cipher is initialized")
.decrypt(&package_nonce, ciphertext, &header[..4])
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("DARE authentication failed: {err}")))?;
Ok((plaintext, *current_ref, is_final_header(header)))
@@ -707,6 +766,42 @@ mod tests {
assert_eq!(decrypted, plaintext);
}
#[tokio::test]
async fn decrypt_reader_accepts_chacha20_poly1305_dare_v2_package() {
let object_key = [0x72u8; 32];
let plaintext = b"minio may emit ChaCha20-Poly1305 DARE packages".to_vec();
let mut header = [0u8; DARE_HEADER_SIZE];
header[0] = DARE_VERSION_20;
header[1] = DARE_CIPHER_CHACHA20_POLY1305;
header[2..4].copy_from_slice(&(u16::try_from(plaintext.len() - 1).expect("test payload fits")).to_le_bytes());
header[4] = 0x80;
header[5..16].copy_from_slice(&[0x27; 11]);
let cipher = ChaCha20Poly1305::new_from_slice(&object_key).expect("valid test key");
let nonce = chacha20poly1305::Nonce::try_from(&header[4..16]).expect("valid test nonce");
let ciphertext = cipher
.encrypt(
&nonce,
Payload {
msg: &plaintext,
aad: &header[..4],
},
)
.expect("encrypt chacha DARE package");
let mut encrypted = header.to_vec();
encrypted.extend_from_slice(&ciphertext);
let mut decrypted = Vec::new();
DecryptReader::new_with_object_key(Cursor::new(encrypted), object_key)
.read_to_end(&mut decrypted)
.await
.expect("decrypt chacha DARE package");
assert_eq!(decrypted, plaintext);
}
#[test]
fn derive_part_key_matches_minio_test_vectors() {
assert_eq!(
rustfs/Cargo.toml
+1
View File
@@ -171,6 +171,7 @@ zip = { workspace = true }
libc = { workspace = true }
rand = { workspace = true }
aes-gcm = { workspace = true }
chacha20poly1305 = { workspace = true }
# Observability and Metrics
metrics = { workspace = true }
rustfs/src/storage/sse.rs
+46 -14
View File
@@ -70,6 +70,8 @@
//! ```
use crate::storage::storage_compat::StorageError;
#[cfg(feature = "rio-v2")]
use aes_gcm::aead::Payload;
use aes_gcm::{
Aes256Gcm, Key, Nonce,
aead::{Aead, KeyInit},
@@ -77,6 +79,8 @@ use aes_gcm::{
use async_trait::async_trait;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
#[cfg(feature = "rio-v2")]
use chacha20poly1305::ChaCha20Poly1305;
#[cfg(feature = "rio-v2")]
use hmac::{Hmac, Mac};
use http::{HeaderMap, HeaderValue};
use rand::Rng;
@@ -117,6 +121,8 @@ const DARE_VERSION_20: u8 = 0x20;
#[cfg(feature = "rio-v2")]
const DARE_CIPHER_AES_256_GCM: u8 = 0x00;
#[cfg(feature = "rio-v2")]
const DARE_CIPHER_CHACHA20_POLY1305: u8 = 0x01;
#[cfg(feature = "rio-v2")]
const DARE_HEADER_SIZE: usize = 16;
#[cfg(feature = "rio-v2")]
const DARE_TAG_SIZE: usize = 16;
@@ -821,6 +827,36 @@ fn decode_minio_kms_context(metadata: &HashMap<String, String>) -> Result<Option
.map_err(|e| ApiError::from(StorageError::other(format!("Failed to parse MinIO KMS context: {e}"))))
}
#[cfg(feature = "rio-v2")]
fn is_supported_sealed_object_key_cipher(cipher: u8) -> bool {
matches!(cipher, DARE_CIPHER_AES_256_GCM | DARE_CIPHER_CHACHA20_POLY1305)
}
#[cfg(feature = "rio-v2")]
fn decrypt_sealed_object_key_payload(sealing_key: [u8; 32], header: &[u8], sealed_key: &[u8]) -> Result<Vec<u8>, ApiError> {
let nonce = &header[4..16];
let ciphertext = &sealed_key[DARE_HEADER_SIZE..];
let aad = &header[..4];
match header[1] {
DARE_CIPHER_AES_256_GCM => {
let cipher = Aes256Gcm::new_from_slice(&sealing_key)
.map_err(|err| ApiError::from(StorageError::other(format!("Invalid AES-GCM sealing key: {err}"))))?;
let nonce = Nonce::try_from(nonce)
.map_err(|_| ApiError::from(StorageError::other("Invalid sealed object-key package nonce")))?;
cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
}
DARE_CIPHER_CHACHA20_POLY1305 => {
let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key)
.map_err(|err| ApiError::from(StorageError::other(format!("Invalid ChaCha20-Poly1305 sealing key: {err}"))))?;
let nonce = chacha20poly1305::Nonce::try_from(nonce)
.map_err(|_| ApiError::from(StorageError::other("Invalid sealed object-key package nonce")))?;
cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
}
_ => return Err(ApiError::from(StorageError::other("Unsupported sealed object-key DARE header"))),
}
.map_err(|err| ApiError::from(StorageError::other(format!("Failed to unseal object key: {err}"))))
}
#[cfg(feature = "rio-v2")]
fn canonical_sse_path(bucket: &str, object: &str) -> String {
let bucket = bucket.trim_matches('/');
@@ -918,7 +954,7 @@ fn unseal_object_key(
object: &str,
) -> Result<[u8; 32], ApiError> {
let header = &sealed.sealed_key[..DARE_HEADER_SIZE];
if header[0] != DARE_VERSION_20 || header[1] != DARE_CIPHER_AES_256_GCM {
if header[0] != DARE_VERSION_20 || !is_supported_sealed_object_key_cipher(header[1]) {
return Err(ApiError::from(StorageError::other("Unsupported sealed object-key DARE header")));
}
if u16::from_le_bytes([header[2], header[3]]) != 31 || header[4] & 0x80 == 0 {
@@ -926,19 +962,7 @@ fn unseal_object_key(
}
let sealing_key = derive_sealing_key(external_key, sealed.iv, managed_sse_domain(sse_type), bucket, object);
let cipher = Aes256Gcm::new_from_slice(&sealing_key)
.map_err(|err| ApiError::from(StorageError::other(format!("Invalid sealing key: {err}"))))?;
let nonce = Nonce::try_from(&header[4..16])
.map_err(|_| ApiError::from(StorageError::other("Invalid sealed object-key package nonce")))?;
let plaintext = cipher
.decrypt(
&nonce,
aes_gcm::aead::Payload {
msg: &sealed.sealed_key[DARE_HEADER_SIZE..],
aad: &header[..4],
},
)
.map_err(|err| ApiError::from(StorageError::other(format!("Failed to unseal object key: {err}"))))?;
let plaintext = decrypt_sealed_object_key_payload(sealing_key, header, &sealed.sealed_key)?;
let object_key: [u8; 32] = plaintext
.as_slice()
@@ -3117,6 +3141,14 @@ mod tests {
reset_sse_dek_provider();
}
#[cfg(feature = "rio-v2")]
#[test]
fn test_supported_sealed_object_key_cipher_accepts_current_minio_fixture_value() {
assert!(is_supported_sealed_object_key_cipher(DARE_CIPHER_AES_256_GCM));
assert!(is_supported_sealed_object_key_cipher(DARE_CIPHER_CHACHA20_POLY1305));
assert!(!is_supported_sealed_object_key_cipher(0x02));
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn test_ssec_rio_v2_uses_sealed_object_key_metadata_roundtrip() {