mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-03 17:45:05 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 唐小鸭
|
20ef9ebf15 | ||
|
Boyi (Elliot) Li
|
cc08e0d11b |
Generated
+40
-39
@@ -40,9 +40,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aes"
|
||||
version = "0.9.4"
|
||||
version = "0.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1"
|
||||
checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32"
|
||||
dependencies = [
|
||||
"cipher 0.5.2",
|
||||
"cpubits",
|
||||
@@ -57,7 +57,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes 0.9.4",
|
||||
"aes 0.9.3",
|
||||
"cipher 0.5.2",
|
||||
"ctr",
|
||||
"ctutils",
|
||||
@@ -772,9 +772,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-executor"
|
||||
version = "1.14.1"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f426a31126c735126297bc5ea7a521263c9faff4f63ca415267ee625651c4f37"
|
||||
checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a"
|
||||
dependencies = [
|
||||
"async-task",
|
||||
"concurrent-queue",
|
||||
@@ -800,9 +800,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-lock"
|
||||
version = "3.5.0"
|
||||
version = "3.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f"
|
||||
checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311"
|
||||
dependencies = [
|
||||
"event-listener",
|
||||
"event-listener-strategy",
|
||||
@@ -2580,9 +2580,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crc32c"
|
||||
version = "0.6.9"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f132a176423ee852e8f0a35d43088d1850966b50745be818a88b76f64b1c6cb"
|
||||
checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
|
||||
dependencies = [
|
||||
"rustc_version",
|
||||
]
|
||||
@@ -2766,9 +2766,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ctutils"
|
||||
version = "0.4.8"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f"
|
||||
checksum = "a2376ffd6f96d8a9db59875b4b6b0237d6900b5baf64279714e05493ce64828a"
|
||||
dependencies = [
|
||||
"cmov",
|
||||
"subtle",
|
||||
@@ -3906,9 +3906,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9"
|
||||
version = "0.5.7"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f3298a54fd82af243c9f122595762f5b0e48bd3d1ea8fe3f3eb5e95c130e881"
|
||||
checksum = "11185c62245c5ff61c753ec8ec3d56341a13fe4195344ecc100ce94f780533cc"
|
||||
dependencies = [
|
||||
"bon",
|
||||
"dial9-core",
|
||||
@@ -3923,9 +3923,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-core"
|
||||
version = "0.5.7"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc7907b149a394404e3075f57bfbc751c7292ef115027f13f4995ff3a8b360c"
|
||||
checksum = "54f1dcb5d20bdc85b9beb7c2864540526c1394c00783e9f3544feed1cfe529e8"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arc-swap",
|
||||
@@ -3957,9 +3957,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-metrique"
|
||||
version = "0.5.7"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35f30e80d69214ed7c684d81afd8e52315b4acbfa5be9ebb84ccee02ebd06f44"
|
||||
checksum = "56645773b4185f51a9528625a5aedcd4bbc5b5919ba68e7a6108cf076f635efc"
|
||||
dependencies = [
|
||||
"dial9-core",
|
||||
"dial9-trace-format",
|
||||
@@ -3971,9 +3971,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-perf-self-profile"
|
||||
version = "0.5.7"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17f09434b7623ef8b9b107062941a8e471e6044d6c946fa4af899130223b7a28"
|
||||
checksum = "00c8c96d18b28ec76adde2107f7d2d5f8bb53b0670fba31bc012841703bc49ca"
|
||||
dependencies = [
|
||||
"blazesym",
|
||||
"bon",
|
||||
@@ -3990,9 +3990,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-tokio-telemetry"
|
||||
version = "0.5.7"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0428cba60fbcf37ab82666f3a7ef3b6d0671e90dfb6a352376813b471319f061"
|
||||
checksum = "7ccc79f3683219471a8758b8ce459589167c55d72a649d27b89068a1a4cfffe9"
|
||||
dependencies = [
|
||||
"bon",
|
||||
"bytes",
|
||||
@@ -4015,9 +4015,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-trace-format"
|
||||
version = "0.5.4"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af753ba562fab9b99024b74234fe3a93b01c32d6474db8ae4f2c3bb562958f8f"
|
||||
checksum = "44026339a4c6cbd4914c3eb85e653e741d5739f1e401695a039127ba49c19731"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"dial9-trace-format-derive",
|
||||
@@ -4027,9 +4027,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-trace-format-derive"
|
||||
version = "0.5.4"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e00786284c5d77b60aac60ee96797b8ac020306da8f02fecb0e86c5e82610aae"
|
||||
checksum = "bbdbf5c76eb6824b0576e6891811caae93fc0bc96c23003556e3f752d069a84d"
|
||||
dependencies = [
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
@@ -5895,9 +5895,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jiff"
|
||||
version = "0.2.39"
|
||||
version = "0.2.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ac703058abac89e5c39b0b879dce2b6b14a4ba36600c7e7e3d0c60a6883ef3c"
|
||||
checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383"
|
||||
dependencies = [
|
||||
"defmt 1.2.0",
|
||||
"jiff-core",
|
||||
@@ -5912,9 +5912,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jiff-core"
|
||||
version = "0.1.2"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aee43309eb53bafb8c36d35d6e444769c8f8e6206a9fbf7546f5db9d7d4a0e93"
|
||||
checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c"
|
||||
dependencies = [
|
||||
"defmt 1.2.0",
|
||||
"log",
|
||||
@@ -5922,9 +5922,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jiff-static"
|
||||
version = "0.2.39"
|
||||
version = "0.2.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dc32470c67eb7d0e9ccb74303fa925313dc04cecb1aa048beaa6f391e36a93d"
|
||||
checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c"
|
||||
dependencies = [
|
||||
"jiff-core",
|
||||
"proc-macro2",
|
||||
@@ -6413,9 +6413,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rust2"
|
||||
version = "0.24.0"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "490750560fa7851fbbf5d3ab1a8750f09350ed361ed8aed85b1e1e40587812ee"
|
||||
checksum = "f1c46f3fcd2e888afdb01a07efc071cb21300e07c68d32c3e8b01b5c8cb1fcaa"
|
||||
dependencies = [
|
||||
"sha2 0.11.1",
|
||||
]
|
||||
@@ -8040,7 +8040,7 @@ version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca"
|
||||
dependencies = [
|
||||
"aes 0.9.4",
|
||||
"aes 0.9.3",
|
||||
"aes-gcm",
|
||||
"cbc 0.2.1",
|
||||
"der 0.8.3",
|
||||
@@ -9353,7 +9353,7 @@ version = "0.65.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3bb77a4b259312b2abbc541bca1c13cb4e9b61c560a7fd894068c3b92e0c9dc3"
|
||||
dependencies = [
|
||||
"aes 0.9.4",
|
||||
"aes 0.9.3",
|
||||
"aws-lc-rs",
|
||||
"bitflags 2.13.2",
|
||||
"block-padding 0.4.2",
|
||||
@@ -11058,6 +11058,7 @@ dependencies = [
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.2"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
"brotli 9.1.1",
|
||||
@@ -12144,7 +12145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes 0.9.4",
|
||||
"aes 0.9.3",
|
||||
"aes-gcm",
|
||||
"chacha20",
|
||||
"cipher 0.5.2",
|
||||
@@ -14198,11 +14199,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.3.0"
|
||||
version = "9.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eaeb7f16d2605b49aef586996ded29e65642fab7259a3077e537351df6d80426"
|
||||
checksum = "e408c6cc4699dd017d53dc080cd51c2cea2b5354af57a1479bd265ed42911487"
|
||||
dependencies = [
|
||||
"aes 0.9.4",
|
||||
"aes 0.9.3",
|
||||
"bzip2",
|
||||
"constant_time_eq",
|
||||
"crc32fast",
|
||||
|
||||
+4
-3
@@ -212,6 +212,7 @@ serde_urlencoded = "0.7.1"
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
aws-lc-rs = { version = "1.18.0", default-features = false }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
@@ -234,7 +235,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.39" }
|
||||
jiff = { version = "0.2.38" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -356,13 +357,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.3.0"
|
||||
zip = "9.2.2"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.7", default-features = false }
|
||||
dial9 = { version = "0.5.6", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
|
||||
@@ -31,7 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips"]
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
|
||||
@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
|
||||
};
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
|
||||
global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
|
||||
use rustfs_utils::get_env_bool;
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
|
||||
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
|
||||
|
||||
@@ -26,6 +26,7 @@ categories = ["web-programming", "development-tools", "cryptography"]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
aws-lc-rs = { workspace = true, optional = true }
|
||||
base64-simd = { workspace = true, optional = true }
|
||||
blake2 = { workspace = true, optional = true }
|
||||
brotli = { workspace = true, optional = true }
|
||||
@@ -89,6 +90,7 @@ path = [] # path manipulation features
|
||||
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
|
||||
string = ["dep:regex"]
|
||||
crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1", "dep:sha2"]
|
||||
fips = ["crypto", "dep:aws-lc-rs", "aws-lc-rs/fips"]
|
||||
hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake2", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:crc-fast"]
|
||||
os = ["dep:rustix", "dep:tempfile", "dep:windows"] # operating system utilities
|
||||
integration = [] # integration test features
|
||||
|
||||
@@ -13,12 +13,77 @@
|
||||
// limitations under the License.
|
||||
|
||||
use hex_simd::{AsOut, AsciiCase};
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use hmac::Hmac;
|
||||
#[cfg(not(feature = "fips"))]
|
||||
pub use hmac::{KeyInit, Mac};
|
||||
use hyper::body::Bytes;
|
||||
#[cfg(not(feature = "fips"))]
|
||||
use sha1::Sha1;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::mem::MaybeUninit;
|
||||
|
||||
#[cfg(not(feature = "fips"))]
|
||||
pub type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
#[cfg(feature = "fips")]
|
||||
pub struct HmacSha256Output([u8; 32]);
|
||||
|
||||
#[cfg(feature = "fips")]
|
||||
impl HmacSha256Output {
|
||||
#[must_use]
|
||||
pub fn into_bytes(self) -> hmac::digest::Output<Hmac<Sha256>> {
|
||||
self.0.into()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fips")]
|
||||
pub trait KeyInit: Sized {
|
||||
fn new_from_slice(key: &[u8]) -> Result<Self, hmac::digest::InvalidLength>;
|
||||
}
|
||||
|
||||
#[cfg(feature = "fips")]
|
||||
pub trait Mac: Sized {
|
||||
fn update(&mut self, data: &[u8]);
|
||||
fn finalize(self) -> HmacSha256Output;
|
||||
fn verify_slice(self, tag: &[u8]) -> Result<(), hmac::digest::MacError>;
|
||||
}
|
||||
|
||||
#[cfg(feature = "fips")]
|
||||
#[derive(Clone)]
|
||||
pub struct HmacSha256(aws_lc_rs::hmac::Context);
|
||||
|
||||
#[cfg(feature = "fips")]
|
||||
impl HmacSha256 {
|
||||
pub fn new_from_slice(key: &[u8]) -> Result<Self, hmac::digest::InvalidLength> {
|
||||
let signing_key = aws_lc_rs::hmac::Key::new(aws_lc_rs::hmac::HMAC_SHA256, key);
|
||||
Ok(Self(aws_lc_rs::hmac::Context::with_key(&signing_key)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fips")]
|
||||
impl KeyInit for HmacSha256 {
|
||||
fn new_from_slice(key: &[u8]) -> Result<Self, hmac::digest::InvalidLength> {
|
||||
Self::new_from_slice(key)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fips")]
|
||||
impl Mac for HmacSha256 {
|
||||
fn update(&mut self, data: &[u8]) {
|
||||
self.0.update(data);
|
||||
}
|
||||
|
||||
fn finalize(self) -> HmacSha256Output {
|
||||
let tag = self.0.sign();
|
||||
HmacSha256Output(tag.as_ref().try_into().expect("HMAC-SHA256 tag is 32 bytes"))
|
||||
}
|
||||
|
||||
fn verify_slice(self, tag: &[u8]) -> Result<(), hmac::digest::MacError> {
|
||||
let computed = self.0.sign();
|
||||
aws_lc_rs::constant_time::verify_slices_are_equal(computed.as_ref(), tag).map_err(|_| hmac::digest::MacError)
|
||||
}
|
||||
}
|
||||
|
||||
/// Base64 URL safe encoding without padding
|
||||
/// `base64_encode_url_safe_no_pad(input)`
|
||||
///
|
||||
@@ -84,9 +149,18 @@ pub fn is_sha256_checksum(s: &str) -> bool {
|
||||
/// A 20-byte array containing the HMAC-SHA1 hash of the input data using the provided key
|
||||
///
|
||||
pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] {
|
||||
let mut m = <Hmac<Sha1>>::new_from_slice(key.as_ref()).expect("operation should succeed");
|
||||
m.update(data.as_ref());
|
||||
m.finalize().into_bytes().into()
|
||||
#[cfg(feature = "fips")]
|
||||
{
|
||||
let signing_key = aws_lc_rs::hmac::Key::new(aws_lc_rs::hmac::HMAC_SHA1_FOR_LEGACY_USE_ONLY, key.as_ref());
|
||||
let tag = aws_lc_rs::hmac::sign(&signing_key, data.as_ref());
|
||||
tag.as_ref().try_into().expect("HMAC-SHA1 tag is 20 bytes")
|
||||
}
|
||||
#[cfg(not(feature = "fips"))]
|
||||
{
|
||||
let mut m = <Hmac<Sha1>>::new_from_slice(key.as_ref()).expect("operation should succeed");
|
||||
m.update(data.as_ref());
|
||||
m.finalize().into_bytes().into()
|
||||
}
|
||||
}
|
||||
|
||||
/// HMAC-SHA256 hashing
|
||||
@@ -100,7 +174,7 @@ pub fn hmac_sha1(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 20] {
|
||||
/// A 32-byte array containing the HMAC-SHA256 hash of the input data using the provided key
|
||||
///
|
||||
pub fn hmac_sha256(key: impl AsRef<[u8]>, data: impl AsRef<[u8]>) -> [u8; 32] {
|
||||
let mut m = Hmac::<Sha256>::new_from_slice(key.as_ref()).expect("operation should succeed");
|
||||
let mut m = HmacSha256::new_from_slice(key.as_ref()).expect("operation should succeed");
|
||||
m.update(data.as_ref());
|
||||
m.finalize().into_bytes().into()
|
||||
}
|
||||
@@ -167,3 +241,20 @@ fn sha256_checksum_rejects_wrong_length_or_non_lowercase_hex() {
|
||||
assert!(!is_sha256_checksum("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdeg"));
|
||||
assert!(!is_sha256_checksum("0123456789ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hmac_sha256_rfc4231() {
|
||||
#[cfg(feature = "fips")]
|
||||
aws_lc_rs::try_fips_mode().expect("aws-lc-rs FIPS mode must be active when `rustfs-utils/fips` is enabled");
|
||||
|
||||
let key = [0x0b_u8; 20];
|
||||
let data = b"Hi There";
|
||||
assert_eq!(hex(hmac_sha1(key, data)), "b617318655057264e28bc0b6fb378c8ef146be00");
|
||||
let mac = hmac_sha256(key, data);
|
||||
assert_eq!(hex(mac), "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7");
|
||||
|
||||
let mut verifier = <HmacSha256 as KeyInit>::new_from_slice(&key).expect("valid key");
|
||||
verifier.update(data);
|
||||
assert!(verifier.clone().verify_slice(&mac).is_ok());
|
||||
assert!(verifier.verify_slice(&[0_u8; 32]).is_err());
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ This document records where RustFS stands on cryptographic module validation and
|
||||
| Layer | Where | Implementation | Primitives | Validated module |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| TLS using the process-wide rustls provider | Provider installed or checked by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the process-wide provider (AWS-LC is installed when none exists) | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange (restricted to FIPS-approved suites and groups when `--features fips` is enabled) | No in the default build (`aws-lc-sys`); opt-in `--features fips` enables `aws-lc-fips-sys` and checks the installed provider at startup; module validation depends on the deployment |
|
||||
| Internode RPC HMAC authentication | `crates/ecstore/src/cluster/rpc/http_auth.rs` via `rustfs_utils::crypto::HmacSha256` | RustCrypto `hmac` + `sha2` in the default build; `aws-lc-rs` when `fips` is enabled | HMAC-SHA256 (`x-rustfs-signature`, `x-rustfs-rpc-signature-v2`, `x-rustfs-rpc-signature-v3`) | No in the default build; opt-in `--features fips` enables `rustfs-utils/fips` (`aws-lc-fips-sys`) |
|
||||
| Object data path AEAD (SSE) | `crates/kms/src/encryption/ciphers.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs` | RustCrypto `aes-gcm`, `chacha20poly1305` | AES-256-GCM, ChaCha20-Poly1305 | No |
|
||||
| DEK wrapping | `crates/kms/src/encryption/dek.rs` | RustCrypto `aes-gcm` | AES-256-GCM | No |
|
||||
| Local KMS backend master key | `crates/kms/src/backends/local.rs` | RustCrypto `argon2`, `aes-gcm` | Argon2id KDF, AES-256-GCM | No |
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ required-features = ["swift"]
|
||||
|
||||
[features]
|
||||
default = ["ftps", "webdav", "gcs"]
|
||||
fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips", "rustfs-ecstore/fips", "rustfs-protocols/fips", "rustfs-s3-client/fips", "rustfs-targets/fips"]
|
||||
fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips", "rustfs-ecstore/fips", "rustfs-protocols/fips", "rustfs-s3-client/fips", "rustfs-targets/fips", "rustfs-utils/fips"]
|
||||
gcs = ["rustfs-ecstore/gcs", "dep:google-cloud-auth"]
|
||||
metrics-gpu = ["rustfs-obs/gpu"]
|
||||
ftps = ["rustfs-protocols/ftps"]
|
||||
|
||||
+119
-13
@@ -295,16 +295,32 @@ fn enrich_delete_replication_state_if_needed(
|
||||
}
|
||||
}
|
||||
|
||||
fn should_schedule_replica_delete_replication(
|
||||
/// The current peer set is read only when a rule has the derived shape. If it
|
||||
/// cannot be read, no rule is skipped: a redundant echo is safer than a chained
|
||||
/// target missing the delete.
|
||||
async fn replica_delete_fanout_state(
|
||||
snapshot: &DeleteReplicationConfigSnapshot,
|
||||
bucket: &str,
|
||||
replication_source: &ObjectInfo,
|
||||
version_id: Option<Uuid>,
|
||||
) -> bool {
|
||||
let Some(config) = snapshot.replication_config() else {
|
||||
return false;
|
||||
) -> Option<ReplicationState> {
|
||||
let config = snapshot.replication_config()?;
|
||||
let site_peer_deployment_ids = if has_site_replication_shaped_rule(config) {
|
||||
match crate::site_replication::site_replication_remote_peer_deployment_ids().await {
|
||||
Ok(peers) => peers,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
error = %error,
|
||||
"failed to load site replication peers; forwarding replica delete along every rule"
|
||||
);
|
||||
HashSet::new()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
HashSet::new()
|
||||
};
|
||||
|
||||
delete_replication_state_from_config(config, replication_source, version_id, true).is_some()
|
||||
replica_delete_fanout_state_from_config(config, replication_source, version_id, &site_peer_deployment_ids)
|
||||
}
|
||||
|
||||
fn validate_undo_delete_version(expected: Option<&str>, requested: Option<&str>) -> S3Result<()> {
|
||||
@@ -1347,12 +1363,21 @@ impl DefaultObjectUsecase {
|
||||
let deleted_delete_marker_version = deleted_replication_info.is_some_and(|info| info.delete_marker);
|
||||
|
||||
let delete_replication_version_id = delete_replication_version_id(deleted_object_source, deleted_delete_marker_version);
|
||||
let schedule_delete_replication = if opts.replication_request && replica {
|
||||
should_schedule_replica_delete_replication(
|
||||
&delete_config_snapshot,
|
||||
replication_state_source,
|
||||
delete_replication_version_id,
|
||||
let replica_fanout_state = if opts.replication_request && replica {
|
||||
Some(
|
||||
replica_delete_fanout_state(
|
||||
&delete_config_snapshot,
|
||||
&bucket,
|
||||
replication_state_source,
|
||||
delete_replication_version_id,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let schedule_delete_replication = if let Some(fanout_state) = &replica_fanout_state {
|
||||
fanout_state.is_some()
|
||||
} else {
|
||||
should_schedule_delete_replication(
|
||||
&opts,
|
||||
@@ -1381,8 +1406,12 @@ impl DefaultObjectUsecase {
|
||||
replication_state: None,
|
||||
..Default::default()
|
||||
};
|
||||
set_deleted_object_replication_state(&mut deleted_object, &replication_state_source.replication_state());
|
||||
enrich_delete_replication_state_if_needed(&delete_config_snapshot, &mut deleted_object, replication_state_source);
|
||||
if let Some(Some(fanout_state)) = &replica_fanout_state {
|
||||
set_deleted_object_replication_state(&mut deleted_object, fanout_state);
|
||||
} else {
|
||||
set_deleted_object_replication_state(&mut deleted_object, &replication_state_source.replication_state());
|
||||
enrich_delete_replication_state_if_needed(&delete_config_snapshot, &mut deleted_object, replication_state_source);
|
||||
}
|
||||
schedule_replication_delete(deleted_object, bucket.clone(), REPLICATE_INCOMING_DELETE.to_string()).await;
|
||||
}
|
||||
|
||||
@@ -3583,6 +3612,83 @@ mod tests {
|
||||
assert!(state.purge_targets.contains_key(&arn));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replica_delete_fanout_skips_site_replication_peers_but_keeps_chain_targets() {
|
||||
let peer_arn = "arn:rustfs:replication::peer-dep:bucket";
|
||||
let chain_arn = "arn:aws:s3:::downstream-bucket".to_string();
|
||||
let site_rule = crate::site_replication::build_site_replication_rule(peer_arn, 10, "site-repl-peer-dep");
|
||||
let mut chain_rule = site_rule.clone();
|
||||
chain_rule.id = Some("chain".to_string());
|
||||
chain_rule.priority = Some(1);
|
||||
chain_rule.destination.bucket = chain_arn.clone();
|
||||
let site_only = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![site_rule.clone()],
|
||||
};
|
||||
let site_and_chain = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![site_rule, chain_rule],
|
||||
};
|
||||
let marker = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
delete_marker: true,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
..Default::default()
|
||||
};
|
||||
let version = ObjectInfo {
|
||||
delete_marker: false,
|
||||
..marker.clone()
|
||||
};
|
||||
// An ordinary bucket target ARN carries its target id, so an operator
|
||||
// rule named `site-repl-<target id>` has the derived shape.
|
||||
let shaped_operator_arn = "arn:minio:replication:us-east-1:target-1:downstream".to_string();
|
||||
let mut shaped_operator_rule =
|
||||
crate::site_replication::build_site_replication_rule(&shaped_operator_arn, 1, "site-repl-target-1");
|
||||
shaped_operator_rule.priority = Some(1);
|
||||
let shaped_operator_only = ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![shaped_operator_rule],
|
||||
};
|
||||
assert!(has_site_replication_shaped_rule(&shaped_operator_only));
|
||||
let site_peers = HashSet::from(["peer-dep".to_string()]);
|
||||
let no_site_peers = HashSet::new();
|
||||
let removed_peer_only = HashSet::from(["other-dep".to_string()]);
|
||||
let purged_version = Some(Uuid::new_v4());
|
||||
let fanout_targets = |state: ReplicationState, version_id: Option<Uuid>| {
|
||||
if version_id.is_some() {
|
||||
state.purge_targets.keys().cloned().collect::<Vec<_>>()
|
||||
} else {
|
||||
state.targets.keys().cloned().collect::<Vec<_>>()
|
||||
}
|
||||
};
|
||||
|
||||
for (source, version_id) in [(&marker, None), (&marker, purged_version), (&version, purged_version)] {
|
||||
assert!(
|
||||
delete_replication_state_from_config(&site_only, source, version_id, true).is_some(),
|
||||
"the site-replication rule alone would forward the replica delete"
|
||||
);
|
||||
assert!(
|
||||
replica_delete_fanout_state_from_config(&site_only, source, version_id, &site_peers).is_none(),
|
||||
"a replica delete must not echo back across the site-replication mesh"
|
||||
);
|
||||
|
||||
let state = replica_delete_fanout_state_from_config(&site_and_chain, source, version_id, &site_peers)
|
||||
.expect("a replica delete must still follow a chained operator rule");
|
||||
assert!(!state.replicate_decision_str.contains(peer_arn));
|
||||
assert_eq!(fanout_targets(state, version_id), vec![chain_arn.clone()]);
|
||||
|
||||
for peers in [&no_site_peers, &removed_peer_only, &site_peers] {
|
||||
let state = replica_delete_fanout_state_from_config(&shaped_operator_only, source, version_id, peers)
|
||||
.expect("an operator rule naming no current site peer must keep forwarding replica deletes");
|
||||
assert_eq!(fanout_targets(state, version_id), vec![shaped_operator_arn.clone()]);
|
||||
}
|
||||
let state = replica_delete_fanout_state_from_config(&site_only, source, version_id, &no_site_peers)
|
||||
.expect("with site replication disabled a site-repl rule is an operator rule");
|
||||
assert_eq!(fanout_targets(state, version_id), vec![peer_arn.to_string()]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replica_delete_enrichment_must_not_reuse_upstream_targets() {
|
||||
let upstream_state = ReplicationState {
|
||||
|
||||
@@ -29,8 +29,9 @@ use super::storage_api::object_usecase::access::{
|
||||
#[cfg(test)]
|
||||
use super::storage_api::object_usecase::bucket::quota::BucketQuota;
|
||||
use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker;
|
||||
use super::storage_api::object_usecase::bucket::replication::ReplicationState;
|
||||
#[cfg(test)]
|
||||
use super::storage_api::object_usecase::bucket::replication::{ReplicationState, replication_statuses_map};
|
||||
use super::storage_api::object_usecase::bucket::replication::replication_statuses_map;
|
||||
use super::storage_api::object_usecase::bucket::{
|
||||
VersioningConfigExt as _,
|
||||
lifecycle::{
|
||||
@@ -49,10 +50,11 @@ use super::storage_api::object_usecase::bucket::{
|
||||
replication::{
|
||||
DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, commit_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, deleted_object_has_pending_replication_delete,
|
||||
force_delete_target_set, get_read_proxy_targets, has_active_delete_rule, load_delete_config_snapshot,
|
||||
must_replicate_object, persist_force_delete_intent, record_replication_proxy, schedule_object_replication,
|
||||
schedule_replication_delete, schedule_replication_deletes, set_deleted_object_replication_state,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
force_delete_target_set, get_read_proxy_targets, has_active_delete_rule, has_site_replication_shaped_rule,
|
||||
load_delete_config_snapshot, must_replicate_object, persist_force_delete_intent, record_replication_proxy,
|
||||
replica_delete_fanout_state_from_config, schedule_object_replication, schedule_replication_delete,
|
||||
schedule_replication_deletes, set_deleted_object_replication_state, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info,
|
||||
},
|
||||
tagging::decode_tags,
|
||||
validate_restore_request,
|
||||
|
||||
@@ -708,7 +708,6 @@ pub(crate) mod bucket {
|
||||
pub(crate) type DeleteReplicationConfigSnapshot =
|
||||
crate::storage::storage_api::ecstore_bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
pub(crate) type ReplicateDecision = replication_contracts::ReplicateDecision;
|
||||
#[cfg(test)]
|
||||
pub(crate) type ReplicationState = replication_contracts::ReplicationState;
|
||||
pub(crate) type ReplicationStatusType = replication_contracts::ReplicationStatusType;
|
||||
pub(crate) type ReplicationTargetValidationError = replication_contracts::ReplicationTargetValidationError;
|
||||
@@ -959,6 +958,52 @@ pub(crate) mod bucket {
|
||||
replication_contracts::delete_replication_state_from_config(config, &source)
|
||||
}
|
||||
|
||||
/// Replication state for forwarding a replica delete (an inbound
|
||||
/// replication request) to this site's own targets.
|
||||
///
|
||||
/// Operator rules with ReplicaModifications keep forwarding, so chained
|
||||
/// replication A -> B -> C propagates deletes (rustfs#2609). Derived
|
||||
/// site-replication rules never do: site replication is a full mesh
|
||||
/// where the origin delivers, and MRF-retries, every delete to each
|
||||
/// peer itself. Forwarding along a `site-repl-<deploymentId>` rule only
|
||||
/// echoes the delete back to the origin, where it removes the origin's
|
||||
/// pending-purge version before its own delivery state is written back,
|
||||
/// and duplicates it to every other peer (rustfs#9121). MinIO never
|
||||
/// re-replicates a replication request at all.
|
||||
///
|
||||
/// Rule ids and target ARN shapes are not reserved: an ordinary bucket
|
||||
/// target ARN also carries a target id, so an operator rule can match
|
||||
/// the derived shape. Only a rule naming a current remote peer in
|
||||
/// `site_peer_deployment_ids` is skipped; with site replication
|
||||
/// disabled the set is empty and every rule forwards.
|
||||
pub(crate) fn replica_delete_fanout_state_from_config(
|
||||
config: &s3s::dto::ReplicationConfiguration,
|
||||
obj_info: &crate::storage::storage_api::StorageObjectInfo,
|
||||
version_id: Option<Uuid>,
|
||||
site_peer_deployment_ids: &HashSet<String>,
|
||||
) -> Option<replication_contracts::ReplicationState> {
|
||||
let is_site_peer_rule = |rule: &s3s::dto::ReplicationRule| {
|
||||
replication_contracts::site_replication_rule_deployment_id(rule)
|
||||
.is_some_and(|deployment_id| site_peer_deployment_ids.contains(deployment_id))
|
||||
};
|
||||
if !config.rules.iter().any(is_site_peer_rule) {
|
||||
return delete_replication_state_from_config(config, obj_info, version_id, true);
|
||||
}
|
||||
|
||||
let mut operator_config = config.clone();
|
||||
operator_config.rules.retain(|rule| !is_site_peer_rule(rule));
|
||||
delete_replication_state_from_config(&operator_config, obj_info, version_id, true)
|
||||
}
|
||||
|
||||
/// Whether `config` has a rule with the derived site-replication shape,
|
||||
/// so the replica-delete fan-out needs the current peer set.
|
||||
pub(crate) fn has_site_replication_shaped_rule(config: &s3s::dto::ReplicationConfiguration) -> bool {
|
||||
config
|
||||
.rules
|
||||
.iter()
|
||||
.any(|rule| replication_contracts::site_replication_rule_deployment_id(rule).is_some())
|
||||
}
|
||||
|
||||
pub(crate) fn replication_target_arns(config: &s3s::dto::ReplicationConfiguration) -> HashSet<String> {
|
||||
replication_contracts::replication_target_arns(config)
|
||||
}
|
||||
|
||||
@@ -368,6 +368,18 @@ pub(crate) async fn site_replication_deployment_ids() -> S3Result<HashSet<String
|
||||
Ok(state.peers.values().map(|peer| peer.deployment_id.clone()).collect())
|
||||
}
|
||||
|
||||
/// Deployment ids of the remote peers this site's reconciler derives a
|
||||
/// `site-repl-<id>` rule for; empty when site replication is not enabled.
|
||||
/// Read on the replica-delete path so only rules naming a current peer are
|
||||
/// treated as reconciler-owned.
|
||||
pub(crate) async fn site_replication_remote_peer_deployment_ids() -> S3Result<HashSet<String>> {
|
||||
let state = load_site_replication_state().await?;
|
||||
if !state.enabled() {
|
||||
return Ok(HashSet::new());
|
||||
}
|
||||
Ok(remote_peer_deployment_ids(&state, ¤t_local_runtime_peer(&state)))
|
||||
}
|
||||
|
||||
pub(crate) async fn load_site_replication_state_no_lock(store: Arc<ECStore>) -> S3Result<SiteReplicationState> {
|
||||
match read_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await {
|
||||
Ok(data) => parse_site_replication_state(&data),
|
||||
|
||||
Reference in New Issue
Block a user