mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 08:27:06 +00:00
refactor(checksums): unify s3-client checksum dispatch in one registry (#6696)
The s3-client ChecksumMode previously duplicated per-algorithm header names, wire names, digest lengths, and checksum-type capability tables in EnumSet-mask matches. ChecksumAlgorithm in rustfs-checksums now owns that metadata behind exhaustive matches (a new variant fails to compile until its metadata is decided), and ChecksumMode delegates through a single algorithm() bridge. Wire behaviour is pinned unchanged by tests on both sides. Refs rustfs/backlog#1844 (PR1 of 3).
This commit is contained in:
Zhengchao An
+160
-153
@@ -19,31 +19,27 @@
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_checksums::ChecksumAlgorithm;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::utils::base64_decode;
|
||||
use crate::utils::base64_encode;
|
||||
use crate::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
|
||||
// s3s::header has no CRC64NVME constant yet; the canonical RustFS copy lives
|
||||
// in rustfs-utils' headers module.
|
||||
use rustfs_utils::http::headers::AMZ_CHECKSUM_CRC64NVME;
|
||||
use s3s::header::{
|
||||
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
|
||||
};
|
||||
|
||||
use enumset::{EnumSet, EnumSetType, enum_set};
|
||||
use enumset::EnumSetType;
|
||||
|
||||
/// One of three deliberately separate checksum registries (backlog#1833):
|
||||
/// this enum is the MinIO-port client's wire vocabulary and stops at the
|
||||
/// standard S3 set (CRC64NVME is its newest member; the RustFS extensions do
|
||||
/// not exist on this client path). The streaming-hash registry lives in
|
||||
/// `rustfs_checksums::ChecksumAlgorithm` (crates/checksums/src/lib.rs) and
|
||||
/// the on-disk xl.meta bitset in `rustfs_rio::ChecksumType`
|
||||
/// (crates/rio/src/checksum.rs, varint bits are append-only). When adding an
|
||||
/// algorithm, extend all three (or record why not) — they do not derive from
|
||||
/// each other.
|
||||
/// The MinIO-port client's checksum vocabulary: the standard S3 algorithm set
|
||||
/// plus the `None`/`FullObject` markers this client's option plumbing needs
|
||||
/// (the RustFS extension algorithms do not exist on this client path). All
|
||||
/// per-algorithm dispatch — header names, wire names, digest lengths,
|
||||
/// checksum-type capabilities, hashers — is delegated through [`Self::algorithm`]
|
||||
/// to the canonical registry in `rustfs_checksums::ChecksumAlgorithm`
|
||||
/// (crates/checksums/src/lib.rs); only the variant-to-algorithm bridge lives
|
||||
/// here. The on-disk xl.meta bitset remains separate in
|
||||
/// `rustfs_rio::ChecksumType` (crates/rio/src/checksum.rs, varint bits are
|
||||
/// append-only). When adding an algorithm: extend `ChecksumAlgorithm` (its
|
||||
/// exhaustive matches force the metadata), add the variant + one `algorithm()`
|
||||
/// arm here, and allocate an xl.meta bit in rio (or record why not).
|
||||
#[derive(Debug, EnumSetType, Default)]
|
||||
#[enumset(repr = "u8")]
|
||||
pub enum ChecksumMode {
|
||||
@@ -57,34 +53,31 @@ pub enum ChecksumMode {
|
||||
ChecksumFullObject,
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
static ref C_ChecksumMask: EnumSet<ChecksumMode> = {
|
||||
let mut s = EnumSet::all();
|
||||
s.remove(ChecksumMode::ChecksumFullObject);
|
||||
s
|
||||
};
|
||||
static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> =
|
||||
enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
|
||||
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
|
||||
enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
|
||||
}
|
||||
impl ChecksumMode {
|
||||
//pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
|
||||
/// The single bridge from this client vocabulary to the canonical
|
||||
/// algorithm registry. Every per-algorithm question below goes through
|
||||
/// here; the marker variants (`ChecksumNone`, bare `ChecksumFullObject`)
|
||||
/// map to `None` and fail closed at each call site.
|
||||
pub fn algorithm(&self) -> Option<ChecksumAlgorithm> {
|
||||
match self {
|
||||
ChecksumMode::ChecksumSHA256 => Some(ChecksumAlgorithm::Sha256),
|
||||
ChecksumMode::ChecksumSHA1 => Some(ChecksumAlgorithm::Sha1),
|
||||
ChecksumMode::ChecksumCRC32 => Some(ChecksumAlgorithm::Crc32),
|
||||
ChecksumMode::ChecksumCRC32C => Some(ChecksumAlgorithm::Crc32c),
|
||||
ChecksumMode::ChecksumCRC64NVME => Some(ChecksumAlgorithm::Crc64Nvme),
|
||||
ChecksumMode::ChecksumNone | ChecksumMode::ChecksumFullObject => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn base(&self) -> ChecksumMode {
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
match s.as_u8() {
|
||||
1_u8 => ChecksumMode::ChecksumNone,
|
||||
2_u8 => ChecksumMode::ChecksumSHA256,
|
||||
4_u8 => ChecksumMode::ChecksumSHA1,
|
||||
8_u8 => ChecksumMode::ChecksumCRC32,
|
||||
16_u8 => ChecksumMode::ChecksumCRC32C,
|
||||
32_u8 => ChecksumMode::ChecksumCRC64NVME,
|
||||
// Fail closed: any mode without a concrete base algorithm (e.g. a
|
||||
// bare ChecksumFullObject flag) is treated as "no checksum" rather
|
||||
// than panicking. Callers already gate real work behind
|
||||
// is_set()/can_composite()/hasher(), so this only removes a crash.
|
||||
_ => ChecksumMode::ChecksumNone,
|
||||
// Fail closed: any mode without a concrete base algorithm (e.g. a
|
||||
// bare ChecksumFullObject flag) is treated as "no checksum" rather
|
||||
// than panicking. Callers already gate real work behind
|
||||
// is_set()/can_composite()/hasher(), so this only removes a crash.
|
||||
if self.algorithm().is_some() {
|
||||
*self
|
||||
} else {
|
||||
ChecksumMode::ChecksumNone
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,58 +86,22 @@ impl ChecksumMode {
|
||||
}
|
||||
|
||||
pub fn key(&self) -> String {
|
||||
//match c & checksumMask {
|
||||
match self {
|
||||
ChecksumMode::ChecksumCRC32 => {
|
||||
return X_AMZ_CHECKSUM_CRC32.to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
return X_AMZ_CHECKSUM_CRC32C.to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA1 => {
|
||||
return X_AMZ_CHECKSUM_SHA1.to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA256 => {
|
||||
return X_AMZ_CHECKSUM_SHA256.to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC64NVME => {
|
||||
return AMZ_CHECKSUM_CRC64NVME.to_string();
|
||||
}
|
||||
_ => {
|
||||
return "".to_string();
|
||||
}
|
||||
}
|
||||
self.algorithm().map(|a| a.http_header_name().to_string()).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn can_composite(&self) -> bool {
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
match s.as_u8() {
|
||||
2_u8 => true,
|
||||
4_u8 => true,
|
||||
8_u8 => true,
|
||||
16_u8 => true,
|
||||
_ => false,
|
||||
}
|
||||
self.algorithm().is_some_and(|a| a.supports_composite())
|
||||
}
|
||||
|
||||
pub fn can_merge_crc(&self) -> bool {
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
match s.as_u8() {
|
||||
8_u8 => true,
|
||||
16_u8 => true,
|
||||
32_u8 => true,
|
||||
_ => false,
|
||||
}
|
||||
self.algorithm().is_some_and(|a| a.supports_full_object())
|
||||
}
|
||||
|
||||
pub fn full_object_requested(&self) -> bool {
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
match s.as_u8() {
|
||||
//C_ChecksumFullObjectCRC32 as u8 => true,
|
||||
//C_ChecksumFullObjectCRC32C as u8 => true,
|
||||
32_u8 => true,
|
||||
_ => false,
|
||||
}
|
||||
// CRC64NVME is FULL_OBJECT-only, so selecting it implies a full-object
|
||||
// checksum even without an explicit request (AWS behaviour).
|
||||
self.algorithm()
|
||||
.is_some_and(|a| a.supports_full_object() && !a.supports_composite())
|
||||
}
|
||||
|
||||
pub fn key_capitalized(&self) -> String {
|
||||
@@ -152,57 +109,23 @@ impl ChecksumMode {
|
||||
}
|
||||
|
||||
pub fn raw_byte_len(&self) -> usize {
|
||||
let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
|
||||
if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
|
||||
4
|
||||
} else if u == ChecksumMode::ChecksumSHA1 as u8 {
|
||||
use sha1::Digest;
|
||||
sha1::Sha1::output_size() as usize
|
||||
} else if u == ChecksumMode::ChecksumSHA256 as u8 {
|
||||
use sha2::Digest;
|
||||
sha2::Sha256::output_size() as usize
|
||||
} else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
|
||||
8
|
||||
} else {
|
||||
0
|
||||
}
|
||||
self.algorithm().map(|a| a.raw_len()).unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn hasher(&self) -> Result<Box<dyn rustfs_checksums::http::HttpChecksum>, std::io::Error> {
|
||||
match /*C_ChecksumMask & **/self {
|
||||
ChecksumMode::ChecksumCRC32 => {
|
||||
return Ok(ChecksumAlgorithm::Crc32.into_impl());
|
||||
}
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
return Ok(ChecksumAlgorithm::Crc32c.into_impl());
|
||||
}
|
||||
ChecksumMode::ChecksumSHA1 => {
|
||||
return Ok(ChecksumAlgorithm::Sha1.into_impl());
|
||||
}
|
||||
ChecksumMode::ChecksumSHA256 => {
|
||||
return Ok(ChecksumAlgorithm::Sha256.into_impl());
|
||||
}
|
||||
ChecksumMode::ChecksumCRC64NVME => {
|
||||
return Ok(ChecksumAlgorithm::Crc64Nvme.into_impl());
|
||||
}
|
||||
_ => return Err(std::io::Error::other("unsupported checksum type")),
|
||||
}
|
||||
self.algorithm()
|
||||
.map(ChecksumAlgorithm::into_impl)
|
||||
.ok_or_else(|| std::io::Error::other("unsupported checksum type"))
|
||||
}
|
||||
|
||||
pub fn is_set(&self) -> bool {
|
||||
// `ChecksumNone` is the zeroth enum variant, so it occupies bit 0 of the
|
||||
// `EnumSet` repr and a naive `len() == 1` check reports "no checksum" as a
|
||||
// configured checksum. A checksum is only "set" when a concrete algorithm
|
||||
// (one with a real hasher) is selected; the bare `ChecksumFullObject` flag
|
||||
// has no base algorithm and is likewise not set. Treating `ChecksumNone`
|
||||
// as set made ILM transitions of >128 MiB objects fail with
|
||||
// "unsupported checksum type" (rustfs/rustfs#4811): the multipart put path
|
||||
// took the checksum branch and called `ChecksumNone.hasher()`.
|
||||
if matches!(self, ChecksumMode::ChecksumNone) {
|
||||
return false;
|
||||
}
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
s.len() == 1
|
||||
// A checksum is only "set" when a concrete algorithm (one with a real
|
||||
// hasher) is selected; `ChecksumNone` and the bare `ChecksumFullObject`
|
||||
// flag are not. Treating `ChecksumNone` as set made ILM transitions of
|
||||
// >128 MiB objects fail with "unsupported checksum type"
|
||||
// (rustfs/rustfs#4811): the multipart put path took the checksum branch
|
||||
// and called `ChecksumNone.hasher()`.
|
||||
self.algorithm().is_some()
|
||||
}
|
||||
|
||||
pub fn set_default(&mut self, t: ChecksumMode) {
|
||||
@@ -222,29 +145,10 @@ impl ChecksumMode {
|
||||
}
|
||||
|
||||
pub fn to_string(&self) -> String {
|
||||
//match c & checksumMask {
|
||||
match self {
|
||||
ChecksumMode::ChecksumCRC32 => {
|
||||
return "CRC32".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
return "CRC32C".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA1 => {
|
||||
return "SHA1".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA256 => {
|
||||
return "SHA256".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumNone => {
|
||||
return "".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC64NVME => {
|
||||
return "CRC64NVME".to_string();
|
||||
}
|
||||
_ => {
|
||||
return "<invalid>".to_string();
|
||||
}
|
||||
match self.algorithm() {
|
||||
Some(algorithm) => algorithm.s3_algorithm_name().to_string(),
|
||||
None if matches!(self, ChecksumMode::ChecksumNone) => "".to_string(),
|
||||
None => "<invalid>".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -344,6 +248,109 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delegated_dispatch_preserves_wire_behaviour() {
|
||||
// Behaviour lock for the backlog#1844 unification: every output that
|
||||
// reaches the wire (header names, algorithm names, digest lengths,
|
||||
// checksum-type capabilities) is pinned to the exact values the
|
||||
// pre-delegation per-algorithm matches produced. If delegation to
|
||||
// rustfs_checksums::ChecksumAlgorithm ever drifts, this fails loudly.
|
||||
struct Expected {
|
||||
mode: ChecksumMode,
|
||||
key: &'static str,
|
||||
name: &'static str,
|
||||
raw_len: usize,
|
||||
composite: bool,
|
||||
merge_crc: bool,
|
||||
full_object: bool,
|
||||
}
|
||||
let table = [
|
||||
Expected {
|
||||
mode: ChecksumMode::ChecksumCRC32,
|
||||
key: "x-amz-checksum-crc32",
|
||||
name: "CRC32",
|
||||
raw_len: 4,
|
||||
composite: true,
|
||||
merge_crc: true,
|
||||
full_object: false,
|
||||
},
|
||||
Expected {
|
||||
mode: ChecksumMode::ChecksumCRC32C,
|
||||
key: "x-amz-checksum-crc32c",
|
||||
name: "CRC32C",
|
||||
raw_len: 4,
|
||||
composite: true,
|
||||
merge_crc: true,
|
||||
full_object: false,
|
||||
},
|
||||
Expected {
|
||||
mode: ChecksumMode::ChecksumSHA1,
|
||||
key: "x-amz-checksum-sha1",
|
||||
name: "SHA1",
|
||||
raw_len: 20,
|
||||
composite: true,
|
||||
merge_crc: false,
|
||||
full_object: false,
|
||||
},
|
||||
Expected {
|
||||
mode: ChecksumMode::ChecksumSHA256,
|
||||
key: "x-amz-checksum-sha256",
|
||||
name: "SHA256",
|
||||
raw_len: 32,
|
||||
composite: true,
|
||||
merge_crc: false,
|
||||
full_object: false,
|
||||
},
|
||||
Expected {
|
||||
mode: ChecksumMode::ChecksumCRC64NVME,
|
||||
key: "x-amz-checksum-crc64nvme",
|
||||
name: "CRC64NVME",
|
||||
raw_len: 8,
|
||||
composite: false,
|
||||
merge_crc: true,
|
||||
full_object: true,
|
||||
},
|
||||
Expected {
|
||||
mode: ChecksumMode::ChecksumNone,
|
||||
key: "",
|
||||
name: "",
|
||||
raw_len: 0,
|
||||
composite: false,
|
||||
merge_crc: false,
|
||||
full_object: false,
|
||||
},
|
||||
Expected {
|
||||
mode: ChecksumMode::ChecksumFullObject,
|
||||
key: "",
|
||||
name: "<invalid>",
|
||||
raw_len: 0,
|
||||
composite: false,
|
||||
merge_crc: false,
|
||||
full_object: false,
|
||||
},
|
||||
];
|
||||
|
||||
for e in table {
|
||||
assert_eq!(e.mode.key(), e.key, "{:?} key()", e.mode);
|
||||
assert_eq!(e.mode.key_capitalized(), e.key, "{:?} key_capitalized()", e.mode);
|
||||
assert_eq!(e.mode.to_string(), e.name, "{:?} to_string()", e.mode);
|
||||
assert_eq!(e.mode.raw_byte_len(), e.raw_len, "{:?} raw_byte_len()", e.mode);
|
||||
assert_eq!(e.mode.can_composite(), e.composite, "{:?} can_composite()", e.mode);
|
||||
assert_eq!(e.mode.can_merge_crc(), e.merge_crc, "{:?} can_merge_crc()", e.mode);
|
||||
assert_eq!(e.mode.full_object_requested(), e.full_object, "{:?} full_object_requested()", e.mode);
|
||||
|
||||
// The hasher, when present, must produce digests of the advertised
|
||||
// length under the advertised header name.
|
||||
if let Ok(mut hasher) = e.mode.hasher() {
|
||||
assert_eq!(e.mode.hasher().unwrap().header_name(), e.key, "{:?} hasher header", e.mode);
|
||||
hasher.update(b"wire behaviour probe");
|
||||
assert_eq!(hasher.finalize().len(), e.raw_len, "{:?} digest length", e.mode);
|
||||
} else {
|
||||
assert_eq!(e.raw_len, 0, "{:?} has no hasher but a nonzero raw_len", e.mode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_default_upgrades_none() {
|
||||
// With `is_set()` fixed, `set_default` must upgrade an unset mode to the
|
||||
|
||||
Reference in New Issue
Block a user