From 52703e0da6e631ccf0994f06479989336c4910be Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 27 Aug 2026 10:08:09 +0800 Subject: [PATCH] docs(rio): record the checksum hasher verdict and pin shared vectors (#6706) --- crates/checksums/src/lib.rs | 12 ++++++---- crates/rio/src/checksum.rs | 46 +++++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/crates/checksums/src/lib.rs b/crates/checksums/src/lib.rs index e3f397384..479ebeed0 100644 --- a/crates/checksums/src/lib.rs +++ b/crates/checksums/src/lib.rs @@ -49,10 +49,14 @@ pub const MD5_NAME: &str = "md5"; /// (crates/s3-client/src/checksum.rs) delegates all per-algorithm dispatch /// here through its `algorithm()` bridge. The on-disk xl.meta bitset remains /// deliberately separate in `rustfs_rio::ChecksumType` -/// (crates/rio/src/checksum.rs, varint bits are append-only). When adding an -/// algorithm: add the variant here (the exhaustive matches force every -/// metadata decision), bridge it in the client, and allocate an xl.meta bit -/// in rio (or record why not). +/// (crates/rio/src/checksum.rs, varint bits are append-only), and rio also +/// keeps its own hot-path hasher shells — equivalence with this crate's +/// hashers is enforced by both test suites pinning the same official +/// known-answer vectors (backlog#1844 PR3 verdict, recorded on +/// `rustfs_rio::ChecksumType`). When adding an algorithm: add the variant +/// here (the exhaustive matches force every metadata decision), bridge it in +/// the client, and allocate an xl.meta bit + hasher + shared vector in rio +/// (or record why not). #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[non_exhaustive] pub enum ChecksumAlgorithm { diff --git a/crates/rio/src/checksum.rs b/crates/rio/src/checksum.rs index 1f211476d..6c177b2c4 100644 --- a/crates/rio/src/checksum.rs +++ b/crates/rio/src/checksum.rs @@ -31,14 +31,28 @@ pub const RUSTFS_MULTIPART_CHECKSUM_TYPE: &str = "x-rustfs-multipart-checksum-ty /// Checksum type enumeration with flags /// -/// One of three deliberately separate checksum registries (backlog#1833): -/// this bitset owns the **on-disk xl.meta encoding** — the raw `u32` is +/// This bitset owns the **on-disk xl.meta encoding** — the raw `u32` is /// varint-serialized into xl.meta (see `append_to`), so bits are append-only -/// and must never be renumbered. `rustfs_checksums::ChecksumAlgorithm` -/// (crates/checksums/src/lib.rs) owns the streaming-hash algorithm registry, -/// and the MinIO-port client keeps its own `ChecksumMode` -/// (crates/ecstore/src/client/checksum.rs). When adding an algorithm, extend -/// all three (or record why not) — they do not derive from each other. +/// and must never be renumbered. Canonical per-algorithm metadata (wire +/// names, headers, digest lengths, checksum-type capabilities) lives in +/// `rustfs_checksums::ChecksumAlgorithm` (crates/checksums/src/lib.rs), which +/// the MinIO-port client's `ChecksumMode` +/// (crates/s3-client/src/checksum.rs) consumes through its `algorithm()` +/// bridge (backlog#1844). +/// +/// The hasher implementations below stay rio-native rather than delegating +/// to rustfs-checksums (backlog#1844 PR3 verdict): `ChecksumHasher` needs a +/// non-consuming `finalize` plus `reset` on the server hot path, while the +/// checksums `Checksum` trait finalizes by consuming the box; MD5 is a base +/// type here (x-amz-checksum-md5, bit 14) but deliberately not a +/// `ChecksumAlgorithm` variant; and both crates already drive the same +/// backends (crc_fast, sha1/sha2, xxhash_rust, md5). Equivalence is enforced +/// instead by pinning both sides to the same official known-answer vectors — +/// see `hashers_match_rustfs_checksums_known_answer_vectors` below and the +/// digest tests in crates/checksums. When adding an algorithm: extend +/// `ChecksumAlgorithm` (exhaustive matches force the metadata), bridge it in +/// the client, allocate a bit + hasher here, and pin the shared vector in +/// both test suites (or record why a surface is skipped). #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct ChecksumType(pub u32); @@ -1646,6 +1660,24 @@ mod tests { } } + // Drift lock for the backlog#1844 PR3 verdict: rio keeps its own hasher + // shells instead of delegating to rustfs-checksums, so both sides pin the + // SAME input and official digests (crates/checksums/src/lib.rs pins these + // values for "test data" in its own tests). If either implementation ever + // changes backend, seed, or byte order, one of the two suites fails loudly. + #[test] + fn hashers_match_rustfs_checksums_known_answer_vectors() { + for (t, want_hex) in [ + (ChecksumType::CRC32, "d308aeb2"), + (ChecksumType::CRC32C, "3379b4ca"), + (ChecksumType::CRC64_NVME, "aecaf3af9c98a855"), + (ChecksumType::SHA1, "f48dd853820860816c75d54d0f584dc863327a7c"), + (ChecksumType::SHA256, "916f0027a575074ce72a331777c3478d6513f786a591bd892da1a577bf2335f9"), + ] { + assert_eq!(raw_hex(t, b"test data"), want_hex, "{t:?} digest drifted from the shared vector"); + } + } + // S11: from_string_with_obj_type dropped to_uppercase() (a per-request heap alloc) // for eq_ignore_ascii_case. Lock that this did not change any behaviour. #[test]