mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 08:36:54 +00:00
feat: improve legacy metadata and admin compatibility (#2202)
This commit is contained in:
weisd
+101
-11
@@ -12,13 +12,30 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use blake2::{Blake2b512, Digest as Blake2Digest};
|
||||
use highway::{HighwayHash, HighwayHasher, Key};
|
||||
use md5::{Digest, Md5};
|
||||
use md5::Md5;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Sha256;
|
||||
|
||||
/// The fixed key for HighwayHash256. DO NOT change for compatibility.
|
||||
const HIGHWAY_HASH256_KEY: [u64; 4] = [3, 4, 2, 1];
|
||||
/// Magic HH-256 key: HH-256 hash of first 100 decimals of π as utf-8 with zero key.
|
||||
const MAGIC_HIGHWAY_HASH256_KEY: [u8; 32] = [
|
||||
0x4b, 0xe7, 0x34, 0xfa, 0x8e, 0x23, 0x8a, 0xcd, 0x26, 0x3e, 0x83, 0xe6, 0xbb, 0x96, 0x85, 0x52, 0x04, 0x0f, 0x93, 0x5d, 0xa3,
|
||||
0x9f, 0x44, 0x14, 0x97, 0xe0, 0x9d, 0x13, 0x22, 0xde, 0x36, 0xa0,
|
||||
];
|
||||
|
||||
/// Legacy HH-256 key (main branch): fixed [3,4,2,1] as u64 LE.
|
||||
const LEGACY_HIGHWAY_HASH256_KEY: [u8; 32] = [
|
||||
3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
|
||||
];
|
||||
|
||||
fn highway_key_from_bytes(bytes: &[u8; 32]) -> [u64; 4] {
|
||||
let mut key = [0u64; 4];
|
||||
for (i, chunk) in bytes.chunks_exact(8).enumerate() {
|
||||
key[i] = u64::from_le_bytes(chunk.try_into().unwrap());
|
||||
}
|
||||
key
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)]
|
||||
/// Supported hash algorithms for bitrot protection.
|
||||
@@ -30,6 +47,8 @@ pub enum HashAlgorithm {
|
||||
// HighwayHash256S represents the Streaming HighwayHash-256 hash function
|
||||
#[default]
|
||||
HighwayHash256S,
|
||||
/// Legacy HighwayHash256S (main branch) with fixed key [3,4,2,1]
|
||||
HighwayHash256SLegacy,
|
||||
// BLAKE2b512 represents the BLAKE2b-512 hash function
|
||||
BLAKE2b512,
|
||||
/// MD5 (128-bit)
|
||||
@@ -43,7 +62,8 @@ enum HashEncoded {
|
||||
Sha256([u8; 32]),
|
||||
HighwayHash256([u8; 32]),
|
||||
HighwayHash256S([u8; 32]),
|
||||
Blake2b512(blake3::Hash),
|
||||
HighwayHash256SLegacy([u8; 32]),
|
||||
Blake2b512([u8; 64]),
|
||||
None,
|
||||
}
|
||||
|
||||
@@ -55,7 +75,8 @@ impl AsRef<[u8]> for HashEncoded {
|
||||
HashEncoded::Sha256(hash) => hash.as_ref(),
|
||||
HashEncoded::HighwayHash256(hash) => hash.as_ref(),
|
||||
HashEncoded::HighwayHash256S(hash) => hash.as_ref(),
|
||||
HashEncoded::Blake2b512(hash) => hash.as_bytes(),
|
||||
HashEncoded::HighwayHash256SLegacy(hash) => hash.as_ref(),
|
||||
HashEncoded::Blake2b512(hash) => hash.as_ref(),
|
||||
HashEncoded::None => &[],
|
||||
}
|
||||
}
|
||||
@@ -83,17 +104,30 @@ impl HashAlgorithm {
|
||||
match self {
|
||||
HashAlgorithm::Md5 => HashEncoded::Md5(Md5::digest(data).into()),
|
||||
HashAlgorithm::HighwayHash256 => {
|
||||
let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY));
|
||||
let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
|
||||
let mut hasher = HighwayHasher::new(key);
|
||||
hasher.append(data);
|
||||
HashEncoded::HighwayHash256(u8x32_from_u64x4(hasher.finalize256()))
|
||||
}
|
||||
HashAlgorithm::SHA256 => HashEncoded::Sha256(Sha256::digest(data).into()),
|
||||
HashAlgorithm::HighwayHash256S => {
|
||||
let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY));
|
||||
let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
|
||||
let mut hasher = HighwayHasher::new(key);
|
||||
hasher.append(data);
|
||||
HashEncoded::HighwayHash256S(u8x32_from_u64x4(hasher.finalize256()))
|
||||
}
|
||||
HashAlgorithm::BLAKE2b512 => HashEncoded::Blake2b512(blake3::hash(data)),
|
||||
HashAlgorithm::HighwayHash256SLegacy => {
|
||||
let key = Key(highway_key_from_bytes(&LEGACY_HIGHWAY_HASH256_KEY));
|
||||
let mut hasher = HighwayHasher::new(key);
|
||||
hasher.append(data);
|
||||
HashEncoded::HighwayHash256SLegacy(u8x32_from_u64x4(hasher.finalize256()))
|
||||
}
|
||||
HashAlgorithm::BLAKE2b512 => {
|
||||
let hash = Blake2b512::digest(data);
|
||||
let mut out = [0u8; 64];
|
||||
out.copy_from_slice(hash.as_ref());
|
||||
HashEncoded::Blake2b512(out)
|
||||
}
|
||||
HashAlgorithm::None => HashEncoded::None,
|
||||
}
|
||||
}
|
||||
@@ -108,7 +142,8 @@ impl HashAlgorithm {
|
||||
HashAlgorithm::SHA256 => 32,
|
||||
HashAlgorithm::HighwayHash256 => 32,
|
||||
HashAlgorithm::HighwayHash256S => 32,
|
||||
HashAlgorithm::BLAKE2b512 => 32, // blake3 outputs 32 bytes by default
|
||||
HashAlgorithm::HighwayHash256SLegacy => 32,
|
||||
HashAlgorithm::BLAKE2b512 => 64,
|
||||
HashAlgorithm::Md5 => 16,
|
||||
HashAlgorithm::None => 0,
|
||||
}
|
||||
@@ -167,7 +202,7 @@ mod tests {
|
||||
assert_eq!(HashAlgorithm::HighwayHash256.size(), 32);
|
||||
assert_eq!(HashAlgorithm::HighwayHash256S.size(), 32);
|
||||
assert_eq!(HashAlgorithm::SHA256.size(), 32);
|
||||
assert_eq!(HashAlgorithm::BLAKE2b512.size(), 32);
|
||||
assert_eq!(HashAlgorithm::BLAKE2b512.size(), 64);
|
||||
assert_eq!(HashAlgorithm::None.size(), 0);
|
||||
}
|
||||
|
||||
@@ -220,13 +255,68 @@ mod tests {
|
||||
let data = b"test data";
|
||||
let hash = HashAlgorithm::BLAKE2b512.hash_encode(data);
|
||||
let hash = hash.as_ref();
|
||||
assert_eq!(hash.len(), 32); // blake3 outputs 32 bytes by default
|
||||
assert_eq!(hash.len(), 64);
|
||||
// BLAKE2b512 should be deterministic
|
||||
let hash2 = HashAlgorithm::BLAKE2b512.hash_encode(data);
|
||||
let hash2 = hash2.as_ref();
|
||||
assert_eq!(hash, hash2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bitrot_selftest() {
|
||||
let checksums: [(HashAlgorithm, &str); 5] = [
|
||||
(HashAlgorithm::SHA256, "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004"),
|
||||
(
|
||||
HashAlgorithm::BLAKE2b512,
|
||||
"e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669",
|
||||
),
|
||||
(
|
||||
HashAlgorithm::HighwayHash256,
|
||||
"39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313",
|
||||
),
|
||||
(
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
"39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313",
|
||||
),
|
||||
(
|
||||
HashAlgorithm::HighwayHash256SLegacy,
|
||||
"a5592a831588836b0f61bff43da4bd957c376d9b6412a9ecbbd144a3ecf34649",
|
||||
),
|
||||
];
|
||||
for (algo, expected_hex) in checksums {
|
||||
let block_size = match algo {
|
||||
HashAlgorithm::SHA256 => 64,
|
||||
HashAlgorithm::BLAKE2b512 => 128,
|
||||
HashAlgorithm::HighwayHash256 | HashAlgorithm::HighwayHash256S | HashAlgorithm::HighwayHash256SLegacy => 32,
|
||||
_ => continue,
|
||||
};
|
||||
let mut msg = Vec::new();
|
||||
let mut sum = Vec::new();
|
||||
for _ in 0..block_size {
|
||||
sum = algo.hash_encode(&msg).as_ref().to_vec();
|
||||
msg.extend_from_slice(&sum);
|
||||
}
|
||||
let got = hex_simd::encode_to_string(&sum, hex_simd::AsciiCase::Lower);
|
||||
assert_eq!(got, expected_hex, "{:?} selftest mismatch: got {} want {}", algo, got, expected_hex);
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates 7557 bytes
|
||||
/// Pattern: (i*7+13)%256 for each byte.
|
||||
fn generate_compat_test_data(size: usize) -> Vec<u8> {
|
||||
(0..size).map(|i| ((i * 7 + 13) % 256) as u8).collect()
|
||||
}
|
||||
|
||||
/// Run: cargo test -p rustfs-utils test_highwayhash_compat
|
||||
#[test]
|
||||
fn test_highwayhash_compat() {
|
||||
let data = generate_compat_test_data(7557);
|
||||
let hash = HashAlgorithm::HighwayHash256S.hash_encode(&data);
|
||||
let got = hex_simd::encode_to_string(hash.as_ref(), hex_simd::AsciiCase::Lower);
|
||||
let expected = "06543bf1c637e67386922a43b71cca08e5faa0f9131105a2bf96dec880529551";
|
||||
assert_eq!(got, expected, "HighwayHash256S must match: got {} want {}", got, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_different_data_different_hashes() {
|
||||
let data1 = b"test data 1";
|
||||
|
||||
Reference in New Issue
Block a user