diff --git a/Cargo.lock b/Cargo.lock index 6797dc035..1877c1c77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,7 +91,6 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", - "serde", "version_check", "zerocopy", ] @@ -9627,7 +9626,6 @@ dependencies = [ name = "rustfs-ecstore" version = "1.0.0-rc.4" dependencies = [ - "ahash", "arc-swap", "async-channel", "async-recursion", @@ -9773,7 +9771,6 @@ dependencies = [ name = "rustfs-filemeta" version = "1.0.0-rc.4" dependencies = [ - "ahash", "arc-swap", "byteorder", "bytes", diff --git a/Cargo.toml b/Cargo.toml index aa9a4bce0..35fb5e512 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -368,9 +368,6 @@ hotpath = { version = "0.24.0", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } -# High-performance hashing -ahash = { version = "0.8", default-features = false, features = ["std", "runtime-rng", "serde"] } - [workspace.metadata.cargo-shear] ignored = ["hotpath", "rustfs"] diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 2af43aa47..10100a435 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -218,9 +218,6 @@ faster-hex = { workspace = true } ratelimit = { workspace = true } aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] } -# High-performance hashing -ahash = { workspace = true, features = ["serde"] } - # Observability and Metrics metrics = { workspace = true } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 2da838626..d0e5cec70 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -69,7 +69,6 @@ use crate::storage_api_contracts::{ range::HTTPRangeSpec, }; use crate::store::ECStore; -use ahash::AHashMap; use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; use http::HeaderMap; use rand::RngExt as _; @@ -2871,7 +2870,7 @@ fn spawn_transition_transaction_recovery_once(api: Arc) { struct StaleMultipartUploadCandidate { path: String, initiated: OffsetDateTime, - metadata: Option>, + metadata: Option>, } fn parse_stale_uploads_duration(env_key: &str, default: StdDuration) -> StdDuration { @@ -2916,9 +2915,9 @@ async fn stale_upload_current_size(set: &Arc, metadata: &HashMap( +async fn stale_upload_current_size_with_opts( set: &Arc, - metadata: &HashMap, + metadata: &HashMap, upload_dir: &str, no_lock: bool, ) -> Option { @@ -2951,9 +2950,9 @@ async fn stale_upload_current_size_with_opts( ) } -async fn stale_upload_lifecycle_due( +async fn stale_upload_lifecycle_due( set: &Arc, - metadata: &HashMap, + metadata: &HashMap, initiated: OffsetDateTime, upload_dir: &str, no_lock: bool, @@ -2979,7 +2978,7 @@ async fn stale_upload_lifecycle_due( .unwrap_or_default(), is_latest: true, delete_marker: false, - user_defined: metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + user_defined: metadata.clone(), ..Default::default() }; @@ -11617,7 +11616,7 @@ mod tests { // Persist the durable backend identity on the transitioned version so the // recovered free version carries it (matching a registered mock tier); // free-version remote cleanup fails closed without it. - let mut transitioned_metadata = AHashMap::new(); + let mut transitioned_metadata = HashMap::new(); if let Some(identity) = backend_identity { rustfs_utils::http::metadata_compat::insert_str( &mut transitioned_metadata, @@ -12354,7 +12353,7 @@ mod tests { StaleMultipartUploadCandidate { path: "sha/upload".to_string(), initiated: OffsetDateTime::UNIX_EPOCH, - metadata: Some(AHashMap::from([("k".to_string(), "v".to_string())])), + metadata: Some(HashMap::from([("k".to_string(), "v".to_string())])), }, ); diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 03c96f65e..f25b42f82 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -5815,7 +5815,7 @@ fn decommission_remote_tiered_opts( versioned: version_id.is_some(), version_id, mod_time: version.mod_time, - user_defined: version.metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + user_defined: version.metadata.clone(), src_pool_idx, data_movement: true, incl_free_versions: version.tier_free_version(), @@ -11894,7 +11894,7 @@ mod tests { let mod_time = OffsetDateTime::now_utc(); let version = rustfs_filemeta::FileInfo { mod_time: Some(mod_time), - metadata: AHashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]), + metadata: HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]), ..Default::default() }; diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index 715f057cc..2a552c186 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -1877,7 +1877,7 @@ mod tests { mod_time: Some(OffsetDateTime::UNIX_EPOCH), data_dir: Some(Uuid::from_u128(100)), checksum: Some(Bytes::from_static(b"object-checksum")), - metadata: AHashMap::from([ + metadata: HashMap::from([ ("etag".to_string(), "etag-value".to_string()), ("x-amz-meta-key".to_string(), metadata_value.to_string()), ]), @@ -1887,7 +1887,7 @@ mod tests { size: 128, actual_size: 128, mod_time: Some(OffsetDateTime::UNIX_EPOCH), - checksums: Some(AHashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])), + checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])), ..Default::default() }], ..Default::default() diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index 6638aeb01..769ade684 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -64,11 +64,7 @@ pub(crate) const ENCRYPTED_FRAME_LAYOUT_FIXED8K_SUFFIX: &str = "encrypted-frame- pub(crate) const ENV_RUSTFS_ENCRYPTED_RANGE_SEEK: &str = "RUSTFS_ENCRYPTED_RANGE_SEEK"; pub(crate) const DEFAULT_RUSTFS_ENCRYPTED_RANGE_SEEK: bool = true; -pub(crate) fn has_encrypted_part_layout_marker( - metadata: &HashMap, - suffix: &str, - expected: &str, -) -> bool { +pub(crate) fn has_encrypted_part_layout_marker(metadata: &HashMap, suffix: &str, expected: &str) -> bool { let mut value = None; for (key, candidate) in metadata { if !rustfs_utils::http::has_internal_suffix(key, suffix) { diff --git a/crates/ecstore/src/object_api/object_api_utils.rs b/crates/ecstore/src/object_api/object_api_utils.rs index 32dc3384a..b4bd3e0b5 100644 --- a/crates/ecstore/src/object_api/object_api_utils.rs +++ b/crates/ecstore/src/object_api/object_api_utils.rs @@ -168,7 +168,7 @@ pub fn to_s3s_etag(etag: &str) -> ETag { ETag::Strong(etag.to_string()) } -pub fn get_raw_etag(metadata: &HashMap) -> String { +pub fn get_raw_etag(metadata: &HashMap) -> String { metadata .get("etag") .cloned() diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 9a63114dd..280dd10cc 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -1008,7 +1008,7 @@ impl ObjectInfo { successor_mod_time: fi.successor_mod_time, etag, inlined, - user_defined: Arc::new(metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect()), + user_defined: Arc::new(metadata), transitioned_object, transition_version_state: fi.transition_version_state, checksum: fi.checksum.clone(), @@ -1316,7 +1316,7 @@ impl ObjectInfo { if part > 0 && let Some(checksums) = self.parts.iter().find(|p| p.number == part).and_then(|p| p.checksums.clone()) { - return Ok((checksums.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), true)); + return Ok((checksums, true)); } if let Some(data) = &self.checksum { @@ -1812,7 +1812,7 @@ mod tests { storageclass::GLACIER, ] { let fi = FileInfo { - metadata: AHashMap::from([(AMZ_STORAGE_CLASS.to_string(), legacy_label.to_string())]), + metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), legacy_label.to_string())]), ..Default::default() }; diff --git a/crates/ecstore/src/services/rebalance/migration.rs b/crates/ecstore/src/services/rebalance/migration.rs index ee4d635f4..cf2dabd8f 100644 --- a/crates/ecstore/src/services/rebalance/migration.rs +++ b/crates/ecstore/src/services/rebalance/migration.rs @@ -57,7 +57,7 @@ fn rebalance_remote_tiered_opts( versioned: version_id.is_some(), version_id, mod_time: version.mod_time, - user_defined: version.metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + user_defined: version.metadata.clone(), src_pool_idx, data_movement: true, include_part_checksums: true, diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index 340e23145..bac56bd0b 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -1454,9 +1454,7 @@ fn tier_backend_identity(config: &TierConfig) -> io::Result { encode_tier_backend_identity(tier_type, endpoint, bucket, prefix, region, routing_account) } -pub(crate) fn tier_destination_id_from_metadata( - metadata: &HashMap, -) -> io::Result> { +pub(crate) fn tier_destination_id_from_metadata(metadata: &HashMap) -> io::Result> { let Some(encoded) = rustfs_utils::http::metadata_compat::get_consistent_str( metadata, rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID, diff --git a/crates/ecstore/src/set_disk/metadata.rs b/crates/ecstore/src/set_disk/metadata.rs index dff3cb1f8..8337b8559 100644 --- a/crates/ecstore/src/set_disk/metadata.rs +++ b/crates/ecstore/src/set_disk/metadata.rs @@ -609,7 +609,7 @@ impl SetDisks { || Self::starts_with_ignore_ascii_case(suffix, http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX) } - fn update_hash_quorum_metadata_map(hasher: &mut Sha256, entries: &HashMap) { + fn update_hash_quorum_metadata_map(hasher: &mut Sha256, entries: &HashMap) { let mut entries = entries .iter() .filter(|(name, _)| !Self::is_replication_quorum_metadata_key(name)) @@ -635,10 +635,7 @@ impl SetDisks { /// so the dual internal prefixes carrying the same mapping share one /// identity, while a genuine disagreement between disks still changes the /// hash and surfaces as a quorum difference. - fn update_hash_target_delete_marker_versions( - hasher: &mut Sha256, - metadata: &HashMap, - ) { + fn update_hash_target_delete_marker_versions(hasher: &mut Sha256, metadata: &HashMap) { let (versions, corrupt) = http::target_delete_marker_versions(metadata); hasher.update([u8::from(corrupt)]); let mut versions = versions.iter().collect::>(); diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 83b86b98b..83398c02e 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -190,9 +190,7 @@ use tracing::error; use tracing::{Instrument, debug, info, warn}; use uuid::Uuid; -pub(super) fn restore_operation_id_from_metadata( - metadata: &HashMap, -) -> Result> { +pub(super) fn restore_operation_id_from_metadata(metadata: &HashMap) -> Result> { let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_RESTORE_OPERATION_ID) else { if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_RESTORE_OPERATION_ID) { return Err(Error::other("invalid restore operation id metadata".to_string())); @@ -206,19 +204,14 @@ pub(super) fn restore_operation_id_from_metadata( Ok(Some(id)) } -pub(super) fn require_restore_operation_id( - metadata: &HashMap, - expected: Uuid, -) -> Result<()> { +pub(super) fn require_restore_operation_id(metadata: &HashMap, expected: Uuid) -> Result<()> { match restore_operation_id_from_metadata(metadata)? { Some(actual) if actual == expected => Ok(()), _ => Err(Error::other("restore operation id changed before copy-back".to_string())), } } -pub(super) fn restore_commit_operation_id_from_metadata( - metadata: &HashMap, -) -> Result> { +pub(super) fn restore_commit_operation_id_from_metadata(metadata: &HashMap) -> Result> { if !metadata.contains_key(X_AMZ_RESTORE.as_str()) { return Ok(None); } @@ -473,13 +466,13 @@ fn release_materialized_read_lock(bucket: &str, object: &str, read_lock_guard: O drop(read_lock_guard); } -pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap) { +pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap) { metadata.remove(RUSTFS_MULTIPART_BUCKET_KEY); metadata.remove(RUSTFS_MULTIPART_OBJECT_KEY); rustfs_utils::http::metadata_compat::remove_str(metadata, SUFFIX_BUCKET_INCARNATION_ID); } -fn should_persist_encryption_original_size(metadata: &HashMap) -> bool { +fn should_persist_encryption_original_size(metadata: &HashMap) -> bool { metadata.keys().any(|key| is_object_encryption_marker(key)) } @@ -8802,7 +8795,7 @@ mod tests { #[test] fn test_list_object_etags() { // Test extracting etags from file info metadata - let mut metadata = AHashMap::new(); + let mut metadata = HashMap::new(); metadata.insert("etag".to_string(), "test-etag".to_string()); let file_info = FileInfo { diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 41eef9135..52e905877 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -449,7 +449,7 @@ fn fence_commit_on_lock_loss(guard: Option<&ObjectLockDiagGuard>, mode: &'static Ok(()) } -fn multipart_bucket_incarnation_id(metadata: &HashMap) -> Result> { +fn multipart_bucket_incarnation_id(metadata: &HashMap) -> Result> { let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) else { if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) { return Err(Error::other("invalid multipart bucket incarnation metadata")); @@ -463,15 +463,12 @@ fn multipart_bucket_incarnation_id(metadata: &HashMap Ok(Some(incarnation)) } -fn multipart_bucket_incarnation_matches( - metadata: &HashMap, - expected: Uuid, -) -> bool { +fn multipart_bucket_incarnation_matches(metadata: &HashMap, expected: Uuid) -> bool { matches!(multipart_bucket_incarnation_id(metadata), Ok(Some(actual)) if actual == expected) } -fn validate_multipart_bucket_incarnation( - metadata: &HashMap, +fn validate_multipart_bucket_incarnation( + metadata: &HashMap, bucket: &str, object: &str, upload_id: &str, @@ -1524,7 +1521,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { mod_time: Some(OffsetDateTime::now_utc()), actual_size, index: index_op, - checksums: if checksums.is_empty() { None } else { Some(checksums.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) }, + checksums: if checksums.is_empty() { None } else { Some(checksums) }, ..Default::default() }; @@ -1719,7 +1716,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { max_parts, part_number_marker, user_defined: { - let mut metadata: HashMap = fi.metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let mut metadata = fi.metadata.clone(); strip_internal_multipart_metadata(&mut metadata); metadata }, @@ -1978,7 +1975,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let mod_time = opts.mod_time.unwrap_or_else(OffsetDateTime::now_utc); for f in parts_metadatas.iter_mut() { - f.metadata = user_defined.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + f.metadata = user_defined.clone(); f.mod_time = Some(mod_time); f.fresh = true; } @@ -2067,7 +2064,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { upload_id: upload_id.to_owned(), user_defined: { strip_internal_multipart_metadata(&mut fi.metadata); - fi.metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + fi.metadata.clone() }, ..Default::default() }) diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index f01f140b3..83604a719 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -72,11 +72,11 @@ use crate::set_disk::runtime_sources; use crate::storage_api_contracts::multipart::MultipartOperations; use crate::storage_api_contracts::object::ObjectIO; use crate::storage_api_contracts::object::ObjectOperations; -use ahash::AHashMap; use rustfs_lock::LockManager; use rustfs_rio::EtagResolvable; use rustfs_rio::HashReaderMut; use rustfs_rio::TryGetIndex; +use rustfs_utils::http::HeaderExt; use tokio::io::AsyncWriteExt; #[cfg(all(test, feature = "test-util"))] @@ -1107,13 +1107,13 @@ fn is_restore_control_metadata(key: &str) -> bool { .is_some_and(|remainder| remainder.is_empty()) } -fn restore_metadata_update_preserves_protected_metadata( - existing: &HashMap, - replacement: &HashMap, +fn restore_metadata_update_preserves_protected_metadata( + existing: &HashMap, + replacement: &HashMap, ) -> bool { - let mut existing: HashMap = existing.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let mut existing = existing.clone(); clean_metadata(&mut existing); - let mut replacement: HashMap = replacement.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let mut replacement = replacement.clone(); clean_metadata(&mut replacement); let existing_count = existing.keys().filter(|key| !is_restore_control_metadata(key)).count(); let replacement_count = replacement.keys().filter(|key| !is_restore_control_metadata(key)).count(); @@ -2211,9 +2211,9 @@ pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &Objec /// /// Returns whether `inbound` was modified. Callers must hold the object write /// lock so the stored values compared here are the ones being replaced. -pub(in crate::set_disk) fn merge_replication_metadata_lww( - inbound: &mut HashMap, - existing: &HashMap, +pub(in crate::set_disk) fn merge_replication_metadata_lww( + inbound: &mut HashMap, + existing: &HashMap, opts: &ObjectOptions, ) -> bool { use rustfs_utils::http::headers::{ @@ -2844,7 +2844,7 @@ impl SetDisks { ))); } - fi.metadata = user_defined.iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + fi.metadata = user_defined; fi.mod_time = mod_time; fi.size = w_size as i64; fi.versioned = opts.versioned || opts.version_suspended; @@ -6051,8 +6051,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { None }; - let mut replacement_metadata: AHashMap = - (*src_info.user_defined).iter().map(|(k, v)| (k.clone(), v.clone())).collect(); + let mut replacement_metadata = (*src_info.user_defined).clone(); if let Some(part_checksums) = preserved_part_checksums { rustfs_utils::http::insert_str(&mut replacement_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums); } @@ -7494,7 +7493,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(), X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(), ] { - if let Some(value) = fi.metadata.get(header).filter(|value| !value.is_empty()) { + if let Some(value) = fi.metadata.lookup(header).filter(|value| !value.is_empty()) { transition_meta.insert(header.to_ascii_lowercase(), value.to_string()); } } diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index e5a99676e..6e3ce4214 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -4780,7 +4780,7 @@ mod tests { purge_targets: version_purge_statuses_map("arn:minio:replication:target=PENDING;"), ..Default::default() })), - metadata: AHashMap::from([ + metadata: HashMap::from([ ("etag".to_string(), "etag-value".to_string()), ("x-amz-meta-key".to_string(), "metadata-value".to_string()), (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), "tag=value".to_string()), @@ -4864,7 +4864,7 @@ mod tests { source.parts = vec![rustfs_filemeta::ObjectPartInfo { number: 1, mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND), - checksums: Some(AHashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])), + checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])), ..Default::default() }]; rustfs_utils::http::insert_str( diff --git a/crates/ecstore/src/store/utils.rs b/crates/ecstore/src/store/utils.rs index 28f3fe907..a39bca8f3 100644 --- a/crates/ecstore/src/store/utils.rs +++ b/crates/ecstore/src/store/utils.rs @@ -26,18 +26,18 @@ static STRICT_BUCKET_NAME_REGEX: LazyLock = static NON_STRICT_BUCKET_NAME_REGEX: LazyLock = LazyLock::new(|| Regex::new(r"^[A-Za-z0-9][A-Za-z0-9\.\-_:]{1,61}[A-Za-z0-9]$").expect("valid non-strict bucket name regex")); -pub fn clean_metadata(metadata: &mut HashMap) { +pub fn clean_metadata(metadata: &mut HashMap) { remove_standard_storage_class(metadata); clean_metadata_keys(metadata, &["md5Sum", "etag", "expires", AMZ_OBJECT_TAGGING, "last-modified"]); } -pub fn remove_standard_storage_class(metadata: &mut HashMap) { +pub fn remove_standard_storage_class(metadata: &mut HashMap) { if metadata.get(AMZ_STORAGE_CLASS) == Some(&STANDARD.to_string()) { metadata.remove(AMZ_STORAGE_CLASS); } } -pub fn clean_metadata_keys(metadata: &mut HashMap, key_names: &[&str]) { +pub fn clean_metadata_keys(metadata: &mut HashMap, key_names: &[&str]) { for key in key_names { metadata.remove(key.to_owned()); } diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index 101dc325e..ef6095d65 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -51,9 +51,6 @@ s3s = { workspace = true, features = ["minio"] } regex.workspace = true arc-swap.workspace = true -# High-performance hashing -ahash = { workspace = true, features = ["serde"] } - [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } tempfile = { workspace = true } diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index ba35e98bc..42613185e 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -13,7 +13,6 @@ // limitations under the License. use crate::{Error, ReplicationState, ReplicationStatusType, Result, TRANSITION_COMPLETE, VersionPurgeStatusType}; -use ahash::AHashMap; use bytes::Bytes; use rmp_serde::Serializer; use rustfs_utils::HashAlgorithm; @@ -68,7 +67,7 @@ pub struct ObjectPartInfo { // Index holds the index of the part in the erasure coding pub index: Option, // Checksums holds checksums of the part - pub checksums: Option>, + pub checksums: Option>, pub error: Option, } @@ -269,7 +268,7 @@ pub struct FileInfo { pub mode: Option, // WrittenByVersion is the unix time stamp of the version that created this version of the object pub written_by_version: Option, - pub metadata: AHashMap, + pub metadata: HashMap, pub parts: Vec, pub erasure: ErasureInfo, // MarkDeleted marks this version as deleted @@ -302,7 +301,7 @@ fn is_sensitive_metadata_key(key: &str) -> bool { .any(|prefix| starts_with_ignore_ascii_case(key, prefix)) } -struct RedactedMetadata<'a>(&'a AHashMap); +struct RedactedMetadata<'a>(&'a HashMap); impl std::fmt::Debug for RedactedMetadata<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -426,7 +425,7 @@ struct FileInfoMapDef { size: i64, mode: Option, written_by_version: Option, - metadata: AHashMap, + metadata: HashMap, parts: Vec, erasure: ErasureInfo, mark_deleted: bool, @@ -1080,7 +1079,7 @@ impl FileInfo { mod_time: Option, actual_size: i64, index: Option, - checksums: Option>, + checksums: Option>, ) { let part = ObjectPartInfo { etag, @@ -1458,7 +1457,7 @@ pub fn parse_restore_obj_status(restore_hdr: &str) -> Result { Err(Error::other(ERR_RESTORE_HDR_MALFORMED)) } -pub fn is_restored_object_on_disk(meta: &HashMap) -> bool { +pub fn is_restored_object_on_disk(meta: &HashMap) -> bool { if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str()) && let Ok(restore_status) = parse_restore_obj_status(restore_hdr) { @@ -2136,10 +2135,7 @@ mod tests { -1_000_000i64..=1_000_000i64, optional_timestamp_strategy(), proptest::option::of(bytes_strategy(16)), - proptest::option::of( - hash_map(small_string_strategy(), small_string_strategy(), 0..=3) - .prop_map(|m| m.into_iter().collect::>()), - ), + proptest::option::of(hash_map(small_string_strategy(), small_string_strategy(), 0..=3)), proptest::option::of(small_string_strategy()), ) .prop_map(|(etag, number, size, actual_size, mod_time, index, checksums, error)| ObjectPartInfo { @@ -2174,8 +2170,7 @@ mod tests { -1_000_000i64..=1_000_000i64, proptest::option::of(any::()), proptest::option::of(any::()), - hash_map(small_string_strategy(), small_string_strategy(), 0..=4) - .prop_map(|m| m.into_iter().collect::>()), + hash_map(small_string_strategy(), small_string_strategy(), 0..=4), vec(object_part_info_strategy(), 0..=3), erasure_info_strategy(), any::(), diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 0ef17adcb..5ce22b3c7 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -174,10 +174,10 @@ fn valid_target_delete_marker_version(arn: &str, version_id: &str) -> bool { /// included in the quorum hash, so such a divergence does surface — but as a /// quorum failure on an otherwise healthy object, which is not a state worth /// reaching. Merge the RPC metadata carrier instead, and only ever insert. -fn persist_target_delete_marker_versions( +fn persist_target_delete_marker_versions( meta_sys: &mut HashMap>, versions: &HashMap, - transport_metadata: &HashMap, + transport_metadata: &HashMap, ) { let mut bounded = BTreeMap::new(); // A corrupt carrier means the dual internal prefixes disagreed. Do not merge diff --git a/crates/filemeta/src/filemeta/inline_data.rs b/crates/filemeta/src/filemeta/inline_data.rs index f99936d4e..98d3bb28f 100644 --- a/crates/filemeta/src/filemeta/inline_data.rs +++ b/crates/filemeta/src/filemeta/inline_data.rs @@ -181,7 +181,7 @@ mod tests { data_dir: Some(data_dir), size: 64 * 1024, mod_time: Some(OffsetDateTime::now_utc()), - metadata: metadata.into_iter().collect(), + metadata, erasure: ErasureInfo { algorithm: ErasureAlgo::ReedSolomon.to_string(), data_blocks: 4, diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index 240b1b616..034e76017 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -26,7 +26,7 @@ use super::msgp_decode::{ PrependByteReader, prealloc_hint, read_exact_vec, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value, }; use super::*; -use crate::{AHashMap, ChecksumInfo, TransitionVersionState}; +use crate::{ChecksumInfo, TransitionVersionState}; use rustfs_utils::HashAlgorithm; use rustfs_utils::http::{ RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PART_CHECKSUMS, SUFFIX_PURGESTATUS, @@ -377,7 +377,7 @@ impl<'a> DerivedInternalMetadata<'a> { } } -struct UniquePartChecksums(AHashMap); +struct UniquePartChecksums(HashMap); impl<'de> serde::Deserialize<'de> for UniquePartChecksums { fn deserialize(deserializer: D) -> std::result::Result @@ -397,7 +397,7 @@ impl<'de> serde::Deserialize<'de> for UniquePartChecksums { where A: serde::de::SeqAccess<'de>, { - let mut checksums = AHashMap::with_capacity(seq.size_hint().unwrap_or_default()); + let mut checksums = HashMap::with_capacity(seq.size_hint().unwrap_or_default()); while let Some((key, value)) = seq.next_element::<(String, String)>()? { if checksums.insert(key, value).is_some() { return Err(serde::de::Error::custom("duplicate part checksum name")); @@ -1477,7 +1477,7 @@ pub struct MetaObjectV1 { #[serde(rename = "Erasure")] pub erasure: MetaObjectV1Erasure, #[serde(rename = "Meta")] - pub meta: AHashMap, + pub meta: HashMap, #[serde(rename = "Parts")] pub parts: Vec, #[serde(rename = "VersionID")] @@ -1543,7 +1543,7 @@ pub struct MetaObjectV1Part { #[serde(rename = "i")] pub index: Option, #[serde(rename = "crc")] - pub checksums: Option>, + pub checksums: Option>, #[serde(rename = "err")] pub error: Option, } @@ -1887,7 +1887,7 @@ impl MetaObjectV1Part { "i" => self.index = Some(Bytes::from(read_msgp_bin(rd)?)), "crc" => { let len = rmp::decode::read_map_len(rd)? as usize; - let mut checksums = AHashMap::with_capacity(prealloc_hint(len)); + let mut checksums = HashMap::with_capacity(prealloc_hint(len)); for _ in 0..len { checksums.insert(read_msgp_string(rd)?, read_msgp_string(rd)?); } @@ -2570,7 +2570,7 @@ impl MetaObject { Vec::new() }; - let mut metadata = AHashMap::with_capacity(self.meta_user.len() + self.meta_sys.len()); + let mut metadata = HashMap::with_capacity(self.meta_user.len() + self.meta_sys.len()); for (k, v) in &self.meta_user { if k == AMZ_META_UNENCRYPTED_CONTENT_LENGTH || k == AMZ_META_UNENCRYPTED_CONTENT_MD5 { continue; @@ -2861,7 +2861,7 @@ impl From for MetaObject { } } -fn get_internal_replication_state(metadata: &HashMap) -> Option { +fn get_internal_replication_state(metadata: &HashMap) -> Option { let mut rs = ReplicationState::default(); let mut has = false; @@ -2942,7 +2942,7 @@ impl MetaDeleteMarker { } pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> Result { - let metadata: AHashMap = self + let metadata = self .meta_sys .clone() .into_iter() @@ -5500,10 +5500,10 @@ mod tests { /// entirely, silently dropping the whole legacy body on re-marshal. #[test] fn legacy_version_body_round_trips_through_encode() { - let mut meta = AHashMap::new(); + let mut meta = HashMap::new(); meta.insert("content-type".to_string(), "application/octet-stream".to_string()); - let mut crc = AHashMap::new(); + let mut crc = HashMap::new(); crc.insert("crc32c".to_string(), "deadbeef".to_string()); let legacy = MetaObjectV1 { diff --git a/crates/filemeta/src/lib.rs b/crates/filemeta/src/lib.rs index 06c38d326..3fca95ea6 100644 --- a/crates/filemeta/src/lib.rs +++ b/crates/filemeta/src/lib.rs @@ -22,12 +22,6 @@ mod replication; pub mod test_data; -/// High-performance HashMap type alias using ahash instead of SipHash. -pub type AHashMap = ahash::AHashMap; - -/// High-performance HashSet type alias using ahash. -pub type AHashSet = ahash::AHashSet; - pub use error::*; pub use fileinfo::*; pub use filemeta::*; diff --git a/crates/filemeta/src/metacache.rs b/crates/filemeta/src/metacache.rs index 1d721c816..0571bc466 100644 --- a/crates/filemeta/src/metacache.rs +++ b/crates/filemeta/src/metacache.rs @@ -1676,7 +1676,7 @@ mod tests { } fn metacache_entry_with_mod_time(mod_time: OffsetDateTime, etag: &str) -> MetaCacheEntry { - let mut metadata = ahash::AHashMap::new(); + let mut metadata = HashMap::new(); metadata.insert("etag".to_string(), etag.to_string()); let mut meta = FileMeta::new(); @@ -1705,7 +1705,7 @@ mod tests { data_blocks: usize, parity_blocks: usize, ) -> MetaCacheEntry { - let mut metadata = ahash::AHashMap::new(); + let mut metadata = HashMap::new(); metadata.insert("etag".to_string(), etag.to_string()); let mut fi = FileInfo::new("object", data_blocks, parity_blocks); @@ -1730,7 +1730,7 @@ mod tests { fn metacache_entry_with_erasure_versions(versions: &[(OffsetDateTime, &str, usize, usize)]) -> MetaCacheEntry { let mut meta = FileMeta::new(); for (idx, (mod_time, etag, data_blocks, parity_blocks)) in versions.iter().enumerate() { - let mut metadata = ahash::AHashMap::new(); + let mut metadata = HashMap::new(); metadata.insert("etag".to_string(), (*etag).to_string()); let mut fi = FileInfo::new("object", *data_blocks, *parity_blocks); @@ -1776,7 +1776,7 @@ mod tests { /// Build an entry holding a single object version with an explicit version id /// and mod_time, so a set of these can model DISJOINT per-disk version sets. fn metacache_entry_single_version(version_u128: u128, mod_time: OffsetDateTime, etag: &str) -> MetaCacheEntry { - let mut metadata = ahash::AHashMap::new(); + let mut metadata = HashMap::new(); metadata.insert("etag".to_string(), etag.to_string()); let mut fi = FileInfo::new("object", 4, 2); diff --git a/crates/utils/src/http/header_compat.rs b/crates/utils/src/http/header_compat.rs index 7f2090082..c36632d8e 100644 --- a/crates/utils/src/http/header_compat.rs +++ b/crates/utils/src/http/header_compat.rs @@ -89,9 +89,7 @@ pub fn is_object_encryption_marker(key: &str) -> bool { } /// Reads the logical object size recorded by encryption metadata. -pub fn get_object_encryption_original_size( - metadata: &std::collections::HashMap, -) -> std::io::Result> { +pub fn get_object_encryption_original_size(metadata: &std::collections::HashMap) -> std::io::Result> { let actual_size = super::get_str(metadata, super::SUFFIX_ACTUAL_SIZE); let size = get_case_insensitive(metadata, RUSTFS_ENCRYPTION_ORIGINAL_SIZE) .or_else(|| get_case_insensitive(metadata, SSEC_ORIGINAL_SIZE)) @@ -105,10 +103,7 @@ pub fn get_object_encryption_original_size( .map_err(|error| std::io::Error::other(format!("Failed to parse encryption original size: {error}"))) } -fn get_case_insensitive<'a, S: std::hash::BuildHasher>( - metadata: &'a std::collections::HashMap, - key: &str, -) -> Option<&'a str> { +fn get_case_insensitive<'a>(metadata: &'a std::collections::HashMap, key: &str) -> Option<&'a str> { metadata.get(key).map(String::as_str).or_else(|| { metadata .iter() diff --git a/crates/utils/src/http/metadata_compat.rs b/crates/utils/src/http/metadata_compat.rs index bbc5cadfd..e1857d354 100644 --- a/crates/utils/src/http/metadata_compat.rs +++ b/crates/utils/src/http/metadata_compat.rs @@ -184,13 +184,13 @@ pub fn internal_key_rustfs(suffix: &str) -> String { // === String type (FileInfo.metadata, user_defined) === -pub fn insert_str(map: &mut HashMap, suffix: &str, value: String) { +pub fn insert_str(map: &mut HashMap, suffix: &str, value: String) { let (k1, k2) = both_keys(suffix); map.insert(k1, value.clone()); map.insert(k2, value); } -pub fn get_str(map: &HashMap, suffix: &str) -> Option { +pub fn get_str(map: &HashMap, suffix: &str) -> Option { if let Some(v) = with_internal_key(RUSTFS_INTERNAL_PREFIX, suffix, |k1| map.get(k1).cloned()) { return Some(v); } @@ -204,10 +204,7 @@ pub fn get_str(map: &HashMap, suff .map(|(_, value)| value.clone()) } -fn get_consistent_value<'a, V: AsRef<[u8]>, S: std::hash::BuildHasher>( - map: &'a HashMap, - suffix: &str, -) -> Option<&'a V> { +fn get_consistent_value<'a, V: AsRef<[u8]>>(map: &'a HashMap, suffix: &str) -> Option<&'a V> { let (rustfs_key, minio_key) = both_keys(suffix); let mut value = None; for (key, candidate) in map { @@ -225,11 +222,11 @@ fn get_consistent_value<'a, V: AsRef<[u8]>, S: std::hash::BuildHasher>( /// Returns a non-empty value when every compatibility key present for `suffix` agrees. /// A single RustFS or MinIO key is accepted for backward compatibility; conflicting or empty /// values return `None` so callers at destructive boundaries can fail closed. -pub fn get_consistent_str<'a, S: std::hash::BuildHasher>(map: &'a HashMap, suffix: &str) -> Option<&'a str> { +pub fn get_consistent_str<'a>(map: &'a HashMap, suffix: &str) -> Option<&'a str> { get_consistent_value(map, suffix).map(String::as_str) } -pub fn contains_key_str(map: &HashMap, suffix: &str) -> bool { +pub fn contains_key_str(map: &HashMap, suffix: &str) -> bool { if with_internal_key(RUSTFS_INTERNAL_PREFIX, suffix, |k1| map.contains_key(k1)) { return true; } @@ -241,7 +238,7 @@ pub fn contains_key_str(map: &HashMap(map: &mut HashMap, suffix: &str) { +pub fn remove_str(map: &mut HashMap, suffix: &str) { with_internal_key(RUSTFS_INTERNAL_PREFIX, suffix, |k1| map.remove(k1)); with_internal_key(MINIO_INTERNAL_PREFIX, suffix, |k2| map.remove(k2)); let (k1, k2) = both_keys(suffix); @@ -290,9 +287,7 @@ pub fn strip_internal_prefix_preserving_case(key: &str) -> Option<&str> { /// Reads the bounded per-target delete-marker version map in one metadata scan. /// The boolean is set when matching metadata is malformed or compatibility keys disagree. -pub fn target_delete_marker_versions( - map: &HashMap, -) -> (HashMap, bool) { +pub fn target_delete_marker_versions(map: &HashMap) -> (HashMap, bool) { const MAX_ENTRIES: usize = 1_000; const MAX_ARN_LEN: usize = 1_024; const MAX_VERSION_ID_LEN: usize = 1_024;