gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-28 16:07:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

revert: rollback AHashMap changes and keep using std HashMap (#6741)

Browse Source 
* Revert "perf(ecstore): use AHashMap for FileInfo metadata fields (#6738)"

This reverts commit 13a2ae212e.

* fix(filemeta): restore standard HashMap metadata (#6742)

Remove the direct ahash dependency added for FileInfo metadata and revert the affected filemeta/ecstore call sites back to std::collections::HashMap.

Co-authored-by: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-27 20:46:08 +08:00
committed by GitHub
parent f1de19fc14
commit 152f110583
26 changed files with 87 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
-3
View File
@@ -91,7 +91,6 @@ dependencies = [
"const-random",
"getrandom 0.3.4",
"once_cell",
"serde",
"version_check",
"zerocopy",
]
@@ -9627,7 +9626,6 @@ dependencies = [
name = "rustfs-ecstore"
version = "1.0.0-rc.4"
dependencies = [
"ahash",
"arc-swap",
"async-channel",
"async-recursion",
@@ -9773,7 +9771,6 @@ dependencies = [
name = "rustfs-filemeta"
version = "1.0.0-rc.4"
dependencies = [
"ahash",
"arc-swap",
"byteorder",
"bytes",
Cargo.toml
-3
View File
@@ -368,9 +368,6 @@ hotpath = { version = "0.24.0", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.48" }
# High-performance hashing
ahash = { version = "0.8", default-features = false, features = ["std", "runtime-rng", "serde"] }
[workspace.metadata.cargo-shear]
ignored = ["hotpath", "rustfs"]
crates/ecstore/Cargo.toml
-3
View File
@@ -218,9 +218,6 @@ faster-hex = { workspace = true }
ratelimit = { workspace = true }
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
# High-performance hashing
ahash = { workspace = true, features = ["serde"] }
# Observability and Metrics
metrics = { workspace = true }
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+8 -9
View File
@@ -69,7 +69,6 @@ use crate::storage_api_contracts::{
range::HTTPRangeSpec,
};
use crate::store::ECStore;
use ahash::AHashMap;
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use http::HeaderMap;
use rand::RngExt as _;
@@ -2871,7 +2870,7 @@ fn spawn_transition_transaction_recovery_once(api: Arc<ECStore>) {
struct StaleMultipartUploadCandidate {
path: String,
initiated: OffsetDateTime,
metadata: Option<AHashMap<String, String>>,
metadata: Option<HashMap<String, String>>,
}
fn parse_stale_uploads_duration(env_key: &str, default: StdDuration) -> StdDuration {
@@ -2916,9 +2915,9 @@ async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<Strin
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
}
async fn stale_upload_current_size_with_opts<S: std::hash::BuildHasher>(
async fn stale_upload_current_size_with_opts(
set: &Arc<SetDisks>,
metadata: &HashMap<String, String, S>,
metadata: &HashMap<String, String>,
upload_dir: &str,
no_lock: bool,
) -> Option<usize> {
@@ -2951,9 +2950,9 @@ async fn stale_upload_current_size_with_opts<S: std::hash::BuildHasher>(
)
}
async fn stale_upload_lifecycle_due<S: std::hash::BuildHasher>(
async fn stale_upload_lifecycle_due(
set: &Arc<SetDisks>,
metadata: &HashMap<String, String, S>,
metadata: &HashMap<String, String>,
initiated: OffsetDateTime,
upload_dir: &str,
no_lock: bool,
@@ -2979,7 +2978,7 @@ async fn stale_upload_lifecycle_due<S: std::hash::BuildHasher>(
.unwrap_or_default(),
is_latest: true,
delete_marker: false,
user_defined: metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
user_defined: metadata.clone(),
..Default::default()
};
@@ -11617,7 +11616,7 @@ mod tests {
// Persist the durable backend identity on the transitioned version so the
// recovered free version carries it (matching a registered mock tier);
// free-version remote cleanup fails closed without it.
let mut transitioned_metadata = AHashMap::new();
let mut transitioned_metadata = HashMap::new();
if let Some(identity) = backend_identity {
rustfs_utils::http::metadata_compat::insert_str(
&mut transitioned_metadata,
@@ -12354,7 +12353,7 @@ mod tests {
StaleMultipartUploadCandidate {
path: "sha/upload".to_string(),
initiated: OffsetDateTime::UNIX_EPOCH,
metadata: Some(AHashMap::from([("k".to_string(), "v".to_string())])),
metadata: Some(HashMap::from([("k".to_string(), "v".to_string())])),
},
);
crates/ecstore/src/core/pools.rs
+2 -2
View File
@@ -5815,7 +5815,7 @@ fn decommission_remote_tiered_opts(
versioned: version_id.is_some(),
version_id,
mod_time: version.mod_time,
user_defined: version.metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
user_defined: version.metadata.clone(),
src_pool_idx,
data_movement: true,
incl_free_versions: version.tier_free_version(),
@@ -11894,7 +11894,7 @@ mod tests {
let mod_time = OffsetDateTime::now_utc();
let version = rustfs_filemeta::FileInfo {
mod_time: Some(mod_time),
metadata: AHashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]),
metadata: HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]),
..Default::default()
};
crates/ecstore/src/data_movement/mod.rs
+2 -2
View File
@@ -1877,7 +1877,7 @@ mod tests {
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
data_dir: Some(Uuid::from_u128(100)),
checksum: Some(Bytes::from_static(b"object-checksum")),
metadata: AHashMap::from([
metadata: HashMap::from([
("etag".to_string(), "etag-value".to_string()),
("x-amz-meta-key".to_string(), metadata_value.to_string()),
]),
@@ -1887,7 +1887,7 @@ mod tests {
size: 128,
actual_size: 128,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
checksums: Some(AHashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])),
checksums: Some(HashMap::from([(ChecksumType::CRC32C.to_string(), "part-checksum".to_string())])),
..Default::default()
}],
..Default::default()
crates/ecstore/src/object_api/mod.rs
+1 -5
View File
@@ -64,11 +64,7 @@ pub(crate) const ENCRYPTED_FRAME_LAYOUT_FIXED8K_SUFFIX: &str = "encrypted-frame-
pub(crate) const ENV_RUSTFS_ENCRYPTED_RANGE_SEEK: &str = "RUSTFS_ENCRYPTED_RANGE_SEEK";
pub(crate) const DEFAULT_RUSTFS_ENCRYPTED_RANGE_SEEK: bool = true;
pub(crate) fn has_encrypted_part_layout_marker<S: std::hash::BuildHasher>(
metadata: &HashMap<String, String, S>,
suffix: &str,
expected: &str,
) -> bool {
pub(crate) fn has_encrypted_part_layout_marker(metadata: &HashMap<String, String>, suffix: &str, expected: &str) -> bool {
let mut value = None;
for (key, candidate) in metadata {
if !rustfs_utils::http::has_internal_suffix(key, suffix) {
crates/ecstore/src/object_api/object_api_utils.rs
+1 -1
View File
@@ -168,7 +168,7 @@ pub fn to_s3s_etag(etag: &str) -> ETag {
ETag::Strong(etag.to_string())
}
pub fn get_raw_etag<S: std::hash::BuildHasher>(metadata: &HashMap<String, String, S>) -> String {
pub fn get_raw_etag(metadata: &HashMap<String, String>) -> String {
metadata
.get("etag")
.cloned()
crates/ecstore/src/object_api/types.rs
+3 -3
View File
@@ -1008,7 +1008,7 @@ impl ObjectInfo {
successor_mod_time: fi.successor_mod_time,
etag,
inlined,
user_defined: Arc::new(metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect()),
user_defined: Arc::new(metadata),
transitioned_object,
transition_version_state: fi.transition_version_state,
checksum: fi.checksum.clone(),
@@ -1316,7 +1316,7 @@ impl ObjectInfo {
if part > 0
&& let Some(checksums) = self.parts.iter().find(|p| p.number == part).and_then(|p| p.checksums.clone())
{
return Ok((checksums.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), true));
return Ok((checksums, true));
}
if let Some(data) = &self.checksum {
@@ -1812,7 +1812,7 @@ mod tests {
storageclass::GLACIER,
] {
let fi = FileInfo {
metadata: AHashMap::from([(AMZ_STORAGE_CLASS.to_string(), legacy_label.to_string())]),
metadata: HashMap::from([(AMZ_STORAGE_CLASS.to_string(), legacy_label.to_string())]),
..Default::default()
};
crates/ecstore/src/services/rebalance/migration.rs
+1 -1
View File
@@ -57,7 +57,7 @@ fn rebalance_remote_tiered_opts(
versioned: version_id.is_some(),
version_id,
mod_time: version.mod_time,
user_defined: version.metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
user_defined: version.metadata.clone(),
src_pool_idx,
data_movement: true,
include_part_checksums: true,
crates/ecstore/src/services/tier/tier.rs
+1 -3
View File
@@ -1454,9 +1454,7 @@ fn tier_backend_identity(config: &TierConfig) -> io::Result<TierDestinationId> {
encode_tier_backend_identity(tier_type, endpoint, bucket, prefix, region, routing_account)
}
pub(crate) fn tier_destination_id_from_metadata<S: std::hash::BuildHasher>(
metadata: &HashMap<String, String, S>,
) -> io::Result<Option<TierDestinationId>> {
pub(crate) fn tier_destination_id_from_metadata(metadata: &HashMap<String, String>) -> io::Result<Option<TierDestinationId>> {
let Some(encoded) = rustfs_utils::http::metadata_compat::get_consistent_str(
metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
crates/ecstore/src/set_disk/metadata.rs
+2 -5
View File
@@ -609,7 +609,7 @@ impl SetDisks {
|| Self::starts_with_ignore_ascii_case(suffix, http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX)
}
fn update_hash_quorum_metadata_map<S: std::hash::BuildHasher>(hasher: &mut Sha256, entries: &HashMap<String, String, S>) {
fn update_hash_quorum_metadata_map(hasher: &mut Sha256, entries: &HashMap<String, String>) {
let mut entries = entries
.iter()
.filter(|(name, _)| !Self::is_replication_quorum_metadata_key(name))
@@ -635,10 +635,7 @@ impl SetDisks {
/// so the dual internal prefixes carrying the same mapping share one
/// identity, while a genuine disagreement between disks still changes the
/// hash and surfaces as a quorum difference.
fn update_hash_target_delete_marker_versions<S: std::hash::BuildHasher>(
hasher: &mut Sha256,
metadata: &HashMap<String, String, S>,
) {
fn update_hash_target_delete_marker_versions(hasher: &mut Sha256, metadata: &HashMap<String, String>) {
let (versions, corrupt) = http::target_delete_marker_versions(metadata);
hasher.update([u8::from(corrupt)]);
let mut versions = versions.iter().collect::<Vec<_>>();
crates/ecstore/src/set_disk/mod.rs
+6 -13
View File
@@ -190,9 +190,7 @@ use tracing::error;
use tracing::{Instrument, debug, info, warn};
use uuid::Uuid;
pub(super) fn restore_operation_id_from_metadata<S: std::hash::BuildHasher>(
metadata: &HashMap<String, String, S>,
) -> Result<Option<Uuid>> {
pub(super) fn restore_operation_id_from_metadata(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_RESTORE_OPERATION_ID) else {
if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_RESTORE_OPERATION_ID) {
return Err(Error::other("invalid restore operation id metadata".to_string()));
@@ -206,19 +204,14 @@ pub(super) fn restore_operation_id_from_metadata<S: std::hash::BuildHasher>(
Ok(Some(id))
}
pub(super) fn require_restore_operation_id<S: std::hash::BuildHasher>(
metadata: &HashMap<String, String, S>,
expected: Uuid,
) -> Result<()> {
pub(super) fn require_restore_operation_id(metadata: &HashMap<String, String>, expected: Uuid) -> Result<()> {
match restore_operation_id_from_metadata(metadata)? {
Some(actual) if actual == expected => Ok(()),
_ => Err(Error::other("restore operation id changed before copy-back".to_string())),
}
}
pub(super) fn restore_commit_operation_id_from_metadata<S: std::hash::BuildHasher>(
metadata: &HashMap<String, String, S>,
) -> Result<Option<Uuid>> {
pub(super) fn restore_commit_operation_id_from_metadata(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
if !metadata.contains_key(X_AMZ_RESTORE.as_str()) {
return Ok(None);
}
@@ -473,13 +466,13 @@ fn release_materialized_read_lock(bucket: &str, object: &str, read_lock_guard: O
drop(read_lock_guard);
}
pub(crate) fn strip_internal_multipart_metadata<S: std::hash::BuildHasher>(metadata: &mut HashMap<String, String, S>) {
pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap<String, String>) {
metadata.remove(RUSTFS_MULTIPART_BUCKET_KEY);
metadata.remove(RUSTFS_MULTIPART_OBJECT_KEY);
rustfs_utils::http::metadata_compat::remove_str(metadata, SUFFIX_BUCKET_INCARNATION_ID);
}
fn should_persist_encryption_original_size<S: std::hash::BuildHasher>(metadata: &HashMap<String, String, S>) -> bool {
fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
metadata.keys().any(|key| is_object_encryption_marker(key))
}
@@ -8802,7 +8795,7 @@ mod tests {
#[test]
fn test_list_object_etags() {
// Test extracting etags from file info metadata
let mut metadata = AHashMap::new();
let mut metadata = HashMap::new();
metadata.insert("etag".to_string(), "test-etag".to_string());
let file_info = FileInfo {
crates/ecstore/src/set_disk/ops/multipart.rs
+8 -11
View File
@@ -449,7 +449,7 @@ fn fence_commit_on_lock_loss(guard: Option<&ObjectLockDiagGuard>, mode: &'static
Ok(())
}
fn multipart_bucket_incarnation_id<S: std::hash::BuildHasher>(metadata: &HashMap<String, String, S>) -> Result<Option<Uuid>> {
fn multipart_bucket_incarnation_id(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) else {
if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) {
return Err(Error::other("invalid multipart bucket incarnation metadata"));
@@ -463,15 +463,12 @@ fn multipart_bucket_incarnation_id<S: std::hash::BuildHasher>(metadata: &HashMap
Ok(Some(incarnation))
}
fn multipart_bucket_incarnation_matches<S: std::hash::BuildHasher>(
metadata: &HashMap<String, String, S>,
expected: Uuid,
) -> bool {
fn multipart_bucket_incarnation_matches(metadata: &HashMap<String, String>, expected: Uuid) -> bool {
matches!(multipart_bucket_incarnation_id(metadata), Ok(Some(actual)) if actual == expected)
}
fn validate_multipart_bucket_incarnation<S: std::hash::BuildHasher>(
metadata: &HashMap<String, String, S>,
fn validate_multipart_bucket_incarnation(
metadata: &HashMap<String, String>,
bucket: &str,
object: &str,
upload_id: &str,
@@ -1524,7 +1521,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
mod_time: Some(OffsetDateTime::now_utc()),
actual_size,
index: index_op,
checksums: if checksums.is_empty() { None } else { Some(checksums.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) },
checksums: if checksums.is_empty() { None } else { Some(checksums) },
..Default::default()
};
@@ -1719,7 +1716,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
max_parts,
part_number_marker,
user_defined: {
let mut metadata: HashMap<String, String> = fi.metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
let mut metadata = fi.metadata.clone();
strip_internal_multipart_metadata(&mut metadata);
metadata
},
@@ -1978,7 +1975,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let mod_time = opts.mod_time.unwrap_or_else(OffsetDateTime::now_utc);
for f in parts_metadatas.iter_mut() {
f.metadata = user_defined.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
f.metadata = user_defined.clone();
f.mod_time = Some(mod_time);
f.fresh = true;
}
@@ -2067,7 +2064,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
upload_id: upload_id.to_owned(),
user_defined: {
strip_internal_multipart_metadata(&mut fi.metadata);
fi.metadata.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
fi.metadata.clone()
},
..Default::default()
})
crates/ecstore/src/set_disk/ops/object.rs
+12 -13
View File
@@ -72,11 +72,11 @@ use crate::set_disk::runtime_sources;
use crate::storage_api_contracts::multipart::MultipartOperations;
use crate::storage_api_contracts::object::ObjectIO;
use crate::storage_api_contracts::object::ObjectOperations;
use ahash::AHashMap;
use rustfs_lock::LockManager;
use rustfs_rio::EtagResolvable;
use rustfs_rio::HashReaderMut;
use rustfs_rio::TryGetIndex;
use rustfs_utils::http::HeaderExt;
use tokio::io::AsyncWriteExt;
#[cfg(all(test, feature = "test-util"))]
@@ -1107,13 +1107,13 @@ fn is_restore_control_metadata(key: &str) -> bool {
.is_some_and(|remainder| remainder.is_empty())
}
fn restore_metadata_update_preserves_protected_metadata<S1: std::hash::BuildHasher, S2: std::hash::BuildHasher>(
existing: &HashMap<String, String, S1>,
replacement: &HashMap<String, String, S2>,
fn restore_metadata_update_preserves_protected_metadata(
existing: &HashMap<String, String>,
replacement: &HashMap<String, String>,
) -> bool {
let mut existing: HashMap<String, String> = existing.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
let mut existing = existing.clone();
clean_metadata(&mut existing);
let mut replacement: HashMap<String, String> = replacement.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
let mut replacement = replacement.clone();
clean_metadata(&mut replacement);
let existing_count = existing.keys().filter(|key| !is_restore_control_metadata(key)).count();
let replacement_count = replacement.keys().filter(|key| !is_restore_control_metadata(key)).count();
@@ -2211,9 +2211,9 @@ pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &Objec
///
/// Returns whether `inbound` was modified. Callers must hold the object write
/// lock so the stored values compared here are the ones being replaced.
pub(in crate::set_disk) fn merge_replication_metadata_lww<S1: std::hash::BuildHasher, S2: std::hash::BuildHasher>(
inbound: &mut HashMap<String, String, S1>,
existing: &HashMap<String, String, S2>,
pub(in crate::set_disk) fn merge_replication_metadata_lww(
inbound: &mut HashMap<String, String>,
existing: &HashMap<String, String>,
opts: &ObjectOptions,
) -> bool {
use rustfs_utils::http::headers::{
@@ -2844,7 +2844,7 @@ impl SetDisks {
)));
}
fi.metadata = user_defined.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
fi.metadata = user_defined;
fi.mod_time = mod_time;
fi.size = w_size as i64;
fi.versioned = opts.versioned || opts.version_suspended;
@@ -6051,8 +6051,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
} else {
None
};
let mut replacement_metadata: AHashMap<String, String> =
(*src_info.user_defined).iter().map(|(k, v)| (k.clone(), v.clone())).collect();
let mut replacement_metadata = (*src_info.user_defined).clone();
if let Some(part_checksums) = preserved_part_checksums {
rustfs_utils::http::insert_str(&mut replacement_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums);
}
@@ -7494,7 +7493,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(),
X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(),
] {
if let Some(value) = fi.metadata.get(header).filter(|value| !value.is_empty()) {
if let Some(value) = fi.metadata.lookup(header).filter(|value| !value.is_empty()) {
transition_meta.insert(header.to_ascii_lowercase(), value.to_string());
}
}
crates/ecstore/src/store/object.rs
+2 -2
View File
@@ -4780,7 +4780,7 @@ mod tests {
purge_targets: version_purge_statuses_map("arn:minio:replication:target=PENDING;"),
..Default::default()
})),
metadata: AHashMap::from([
metadata: HashMap::from([
("etag".to_string(), "etag-value".to_string()),
("x-amz-meta-key".to_string(), "metadata-value".to_string()),
(rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), "tag=value".to_string()),
@@ -4864,7 +4864,7 @@ mod tests {
source.parts = vec![rustfs_filemeta::ObjectPartInfo {
number: 1,
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
checksums: Some(AHashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])),
checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])),
..Default::default()
}];
rustfs_utils::http::insert_str(
crates/ecstore/src/store/utils.rs
+3 -3
View File
@@ -26,18 +26,18 @@ static STRICT_BUCKET_NAME_REGEX: LazyLock<Regex> =
static NON_STRICT_BUCKET_NAME_REGEX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^[A-Za-z0-9][A-Za-z0-9\.\-_:]{1,61}[A-Za-z0-9]$").expect("valid non-strict bucket name regex"));
pub fn clean_metadata<S: std::hash::BuildHasher>(metadata: &mut HashMap<String, String, S>) {
pub fn clean_metadata(metadata: &mut HashMap<String, String>) {
remove_standard_storage_class(metadata);
clean_metadata_keys(metadata, &["md5Sum", "etag", "expires", AMZ_OBJECT_TAGGING, "last-modified"]);
}
pub fn remove_standard_storage_class<S: std::hash::BuildHasher>(metadata: &mut HashMap<String, String, S>) {
pub fn remove_standard_storage_class(metadata: &mut HashMap<String, String>) {
if metadata.get(AMZ_STORAGE_CLASS) == Some(&STANDARD.to_string()) {
metadata.remove(AMZ_STORAGE_CLASS);
}
}
pub fn clean_metadata_keys<S: std::hash::BuildHasher>(metadata: &mut HashMap<String, String, S>, key_names: &[&str]) {
pub fn clean_metadata_keys(metadata: &mut HashMap<String, String>, key_names: &[&str]) {
for key in key_names {
metadata.remove(key.to_owned());
}
crates/filemeta/Cargo.toml
-3
View File
@@ -51,9 +51,6 @@ s3s = { workspace = true, features = ["minio"] }
regex.workspace = true
arc-swap.workspace = true
# High-performance hashing
ahash = { workspace = true, features = ["serde"] }
[dev-dependencies]
criterion = { workspace = true, features = ["html_reports"] }
tempfile = { workspace = true }
crates/filemeta/src/fileinfo.rs
+8 -13
View File
@@ -13,7 +13,6 @@
// limitations under the License.
use crate::{Error, ReplicationState, ReplicationStatusType, Result, TRANSITION_COMPLETE, VersionPurgeStatusType};
use ahash::AHashMap;
use bytes::Bytes;
use rmp_serde::Serializer;
use rustfs_utils::HashAlgorithm;
@@ -68,7 +67,7 @@ pub struct ObjectPartInfo {
// Index holds the index of the part in the erasure coding
pub index: Option<Bytes>,
// Checksums holds checksums of the part
pub checksums: Option<AHashMap<String, String>>,
pub checksums: Option<HashMap<String, String>>,
pub error: Option<String>,
}
@@ -269,7 +268,7 @@ pub struct FileInfo {
pub mode: Option<u32>,
// WrittenByVersion is the unix time stamp of the version that created this version of the object
pub written_by_version: Option<u64>,
pub metadata: AHashMap<String, String>,
pub metadata: HashMap<String, String>,
pub parts: Vec<ObjectPartInfo>,
pub erasure: ErasureInfo,
// MarkDeleted marks this version as deleted
@@ -302,7 +301,7 @@ fn is_sensitive_metadata_key(key: &str) -> bool {
.any(|prefix| starts_with_ignore_ascii_case(key, prefix))
}
struct RedactedMetadata<'a>(&'a AHashMap<String, String>);
struct RedactedMetadata<'a>(&'a HashMap<String, String>);
impl std::fmt::Debug for RedactedMetadata<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -426,7 +425,7 @@ struct FileInfoMapDef {
size: i64,
mode: Option<u32>,
written_by_version: Option<u64>,
metadata: AHashMap<String, String>,
metadata: HashMap<String, String>,
parts: Vec<ObjectPartInfo>,
erasure: ErasureInfo,
mark_deleted: bool,
@@ -1080,7 +1079,7 @@ impl FileInfo {
mod_time: Option<OffsetDateTime>,
actual_size: i64,
index: Option<Bytes>,
checksums: Option<AHashMap<String, String>>,
checksums: Option<HashMap<String, String>>,
) {
let part = ObjectPartInfo {
etag,
@@ -1458,7 +1457,7 @@ pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
Err(Error::other(ERR_RESTORE_HDR_MALFORMED))
}
pub fn is_restored_object_on_disk<S: std::hash::BuildHasher>(meta: &HashMap<String, String, S>) -> bool {
pub fn is_restored_object_on_disk(meta: &HashMap<String, String>) -> bool {
if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str())
&& let Ok(restore_status) = parse_restore_obj_status(restore_hdr)
{
@@ -2136,10 +2135,7 @@ mod tests {
-1_000_000i64..=1_000_000i64,
optional_timestamp_strategy(),
proptest::option::of(bytes_strategy(16)),
proptest::option::of(
hash_map(small_string_strategy(), small_string_strategy(), 0..=3)
.prop_map(|m| m.into_iter().collect::<AHashMap<String, String>>()),
),
proptest::option::of(hash_map(small_string_strategy(), small_string_strategy(), 0..=3)),
proptest::option::of(small_string_strategy()),
)
.prop_map(|(etag, number, size, actual_size, mod_time, index, checksums, error)| ObjectPartInfo {
@@ -2174,8 +2170,7 @@ mod tests {
-1_000_000i64..=1_000_000i64,
proptest::option::of(any::<u32>()),
proptest::option::of(any::<u64>()),
hash_map(small_string_strategy(), small_string_strategy(), 0..=4)
.prop_map(|m| m.into_iter().collect::<AHashMap<String, String>>()),
hash_map(small_string_strategy(), small_string_strategy(), 0..=4),
vec(object_part_info_strategy(), 0..=3),
erasure_info_strategy(),
any::<bool>(),
crates/filemeta/src/filemeta.rs
+2 -2
View File
@@ -174,10 +174,10 @@ fn valid_target_delete_marker_version(arn: &str, version_id: &str) -> bool {
/// included in the quorum hash, so such a divergence does surface — but as a
/// quorum failure on an otherwise healthy object, which is not a state worth
/// reaching. Merge the RPC metadata carrier instead, and only ever insert.
fn persist_target_delete_marker_versions<S: std::hash::BuildHasher>(
fn persist_target_delete_marker_versions(
meta_sys: &mut HashMap<String, Vec<u8>>,
versions: &HashMap<String, String>,
transport_metadata: &HashMap<String, String, S>,
transport_metadata: &HashMap<String, String>,
) {
let mut bounded = BTreeMap::new();
// A corrupt carrier means the dual internal prefixes disagreed. Do not merge
crates/filemeta/src/filemeta/inline_data.rs
+1 -1
View File
@@ -181,7 +181,7 @@ mod tests {
data_dir: Some(data_dir),
size: 64 * 1024,
mod_time: Some(OffsetDateTime::now_utc()),
metadata: metadata.into_iter().collect(),
metadata,
erasure: ErasureInfo {
algorithm: ErasureAlgo::ReedSolomon.to_string(),
data_blocks: 4,
crates/filemeta/src/filemeta/version.rs
+11 -11
View File
@@ -26,7 +26,7 @@ use super::msgp_decode::{
PrependByteReader, prealloc_hint, read_exact_vec, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value,
};
use super::*;
use crate::{AHashMap, ChecksumInfo, TransitionVersionState};
use crate::{ChecksumInfo, TransitionVersionState};
use rustfs_utils::HashAlgorithm;
use rustfs_utils::http::{
RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PART_CHECKSUMS, SUFFIX_PURGESTATUS,
@@ -377,7 +377,7 @@ impl<'a> DerivedInternalMetadata<'a> {
}
}
struct UniquePartChecksums(AHashMap<String, String>);
struct UniquePartChecksums(HashMap<String, String>);
impl<'de> serde::Deserialize<'de> for UniquePartChecksums {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
@@ -397,7 +397,7 @@ impl<'de> serde::Deserialize<'de> for UniquePartChecksums {
where
A: serde::de::SeqAccess<'de>,
{
let mut checksums = AHashMap::with_capacity(seq.size_hint().unwrap_or_default());
let mut checksums = HashMap::with_capacity(seq.size_hint().unwrap_or_default());
while let Some((key, value)) = seq.next_element::<(String, String)>()? {
if checksums.insert(key, value).is_some() {
return Err(serde::de::Error::custom("duplicate part checksum name"));
@@ -1477,7 +1477,7 @@ pub struct MetaObjectV1 {
#[serde(rename = "Erasure")]
pub erasure: MetaObjectV1Erasure,
#[serde(rename = "Meta")]
pub meta: AHashMap<String, String>,
pub meta: HashMap<String, String>,
#[serde(rename = "Parts")]
pub parts: Vec<MetaObjectV1Part>,
#[serde(rename = "VersionID")]
@@ -1543,7 +1543,7 @@ pub struct MetaObjectV1Part {
#[serde(rename = "i")]
pub index: Option<Bytes>,
#[serde(rename = "crc")]
pub checksums: Option<AHashMap<String, String>>,
pub checksums: Option<HashMap<String, String>>,
#[serde(rename = "err")]
pub error: Option<String>,
}
@@ -1887,7 +1887,7 @@ impl MetaObjectV1Part {
"i" => self.index = Some(Bytes::from(read_msgp_bin(rd)?)),
"crc" => {
let len = rmp::decode::read_map_len(rd)? as usize;
let mut checksums = AHashMap::with_capacity(prealloc_hint(len));
let mut checksums = HashMap::with_capacity(prealloc_hint(len));
for _ in 0..len {
checksums.insert(read_msgp_string(rd)?, read_msgp_string(rd)?);
}
@@ -2570,7 +2570,7 @@ impl MetaObject {
Vec::new()
};
let mut metadata = AHashMap::with_capacity(self.meta_user.len() + self.meta_sys.len());
let mut metadata = HashMap::with_capacity(self.meta_user.len() + self.meta_sys.len());
for (k, v) in &self.meta_user {
if k == AMZ_META_UNENCRYPTED_CONTENT_LENGTH || k == AMZ_META_UNENCRYPTED_CONTENT_MD5 {
continue;
@@ -2861,7 +2861,7 @@ impl From<FileInfo> for MetaObject {
}
}
fn get_internal_replication_state<S: std::hash::BuildHasher>(metadata: &HashMap<String, String, S>) -> Option<ReplicationState> {
fn get_internal_replication_state(metadata: &HashMap<String, String>) -> Option<ReplicationState> {
let mut rs = ReplicationState::default();
let mut has = false;
@@ -2942,7 +2942,7 @@ impl MetaDeleteMarker {
}
pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> Result<FileInfo> {
let metadata: AHashMap<String, String> = self
let metadata = self
.meta_sys
.clone()
.into_iter()
@@ -5500,10 +5500,10 @@ mod tests {
/// entirely, silently dropping the whole legacy body on re-marshal.
#[test]
fn legacy_version_body_round_trips_through_encode() {
let mut meta = AHashMap::new();
let mut meta = HashMap::new();
meta.insert("content-type".to_string(), "application/octet-stream".to_string());
let mut crc = AHashMap::new();
let mut crc = HashMap::new();
crc.insert("crc32c".to_string(), "deadbeef".to_string());
let legacy = MetaObjectV1 {
crates/filemeta/src/lib.rs
-6
View File
@@ -22,12 +22,6 @@ mod replication;
pub mod test_data;
/// High-performance HashMap type alias using ahash instead of SipHash.
pub type AHashMap<K, V> = ahash::AHashMap<K, V>;
/// High-performance HashSet type alias using ahash.
pub type AHashSet<K> = ahash::AHashSet<K>;
pub use error::*;
pub use fileinfo::*;
pub use filemeta::*;
crates/filemeta/src/metacache.rs
+4 -4
View File
@@ -1676,7 +1676,7 @@ mod tests {
}
fn metacache_entry_with_mod_time(mod_time: OffsetDateTime, etag: &str) -> MetaCacheEntry {
let mut metadata = ahash::AHashMap::new();
let mut metadata = HashMap::new();
metadata.insert("etag".to_string(), etag.to_string());
let mut meta = FileMeta::new();
@@ -1705,7 +1705,7 @@ mod tests {
data_blocks: usize,
parity_blocks: usize,
) -> MetaCacheEntry {
let mut metadata = ahash::AHashMap::new();
let mut metadata = HashMap::new();
metadata.insert("etag".to_string(), etag.to_string());
let mut fi = FileInfo::new("object", data_blocks, parity_blocks);
@@ -1730,7 +1730,7 @@ mod tests {
fn metacache_entry_with_erasure_versions(versions: &[(OffsetDateTime, &str, usize, usize)]) -> MetaCacheEntry {
let mut meta = FileMeta::new();
for (idx, (mod_time, etag, data_blocks, parity_blocks)) in versions.iter().enumerate() {
let mut metadata = ahash::AHashMap::new();
let mut metadata = HashMap::new();
metadata.insert("etag".to_string(), (*etag).to_string());
let mut fi = FileInfo::new("object", *data_blocks, *parity_blocks);
@@ -1776,7 +1776,7 @@ mod tests {
/// Build an entry holding a single object version with an explicit version id
/// and mod_time, so a set of these can model DISJOINT per-disk version sets.
fn metacache_entry_single_version(version_u128: u128, mod_time: OffsetDateTime, etag: &str) -> MetaCacheEntry {
let mut metadata = ahash::AHashMap::new();
let mut metadata = HashMap::new();
metadata.insert("etag".to_string(), etag.to_string());
let mut fi = FileInfo::new("object", 4, 2);
crates/utils/src/http/header_compat.rs
+2 -7
View File
@@ -89,9 +89,7 @@ pub fn is_object_encryption_marker(key: &str) -> bool {
}
/// Reads the logical object size recorded by encryption metadata.
pub fn get_object_encryption_original_size<S: std::hash::BuildHasher>(
metadata: &std::collections::HashMap<String, String, S>,
) -> std::io::Result<Option<i64>> {
pub fn get_object_encryption_original_size(metadata: &std::collections::HashMap<String, String>) -> std::io::Result<Option<i64>> {
let actual_size = super::get_str(metadata, super::SUFFIX_ACTUAL_SIZE);
let size = get_case_insensitive(metadata, RUSTFS_ENCRYPTION_ORIGINAL_SIZE)
.or_else(|| get_case_insensitive(metadata, SSEC_ORIGINAL_SIZE))
@@ -105,10 +103,7 @@ pub fn get_object_encryption_original_size<S: std::hash::BuildHasher>(
.map_err(|error| std::io::Error::other(format!("Failed to parse encryption original size: {error}")))
}
fn get_case_insensitive<'a, S: std::hash::BuildHasher>(
metadata: &'a std::collections::HashMap<String, String, S>,
key: &str,
) -> Option<&'a str> {
fn get_case_insensitive<'a>(metadata: &'a std::collections::HashMap<String, String>, key: &str) -> Option<&'a str> {
metadata.get(key).map(String::as_str).or_else(|| {
metadata
.iter()
crates/utils/src/http/metadata_compat.rs
+7 -12
View File
@@ -184,13 +184,13 @@ pub fn internal_key_rustfs(suffix: &str) -> String {
// === String type (FileInfo.metadata, user_defined) ===
pub fn insert_str<S: std::hash::BuildHasher>(map: &mut HashMap<String, String, S>, suffix: &str, value: String) {
pub fn insert_str(map: &mut HashMap<String, String>, suffix: &str, value: String) {
let (k1, k2) = both_keys(suffix);
map.insert(k1, value.clone());
map.insert(k2, value);
}
pub fn get_str<S: std::hash::BuildHasher>(map: &HashMap<String, String, S>, suffix: &str) -> Option<String> {
pub fn get_str(map: &HashMap<String, String>, suffix: &str) -> Option<String> {
if let Some(v) = with_internal_key(RUSTFS_INTERNAL_PREFIX, suffix, |k1| map.get(k1).cloned()) {
return Some(v);
}
@@ -204,10 +204,7 @@ pub fn get_str<S: std::hash::BuildHasher>(map: &HashMap<String, String, S>, suff
.map(|(_, value)| value.clone())
}
fn get_consistent_value<'a, V: AsRef<[u8]>, S: std::hash::BuildHasher>(
map: &'a HashMap<String, V, S>,
suffix: &str,
) -> Option<&'a V> {
fn get_consistent_value<'a, V: AsRef<[u8]>>(map: &'a HashMap<String, V>, suffix: &str) -> Option<&'a V> {
let (rustfs_key, minio_key) = both_keys(suffix);
let mut value = None;
for (key, candidate) in map {
@@ -225,11 +222,11 @@ fn get_consistent_value<'a, V: AsRef<[u8]>, S: std::hash::BuildHasher>(
/// Returns a non-empty value when every compatibility key present for `suffix` agrees.
/// A single RustFS or MinIO key is accepted for backward compatibility; conflicting or empty
/// values return `None` so callers at destructive boundaries can fail closed.
pub fn get_consistent_str<'a, S: std::hash::BuildHasher>(map: &'a HashMap<String, String, S>, suffix: &str) -> Option<&'a str> {
pub fn get_consistent_str<'a>(map: &'a HashMap<String, String>, suffix: &str) -> Option<&'a str> {
get_consistent_value(map, suffix).map(String::as_str)
}
pub fn contains_key_str<S: std::hash::BuildHasher>(map: &HashMap<String, String, S>, suffix: &str) -> bool {
pub fn contains_key_str(map: &HashMap<String, String>, suffix: &str) -> bool {
if with_internal_key(RUSTFS_INTERNAL_PREFIX, suffix, |k1| map.contains_key(k1)) {
return true;
}
@@ -241,7 +238,7 @@ pub fn contains_key_str<S: std::hash::BuildHasher>(map: &HashMap<String, String,
.any(|key| key.eq_ignore_ascii_case(&k1) || key.eq_ignore_ascii_case(&k2))
}
pub fn remove_str<S: std::hash::BuildHasher>(map: &mut HashMap<String, String, S>, suffix: &str) {
pub fn remove_str(map: &mut HashMap<String, String>, suffix: &str) {
with_internal_key(RUSTFS_INTERNAL_PREFIX, suffix, |k1| map.remove(k1));
with_internal_key(MINIO_INTERNAL_PREFIX, suffix, |k2| map.remove(k2));
let (k1, k2) = both_keys(suffix);
@@ -290,9 +287,7 @@ pub fn strip_internal_prefix_preserving_case(key: &str) -> Option<&str> {
/// Reads the bounded per-target delete-marker version map in one metadata scan.
/// The boolean is set when matching metadata is malformed or compatibility keys disagree.
pub fn target_delete_marker_versions<S: std::hash::BuildHasher>(
map: &HashMap<String, String, S>,
) -> (HashMap<String, String>, bool) {
pub fn target_delete_marker_versions(map: &HashMap<String, String>) -> (HashMap<String, String>, bool) {
const MAX_ENTRIES: usize = 1_000;
const MAX_ARN_LEN: usize = 1_024;
const MAX_VERSION_ID_LEN: usize = 1_024;