// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //! System metadata compatibility: write both x-rustfs-internal-* and x-minio-internal-* //! for MinIO interoperability. Read prefers RustFS, fallback to MinIO. use std::collections::HashMap; pub const RUSTFS_INTERNAL_PREFIX: &str = "x-rustfs-internal-"; pub const MINIO_INTERNAL_PREFIX: &str = "x-minio-internal-"; // Key suffixes (lowercase, no prefix) pub const SUFFIX_INLINE_DATA: &str = "inline-data"; pub const SUFFIX_DATA_MOVED: &str = "data-moved"; /// Transient flag for data movement pub const SUFFIX_DATA_MOV: &str = "data-mov"; /// Transient flag for healing pub const SUFFIX_HEALING: &str = "healing"; pub const SUFFIX_COMPRESSION: &str = "compression"; pub const SUFFIX_COMPRESSION_SIZE: &str = "compression-size"; pub const SUFFIX_ACTUAL_SIZE: &str = "actual-size"; pub const SUFFIX_ACTUAL_OBJECT_SIZE: &str = "actual-object-size"; /// Used by replication; key stored with capital A pub const SUFFIX_ACTUAL_OBJECT_SIZE_CAP: &str = "Actual-Object-Size"; pub const SUFFIX_CRC: &str = "crc"; pub const SUFFIX_TRANSITION_STATUS: &str = "transition-status"; pub const SUFFIX_TRANSITIONED_OBJECTNAME: &str = "transitioned-object"; pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID"; pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier"; pub const SUFFIX_FREE_VERSION: &str = "free-version"; pub const SUFFIX_PURGESTATUS: &str = "purgestatus"; pub const SUFFIX_REPLICA_STATUS: &str = "replica-status"; pub const SUFFIX_REPLICA_TIMESTAMP: &str = "replica-timestamp"; pub const SUFFIX_REPLICATION_STATUS: &str = "replication-status"; pub const SUFFIX_REPLICATION_TIMESTAMP: &str = "replication-timestamp"; pub const SUFFIX_TAGGING_TIMESTAMP: &str = "tagging-timestamp"; pub const SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP: &str = "objectlock-retention-timestamp"; pub const SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP: &str = "objectlock-legalhold-timestamp"; pub const SUFFIX_REPLICATION_RESET: &str = "replication-reset"; /// Prefix for replication-reset-{arn} keys; use with internal_key_strip_suffix_prefix to extract arn. pub const SUFFIX_REPLICATION_RESET_ARN_PREFIX: &str = "replication-reset-"; pub const SUFFIX_TIER_FV_ID: &str = "tier-free-versionID"; pub const SUFFIX_TIER_FV_MARKER: &str = "tier-free-marker"; pub const SUFFIX_TIER_SKIP_FV_ID: &str = "tier-skip-fvid"; /// Returns true if the key is an internal metadata key (x-rustfs-internal-* or x-minio-internal-*) /// for xl.meta compatibility. Case-insensitive. pub fn is_internal_key(key: &str) -> bool { let lower = key.to_lowercase(); lower.starts_with(RUSTFS_INTERNAL_PREFIX) || lower.starts_with(MINIO_INTERNAL_PREFIX) } /// Returns true if the key matches the given suffix for either x-rustfs-internal-* or x-minio-internal-*. pub fn has_internal_suffix(key: &str, suffix: &str) -> bool { let lower = key.to_lowercase(); let rustfs_key = format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"); let minio_key = format!("{MINIO_INTERNAL_PREFIX}{suffix}"); lower == rustfs_key || lower == minio_key } /// Strips x-rustfs-internal- or x-minio-internal- prefix from key. Returns the suffix part. /// Case-insensitive. Returns None if key is not an internal key. pub fn strip_internal_prefix(key: &str) -> Option { let lower = key.to_lowercase(); lower .strip_prefix(RUSTFS_INTERNAL_PREFIX) .or_else(|| lower.strip_prefix(MINIO_INTERNAL_PREFIX)) .map(|s| s.to_string()) } /// Returns true if key is internal and its suffix part starts with the given suffix_prefix. /// E.g. internal_key_starts_with("x-rustfs-internal-replication-reset-arn1", "replication-reset") == true. pub fn internal_key_starts_with(key: &str, suffix_prefix: &str) -> bool { strip_internal_prefix(key).is_some_and(|s| s.starts_with(suffix_prefix)) } /// For keys like x-rustfs-internal-replication-reset-{arn}, strips the internal prefix and suffix_prefix, /// returning the remainder (e.g. "arn1"). Returns None if key does not match. pub fn internal_key_strip_suffix_prefix(key: &str, suffix_prefix: &str) -> Option { let rest = strip_internal_prefix(key)?; rest.strip_prefix(suffix_prefix).map(|s| s.to_string()) } fn both_keys(suffix: &str) -> (String, String) { (format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"), format!("{MINIO_INTERNAL_PREFIX}{suffix}")) } /// Builds the RustFS internal key for the given suffix. Use when a single key is needed (e.g. for /// backward compat). Prefer insert_str/get_str when both keys should be written/read. pub fn internal_key_rustfs(suffix: &str) -> String { format!("{RUSTFS_INTERNAL_PREFIX}{suffix}") } // === String type (FileInfo.metadata, user_defined) === pub fn insert_str(map: &mut HashMap, suffix: &str, value: String) { let (k1, k2) = both_keys(suffix); map.insert(k1, value.clone()); map.insert(k2, value); } pub fn get_str(map: &HashMap, suffix: &str) -> Option { let (k1, k2) = both_keys(suffix); map.get(&k1).cloned().or_else(|| map.get(&k2).cloned()).or_else(|| { map.iter() .find(|(key, _)| key.eq_ignore_ascii_case(&k1) || key.eq_ignore_ascii_case(&k2)) .map(|(_, value)| value.clone()) }) } pub fn contains_key_str(map: &HashMap, suffix: &str) -> bool { let (k1, k2) = both_keys(suffix); map.contains_key(&k1) || map.contains_key(&k2) || map .keys() .any(|key| key.eq_ignore_ascii_case(&k1) || key.eq_ignore_ascii_case(&k2)) } pub fn remove_str(map: &mut HashMap, suffix: &str) { let (k1, k2) = both_keys(suffix); map.remove(&k1); map.remove(&k2); map.retain(|key, _| !key.eq_ignore_ascii_case(&k1) && !key.eq_ignore_ascii_case(&k2)); } // === Vec type (meta_sys) === pub fn insert_bytes(map: &mut HashMap>, suffix: &str, value: Vec) { let (k1, k2) = both_keys(suffix); let v = value.clone(); map.insert(k1, value); map.insert(k2, v); } pub fn get_bytes(map: &HashMap>, suffix: &str) -> Option> { let (k1, k2) = both_keys(suffix); map.get(&k1).cloned().or_else(|| map.get(&k2).cloned()) } pub fn contains_key_bytes(map: &HashMap>, suffix: &str) -> bool { let (k1, k2) = both_keys(suffix); map.contains_key(&k1) || map.contains_key(&k2) } pub fn remove_bytes(map: &mut HashMap>, suffix: &str) { let (k1, k2) = both_keys(suffix); map.remove(&k1); map.remove(&k2); } #[cfg(test)] mod tests { use super::*; #[test] fn test_is_internal_key() { assert!(is_internal_key("x-rustfs-internal-healing")); assert!(is_internal_key("x-rustfs-internal-purgestatus")); assert!(is_internal_key("X-RustFS-Internal-purgestatus")); assert!(is_internal_key("x-minio-internal-compression")); assert!(is_internal_key("x-minio-internal-replication-status")); assert!(is_internal_key("X-Minio-Internal-Compression")); assert!(!is_internal_key("x-amz-meta-custom")); assert!(!is_internal_key("content-type")); assert!(!is_internal_key("x-rustfs-meta-custom")); } #[test] fn test_has_internal_suffix() { assert!(has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_PURGESTATUS)); assert!(has_internal_suffix("X-Minio-Internal-purgestatus", SUFFIX_PURGESTATUS)); assert!(has_internal_suffix("x-minio-internal-compression", SUFFIX_COMPRESSION)); assert!(has_internal_suffix("x-rustfs-internal-healing", SUFFIX_HEALING)); assert!(has_internal_suffix("x-minio-internal-data-mov", SUFFIX_DATA_MOV)); assert!(!has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_HEALING)); assert!(!has_internal_suffix("x-amz-meta-custom", SUFFIX_PURGESTATUS)); } #[test] fn test_str_lookup_accepts_minio_metadata_case() { let mut metadata = HashMap::from([ ("X-Minio-Internal-compression".to_string(), "klauspost/compress/s2".to_string()), ("X-Minio-Internal-actual-size".to_string(), "268435456".to_string()), ]); assert!(contains_key_str(&metadata, SUFFIX_COMPRESSION)); assert_eq!(get_str(&metadata, SUFFIX_COMPRESSION).as_deref(), Some("klauspost/compress/s2")); assert_eq!(get_str(&metadata, SUFFIX_ACTUAL_SIZE).as_deref(), Some("268435456")); remove_str(&mut metadata, SUFFIX_COMPRESSION); assert!(!contains_key_str(&metadata, SUFFIX_COMPRESSION)); assert!(!metadata.contains_key("X-Minio-Internal-compression")); assert!(contains_key_str(&metadata, SUFFIX_ACTUAL_SIZE)); } #[test] fn test_str_prefers_rustfs_key_over_minio() { let metadata = HashMap::from([ (internal_key_rustfs(SUFFIX_TRANSITION_TIER), "rustfs-tier".to_string()), (format!("{MINIO_INTERNAL_PREFIX}{SUFFIX_TRANSITION_TIER}"), "minio-tier".to_string()), ]); assert_eq!(get_str(&metadata, SUFFIX_TRANSITION_TIER).as_deref(), Some("rustfs-tier")); } #[test] fn test_bytes_lookup_falls_back_to_minio_key() { let mut meta_sys = HashMap::from([(format!("{MINIO_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}"), b"version-1".to_vec())]); assert!(contains_key_bytes(&meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)); assert_eq!(get_bytes(&meta_sys, SUFFIX_TRANSITIONED_VERSION_ID), Some(b"version-1".to_vec())); remove_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID); assert!(!contains_key_bytes(&meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)); } #[test] fn test_bytes_prefers_rustfs_key_over_minio() { let meta_sys = HashMap::from([ (internal_key_rustfs(SUFFIX_TRANSITIONED_VERSION_ID), b"rustfs-version".to_vec()), ( format!("{MINIO_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}"), b"minio-version".to_vec(), ), ]); assert_eq!(get_bytes(&meta_sys, SUFFIX_TRANSITIONED_VERSION_ID), Some(b"rustfs-version".to_vec())); } }