feat: improve legacy metadata and admin compatibility (#2202)

crates/utils/src/http/header_compat.rs

@@ -0,0 +1,123 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! HTTP header compatibility: read both x-rustfs-* and x-minio-* headers for MinIO
+//! interoperability. Write both when sending replication requests.
+//!
+//! Use suffix-based API: `get_header(headers, SUFFIX_FORCE_DELETE)` queries both
+//! x-rustfs-force-delete and x-minio-force-delete.
+
+use http::{HeaderMap, HeaderValue};
+use std::borrow::Cow;
+
+const RUSTFS_PREFIX: &str = "x-rustfs-";
+const MINIO_PREFIX: &str = "x-minio-";
+const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
+const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
+
+// Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header.
+pub const SUFFIX_FORCE_DELETE: &str = "force-delete";
+pub const SUFFIX_INCLUDE_DELETED: &str = "include-deleted";
+pub const SUFFIX_REPLICATION_RESET_STATUS: &str = "replication-reset-status";
+pub const SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE: &str = "replication-actual-object-size";
+pub const SUFFIX_SOURCE_VERSION_ID: &str = "source-version-id";
+pub const SUFFIX_SOURCE_MTIME: &str = "source-mtime";
+pub const SUFFIX_SOURCE_ETAG: &str = "source-etag";
+pub const SUFFIX_SOURCE_DELETEMARKER: &str = "source-deletemarker";
+pub const SUFFIX_SOURCE_PROXY_REQUEST: &str = "source-proxy-request";
+pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request";
+pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check";
+pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc";
+
+/// Returns true if the key is an internal encryption metadata key (x-rustfs-encryption-* or
+/// x-minio-encryption-*). Case-insensitive for metadata filtering.
+pub fn is_encryption_metadata_key(key: &str) -> bool {
+    let lower = key.to_lowercase();
+    lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) || lower.starts_with(MINIO_ENCRYPTION_PREFIX)
+}
+
+fn rustfs_key(suffix: &str) -> String {
+    format!("{RUSTFS_PREFIX}{suffix}")
+}
+
+fn minio_key(suffix: &str) -> String {
+    format!("{MINIO_PREFIX}{suffix}")
+}
+
+/// Get header value: tries x-rustfs-{suffix} first, then x-minio-{suffix}. Case-insensitive.
+pub fn get_header<'a>(headers: &'a HeaderMap, suffix: &str) -> Option<Cow<'a, str>> {
+    let rk = rustfs_key(suffix);
+    let mk = minio_key(suffix);
+    headers
+        .get(&rk)
+        .or_else(|| headers.get(&mk))
+        .and_then(|v| v.to_str().ok().map(Cow::Borrowed))
+}
+
+/// Insert header with both x-rustfs-{suffix} and x-minio-{suffix}.
+pub fn insert_header(headers: &mut HeaderMap, suffix: &str, value: impl AsRef<[u8]>) {
+    if let Ok(v) = HeaderValue::from_bytes(value.as_ref()) {
+        if let Ok(k1) = rustfs_key(suffix).parse::<http::HeaderName>() {
+            headers.insert(k1, v.clone());
+        }
+        if let Ok(k2) = minio_key(suffix).parse::<http::HeaderName>() {
+            headers.insert(k2, v);
+        }
+    }
+}
+
+/// Get from HashMap: tries x-rustfs-{suffix} first, then x-minio-{suffix}.
+pub fn get_header_map(map: &std::collections::HashMap<String, String>, suffix: &str) -> Option<String> {
+    let rk = rustfs_key(suffix);
+    let mk = minio_key(suffix);
+    map.get(&rk).cloned().or_else(|| map.get(&mk).cloned())
+}
+
+/// Insert into HashMap with both x-rustfs-{suffix} and x-minio-{suffix}.
+pub fn insert_header_map(map: &mut std::collections::HashMap<String, String>, suffix: &str, value: impl Into<String>) {
+    let v = value.into();
+    map.insert(rustfs_key(suffix), v.clone());
+    map.insert(minio_key(suffix), v);
+}
+
+/// Remove from HashMap both x-rustfs-{suffix} and x-minio-{suffix}.
+pub fn remove_header_map(map: &mut std::collections::HashMap<String, String>, suffix: &str) {
+    map.remove(&rustfs_key(suffix));
+    map.remove(&minio_key(suffix));
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_is_encryption_metadata_key() {
+        assert!(is_encryption_metadata_key("x-rustfs-encryption-iv"));
+        assert!(is_encryption_metadata_key("X-Rustfs-Encryption-Key"));
+        assert!(is_encryption_metadata_key("x-minio-encryption-iv"));
+        assert!(!is_encryption_metadata_key("x-amz-meta-custom"));
+        assert!(!is_encryption_metadata_key("x-rustfs-internal-healing"));
+    }
+
+    #[test]
+    fn test_get_header() {
+        let mut headers = HeaderMap::new();
+        headers.insert("x-minio-force-delete", HeaderValue::from_static("true"));
+        assert_eq!(get_header(&headers, SUFFIX_FORCE_DELETE).as_deref(), Some("true"));
+
+        let mut headers2 = HeaderMap::new();
+        headers2.insert("X-Rustfs-Force-Delete", HeaderValue::from_static("true"));
+        assert_eq!(get_header(&headers2, SUFFIX_FORCE_DELETE).as_deref(), Some("true"));
+    }
+}

crates/utils/src/http/headers.rs

@@ -148,41 +148,9 @@ pub const AMZ_META_NAME: &str = "X-Amz-Meta-Name";
 
 pub const AMZ_META_UNENCRYPTED_CONTENT_LENGTH: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length";
 pub const AMZ_META_UNENCRYPTED_CONTENT_MD5: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5";
-pub const RUSTFS_ENCRYPTION: &str = "X-Rustfs-Encryption-";
-pub const RUSTFS_ENCRYPTION_LOWER: &str = "x-rustfs-encryption-";
-
-pub const RESERVED_METADATA_PREFIX: &str = "X-RustFS-Internal-";
-pub const RESERVED_METADATA_PREFIX_LOWER: &str = "x-rustfs-internal-";
-
-pub const RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing";
-// pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
-
-// pub const X_RUSTFS_INLINE_DATA: &str = "x-rustfs-inline-data";
-
-pub const VERSION_PURGE_STATUS_KEY: &str = "X-Rustfs-Internal-purgestatus";
-
-pub const X_RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing";
-pub const X_RUSTFS_DATA_MOV: &str = "X-Rustfs-Internal-data-mov";
 
 pub const AMZ_TAGGING_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
 
-pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
-
-pub const RUSTFS_FORCE_DELETE: &str = "X-Rustfs-Force-Delete";
-pub const RUSTFS_INCLUDE_DELETED: &str = "X-Rustfs-Include-Deleted";
-
-pub const RUSTFS_REPLICATION_RESET_STATUS: &str = "X-Rustfs-Replication-Reset-Status";
-pub const RUSTFS_REPLICATION_ACTUAL_OBJECT_SIZE: &str = "X-Rustfs-Replication-Actual-Object-Size";
-
-pub const RUSTFS_BUCKET_SOURCE_VERSION_ID: &str = "X-Rustfs-Source-Version-Id";
-pub const RUSTFS_BUCKET_SOURCE_MTIME: &str = "X-RustFS-Source-Mtime";
-pub const RUSTFS_BUCKET_SOURCE_ETAG: &str = "X-Rustfs-Source-Etag";
-pub const RUSTFS_BUCKET_REPLICATION_DELETE_MARKER: &str = "X-Rustfs-Source-DeleteMarker";
-pub const RUSTFS_BUCKET_REPLICATION_PROXY_REQUEST: &str = "X-Rustfs-Source-Proxy-Request";
-pub const RUSTFS_BUCKET_REPLICATION_REQUEST: &str = "X-Rustfs-Source-Replication-Request";
-pub const RUSTFS_BUCKET_REPLICATION_CHECK: &str = "X-Rustfs-Source-Replication-Check";
-pub const RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM: &str = "X-Rustfs-Source-Replication-Ssec-Crc";
-
 // SSEC encryption header constants
 pub const SSEC_ALGORITHM_HEADER: &str = "x-amz-server-side-encryption-customer-algorithm";
 pub const SSEC_KEY_HEADER: &str = "x-amz-server-side-encryption-customer-key";

crates/utils/src/http/metadata_compat.rs

@@ -0,0 +1,181 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! System metadata compatibility: write both x-rustfs-internal-* and x-minio-internal-*
+//! for MinIO interoperability. Read prefers RustFS, fallback to MinIO.
+
+use std::collections::HashMap;
+
+pub const RUSTFS_INTERNAL_PREFIX: &str = "x-rustfs-internal-";
+pub const MINIO_INTERNAL_PREFIX: &str = "x-minio-internal-";
+
+// Key suffixes (lowercase, no prefix)
+pub const SUFFIX_INLINE_DATA: &str = "inline-data";
+pub const SUFFIX_DATA_MOVED: &str = "data-moved";
+/// Transient flag for data movement
+pub const SUFFIX_DATA_MOV: &str = "data-mov";
+/// Transient flag for healing
+pub const SUFFIX_HEALING: &str = "healing";
+pub const SUFFIX_COMPRESSION: &str = "compression";
+pub const SUFFIX_COMPRESSION_SIZE: &str = "compression-size";
+pub const SUFFIX_ACTUAL_SIZE: &str = "actual-size";
+pub const SUFFIX_ACTUAL_OBJECT_SIZE: &str = "actual-object-size";
+/// Used by replication; key stored with capital A
+pub const SUFFIX_ACTUAL_OBJECT_SIZE_CAP: &str = "Actual-Object-Size";
+pub const SUFFIX_CRC: &str = "crc";
+pub const SUFFIX_TRANSITION_STATUS: &str = "transition-status";
+pub const SUFFIX_TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
+pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
+pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier";
+pub const SUFFIX_FREE_VERSION: &str = "free-version";
+pub const SUFFIX_PURGESTATUS: &str = "purgestatus";
+pub const SUFFIX_REPLICA_STATUS: &str = "replica-status";
+pub const SUFFIX_REPLICA_TIMESTAMP: &str = "replica-timestamp";
+pub const SUFFIX_REPLICATION_STATUS: &str = "replication-status";
+pub const SUFFIX_REPLICATION_TIMESTAMP: &str = "replication-timestamp";
+pub const SUFFIX_TAGGING_TIMESTAMP: &str = "tagging-timestamp";
+pub const SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP: &str = "objectlock-retention-timestamp";
+pub const SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP: &str = "objectlock-legalhold-timestamp";
+pub const SUFFIX_REPLICATION_RESET: &str = "replication-reset";
+/// Prefix for replication-reset-{arn} keys; use with internal_key_strip_suffix_prefix to extract arn.
+pub const SUFFIX_REPLICATION_RESET_ARN_PREFIX: &str = "replication-reset-";
+pub const SUFFIX_TIER_FV_ID: &str = "tier-free-versionID";
+pub const SUFFIX_TIER_FV_MARKER: &str = "tier-free-marker";
+pub const SUFFIX_TIER_SKIP_FV_ID: &str = "tier-skip-fvid";
+
+/// Returns true if the key is an internal metadata key (x-rustfs-internal-* or x-minio-internal-*)
+/// for xl.meta compatibility. Case-insensitive.
+pub fn is_internal_key(key: &str) -> bool {
+    let lower = key.to_lowercase();
+    lower.starts_with(RUSTFS_INTERNAL_PREFIX) || lower.starts_with(MINIO_INTERNAL_PREFIX)
+}
+
+/// Returns true if the key matches the given suffix for either x-rustfs-internal-* or x-minio-internal-*.
+pub fn has_internal_suffix(key: &str, suffix: &str) -> bool {
+    let lower = key.to_lowercase();
+    let rustfs_key = format!("{RUSTFS_INTERNAL_PREFIX}{suffix}");
+    let minio_key = format!("{MINIO_INTERNAL_PREFIX}{suffix}");
+    lower == rustfs_key || lower == minio_key
+}
+
+/// Strips x-rustfs-internal- or x-minio-internal- prefix from key. Returns the suffix part.
+/// Case-insensitive. Returns None if key is not an internal key.
+pub fn strip_internal_prefix(key: &str) -> Option<String> {
+    let lower = key.to_lowercase();
+    lower
+        .strip_prefix(RUSTFS_INTERNAL_PREFIX)
+        .or_else(|| lower.strip_prefix(MINIO_INTERNAL_PREFIX))
+        .map(|s| s.to_string())
+}
+
+/// Returns true if key is internal and its suffix part starts with the given suffix_prefix.
+/// E.g. internal_key_starts_with("x-rustfs-internal-replication-reset-arn1", "replication-reset") == true.
+pub fn internal_key_starts_with(key: &str, suffix_prefix: &str) -> bool {
+    strip_internal_prefix(key).is_some_and(|s| s.starts_with(suffix_prefix))
+}
+
+/// For keys like x-rustfs-internal-replication-reset-{arn}, strips the internal prefix and suffix_prefix,
+/// returning the remainder (e.g. "arn1"). Returns None if key does not match.
+pub fn internal_key_strip_suffix_prefix(key: &str, suffix_prefix: &str) -> Option<String> {
+    let rest = strip_internal_prefix(key)?;
+    rest.strip_prefix(suffix_prefix).map(|s| s.to_string())
+}
+
+fn both_keys(suffix: &str) -> (String, String) {
+    (format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"), format!("{MINIO_INTERNAL_PREFIX}{suffix}"))
+}
+
+/// Builds the RustFS internal key for the given suffix. Use when a single key is needed (e.g. for
+/// backward compat). Prefer insert_str/get_str when both keys should be written/read.
+pub fn internal_key_rustfs(suffix: &str) -> String {
+    format!("{RUSTFS_INTERNAL_PREFIX}{suffix}")
+}
+
+// === String type (FileInfo.metadata, user_defined) ===
+
+pub fn insert_str(map: &mut HashMap<String, String>, suffix: &str, value: String) {
+    let (k1, k2) = both_keys(suffix);
+    map.insert(k1, value.clone());
+    map.insert(k2, value);
+}
+
+pub fn get_str(map: &HashMap<String, String>, suffix: &str) -> Option<String> {
+    let (k1, k2) = both_keys(suffix);
+    map.get(&k1).cloned().or_else(|| map.get(&k2).cloned())
+}
+
+pub fn contains_key_str(map: &HashMap<String, String>, suffix: &str) -> bool {
+    let (k1, k2) = both_keys(suffix);
+    map.contains_key(&k1) || map.contains_key(&k2)
+}
+
+pub fn remove_str(map: &mut HashMap<String, String>, suffix: &str) {
+    let (k1, k2) = both_keys(suffix);
+    map.remove(&k1);
+    map.remove(&k2);
+}
+
+// === Vec<u8> type (meta_sys) ===
+
+pub fn insert_bytes(map: &mut HashMap<String, Vec<u8>>, suffix: &str, value: Vec<u8>) {
+    let (k1, k2) = both_keys(suffix);
+    let v = value.clone();
+    map.insert(k1, value);
+    map.insert(k2, v);
+}
+
+pub fn get_bytes(map: &HashMap<String, Vec<u8>>, suffix: &str) -> Option<Vec<u8>> {
+    let (k1, k2) = both_keys(suffix);
+    map.get(&k1).cloned().or_else(|| map.get(&k2).cloned())
+}
+
+pub fn contains_key_bytes(map: &HashMap<String, Vec<u8>>, suffix: &str) -> bool {
+    let (k1, k2) = both_keys(suffix);
+    map.contains_key(&k1) || map.contains_key(&k2)
+}
+
+pub fn remove_bytes(map: &mut HashMap<String, Vec<u8>>, suffix: &str) {
+    let (k1, k2) = both_keys(suffix);
+    map.remove(&k1);
+    map.remove(&k2);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_is_internal_key() {
+        assert!(is_internal_key("x-rustfs-internal-healing"));
+        assert!(is_internal_key("x-rustfs-internal-purgestatus"));
+        assert!(is_internal_key("X-RustFS-Internal-purgestatus"));
+        assert!(is_internal_key("x-minio-internal-compression"));
+        assert!(is_internal_key("x-minio-internal-replication-status"));
+        assert!(is_internal_key("X-Minio-Internal-Compression"));
+        assert!(!is_internal_key("x-amz-meta-custom"));
+        assert!(!is_internal_key("content-type"));
+        assert!(!is_internal_key("x-rustfs-meta-custom"));
+    }
+
+    #[test]
+    fn test_has_internal_suffix() {
+        assert!(has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_PURGESTATUS));
+        assert!(has_internal_suffix("X-Minio-Internal-purgestatus", SUFFIX_PURGESTATUS));
+        assert!(has_internal_suffix("x-minio-internal-compression", SUFFIX_COMPRESSION));
+        assert!(has_internal_suffix("x-rustfs-internal-healing", SUFFIX_HEALING));
+        assert!(has_internal_suffix("x-minio-internal-data-mov", SUFFIX_DATA_MOV));
+        assert!(!has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_HEALING));
+        assert!(!has_internal_suffix("x-amz-meta-custom", SUFFIX_PURGESTATUS));
+    }
+}

crates/utils/src/http/mod.rs

@@ -12,7 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod header_compat;
 pub mod headers;
 pub mod ip;
+pub mod metadata_compat;
+pub use header_compat::*;
 pub use headers::*;
 pub use ip::*;
+pub use metadata_compat::*;