Merge remote-tracking branch 'origin/main' into cxymds/fix-rebalance-multipart-retry

# Conflicts:
#	crates/ecstore/src/set_disk/ops/object.rs
#	crates/ecstore/src/set_disk/read.rs

crates/utils/src/http/header_compat.rs

@@ -17,14 +17,21 @@
 //!
 //! Use suffix-based API: `get_header(headers, SUFFIX_FORCE_DELETE)` queries both
 //! x-rustfs-force-delete and x-minio-force-delete.
+//!
+//! This module is the canonical owner of these interop values. One deliberate
+//! copy exists: `crates/replication/src/http.rs` re-declares the subset it
+//! needs because the wire-contract crate must stay free of internal
+//! dependencies (arch guard in `scripts/check_architecture_migration_rules.sh`
+//! bans replication -> rustfs-utils). When changing a value here, check the
+//! pinned copy there; its tests pin the shared wire values byte-for-byte.
 
 use http::{HeaderMap, HeaderValue};
 use std::borrow::Cow;
 
 const RUSTFS_PREFIX: &str = "x-rustfs-";
 const MINIO_PREFIX: &str = "x-minio-";
-const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
-const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
+pub const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
+pub const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
 const MINIO_INTERNAL_ENCRYPTION_PREFIX: &str = "x-minio-internal-server-side-encryption-";
 const MINIO_INTERNAL_ENCRYPTED_MULTIPART: &str = "x-minio-internal-encrypted-multipart";
 const RUSTFS_ENCRYPTION_ORIGINAL_SIZE: &str = super::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER;

crates/utils/src/http/headers.rs

@@ -156,7 +156,11 @@ pub const REQUEST_ID_HEADER: &str = "x-request-id";
 pub const AMZ_REQUEST_ID: &str = "x-amz-request-id";
 pub const AMZ_REQUEST_HOST_ID: &str = "x-amz-id-2";
 
-// Content Checksums
+// Content Checksums. The standard five x-amz-checksum-* names also exist in
+// the zero-internal-dependency rustfs-checksums leaf crate
+// (crates/checksums/src/http.rs, which additionally owns the RustFS
+// extension names); values are pinned by the S3 wire protocol — keep both
+// sides in sync (backlog#1833).
 pub const AMZ_CHECKSUM_ALGO: &str = "x-amz-checksum-algorithm";
 pub const AMZ_CHECKSUM_CRC32: &str = "x-amz-checksum-crc32";
 pub const AMZ_CHECKSUM_CRC32C: &str = "x-amz-checksum-crc32c";

crates/utils/src/http/metadata_compat.rs

@@ -14,6 +14,13 @@
 
 //! System metadata compatibility: write both x-rustfs-internal-* and x-minio-internal-*
 //! for MinIO interoperability. Read prefers RustFS, fallback to MinIO.
+//!
+//! This module is the canonical owner of these interop values. One deliberate
+//! copy exists: `crates/replication/src/http.rs` re-declares the subset it
+//! needs because the wire-contract crate must stay free of internal
+//! dependencies (arch guard in `scripts/check_architecture_migration_rules.sh`
+//! bans replication -> rustfs-utils). When changing a value here, check the
+//! pinned copy there; its tests pin the shared wire values byte-for-byte.
 
 use std::collections::{BTreeMap, HashMap};
 

crates/utils/src/http/object_encryption_keys.rs

@@ -30,6 +30,13 @@ use super::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER};
 pub const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
 pub const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key";
 pub const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
+/// Carries the AEAD algorithm the object was sealed with.
+///
+/// The S3 `x-amz-server-side-encryption` header records the *SSE mode*
+/// (`AES256` / `aws:kms`), not the cipher, so it cannot round-trip
+/// `ChaCha20Poly1305`. Without this header a ChaCha-sealed object comes back
+/// from the projection claiming `aws:kms` and is then opened with the wrong
+/// cipher.
 pub const INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "x-rustfs-encryption-algorithm";
 pub const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size";
 pub const INTERNAL_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context";
@@ -45,6 +52,15 @@ pub const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-
 pub const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key";
 pub const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context";
 
+/// Reserved RustFS-branded twin of the MinIO-internal SSE key family.
+///
+/// No RustFS writer emits these keys today — the SSE writer persists the
+/// MinIO-branded `X-Minio-Internal-Server-Side-Encryption-*` keys verbatim for
+/// interoperability — but redaction (`rustfs_filemeta`) and replication
+/// stripping treat the family as sensitive so that a future or third-party
+/// writer cannot leak sealed material through the reserved names.
+pub const RUSTFS_INTERNAL_ENCRYPTION_PREFIX: &str = "x-rustfs-internal-server-side-encryption-";
+
 pub const REPLICATION_SSEC_ALGORITHM_HEADER: &str = "X-Rustfs-Replication-Ssec-Algorithm";
 pub const REPLICATION_SSEC_KEY_MD5_HEADER: &str = "X-Rustfs-Replication-Ssec-Key-Md5";
 pub const REPLICATION_SSEC_ORIGINAL_SIZE_HEADER: &str = "X-Rustfs-Replication-Ssec-Original-Size";
@@ -125,13 +141,14 @@ pub fn ssec_replication_transport_header(stored_key: &str) -> Option<&'static st
 /// SSE-C material. SSE-C passthrough re-adds its keys through the transport
 /// mapping instead.
 pub fn is_replication_stripped_encryption_key(key: &str) -> bool {
-    // The dual-key invariant writes an x-rustfs-internal- twin next to every
-    // x-minio-internal- SSE key; cover it here so this predicate is safe to
-    // use standalone, without an is_internal_key backstop.
+    // The x-rustfs-internal- SSE prefix is a reserved name family with no
+    // writer today (see RUSTFS_INTERNAL_ENCRYPTION_PREFIX); cover it here so
+    // this predicate is safe to use standalone, without an is_internal_key
+    // backstop.
     super::is_encryption_metadata_key(key)
         || super::is_sse_header(key)
         || key.eq_ignore_ascii_case(SSEC_ORIGINAL_SIZE_HEADER)
-        || super::starts_with_ignore_ascii_case(key, "x-rustfs-internal-server-side-encryption-")
+        || super::starts_with_ignore_ascii_case(key, RUSTFS_INTERNAL_ENCRYPTION_PREFIX)
 }
 
 #[cfg(test)]

crates/utils/src/path.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::borrow::Cow;
 use std::path::Component;
 use std::path::Path;
 use std::path::PathBuf;
@@ -44,14 +45,19 @@ pub fn has_suffix(s: &str, suffix: &str) -> bool {
 /// If the object name ends with a slash, it is considered a directory object.
 /// The trailing slash is removed and `GLOBAL_DIR_SUFFIX` is appended.
 /// If it does not end with a slash, the name is returned as is.
-pub fn encode_dir_object(object: &str) -> String {
+pub fn encode_dir_object_ref(object: &str) -> Cow<'_, str> {
     if has_suffix(object, SLASH_SEPARATOR) {
-        format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX)
+        Cow::Owned(format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX))
     } else {
-        object.to_string()
+        Cow::Borrowed(object)
     }
 }
 
+/// Owned compatibility wrapper for callers that retain or mutate the encoded name.
+pub fn encode_dir_object(object: &str) -> String {
+    encode_dir_object_ref(object).into_owned()
+}
+
 /// Checks if the given object name represents a directory object.
 ///
 /// Returns true if the object name ends with `GLOBAL_DIR_SUFFIX`.
@@ -438,6 +444,12 @@ impl LazyBuf {
 /// The returned path ends in a slash only if it represents a root directory, such as `/` on Unix or `C:/` on Windows.
 ///
 /// If the result of this process is an empty string, `clean` returns the string `.`.
+///
+/// Note: `crates/policy/src/policy/utils/path.rs` deliberately keeps its own
+/// slash-only Go `path.Clean` port instead of using this function — S3
+/// ARN/resource matching must not treat backslashes as separators, and this
+/// Windows-aware version would change policy evaluation semantics on Windows.
+/// Do not consolidate the two (backlog#1833).
 pub fn clean(path: &str) -> String {
     if path.is_empty() {
         return ".".to_string();
@@ -602,6 +614,17 @@ mod tests {
     use super::*;
     use proptest::prelude::*;
 
+    #[test]
+    fn encode_dir_object_ref_borrows_objects_and_encodes_directories() {
+        let object = "prefix/object";
+        let encoded = encode_dir_object_ref(object);
+        assert!(matches!(encoded, Cow::Borrowed(value) if value == object));
+
+        let encoded = encode_dir_object_ref("prefix/directory/");
+        assert!(matches!(encoded, Cow::Owned(ref value) if value == "prefix/directory__XLDIR__"));
+        assert_eq!(encode_dir_object("prefix/directory/"), encoded);
+    }
+
     #[test]
     fn test_trim_etag() {
         // Test with quoted ETag

crates/utils/src/retry.rs

@@ -101,6 +101,56 @@ impl Stream for RetryTimer {
     }
 }
 
+/// Drives `operation` with capped, jittered exponential backoff, returning the
+/// first success or the last error once `max_attempts` attempts are exhausted.
+///
+/// The sleep before retry `n` (1-based) is `min(base_delay * 2^(n-1), max_delay)`,
+/// reduced by up to half through a cheap clock-derived jitter so concurrent
+/// retriers decorrelate — the same backoff shape as [`RetryTimer`] without
+/// needing a caller-supplied random seed or the Stream API. `max_attempts` is
+/// clamped to at least 1.
+pub async fn retry_with_backoff<F, Fut, T, E>(
+    mut operation: F,
+    max_attempts: usize,
+    base_delay: Duration,
+    max_delay: Duration,
+) -> Result<T, E>
+where
+    F: FnMut() -> Fut,
+    Fut: std::future::Future<Output = Result<T, E>>,
+{
+    let max_attempts = max_attempts.max(1);
+    let mut last_err = None;
+
+    for attempt in 0..max_attempts {
+        match operation().await {
+            Ok(value) => return Ok(value),
+            Err(err) => {
+                last_err = Some(err);
+                if attempt + 1 < max_attempts {
+                    // Cap the shift so the multiplier cannot overflow; the cap
+                    // below bounds the result anyway.
+                    let exp = base_delay.saturating_mul(1u32 << attempt.min(16));
+                    let mut sleep_duration = exp.min(max_delay);
+                    // Up to 50% reduction, derived from the clock's sub-second
+                    // nanoseconds — cheap decorrelation without a rand dependency.
+                    let nanos = std::time::SystemTime::now()
+                        .duration_since(std::time::UNIX_EPOCH)
+                        .map(|d| d.subsec_nanos())
+                        .unwrap_or(0);
+                    let reduction_percent = u64::from(nanos % 50);
+                    let sleep_ms = sleep_duration.as_millis() as u64;
+                    let jittered_ms = sleep_ms.saturating_sub(sleep_ms * reduction_percent / 100).max(1);
+                    sleep_duration = Duration::from_millis(jittered_ms);
+                    tokio::time::sleep(sleep_duration).await;
+                }
+            }
+        }
+    }
+
+    Err(last_err.expect("max_attempts is clamped to at least 1, so at least one attempt ran"))
+}
+
 static RETRYABLE_S3CODES: LazyLock<Vec<String>> = LazyLock::new(|| {
     vec![
         "RequestError".to_string(),
@@ -241,6 +291,87 @@ mod tests {
         assert!(!is_s3code_in_message_retryable(""));
     }
 
+    #[tokio::test]
+    async fn retry_with_backoff_returns_first_success_without_retrying() {
+        let mut calls = 0;
+        let result: Result<i32, std::io::Error> = retry_with_backoff(
+            || {
+                calls += 1;
+                async { Ok(42) }
+            },
+            3,
+            Duration::from_millis(1),
+            Duration::from_millis(2),
+        )
+        .await;
+
+        assert_eq!(result.expect("first attempt succeeds"), 42);
+        assert_eq!(calls, 1, "a success must not trigger further attempts");
+    }
+
+    #[tokio::test]
+    async fn retry_with_backoff_retries_until_success() {
+        let mut calls = 0;
+        let result: Result<i32, std::io::Error> = retry_with_backoff(
+            || {
+                calls += 1;
+                let attempt = calls;
+                async move {
+                    if attempt < 3 {
+                        Err(std::io::Error::other("transient"))
+                    } else {
+                        Ok(7)
+                    }
+                }
+            },
+            5,
+            Duration::from_millis(1),
+            Duration::from_millis(2),
+        )
+        .await;
+
+        assert_eq!(result.expect("third attempt succeeds"), 7);
+        assert_eq!(calls, 3);
+    }
+
+    #[tokio::test]
+    async fn retry_with_backoff_exhausts_attempts_and_returns_last_error() {
+        let mut calls = 0;
+        let result: Result<(), std::io::Error> = retry_with_backoff(
+            || {
+                calls += 1;
+                let attempt = calls;
+                async move { Err(std::io::Error::other(format!("attempt {attempt}"))) }
+            },
+            3,
+            Duration::from_millis(1),
+            Duration::from_millis(2),
+        )
+        .await;
+
+        let err = result.expect_err("all attempts fail");
+        assert_eq!(err.to_string(), "attempt 3", "the LAST error must be returned");
+        assert_eq!(calls, 3);
+    }
+
+    #[tokio::test]
+    async fn retry_with_backoff_clamps_zero_attempts_to_one() {
+        let mut calls = 0;
+        let result: Result<(), std::io::Error> = retry_with_backoff(
+            || {
+                calls += 1;
+                async { Err(std::io::Error::other("always")) }
+            },
+            0,
+            Duration::from_millis(1),
+            Duration::from_millis(2),
+        )
+        .await;
+
+        assert!(result.is_err());
+        assert_eq!(calls, 1, "zero attempts clamps to a single attempt instead of panicking");
+    }
+
     #[test]
     fn is_s3code_in_message_retryable_is_case_sensitive() {
         // Pin the contract: a backend that down-cases its error