fix(s3): normalize root double-slash ListBuckets requests (#4336)

test(s3): MinIO parity for double-slash ListBuckets, invalid copy-source date, region-aware CreateBucket

- Add DoubleSlashListBucketsCompatLayer: rewrite `GET //` to `GET /` so
  it routes to ListBuckets, matching MinIO browser-client behavior (#601)
- Add e2e regression coverage for invalid copy-source conditional date
  headers (#618) and region-aware MakeBucket(us-east-1) (#629)

Refs rustfs/backlog#601 #618 #629

crates/e2e_test/src/copy_source_invalid_date_test.rs

@@ -0,0 +1,142 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression coverage for rustfs/backlog#618 item 8: `CopyObject` with an
+//! unparsable `x-amz-copy-source-if-unmodified-since` value.
+//!
+//! `minio-js` `CopyConditions.setUnmodified(new Date(undefined))` serialises the
+//! header value as the literal string `"Invalid Date"` (via `Date.toUTCString()`)
+//! and signs it. MinIO parses this header from the raw request and *ignores* it
+//! when it fails to parse (`amztime.ParseHeader` error is swallowed), so the copy
+//! proceeds.
+//!
+//! RustFS routes S3 through the `s3s` crate, which models the copy-source
+//! conditional-date headers as typed `Timestamp` fields and rejects an
+//! unparsable value with `400 InvalidArgument` during request deserialization —
+//! this happens *after* SigV4 verification but *before* the RustFS `CopyObject`
+//! handler runs. Because the header is part of the client's `SignedHeaders`, the
+//! value cannot be stripped or rewritten by a pre-`s3s` layer without breaking the
+//! signature, and `s3s` exposes no hook between auth and typed deserialization.
+//! Matching MinIO's tolerant behavior therefore requires a change in `s3s`
+//! (lenient parsing of conditional-date headers). See the PR description for
+//! details.
+//!
+//! This test pins the two ends of the current behavior:
+//!   * a *valid* HTTP-date copy-source conditional header is accepted and the copy
+//!     succeeds (the common, real-SDK path), and
+//!   * an *invalid* date value is rejected with `400 InvalidArgument` at the `s3s`
+//!     layer (the documented gap). When `s3s` gains lenient parsing, the second
+//!     assertion should be flipped to expect a successful copy.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+    use aws_sdk_s3::primitives::ByteStream;
+    use http::header::HOST;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
+    use serial_test::serial;
+    use std::error::Error;
+
+    /// Signed raw `PUT` copy request with an explicit copy-source conditional
+    /// header. The header is included in the SigV4 signature (as real SDKs do).
+    async fn signed_copy(
+        base_url: &str,
+        dst_path: &str,
+        copy_source: &str,
+        if_unmodified_since: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
+        let url = format!("{base_url}{dst_path}");
+        let uri = url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::PUT)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+            .header("x-amz-copy-source", copy_source)
+            .header("x-amz-copy-source-if-unmodified-since", if_unmodified_since);
+        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut rb = client.request(reqwest::Method::PUT, url);
+        for (name, value) in signed.headers() {
+            rb = rb.header(name, value);
+        }
+        let resp = rb.send().await?;
+        let status = resp.status();
+        let body = resp.text().await?;
+        Ok((status, body))
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_copy_source_if_unmodified_since_valid_and_invalid() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        let bucket = "copy-cond-bucket";
+        client.create_bucket().bucket(bucket).send().await?;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("src.txt")
+            .body(ByteStream::from_static(b"hello"))
+            .send()
+            .await?;
+
+        let copy_source = format!("/{bucket}/src.txt");
+
+        // A valid future HTTP-date is accepted by s3s and the copy succeeds.
+        let (status, body) = signed_copy(
+            &env.url,
+            &format!("/{bucket}/dst-valid.txt"),
+            &copy_source,
+            "Wed, 01 Jan 2031 00:00:00 GMT",
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await?;
+        assert_eq!(status, reqwest::StatusCode::OK, "valid copy-source date must succeed: {body}");
+        assert!(body.contains("CopyObjectResult"), "expected CopyObjectResult XML: {body}");
+
+        // An unparsable date ("Invalid Date") is currently rejected by s3s at the
+        // deserialization layer with 400 InvalidArgument. This documents the s3s
+        // limitation described in backlog#618 item 8. When s3s parses these headers
+        // leniently (matching MinIO), this should become a successful copy.
+        let (status, body) = signed_copy(
+            &env.url,
+            &format!("/{bucket}/dst-invalid.txt"),
+            &copy_source,
+            "Invalid Date",
+            &env.access_key,
+            &env.secret_key,
+        )
+        .await?;
+        assert_eq!(
+            status,
+            reqwest::StatusCode::BAD_REQUEST,
+            "invalid copy-source date currently rejected by s3s (backlog#618): {body}"
+        );
+        assert!(body.contains("InvalidArgument"), "expected InvalidArgument for invalid date: {body}");
+
+        env.stop_server();
+        Ok(())
+    }
+}

crates/e2e_test/src/create_bucket_region_test.rs

@@ -0,0 +1,82 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for rustfs/backlog#629(b): region-aware `CreateBucket`.
+//!
+//! `minio-go`'s `MakeBucket(bucket, "us-east-1")` was reported to fail SigV4
+//! validation. The SigV4 signature is verified by `s3s` over the raw request
+//! body bytes (compared against `x-amz-content-sha256`); a
+//! `CreateBucketConfiguration`/`LocationConstraint` body therefore does not
+//! change signature canonicalization. This test proves that both a
+//! region-body `CreateBucket` and a plain `CreateBucket` succeed under SigV4.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration};
+    use serial_test::serial;
+    use std::error::Error;
+
+    /// `CreateBucket` with a `LocationConstraint` body must pass SigV4 validation
+    /// and create the bucket, mirroring `minio-go` `MakeBucket(bucket, "us-east-1")`.
+    #[tokio::test]
+    #[serial]
+    async fn test_create_bucket_with_us_east_1_location_constraint() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+        let client = env.create_s3_client();
+
+        client
+            .create_bucket()
+            .bucket("region-aware-bucket")
+            .create_bucket_configuration(
+                CreateBucketConfiguration::builder()
+                    .location_constraint(BucketLocationConstraint::from("us-east-1"))
+                    .build(),
+            )
+            .send()
+            .await
+            .expect("region-aware CreateBucket must pass SigV4 validation and succeed");
+
+        // The bucket exists and is listed.
+        let listed = client.list_buckets().send().await?;
+        let names: Vec<&str> = listed.buckets().iter().filter_map(|b| b.name()).collect();
+        assert!(names.contains(&"region-aware-bucket"), "created bucket must be listed: {names:?}");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    /// A plain `CreateBucket` (no body) must also succeed; guards against a
+    /// regression where an empty body would be hashed incorrectly during SigV4.
+    #[tokio::test]
+    #[serial]
+    async fn test_create_bucket_without_location_constraint() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+        let client = env.create_s3_client();
+
+        client
+            .create_bucket()
+            .bucket("plain-region-bucket")
+            .send()
+            .await
+            .expect("plain CreateBucket must succeed");
+
+        env.stop_server();
+        Ok(())
+    }
+}

crates/e2e_test/src/lib.rs

@@ -175,4 +175,16 @@ mod admin_timeout_regression_test;
 #[cfg(test)]
 mod overwrite_cleanup_regression_test;
 
+// Regression test for backlog#601: `GET //` ListBuckets browser compatibility.
+#[cfg(test)]
+mod list_buckets_double_slash_test;
+
+// Regression test for backlog#629(b): region-aware CreateBucket SigV4.
+#[cfg(test)]
+mod create_bucket_region_test;
+
+// Regression coverage for backlog#618 item 8: copy-source invalid-date header.
+#[cfg(test)]
+mod copy_source_invalid_date_test;
+
 pub mod tls_gen;

crates/e2e_test/src/list_buckets_double_slash_test.rs

@@ -0,0 +1,137 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regression test for rustfs/backlog#601: `GET //` ListBuckets browser compat.
+//!
+//! The AWS S3 browser (SigV4) concatenates the endpoint with a leading slash and
+//! emits `GET //` for `ListBuckets`. The `s3s` path parser rejects `//` with
+//! `InvalidBucketName` (empty bucket) before routing. `DoubleSlashListBucketsCompatLayer`
+//! rewrites `GET //` to `GET /` up front so it routes to `ListBuckets`, matching
+//! MinIO's explicit `//` route.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+    use http::header::HOST;
+    use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+    use rustfs_signer::sign_v4;
+    use s3s::Body;
+    use serial_test::serial;
+    use std::error::Error;
+
+    /// Sends a SigV4-signed `GET` where the signature is computed over `sign_path`
+    /// but the on-the-wire request target is `wire_path`. Returns (status, body).
+    async fn signed_get(
+        base_url: &str,
+        sign_path: &str,
+        wire_path: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
+        let sign_url = format!("{base_url}{sign_path}");
+        let uri = sign_url.parse::<http::Uri>()?;
+        let authority = uri.authority().ok_or("missing authority")?.to_string();
+        let request = http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+        let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
+
+        let client = local_http_client();
+        let mut rb = client.request(reqwest::Method::GET, format!("{base_url}{wire_path}"));
+        for (name, value) in signed.headers() {
+            rb = rb.header(name, value);
+        }
+        let resp = rb.send().await?;
+        let status = resp.status();
+        let body = resp.text().await?;
+        Ok((status, body))
+    }
+
+    fn assert_list_buckets_ok(status: reqwest::StatusCode, body: &str, label: &str) {
+        assert_eq!(status, reqwest::StatusCode::OK, "{label}: expected 200, body: {body}");
+        assert!(body.contains("ListAllMyBucketsResult"), "{label}: expected ListBuckets XML, body: {body}");
+    }
+
+    /// `GET /` (path-style service call) returns `ListBuckets`.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_buckets_single_slash() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let (status, body) = signed_get(&env.url, "/", "/", &env.access_key, &env.secret_key).await?;
+        assert_list_buckets_ok(status, &body, "GET /");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    /// `GET //` returns `ListBuckets` for the S3 browser SigV4 case: the client
+    /// concatenates the endpoint with a leading `/` so the request target becomes
+    /// `//`, while the signature was computed over the canonical `/` path. The
+    /// compat layer rewrites `//` to `/` before `s3s` parses/verifies the request,
+    /// so both routing and signature verification operate on `/`.
+    #[tokio::test]
+    #[serial]
+    async fn test_list_buckets_double_slash_browser_compat() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let (status, body) = signed_get(&env.url, "/", "//", &env.access_key, &env.secret_key).await?;
+        assert_list_buckets_ok(status, &body, "GET // (signature over /)");
+
+        env.stop_server();
+        Ok(())
+    }
+
+    /// The rewrite is confined to a request target of exactly `//`. A normal
+    /// path-style bucket listing (`GET /bucket`) must keep working (no routing
+    /// regression), and a leading double slash before a bucket name
+    /// (`GET //bucket`) must be left untouched by the compat layer — it is not a
+    /// `ListBuckets` request and s3s continues to reject the empty bucket name.
+    #[tokio::test]
+    #[serial]
+    async fn test_double_slash_rewrite_is_narrowly_scoped() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server(vec![]).await?;
+
+        let client = env.create_s3_client();
+        client.create_bucket().bucket("compat-bucket").send().await?;
+
+        // `GET /compat-bucket` still lists the bucket (routing unchanged).
+        let (status, body) = signed_get(&env.url, "/compat-bucket", "/compat-bucket", &env.access_key, &env.secret_key).await?;
+        assert_eq!(status, reqwest::StatusCode::OK, "GET /compat-bucket body: {body}");
+        assert!(
+            body.contains("ListBucketResult") && !body.contains("ListAllMyBucketsResult"),
+            "GET /compat-bucket must be a bucket listing: {body}"
+        );
+
+        // `GET //compat-bucket` is not rewritten (path is not exactly `//`); s3s
+        // still rejects the empty leading bucket segment with InvalidBucketName.
+        let (status, body) = signed_get(&env.url, "//compat-bucket", "//compat-bucket", &env.access_key, &env.secret_key).await?;
+        assert_eq!(status, reqwest::StatusCode::BAD_REQUEST, "GET //compat-bucket body: {body}");
+        assert!(
+            body.contains("InvalidBucketName"),
+            "GET //compat-bucket must remain InvalidBucketName (not rewritten): {body}"
+        );
+
+        env.stop_server();
+        Ok(())
+    }
+}