diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 1e7b9f7a6..3e4d0bb22 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -23,7 +23,7 @@ use crate::storage::head_prefix::{head_prefix_not_found_message, probe_prefix_ha use crate::storage::helper::OperationHelper; use crate::storage::options::{filter_object_metadata, get_content_sha256}; use crate::storage::readers::InMemoryAsyncReader; -use crate::storage::s3_api::bucket::build_list_objects_output; +use crate::storage::s3_api::bucket::{build_list_objects_output, build_list_objects_v2_output}; use crate::storage::sse::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, check_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, @@ -151,7 +151,6 @@ use tokio_stream::wrappers::ReceiverStream; use tokio_tar::Archive; use tokio_util::io::{ReaderStream, StreamReader}; use tracing::{debug, error, info, instrument, warn}; -use urlencoding::encode; use uuid::Uuid; macro_rules! try_ { @@ -3801,84 +3800,18 @@ impl S3 for FS { .await .map_err(ApiError::from)?; - // warn!("object_infos objects {:?}", object_infos.objects); - - // Apply URL encoding if encoding_type is "url" - // Note: S3 URL encoding should encode special characters but preserve path separators (/) - let should_encode = encoding_type.as_ref().map(|e| e.as_str() == "url").unwrap_or(false); - - // Helper function to encode S3 keys/prefixes (preserving /) - // S3 URL encoding encodes special characters but keeps '/' unencoded - let encode_s3_name = |name: &str| -> String { - name.split('/') - .map(|part| encode(part).to_string()) - .collect::>() - .join("/") - }; - - let objects: Vec = object_infos - .objects - .iter() - .filter(|v| !v.name.is_empty()) - .map(|v| { - let key = if should_encode { - encode_s3_name(&v.name) - } else { - v.name.to_owned() - }; - let mut obj = Object { - key: Some(key), - last_modified: v.mod_time.map(Timestamp::from), - size: Some(v.get_actual_size().unwrap_or_default()), - e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)), - storage_class: v.storage_class.clone().map(ObjectStorageClass::from), - ..Default::default() - }; - - if fetch_owner.is_some_and(|v| v) { - obj.owner = Some(Owner { - display_name: Some("rustfs".to_owned()), - id: Some("v0.1".to_owned()), - }); - } - obj - }) - .collect(); - - let common_prefixes: Vec = object_infos - .prefixes - .into_iter() - .map(|v| { - let prefix = if should_encode { encode_s3_name(&v) } else { v }; - CommonPrefix { prefix: Some(prefix) } - }) - .collect(); - - // KeyCount should include both objects and common prefixes per S3 API spec - let key_count = (objects.len() + common_prefixes.len()) as i32; - - // Encode next_continuation_token to base64 - let next_continuation_token = object_infos - .next_continuation_token - .map(|token| base64_simd::STANDARD.encode_to_string(token.as_bytes())); - - let output = ListObjectsV2Output { - is_truncated: Some(object_infos.is_truncated), - continuation_token: response_continuation_token, - next_continuation_token, - start_after: response_start_after, - key_count: Some(key_count), - max_keys: Some(max_keys), - contents: Some(objects), + let output = build_list_objects_v2_output( + object_infos, + fetch_owner.unwrap_or_default(), + max_keys, + bucket, + prefix, delimiter, - encoding_type: encoding_type.clone(), - name: Some(bucket), - prefix: Some(prefix), - common_prefixes: Some(common_prefixes), - ..Default::default() - }; + encoding_type, + response_continuation_token, + response_start_after, + ); - // let output = ListObjectsV2Output { ..Default::default() }; Ok(S3Response::new(output)) } diff --git a/rustfs/src/storage/s3_api/bucket.rs b/rustfs/src/storage/s3_api/bucket.rs index 19fa3b1b9..ba73776c2 100644 --- a/rustfs/src/storage/s3_api/bucket.rs +++ b/rustfs/src/storage/s3_api/bucket.rs @@ -12,7 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use s3s::dto::{ListObjectsOutput, ListObjectsV2Output}; +use rustfs_ecstore::client::object_api_utils::to_s3s_etag; +use rustfs_ecstore::store_api::ListObjectsV2Info; +use s3s::dto::{ + CommonPrefix, EncodingType, ListObjectsOutput, ListObjectsV2Output, Object, ObjectStorageClass, Owner, Timestamp, +}; +use urlencoding::encode; pub(crate) fn build_list_objects_output(v2: ListObjectsV2Output, request_marker: Option) -> ListObjectsOutput { let next_marker = calculate_next_marker(&v2); @@ -36,6 +41,93 @@ pub(crate) fn build_list_objects_output(v2: ListObjectsV2Output, request_marker: } } +#[allow(clippy::too_many_arguments)] +pub(crate) fn build_list_objects_v2_output( + object_infos: ListObjectsV2Info, + fetch_owner: bool, + max_keys: i32, + bucket: String, + prefix: String, + delimiter: Option, + encoding_type: Option, + response_continuation_token: Option, + response_start_after: Option, +) -> ListObjectsV2Output { + // Apply URL encoding if encoding_type is "url". + // S3 URL encoding encodes special characters but keeps '/' unencoded. + let should_encode = encoding_type.as_ref().is_some_and(|e| e.as_str() == EncodingType::URL); + + let encode_s3_name = |name: &str| -> String { + name.split('/') + .map(|part| encode(part).to_string()) + .collect::>() + .join("/") + }; + + let objects: Vec = object_infos + .objects + .iter() + .filter(|v| !v.name.is_empty()) + .map(|v| { + let key = if should_encode { + encode_s3_name(&v.name) + } else { + v.name.to_owned() + }; + let mut obj = Object { + key: Some(key), + last_modified: v.mod_time.map(Timestamp::from), + size: Some(v.get_actual_size().unwrap_or_default()), + e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)), + storage_class: v.storage_class.clone().map(ObjectStorageClass::from), + ..Default::default() + }; + + if fetch_owner { + obj.owner = Some(Owner { + display_name: Some("rustfs".to_owned()), + id: Some("v0.1".to_owned()), + }); + } + + obj + }) + .collect(); + + let common_prefixes: Vec = object_infos + .prefixes + .into_iter() + .map(|v| { + let prefix = if should_encode { encode_s3_name(&v) } else { v }; + CommonPrefix { prefix: Some(prefix) } + }) + .collect(); + + // KeyCount should include both objects and common prefixes per S3 API spec. + let key_count = (objects.len() + common_prefixes.len()) as i32; + + // Encode next_continuation_token to base64. + let next_continuation_token = object_infos + .next_continuation_token + .map(|token| base64_simd::STANDARD.encode_to_string(token.as_bytes())); + + ListObjectsV2Output { + is_truncated: Some(object_infos.is_truncated), + continuation_token: response_continuation_token, + next_continuation_token, + start_after: response_start_after, + key_count: Some(key_count), + max_keys: Some(max_keys), + contents: Some(objects), + delimiter, + encoding_type, + name: Some(bucket), + prefix: Some(prefix), + common_prefixes: Some(common_prefixes), + ..Default::default() + } +} + fn calculate_next_marker(v2: &ListObjectsV2Output) -> Option { // For ListObjects (v1) API, NextMarker should be the last item returned when truncated. // When both Contents and CommonPrefixes are present, NextMarker should be the @@ -76,8 +168,9 @@ fn calculate_next_marker(v2: &ListObjectsV2Output) -> Option { #[cfg(test)] mod tests { - use super::build_list_objects_output; - use s3s::dto::{CommonPrefix, ListObjectsV2Output, Object}; + use super::{build_list_objects_output, build_list_objects_v2_output}; + use rustfs_ecstore::store_api::{ListObjectsV2Info, ObjectInfo}; + use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object}; #[test] fn test_list_objects_marker_echoes_request_value() { @@ -123,4 +216,92 @@ mod tests { let output = build_list_objects_output(v2, None); assert_eq!(output.next_marker, None); } + + #[test] + fn test_list_objects_v2_key_count_includes_objects_and_prefixes() { + let object_infos = ListObjectsV2Info { + objects: vec![object_info("obj-a"), object_info("")], + prefixes: vec!["p1/".to_string(), "p2/".to_string()], + ..Default::default() + }; + + let output = build_list_objects_v2_output( + object_infos, + false, + 1000, + "bucket-a".to_string(), + "prefix-a".to_string(), + Some("/".to_string()), + None, + None, + None, + ); + + assert_eq!(output.key_count, Some(3)); + assert_eq!(output.contents.as_ref().map(std::vec::Vec::len), Some(1)); + assert_eq!(output.common_prefixes.as_ref().map(std::vec::Vec::len), Some(2)); + } + + #[test] + fn test_list_objects_v2_url_encoding_preserves_slash() { + let object_infos = ListObjectsV2Info { + objects: vec![object_info("dir a/file+b%.txt")], + prefixes: vec!["prefix a/sub+".to_string()], + ..Default::default() + }; + + let output = build_list_objects_v2_output( + object_infos, + true, + 1000, + "bucket-b".to_string(), + "prefix-b".to_string(), + Some("/".to_string()), + Some(EncodingType::from_static(EncodingType::URL)), + None, + None, + ); + + let contents = output.contents.as_ref().expect("contents should exist"); + let common_prefixes = output.common_prefixes.as_ref().expect("common prefixes should exist"); + + assert_eq!(contents[0].key.as_deref(), Some("dir%20a/file%2Bb%25.txt")); + assert_eq!(common_prefixes[0].prefix.as_deref(), Some("prefix%20a/sub%2B")); + assert!(contents[0].owner.is_some()); + assert_eq!(output.encoding_type.as_ref().map(EncodingType::as_str), Some(EncodingType::URL)); + } + + #[test] + fn test_list_objects_v2_next_continuation_token_is_base64_encoded() { + let object_infos = ListObjectsV2Info { + next_continuation_token: Some("token-123".to_string()), + ..Default::default() + }; + + let output = build_list_objects_v2_output( + object_infos, + false, + 1000, + "bucket-c".to_string(), + "prefix-c".to_string(), + None, + None, + Some(String::new()), + Some("start-after".to_string()), + ); + + assert_eq!(output.continuation_token, Some(String::new())); + assert_eq!(output.start_after, Some("start-after".to_string())); + assert_eq!( + output.next_continuation_token, + Some(base64_simd::STANDARD.encode_to_string("token-123".as_bytes())) + ); + } + + fn object_info(name: &str) -> ObjectInfo { + ObjectInfo { + name: name.to_string(), + ..Default::default() + } + } }