gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-27 16:48:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(storage): extract ListObjectsV2 response assembly helper (#1811)

Browse Source
This commit is contained in:
安正超
2026-02-14 12:31:10 +08:00
committed by GitHub
parent 22ae004205
commit 257e31a4b4
2 changed files with 195 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/storage/ecfs.rs
+11 -78
View File
@@ -23,7 +23,7 @@ use crate::storage::head_prefix::{head_prefix_not_found_message, probe_prefix_ha
use crate::storage::helper::OperationHelper;
use crate::storage::options::{filter_object_metadata, get_content_sha256};
use crate::storage::readers::InMemoryAsyncReader;
use crate::storage::s3_api::bucket::build_list_objects_output;
use crate::storage::s3_api::bucket::{build_list_objects_output, build_list_objects_v2_output};
use crate::storage::sse::{
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, check_encryption_metadata, sse_decryption, sse_encryption,
sse_prepare_encryption, strip_managed_encryption_metadata,
@@ -151,7 +151,6 @@ use tokio_stream::wrappers::ReceiverStream;
use tokio_tar::Archive;
use tokio_util::io::{ReaderStream, StreamReader};
use tracing::{debug, error, info, instrument, warn};
use urlencoding::encode;
use uuid::Uuid;
macro_rules! try_ {
@@ -3801,84 +3800,18 @@ impl S3 for FS {
.await
.map_err(ApiError::from)?;
// warn!("object_infos objects {:?}", object_infos.objects);
// Apply URL encoding if encoding_type is "url"
// Note: S3 URL encoding should encode special characters but preserve path separators (/)
let should_encode = encoding_type.as_ref().map(|e| e.as_str() == "url").unwrap_or(false);
// Helper function to encode S3 keys/prefixes (preserving /)
// S3 URL encoding encodes special characters but keeps '/' unencoded
let encode_s3_name = |name: &str| -> String {
name.split('/')
.map(|part| encode(part).to_string())
.collect::<Vec<_>>()
.join("/")
};
let objects: Vec<Object> = object_infos
.objects
.iter()
.filter(|v| !v.name.is_empty())
.map(|v| {
let key = if should_encode {
encode_s3_name(&v.name)
} else {
v.name.to_owned()
};
let mut obj = Object {
key: Some(key),
last_modified: v.mod_time.map(Timestamp::from),
size: Some(v.get_actual_size().unwrap_or_default()),
e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)),
storage_class: v.storage_class.clone().map(ObjectStorageClass::from),
..Default::default()
};
if fetch_owner.is_some_and(|v| v) {
obj.owner = Some(Owner {
display_name: Some("rustfs".to_owned()),
id: Some("v0.1".to_owned()),
});
}
obj
})
.collect();
let common_prefixes: Vec<CommonPrefix> = object_infos
.prefixes
.into_iter()
.map(|v| {
let prefix = if should_encode { encode_s3_name(&v) } else { v };
CommonPrefix { prefix: Some(prefix) }
})
.collect();
// KeyCount should include both objects and common prefixes per S3 API spec
let key_count = (objects.len() + common_prefixes.len()) as i32;
// Encode next_continuation_token to base64
let next_continuation_token = object_infos
.next_continuation_token
.map(|token| base64_simd::STANDARD.encode_to_string(token.as_bytes()));
let output = ListObjectsV2Output {
is_truncated: Some(object_infos.is_truncated),
continuation_token: response_continuation_token,
next_continuation_token,
start_after: response_start_after,
key_count: Some(key_count),
max_keys: Some(max_keys),
contents: Some(objects),
let output = build_list_objects_v2_output(
object_infos,
fetch_owner.unwrap_or_default(),
max_keys,
bucket,
prefix,
delimiter,
encoding_type: encoding_type.clone(),
name: Some(bucket),
prefix: Some(prefix),
common_prefixes: Some(common_prefixes),
..Default::default()
};
encoding_type,
response_continuation_token,
response_start_after,
);
// let output = ListObjectsV2Output { ..Default::default() };
Ok(S3Response::new(output))
}
rustfs/src/storage/s3_api/bucket.rs
+184 -3
View File
@@ -12,7 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use s3s::dto::{ListObjectsOutput, ListObjectsV2Output};
use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
use rustfs_ecstore::store_api::ListObjectsV2Info;
use s3s::dto::{
CommonPrefix, EncodingType, ListObjectsOutput, ListObjectsV2Output, Object, ObjectStorageClass, Owner, Timestamp,
};
use urlencoding::encode;
pub(crate) fn build_list_objects_output(v2: ListObjectsV2Output, request_marker: Option<String>) -> ListObjectsOutput {
let next_marker = calculate_next_marker(&v2);
@@ -36,6 +41,93 @@ pub(crate) fn build_list_objects_output(v2: ListObjectsV2Output, request_marker:
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn build_list_objects_v2_output(
object_infos: ListObjectsV2Info,
fetch_owner: bool,
max_keys: i32,
bucket: String,
prefix: String,
delimiter: Option<String>,
encoding_type: Option<EncodingType>,
response_continuation_token: Option<String>,
response_start_after: Option<String>,
) -> ListObjectsV2Output {
// Apply URL encoding if encoding_type is "url".
// S3 URL encoding encodes special characters but keeps '/' unencoded.
let should_encode = encoding_type.as_ref().is_some_and(|e| e.as_str() == EncodingType::URL);
let encode_s3_name = |name: &str| -> String {
name.split('/')
.map(|part| encode(part).to_string())
.collect::<Vec<_>>()
.join("/")
};
let objects: Vec<Object> = object_infos
.objects
.iter()
.filter(|v| !v.name.is_empty())
.map(|v| {
let key = if should_encode {
encode_s3_name(&v.name)
} else {
v.name.to_owned()
};
let mut obj = Object {
key: Some(key),
last_modified: v.mod_time.map(Timestamp::from),
size: Some(v.get_actual_size().unwrap_or_default()),
e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)),
storage_class: v.storage_class.clone().map(ObjectStorageClass::from),
..Default::default()
};
if fetch_owner {
obj.owner = Some(Owner {
display_name: Some("rustfs".to_owned()),
id: Some("v0.1".to_owned()),
});
}
obj
})
.collect();
let common_prefixes: Vec<CommonPrefix> = object_infos
.prefixes
.into_iter()
.map(|v| {
let prefix = if should_encode { encode_s3_name(&v) } else { v };
CommonPrefix { prefix: Some(prefix) }
})
.collect();
// KeyCount should include both objects and common prefixes per S3 API spec.
let key_count = (objects.len() + common_prefixes.len()) as i32;
// Encode next_continuation_token to base64.
let next_continuation_token = object_infos
.next_continuation_token
.map(|token| base64_simd::STANDARD.encode_to_string(token.as_bytes()));
ListObjectsV2Output {
is_truncated: Some(object_infos.is_truncated),
continuation_token: response_continuation_token,
next_continuation_token,
start_after: response_start_after,
key_count: Some(key_count),
max_keys: Some(max_keys),
contents: Some(objects),
delimiter,
encoding_type,
name: Some(bucket),
prefix: Some(prefix),
common_prefixes: Some(common_prefixes),
..Default::default()
}
}
fn calculate_next_marker(v2: &ListObjectsV2Output) -> Option<String> {
// For ListObjects (v1) API, NextMarker should be the last item returned when truncated.
// When both Contents and CommonPrefixes are present, NextMarker should be the
@@ -76,8 +168,9 @@ fn calculate_next_marker(v2: &ListObjectsV2Output) -> Option<String> {
#[cfg(test)]
mod tests {
use super::build_list_objects_output;
use s3s::dto::{CommonPrefix, ListObjectsV2Output, Object};
use super::{build_list_objects_output, build_list_objects_v2_output};
use rustfs_ecstore::store_api::{ListObjectsV2Info, ObjectInfo};
use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object};
#[test]
fn test_list_objects_marker_echoes_request_value() {
@@ -123,4 +216,92 @@ mod tests {
let output = build_list_objects_output(v2, None);
assert_eq!(output.next_marker, None);
}
#[test]
fn test_list_objects_v2_key_count_includes_objects_and_prefixes() {
let object_infos = ListObjectsV2Info {
objects: vec![object_info("obj-a"), object_info("")],
prefixes: vec!["p1/".to_string(), "p2/".to_string()],
..Default::default()
};
let output = build_list_objects_v2_output(
object_infos,
false,
1000,
"bucket-a".to_string(),
"prefix-a".to_string(),
Some("/".to_string()),
None,
None,
None,
);
assert_eq!(output.key_count, Some(3));
assert_eq!(output.contents.as_ref().map(std::vec::Vec::len), Some(1));
assert_eq!(output.common_prefixes.as_ref().map(std::vec::Vec::len), Some(2));
}
#[test]
fn test_list_objects_v2_url_encoding_preserves_slash() {
let object_infos = ListObjectsV2Info {
objects: vec![object_info("dir a/file+b%.txt")],
prefixes: vec!["prefix a/sub+".to_string()],
..Default::default()
};
let output = build_list_objects_v2_output(
object_infos,
true,
1000,
"bucket-b".to_string(),
"prefix-b".to_string(),
Some("/".to_string()),
Some(EncodingType::from_static(EncodingType::URL)),
None,
None,
);
let contents = output.contents.as_ref().expect("contents should exist");
let common_prefixes = output.common_prefixes.as_ref().expect("common prefixes should exist");
assert_eq!(contents[0].key.as_deref(), Some("dir%20a/file%2Bb%25.txt"));
assert_eq!(common_prefixes[0].prefix.as_deref(), Some("prefix%20a/sub%2B"));
assert!(contents[0].owner.is_some());
assert_eq!(output.encoding_type.as_ref().map(EncodingType::as_str), Some(EncodingType::URL));
}
#[test]
fn test_list_objects_v2_next_continuation_token_is_base64_encoded() {
let object_infos = ListObjectsV2Info {
next_continuation_token: Some("token-123".to_string()),
..Default::default()
};
let output = build_list_objects_v2_output(
object_infos,
false,
1000,
"bucket-c".to_string(),
"prefix-c".to_string(),
None,
None,
Some(String::new()),
Some("start-after".to_string()),
);
assert_eq!(output.continuation_token, Some(String::new()));
assert_eq!(output.start_after, Some("start-after".to_string()));
assert_eq!(
output.next_continuation_token,
Some(base64_simd::STANDARD.encode_to_string("token-123".as_bytes()))
);
}
fn object_info(name: &str) -> ObjectInfo {
ObjectInfo {
name: name.to_string(),
..Default::default()
}
}
}