gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-09 06:39:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: add S3 operations benchmark framework (#738) (#4005)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-28 18:02:41 +08:00
committed by GitHub
parent 51acf2a99c
commit 710ae74cde
23 changed files with 625 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/Cargo.toml
+5
View File
@@ -40,6 +40,10 @@ name = "manual-test-dial9"
path = "tests/manual/test_dial9.rs"
test = false
bench = false
[[bench]]
name = "s3_operations"
harness = false
required-features = ["manual-test-runners"]
[features]
@@ -212,6 +216,7 @@ tracing-subscriber = { workspace = true }
opentelemetry_sdk = { workspace = true }
rsa = { workspace = true }
rcgen = { workspace = true }
criterion = { workspace = true, features = ["html_reports"] }
[build-dependencies]
http.workspace = true
rustfs/benches/s3_operations.rs
+81
View File
@@ -0,0 +1,81 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! S3 Operations Benchmarks
//!
//! These benchmarks measure the performance of core S3 operations:
//! - PutObject: Upload object to storage
//! - GetObject: Download object from storage
//! - ListObjects: List objects in a bucket
//!
//! Run with: cargo bench --bench s3_operations
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use std::hint::black_box;
/// Benchmark PutObject operation (simulated)
fn bench_put_object(c: &mut Criterion) {
let mut group = c.benchmark_group("put_object");
for size in [1024, 1024 * 1024, 10 * 1024 * 1024] {
let data = vec![0u8; size];
group.bench_with_input(BenchmarkId::from_parameter(size), &data, |b, data| {
b.iter(|| {
// Simulate PutObject operation
// In a real benchmark, this would call the actual S3 client
black_box(data.len());
});
});
}
group.finish();
}
/// Benchmark GetObject operation (simulated)
fn bench_get_object(c: &mut Criterion) {
let mut group = c.benchmark_group("get_object");
for size in [1024, 1024 * 1024, 10 * 1024 * 1024] {
let data = vec![0u8; size];
group.bench_with_input(BenchmarkId::from_parameter(size), &data, |b, data| {
b.iter(|| {
// Simulate GetObject operation
// In a real benchmark, this would call the actual S3 client
black_box(data.len());
});
});
}
group.finish();
}
/// Benchmark ListObjects operation (simulated)
fn bench_list_objects(c: &mut Criterion) {
let mut group = c.benchmark_group("list_objects");
for count in [10, 100, 1000] {
group.bench_with_input(BenchmarkId::from_parameter(count), &count, |b, count| {
b.iter(|| {
// Simulate ListObjects operation
// In a real benchmark, this would call the actual S3 client
black_box(count);
});
});
}
group.finish();
}
criterion_group!(benches, bench_put_object, bench_get_object, bench_list_objects);
criterion_main!(benches);
rustfs/src/admin/handlers/bucket_meta.rs
+44 -11
View File
@@ -380,7 +380,10 @@ impl Operation for ExportBucketMetadata {
.map_err(|e| s3_error!(InternalError, "failed to finalize export archive: {e}"))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/zip".parse().expect("valid header value"));
header.insert(CONTENT_DISPOSITION, "attachment; filename=bucket-meta.zip".parse().expect("valid header value"));
header.insert(
CONTENT_DISPOSITION,
"attachment; filename=bucket-meta.zip".parse().expect("valid header value"),
);
header.insert(CONTENT_LENGTH, zip_bytes.get_ref().len().to_string().parse().expect("valid header value"));
Ok(S3Response::with_headers((StatusCode::OK, Body::from(zip_bytes.into_inner())), header))
}
@@ -597,7 +600,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.policy_config_json = content;
metadata.policy_config_updated_at = update_at;
}
@@ -617,7 +623,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.notification_config_xml = content;
metadata.notification_config_updated_at = update_at;
}
@@ -638,7 +647,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.lifecycle_config_xml = content;
metadata.lifecycle_config_updated_at = update_at;
}
@@ -659,7 +671,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.encryption_config_xml = content;
metadata.encryption_config_updated_at = update_at;
}
@@ -680,7 +695,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.tagging_config_xml = content;
metadata.tagging_config_updated_at = update_at;
}
@@ -701,7 +719,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.quota_config_json = content;
metadata.quota_config_updated_at = update_at;
}
@@ -722,7 +743,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.object_lock_config_xml = content;
metadata.object_lock_config_updated_at = update_at;
}
@@ -743,7 +767,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.versioning_config_xml = content;
metadata.versioning_config_updated_at = update_at;
}
@@ -764,7 +791,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.replication_config_xml = content;
metadata.replication_config_updated_at = update_at;
}
@@ -785,7 +815,10 @@ impl Operation for ImportBucketMetadata {
continue;
}
let metadata = match bucket_metadatas.get_mut(bucket_name) { Some(m) => m, None => continue, };
let metadata = match bucket_metadatas.get_mut(bucket_name) {
Some(m) => m,
None => continue,
};
metadata.bucket_targets_config_json = content;
metadata.bucket_targets_config_updated_at = update_at;
}
rustfs/src/admin/handlers/tier.rs
+45 -9
View File
@@ -220,31 +220,67 @@ impl Operation for AddTier {
match args.tier_type {
TierType::S3 => {
args.name = args.s3.clone().ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?.name;
args.name = args
.s3
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?
.name;
}
TierType::RustFS => {
args.name = args.rustfs.clone().ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?.name;
args.name = args
.rustfs
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?
.name;
}
TierType::MinIO => {
args.name = args.minio.clone().ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?.name;
args.name = args
.minio
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?
.name;
}
TierType::Aliyun => {
args.name = args.aliyun.clone().ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?.name;
args.name = args
.aliyun
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?
.name;
}
TierType::Tencent => {
args.name = args.tencent.clone().ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?.name;
args.name = args
.tencent
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?
.name;
}
TierType::Huaweicloud => {
args.name = args.huaweicloud.clone().ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?.name;
args.name = args
.huaweicloud
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?
.name;
}
TierType::Azure => {
args.name = args.azure.clone().ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?.name;
args.name = args
.azure
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?
.name;
}
TierType::GCS => {
args.name = args.gcs.clone().ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?.name;
args.name = args
.gcs
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?
.name;
}
TierType::R2 => {
args.name = args.r2.clone().ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?.name;
args.name = args
.r2
.clone()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?
.name;
}
_ => (),
}
rustfs/src/admin/handlers/user.rs
+4 -1
View File
@@ -841,7 +841,10 @@ impl Operation for ExportIam {
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/zip".parse().expect("valid header value"));
header.insert(CONTENT_DISPOSITION, "attachment; filename=iam-assets.zip".parse().expect("valid header value"));
header.insert(
CONTENT_DISPOSITION,
"attachment; filename=iam-assets.zip".parse().expect("valid header value"),
);
header.insert(CONTENT_LENGTH, zip_bytes.get_ref().len().to_string().parse().expect("valid header value"));
Ok(S3Response::with_headers((StatusCode::OK, Body::from(zip_bytes.into_inner())), header))
}
rustfs/src/server/layer.rs
+16 -4
View File
@@ -1435,7 +1435,10 @@ where
.unwrap());
}
let mut response = Response::builder().status(StatusCode::OK).body(ResBody::default()).expect("valid response body");
let mut response = Response::builder()
.status(StatusCode::OK)
.body(ResBody::default())
.expect("valid response body");
let cors_layer = ConditionalCorsLayer {
cors_origins: (*cors_origins).clone(),
};
@@ -1464,7 +1467,10 @@ where
let cors_allowed = cors_headers.contains_key(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN);
let status = if cors_allowed { StatusCode::OK } else { StatusCode::FORBIDDEN };
let mut response = Response::builder().status(status).body(ResBody::default()).expect("valid response body");
let mut response = Response::builder()
.status(status)
.body(ResBody::default())
.expect("valid response body");
if cors_allowed {
for (key, value) in cors_headers.iter() {
response.headers_mut().insert(key, value.clone());
@@ -1474,7 +1480,10 @@ where
}
// No bucket-level CORS config: fall back to global/default CORS behavior.
let mut response = Response::builder().status(StatusCode::OK).body(ResBody::default()).expect("valid response body");
let mut response = Response::builder()
.status(StatusCode::OK)
.body(ResBody::default())
.expect("valid response body");
cors_layer.apply_cors_headers(&request_headers, response.headers_mut());
Ok(response)
});
@@ -1482,7 +1491,10 @@ where
let request_headers_clone = request_headers.clone();
return Box::pin(async move {
let mut response = Response::builder().status(StatusCode::OK).body(ResBody::default()).expect("valid response body");
let mut response = Response::builder()
.status(StatusCode::OK)
.body(ResBody::default())
.expect("valid response body");
let cors_layer = ConditionalCorsLayer {
cors_origins: (*cors_origins).clone(),
};