mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-04 18:13:57 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| faf636e1c7 | |||
| 2401a8d8ac | |||
| d15dde7bb5 |
@@ -360,12 +360,26 @@ impl OpaMock {
|
||||
|
||||
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, action, bucket, department);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
|
||||
assert_eq!(payload["input"]["resource"]["object"], object);
|
||||
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
|
||||
let input = &payload["input"];
|
||||
assert_eq!(input["identity"]["account"], "opabuckettags");
|
||||
assert_eq!(input["identity"]["is_owner"], false);
|
||||
assert_eq!(input["action"], action);
|
||||
assert_eq!(input["resource"]["bucket"], bucket);
|
||||
let expected = department.map(|value| serde_json::json!([value]));
|
||||
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_validation(&mut self) -> TestResult {
|
||||
@@ -720,7 +734,7 @@ async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, buck
|
||||
async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
|
||||
};
|
||||
|
||||
init_logging();
|
||||
@@ -772,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
client
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
@@ -780,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
assert_eq!(head.tag_count(), Some(1));
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
client.list_objects_v2().bucket(source).send().await?;
|
||||
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
|
||||
client.list_object_versions().bucket(source).send().await?;
|
||||
@@ -789,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
admin.create_bucket().bucket(destination).send().await?;
|
||||
set_department(&admin, destination, "engineering").await?;
|
||||
let copy_source = format!("{source}/report?versionId={version}");
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("denied COPY and REPLACE must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
set_department(&admin, destination, "finance").await?;
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
// The access hook checks preliminary tags, then the copy use case checks
|
||||
// the exact source reader's tags before publishing the destination.
|
||||
for _ in 0..2 {
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
}
|
||||
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
|
||||
assert_eq!(tags.tag_set().len(), 1);
|
||||
assert_eq!(tags.tag_set()[0].key(), "department");
|
||||
assert_eq!(
|
||||
tags.tag_set()[0].value(),
|
||||
if directive == TaggingDirective::Copy {
|
||||
"engineering"
|
||||
} else {
|
||||
"legal"
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
@@ -886,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("denied-source")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied source must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
set_department(&client, source, "finance")
|
||||
.await
|
||||
.expect_err("requested tags must not authorize their own mutation");
|
||||
@@ -924,8 +985,6 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
// Duplicate keys are refused on write; the ambiguous record below is seeded
|
||||
// offline the way an upgraded deployment may still hold it on disk.
|
||||
let invalid_write = admin
|
||||
.put_bucket_tagging()
|
||||
.bucket(destination)
|
||||
@@ -938,6 +997,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.send()
|
||||
.await
|
||||
.expect_err("new duplicate tags must be rejected before publication");
|
||||
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
|
||||
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
|
||||
set_department(&admin, source, "finance").await?;
|
||||
|
||||
+749
-116
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,11 @@ use s3s::header::CONTENT_TYPE;
|
||||
use s3s::{Body, S3Error, S3ErrorCode, S3Response, S3Result, s3_error};
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
// Each compatible request can run two 64 MiB derivations. Keep both CPU and
|
||||
// memory bounded without occupying runtime workers or queueing unbounded jobs.
|
||||
static ADMIN_DECRYPT_PERMITS: Semaphore = Semaphore::const_new(2);
|
||||
|
||||
/// Returns `true` if `s` contains any whitespace character.
|
||||
///
|
||||
@@ -45,9 +50,20 @@ pub(crate) async fn read_compatible_admin_body(
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
|
||||
|
||||
if is_compat_admin_request(path) {
|
||||
decrypt_stream_io(secret_key.as_bytes(), body.as_ref())
|
||||
.or_else(|_| decrypt_data(secret_key.as_bytes(), body.as_ref()))
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to decrypt MinIO admin payload: {}", e))
|
||||
let permit = ADMIN_DECRYPT_PERMITS
|
||||
.try_acquire()
|
||||
.map_err(|_| s3_error!(SlowDown, "Admin payload decryption is busy"))?;
|
||||
let secret_key = secret_key.to_owned();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// Cancellation of the request must not release admission while the
|
||||
// non-cancellable derivation is still running.
|
||||
let _permit = permit;
|
||||
decrypt_stream_io(secret_key.as_bytes(), body.as_ref())
|
||||
.or_else(|_| decrypt_data(secret_key.as_bytes(), body.as_ref()))
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to decrypt MinIO admin payload: {}", e))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| s3_error!(InternalError, "Admin payload decryption failed"))?
|
||||
} else {
|
||||
Ok(body.to_vec())
|
||||
}
|
||||
@@ -155,6 +171,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(admin_decrypt)]
|
||||
async fn reads_legacy_compat_payload_as_fallback() {
|
||||
let payload = b"{\"ok\":true}".to_vec();
|
||||
let encrypted = encrypt_data(b"secret", &payload).expect("encrypt payload");
|
||||
@@ -166,6 +183,71 @@ mod tests {
|
||||
assert_eq!(decoded, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(admin_decrypt)]
|
||||
async fn compatible_admin_decryption_rejects_excess_work_without_blocking_native_requests() {
|
||||
let permits = ADMIN_DECRYPT_PERMITS
|
||||
.acquire_many(2)
|
||||
.await
|
||||
.expect("reserve decryption capacity");
|
||||
let error = read_compatible_admin_body(Body::from(vec![0; 64]), 1024, "/minio/admin/v3/add-user", "secret")
|
||||
.await
|
||||
.expect_err("excess KDF work must not queue");
|
||||
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
|
||||
let plain = read_compatible_admin_body(Body::from(String::from("{}")), 1024, "/rustfs/admin/v3/add-user", "secret")
|
||||
.await
|
||||
.expect("native admin does not use KDF capacity");
|
||||
assert_eq!(plain, b"{}");
|
||||
drop(permits);
|
||||
let encrypted = encrypt_stream_io(b"secret", b"{}").expect("encrypt compatible request");
|
||||
assert_eq!(
|
||||
read_compatible_admin_body(Body::from(encrypted), 1024, "/minio/admin/v3/add-user", "secret")
|
||||
.await
|
||||
.expect("capacity returns after work finishes"),
|
||||
b"{}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(admin_decrypt)]
|
||||
fn cancelled_admin_decryption_keeps_capacity_until_blocking_work_finishes() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.max_blocking_threads(1)
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build single-worker runtime");
|
||||
runtime.block_on(async {
|
||||
let (release, wait) = std::sync::mpsc::channel();
|
||||
let blocker = tokio::task::spawn_blocking(move || wait.recv().expect("release blocker"));
|
||||
let request = tokio::spawn(read_compatible_admin_body(
|
||||
Body::from(vec![0; 64]),
|
||||
1024,
|
||||
"/minio/admin/v3/add-user",
|
||||
"secret",
|
||||
));
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
ADMIN_DECRYPT_PERMITS.available_permits(),
|
||||
1,
|
||||
"decryption was queued off the runtime worker"
|
||||
);
|
||||
request.abort();
|
||||
assert!(request.await.expect_err("request cancelled").is_cancelled());
|
||||
assert_eq!(
|
||||
ADMIN_DECRYPT_PERMITS.available_permits(),
|
||||
1,
|
||||
"cancellation must not release queued work admission"
|
||||
);
|
||||
release.send(()).expect("unblock worker");
|
||||
blocker.await.expect("blocking task finished");
|
||||
let all = ADMIN_DECRYPT_PERMITS
|
||||
.acquire_many(2)
|
||||
.await
|
||||
.expect("decryption finished and released capacity");
|
||||
drop(all);
|
||||
});
|
||||
}
|
||||
|
||||
async fn body_bytes(mut body: Body) -> Vec<u8> {
|
||||
body.store_all_limited(64 * 1024).await.expect("body should read").to_vec()
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ use super::storage_api::bucket_usecase::contract::list::{
|
||||
};
|
||||
use super::storage_api::bucket_usecase::error::StorageError;
|
||||
use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context};
|
||||
use super::storage_api::bucket_usecase::object_utils::to_s3s_etag;
|
||||
use super::storage_api::bucket_usecase::object_utils::to_s3s_xml_etag;
|
||||
use super::storage_api::bucket_usecase::s3_api::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
@@ -966,6 +966,12 @@ fn encode_list_objects_v2_value(value: &str, encoding_type: Option<&EncodingType
|
||||
}
|
||||
|
||||
fn build_metadata_extension_user_metadata(user_defined: &HashMap<String, String>) -> Option<UserMetadata> {
|
||||
// Even builds unable to decrypt MinIO SSE-C must not expose its key-protected metadata.
|
||||
if super::storage_api::bucket_usecase::sse::stored_ssec_metadata(user_defined)
|
||||
|| user_defined.contains_key(rustfs_utils::http::object_encryption_keys::MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut items = extract_user_defined_metadata(user_defined)
|
||||
.into_iter()
|
||||
.filter(|(key, _)| !key.is_empty())
|
||||
@@ -1125,7 +1131,7 @@ fn build_list_object_versions_metadata_output(
|
||||
.etag
|
||||
.as_ref()
|
||||
.filter(|_| rustfs_utils::http::etag_can_be_listed(&object.user_defined))
|
||||
.map(|etag| to_s3s_etag(etag)),
|
||||
.map(|etag| to_s3s_xml_etag(etag)),
|
||||
storage_class: Some(ObjectVersionStorageClass::from(
|
||||
object
|
||||
.storage_class
|
||||
@@ -1216,7 +1222,7 @@ fn build_list_objects_v2_metadata_output(
|
||||
.etag
|
||||
.as_ref()
|
||||
.filter(|_| rustfs_utils::http::etag_can_be_listed(&object.user_defined))
|
||||
.map(|etag| to_s3s_etag(etag)),
|
||||
.map(|etag| to_s3s_xml_etag(etag)),
|
||||
storage_class: Some(ObjectStorageClass::from(
|
||||
object
|
||||
.storage_class
|
||||
@@ -1732,7 +1738,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(GetBucketLocationOutput { location_constraint }))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_list_buckets(&self, req: S3Request<ListBucketsInput>) -> S3Result<S3Response<ListBucketsOutput>> {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
@@ -1799,7 +1805,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::with_status(DeleteBucketEncryptionOutput::default(), StatusCode::NO_CONTENT))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_bucket_cors(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketCorsInput>,
|
||||
@@ -1831,7 +1837,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(DeleteBucketCorsOutput {}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_bucket_lifecycle(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketLifecycleInput>,
|
||||
@@ -1976,7 +1982,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(DeleteBucketReplicationOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_bucket_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketTaggingInput>,
|
||||
@@ -2003,7 +2009,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(DeleteBucketTaggingOutput {}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_public_access_block(
|
||||
&self,
|
||||
req: S3Request<DeletePublicAccessBlockInput>,
|
||||
@@ -2068,7 +2074,7 @@ impl DefaultBucketUsecase {
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_get_bucket_cors(&self, req: S3Request<GetBucketCorsInput>) -> S3Result<S3Response<GetBucketCorsOutput>> {
|
||||
let GetBucketCorsInput { bucket, .. } = req.input;
|
||||
|
||||
@@ -2107,7 +2113,7 @@ impl DefaultBucketUsecase {
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_get_bucket_lifecycle_configuration(
|
||||
&self,
|
||||
req: S3Request<GetBucketLifecycleConfigurationInput>,
|
||||
@@ -2348,7 +2354,7 @@ impl DefaultBucketUsecase {
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_get_bucket_tagging(
|
||||
&self,
|
||||
req: S3Request<GetBucketTaggingInput>,
|
||||
@@ -2385,7 +2391,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(GetBucketTaggingOutput { tag_set }))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_get_public_access_block(
|
||||
&self,
|
||||
req: S3Request<GetPublicAccessBlockInput>,
|
||||
@@ -2419,7 +2425,7 @@ impl DefaultBucketUsecase {
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_get_bucket_versioning(
|
||||
&self,
|
||||
req: S3Request<GetBucketVersioningInput>,
|
||||
@@ -2504,7 +2510,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(PutBucketEncryptionOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_put_bucket_lifecycle_configuration(
|
||||
&self,
|
||||
req: S3Request<PutBucketLifecycleConfigurationInput>,
|
||||
@@ -2752,7 +2758,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(PutBucketPolicyOutput {}))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_put_bucket_cors(&self, req: S3Request<PutBucketCorsInput>) -> S3Result<S3Response<PutBucketCorsOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
@@ -2844,7 +2850,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(PutBucketReplicationOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_put_public_access_block(
|
||||
&self,
|
||||
req: S3Request<PutPublicAccessBlockInput>,
|
||||
@@ -2876,7 +2882,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(PutPublicAccessBlockOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_put_bucket_tagging(
|
||||
&self,
|
||||
req: S3Request<PutBucketTaggingInput>,
|
||||
@@ -2884,6 +2890,7 @@ impl DefaultBucketUsecase {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
|
||||
let PutBucketTaggingInput { bucket, tagging, .. } = req.input;
|
||||
super::storage_api::bucket_usecase::s3_api::tagging::validate_bucket_tag_set(&tagging.tag_set)?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
@@ -2915,7 +2922,7 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(PutBucketTaggingOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_put_bucket_versioning(
|
||||
&self,
|
||||
req: S3Request<PutBucketVersioningInput>,
|
||||
@@ -5148,7 +5155,10 @@ mod tests {
|
||||
assert!(listings.contents[0].object.e_tag.is_none());
|
||||
assert!(listings.contents[1].object.e_tag.is_some());
|
||||
assert!(listings.contents[2].object.e_tag.is_some());
|
||||
assert_eq!(listings.contents[3].object.e_tag, Some(to_s3s_etag("1b8a77dff5845270a8d81d86a28f38ea-2")));
|
||||
assert_eq!(
|
||||
listings.contents[3].object.e_tag,
|
||||
Some(to_s3s_xml_etag("1b8a77dff5845270a8d81d86a28f38ea-2"))
|
||||
);
|
||||
let xml = String::from_utf8(serialize_config(&listings).expect("serialize objects")).expect("UTF-8 objects");
|
||||
assert!(!xml.contains(legacy_md5));
|
||||
|
||||
@@ -5178,6 +5188,159 @@ mod tests {
|
||||
assert!(!xml.contains(legacy_md5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn put_bucket_tagging_rejects_invalid_shapes_before_storage_mutation() {
|
||||
crate::app::gating_test_env::run_large_stack_test("bucket-tag-validation", || async {
|
||||
use crate::app::runtime_sources::{AppContext, ServerContextSlot};
|
||||
use crate::app::storage_api::test::ReqInfo;
|
||||
use crate::app::storage_api::test::ecfs::FS;
|
||||
use s3s::access::S3Access as _;
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let context = Arc::new(AppContext::new(store.clone(), ambient.iam(), ambient.kms()));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context.clone()));
|
||||
let fs = FS::with_server_ctx(slot.clone());
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(context));
|
||||
let bucket = format!("tag-validation-{}", uuid::Uuid::new_v4().simple());
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
BUCKET_POLICY_CONFIG,
|
||||
serde_json::to_vec(
|
||||
&serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},
|
||||
"Action":"s3:PutBucketTagging","Resource":format!("arn:aws:s3:::{bucket}")}]}),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let tag = s3s::dto::Tag {
|
||||
key: Some("team".to_owned()),
|
||||
value: Some("storage".to_owned()),
|
||||
};
|
||||
for (tag_set, valid) in [
|
||||
(vec![tag.clone()], true),
|
||||
(
|
||||
vec![s3s::dto::Tag {
|
||||
key: Some("team".to_owned()),
|
||||
value: None,
|
||||
}],
|
||||
false,
|
||||
),
|
||||
(vec![tag.clone(), tag], false),
|
||||
] {
|
||||
let input = PutBucketTaggingInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.tagging(s3s::dto::Tagging { tag_set })
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut req = build_request(input, Method::PUT);
|
||||
req.extensions.insert(ReqInfo::default());
|
||||
req.extensions.insert(slot.clone());
|
||||
fs.put_bucket_tagging(&mut req).await.expect("permission allows tag updates");
|
||||
let result = usecase.execute_put_bucket_tagging(req).await;
|
||||
if valid {
|
||||
result.expect("valid tags are persisted");
|
||||
} else {
|
||||
assert_eq!(result.unwrap_err().code(), &S3ErrorCode::InvalidTag);
|
||||
}
|
||||
assert_eq!(
|
||||
store.get_bucket_tags_for_policy(&bucket).await.unwrap(),
|
||||
HashMap::from([("team".to_owned(), "storage".to_owned())])
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_listings_hide_ssec_user_metadata_even_with_read_permission() {
|
||||
let objects = [
|
||||
("customer-key", Some(("x-amz-server-side-encryption-customer-algorithm", "AES256"))),
|
||||
(
|
||||
"minio-customer-key",
|
||||
Some((
|
||||
rustfs_utils::http::object_encryption_keys::MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
"sealed-key",
|
||||
)),
|
||||
),
|
||||
("server-key", Some(("x-amz-server-side-encryption", "AES256"))),
|
||||
("plain", None),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(name, encryption)| {
|
||||
let mut metadata = HashMap::from([("project".to_owned(), "private-project".to_owned())]);
|
||||
if let Some((key, value)) = encryption {
|
||||
metadata.insert(key.to_owned(), value.to_owned());
|
||||
}
|
||||
ObjectInfo {
|
||||
name: name.to_owned(),
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let permissions = objects
|
||||
.iter()
|
||||
.map(|object| {
|
||||
(
|
||||
object.name.clone(),
|
||||
ObjectMetadataPermissions {
|
||||
metadata_allowed: true,
|
||||
tags_allowed: true,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
let listings = build_list_objects_v2_metadata_output(
|
||||
ListObjectsV2Info {
|
||||
objects: objects.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
"demo-bucket",
|
||||
&ListObjectsV2Params {
|
||||
prefix: String::new(),
|
||||
max_keys: 1000,
|
||||
delimiter: None,
|
||||
response_start_after: None,
|
||||
start_after_for_query: None,
|
||||
response_continuation_token: None,
|
||||
decoded_continuation_token: None,
|
||||
},
|
||||
None,
|
||||
false,
|
||||
&permissions,
|
||||
);
|
||||
assert!(listings.contents[0].extension.user_metadata.is_none());
|
||||
assert!(listings.contents[1].extension.user_metadata.is_none());
|
||||
assert!(listings.contents[2].extension.user_metadata.is_some());
|
||||
assert!(listings.contents[3].extension.user_metadata.is_some());
|
||||
let versions = build_list_object_versions_metadata_output(
|
||||
ListObjectVersionsInfo {
|
||||
objects,
|
||||
..Default::default()
|
||||
},
|
||||
"demo-bucket",
|
||||
&ListObjectVersionsParams {
|
||||
prefix: String::new(),
|
||||
max_keys: 1000,
|
||||
delimiter: None,
|
||||
key_marker: None,
|
||||
version_id_marker: None,
|
||||
},
|
||||
None,
|
||||
&permissions,
|
||||
);
|
||||
for (index, entry) in versions.entries.iter().enumerate() {
|
||||
let ListObjectVersionMetadataEntry::Version(_, extension) = entry else {
|
||||
panic!("expected version");
|
||||
};
|
||||
assert_eq!(extension.user_metadata.is_some(), index >= 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_list_object_versions_metadata_output_maps_metadata_and_preserves_entry_order() {
|
||||
use time::macros::datetime;
|
||||
|
||||
@@ -158,6 +158,9 @@ pub(crate) async fn durable_quota_test_bucket(prefix: &str, limit: u64) -> (Arc<
|
||||
.set_quota_config(&bucket, BucketQuota::new(Some(limit)))
|
||||
.await
|
||||
.expect("configure durable quota test bucket");
|
||||
super::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative quota usage");
|
||||
(store, bucket)
|
||||
}
|
||||
|
||||
|
||||
@@ -18,13 +18,13 @@ use super::bucket_usecase::DefaultBucketUsecase;
|
||||
use super::object_usecase::DefaultObjectUsecase;
|
||||
use crate::app::runtime_sources::ServerContextSlot;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::storage::access::{ReqInfo, authorize_request, req_info_mut};
|
||||
use crate::storage::access::{ReqInfo, authorize_bucket_listing, req_info_mut};
|
||||
use crate::storage::{get_bucket_website_config_for_store, validate_website_configuration};
|
||||
use async_trait::async_trait;
|
||||
use http::header::CONTENT_TYPE;
|
||||
use http::header::HOST;
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Method, StatusCode, Uri};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_policy::policy::action::S3Action;
|
||||
use s3s::access::S3Access;
|
||||
use s3s::dto::{
|
||||
ETagCondition, EncodingType, GetObjectInput, GetObjectOutput, HeadObjectInput, ListObjectVersionsInput, ListObjectsV2Input,
|
||||
@@ -674,14 +674,14 @@ async fn check_metadata_access(req: &mut S3Request<Body>, target: BucketTarget)
|
||||
let req_info = req_info_mut(req)?;
|
||||
req_info.cred = cred;
|
||||
req_info.is_owner = is_owner;
|
||||
req_info.bucket = Some(target.bucket);
|
||||
req_info.bucket = Some(target.bucket.clone());
|
||||
}
|
||||
|
||||
let action = match target.operation {
|
||||
MetadataOperation::ListObjectVersions => Action::S3Action(S3Action::ListBucketVersionsAction),
|
||||
MetadataOperation::ListObjectsV2 => Action::S3Action(S3Action::ListBucketAction),
|
||||
MetadataOperation::ListObjectVersions => S3Action::ListBucketVersionsAction,
|
||||
MetadataOperation::ListObjectsV2 => S3Action::ListBucketAction,
|
||||
};
|
||||
authorize_request(req, action).await
|
||||
authorize_bucket_listing(req, &target.bucket, action).await
|
||||
}
|
||||
|
||||
async fn call_list_object_versions(req: S3Request<Body>, bucket: String) -> S3Result<S3Response<Body>> {
|
||||
@@ -838,6 +838,197 @@ fn header_value(headers: &HeaderMap, name: &str) -> S3Result<Option<String>> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[tokio::test]
|
||||
async fn metadata_route_prepares_and_applies_table_listing_isolation() {
|
||||
use crate::app::runtime_sources::{AppContext, IamInterface, KmsInterface, ServerContextSlot};
|
||||
use crate::app::storage_api::bucket_usecase::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::app::storage_api::object_usecase::access::TableDataPlaneListAccess;
|
||||
use crate::app::storage_api::object_usecase::contract::object::ObjectIO as _;
|
||||
use crate::app::storage_api::object_usecase::{StorageObjectOptions, StoragePutObjReader};
|
||||
use crate::app::table_list_isolation;
|
||||
use crate::table_catalog::{
|
||||
ConfiguredTableCatalogStore, EcStoreTableCatalogObjectBackend, NamespaceEntry, TABLE_BUCKET_CATALOG_TYPE,
|
||||
TABLE_BUCKET_MARKER_CONFIG, TABLE_CATALOG_ENTRY_VERSION, TableBucketEntry, TableCatalogBackingMode,
|
||||
TableCatalogEntryState, TableCatalogStore, TableEntry,
|
||||
};
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
struct UnreadyIam;
|
||||
impl IamInterface for UnreadyIam {
|
||||
fn handle(&self) -> Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>> {
|
||||
panic!("anonymous listing must not resolve IAM credentials")
|
||||
}
|
||||
fn is_ready(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
struct TestKms;
|
||||
impl KmsInterface for TestKms {
|
||||
fn handle(&self) -> Arc<rustfs_kms::KmsServiceManager> {
|
||||
Arc::new(rustfs_kms::KmsServiceManager::new())
|
||||
}
|
||||
}
|
||||
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let bucket = format!("metadata-isolation-{}", uuid::Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create table bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
TABLE_BUCKET_MARKER_CONFIG,
|
||||
crate::table_catalog::table_bucket_marker_json().expect("table bucket marker"),
|
||||
)
|
||||
.await
|
||||
.expect("enable table bucket");
|
||||
let policy = serde_json::json!({"Version": "2012-10-17", "Statement": [{
|
||||
"Effect": "Allow", "Principal": {"AWS": "*"},
|
||||
"Action": ["s3:ListBucket", "s3:ListBucketVersions"], "Resource": format!("arn:aws:s3:::{bucket}")
|
||||
}]});
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
|
||||
serde_json::to_vec(&policy).expect("listing policy"),
|
||||
)
|
||||
.await
|
||||
.expect("allow anonymous object listings");
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)));
|
||||
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
|
||||
access_key: "table-list-root".to_owned(),
|
||||
secret_key: "table-list-root-secret".to_owned(),
|
||||
status: "on".to_owned(),
|
||||
..Default::default()
|
||||
}));
|
||||
let backend =
|
||||
EcStoreTableCatalogObjectBackend::new_with_strong_runtime(Arc::clone(&store), context.table_catalog_strong_runtime());
|
||||
let catalog =
|
||||
ConfiguredTableCatalogStore::new_for_test(backend, TableCatalogBackingMode::from_env().expect("catalog mode"));
|
||||
catalog
|
||||
.put_table_bucket(TableBucketEntry {
|
||||
version: TABLE_CATALOG_ENTRY_VERSION,
|
||||
table_bucket: bucket.clone(),
|
||||
catalog_type: TABLE_BUCKET_CATALOG_TYPE.to_owned(),
|
||||
warehouse_root: format!("s3://{bucket}/"),
|
||||
state: TableCatalogEntryState::Active,
|
||||
properties: BTreeMap::new(),
|
||||
active_rename_id: None,
|
||||
created_at: None,
|
||||
updated_at: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed catalog bucket");
|
||||
catalog
|
||||
.create_namespace(NamespaceEntry {
|
||||
version: TABLE_CATALOG_ENTRY_VERSION,
|
||||
table_bucket: bucket.clone(),
|
||||
namespace: "sales".to_owned(),
|
||||
namespace_id: "sales".to_owned(),
|
||||
state: TableCatalogEntryState::Active,
|
||||
properties: BTreeMap::new(),
|
||||
created_at: None,
|
||||
updated_at: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed catalog namespace");
|
||||
catalog
|
||||
.create_table(TableEntry {
|
||||
version: TABLE_CATALOG_ENTRY_VERSION,
|
||||
table_bucket: bucket.clone(),
|
||||
namespace: "sales".to_owned(),
|
||||
table: "orders".to_owned(),
|
||||
table_id: "table-id".to_owned(),
|
||||
table_uuid: uuid::Uuid::new_v4().to_string(),
|
||||
format: "ICEBERG".to_owned(),
|
||||
format_version: 2,
|
||||
warehouse_location: format!("s3://{bucket}/tables/table-id"),
|
||||
metadata_location: format!("s3://{bucket}/tables/table-id/metadata.json"),
|
||||
version_token: "v1".to_owned(),
|
||||
generation: 1,
|
||||
state: TableCatalogEntryState::Active,
|
||||
properties: BTreeMap::new(),
|
||||
created_at: None,
|
||||
updated_at: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed catalog table");
|
||||
for key in ["ordinary.txt", "tables/table-id/data.parquet", ".rustfs-table/private.json"] {
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
key,
|
||||
&mut StoragePutObjReader::from_vec(b"private metadata".to_vec()),
|
||||
&StorageObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("seed objects for listing");
|
||||
}
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
for operation in [MetadataOperation::ListObjectsV2, MetadataOperation::ListObjectVersions] {
|
||||
let mut req = website_request(&format!("/{bucket}?list-type=2&metadata=true"));
|
||||
req.extensions.insert(Arc::clone(&slot));
|
||||
super::check_metadata_access(
|
||||
&mut req,
|
||||
super::BucketTarget {
|
||||
operation,
|
||||
bucket: bucket.clone(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("bucket listing is authorized");
|
||||
let access = req
|
||||
.extensions
|
||||
.get::<TableDataPlaneListAccess>()
|
||||
.expect("metadata-route authorization must attach table isolation");
|
||||
assert!(access.allows_object("ordinary.txt"));
|
||||
assert!(!access.allows_object("tables/table-id/data.parquet"));
|
||||
assert!(!access.allows_object(".rustfs-table/private.json"));
|
||||
let objects = match operation {
|
||||
MetadataOperation::ListObjectsV2 => {
|
||||
table_list_isolation::list_objects_v2(
|
||||
Arc::clone(&store),
|
||||
access,
|
||||
table_list_isolation::ListObjectsV2Request {
|
||||
bucket: &bucket,
|
||||
prefix: "",
|
||||
continuation_token: None,
|
||||
delimiter: None,
|
||||
max_keys: 1000,
|
||||
start_after: None,
|
||||
incl_deleted: false,
|
||||
opaque_cursor_supported: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("protected object listing succeeds")
|
||||
.objects
|
||||
}
|
||||
MetadataOperation::ListObjectVersions => {
|
||||
table_list_isolation::list_object_versions(
|
||||
Arc::clone(&store),
|
||||
access,
|
||||
table_list_isolation::ListObjectVersionsRequest {
|
||||
bucket: &bucket,
|
||||
prefix: "",
|
||||
key_marker: None,
|
||||
version_id_marker: None,
|
||||
delimiter: None,
|
||||
max_keys: 1000,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("protected version listing succeeds")
|
||||
.objects
|
||||
}
|
||||
};
|
||||
let keys: Vec<_> = objects.into_iter().map(|object| object.name).collect();
|
||||
assert_eq!(keys, vec!["ordinary.txt"], "catalog and unauthorized warehouse objects must be hidden");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn website_304_preserves_cache_headers_without_content() {
|
||||
use http_body_util::BodyExt;
|
||||
|
||||
@@ -44,11 +44,11 @@ use super::storage_api::multipart_usecase::data_usage::{
|
||||
use super::storage_api::multipart_usecase::error::{
|
||||
StorageError, is_err_invalid_upload_id, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
use super::storage_api::multipart_usecase::helper::OperationHelper;
|
||||
use super::storage_api::multipart_usecase::helper::{OperationHelper, run_owned_mutation};
|
||||
#[cfg(test)]
|
||||
use super::storage_api::multipart_usecase::io::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
|
||||
use super::storage_api::multipart_usecase::io::{HashReader, WriteEncryption, WritePlan, compression_metadata_value};
|
||||
use super::storage_api::multipart_usecase::object_utils::to_s3s_etag;
|
||||
use super::storage_api::multipart_usecase::object_utils::{to_s3s_etag, to_s3s_xml_etag};
|
||||
use super::storage_api::multipart_usecase::options::{
|
||||
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
|
||||
get_content_sha256_with_query, get_opts, has_replication_retention_update, namespace_reserved_user_metadata,
|
||||
@@ -62,20 +62,20 @@ use super::storage_api::multipart_usecase::s3_api::multipart::{
|
||||
use super::storage_api::multipart_usecase::set_disk::is_valid_storage_class;
|
||||
use super::storage_api::multipart_usecase::sse::{
|
||||
DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal,
|
||||
apply_bucket_default_lock_retention, authorize_sse_kms_object_read, build_ssec_read_headers, encryption_material_to_metadata,
|
||||
extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
|
||||
get_buffer_size_opt_in, load_bucket_object_lock_config_state, map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
||||
sse_decryption, sse_prepare_encryption,
|
||||
apply_bucket_default_lock_retention, authorize_sse_kms_multipart_write, authorize_sse_kms_object_read,
|
||||
build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in,
|
||||
load_bucket_object_lock_config_state, map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
||||
multipart_completion_authorization, sse_decryption, sse_prepare_encryption,
|
||||
};
|
||||
use super::storage_api::multipart_usecase::{
|
||||
StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader,
|
||||
validate_object_key,
|
||||
validate_new_object_key,
|
||||
};
|
||||
use super::trailer_adapter::trailer_source;
|
||||
use crate::app::object::{
|
||||
ConcurrencyManager, ForegroundWriteAdmission, get_concurrency_manager, guard_put_object_body_read_timeout,
|
||||
put_object_body_read_timeout, reject_non_wildcard_write_if_none_match, reject_oversize_single_upload,
|
||||
reject_veeam_sosapi_mutation,
|
||||
ConcurrencyManager, ForegroundWriteAdmission, get_concurrency_manager, put_object_body_read_timeout,
|
||||
reject_non_wildcard_write_if_none_match, reject_oversize_single_upload, reject_veeam_sosapi_mutation,
|
||||
};
|
||||
use crate::app::object_data_cache::{
|
||||
ObjectDataCacheAdapter, invalidate_object_data_cache_after_complete_multipart_success,
|
||||
@@ -523,7 +523,7 @@ impl DefaultMultipartUsecase {
|
||||
get_concurrency_manager()
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_abort_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<AbortMultipartUploadInput>,
|
||||
@@ -582,11 +582,18 @@ impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
#[hotpath::measure(impl_type = "MultipartUsecase")]
|
||||
pub async fn execute_complete_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<CompleteMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CompleteMultipartUploadOutput>> {
|
||||
let owner = self.clone();
|
||||
run_owned_mutation(async move { Box::pin(owner.execute_complete_multipart_upload_owned(req)).await }).await
|
||||
}
|
||||
|
||||
async fn execute_complete_multipart_upload_owned(
|
||||
&self,
|
||||
mut req: S3Request<CompleteMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CompleteMultipartUploadOutput>> {
|
||||
// Uploads created before the key was reserved must not publish it.
|
||||
validate_object_key(&req.input.key, "CompleteMultipartUpload")?;
|
||||
validate_new_object_key(&req.input.key, "CompleteMultipartUpload")?;
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
@@ -617,6 +624,7 @@ impl DefaultMultipartUsecase {
|
||||
..
|
||||
} = input;
|
||||
|
||||
let customer_key_provided = sse_customer_key.is_some();
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
reject_non_wildcard_write_if_none_match(if_none_match.as_ref(), &req.headers)?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
@@ -702,25 +710,28 @@ impl DefaultMultipartUsecase {
|
||||
Err(err) if is_err_invalid_upload_id(&err) => None,
|
||||
Err(err) => return Err(ApiError::from(err).into()),
|
||||
};
|
||||
// A ciphertext-passthrough session stores encrypted parts verbatim and
|
||||
// completes without the customer key (the replication client has none),
|
||||
// so the SSE-C completion check must be skipped for it.
|
||||
let completion_principal = SseKmsPrincipal::from_request(&req);
|
||||
let completion_authorization = multipart_completion_authorization(
|
||||
EncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption: None,
|
||||
ssekms_key_id: None,
|
||||
ssekms_context: None,
|
||||
sse_customer_algorithm: sse_customer_algorithm.clone(),
|
||||
sse_customer_key: sse_customer_key.clone(),
|
||||
sse_customer_key_md5: sse_customer_key_md5.clone(),
|
||||
content_size: 0,
|
||||
principal: completion_principal.as_ref(),
|
||||
},
|
||||
replication_authorized,
|
||||
);
|
||||
opts.multipart_completion_authorization = Some(completion_authorization.clone());
|
||||
if let Some(info) = multipart_info.as_ref() {
|
||||
if !contains_key_str(&info.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT) {
|
||||
EncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption: None,
|
||||
ssekms_key_id: None,
|
||||
ssekms_context: None,
|
||||
sse_customer_algorithm: sse_customer_algorithm.clone(),
|
||||
sse_customer_key: sse_customer_key.clone(),
|
||||
sse_customer_key_md5: sse_customer_key_md5.clone(),
|
||||
content_size: 0,
|
||||
principal: None,
|
||||
}
|
||||
.validate_complete_multipart_ssec(&info.user_defined)?;
|
||||
}
|
||||
completion_authorization
|
||||
.authorize(&info.user_defined)
|
||||
.await
|
||||
.map_err(map_get_object_reader_error)?;
|
||||
validate_complete_multipart_checksum_type(&req.headers, &info.user_defined)?;
|
||||
}
|
||||
|
||||
@@ -827,7 +838,7 @@ impl DefaultMultipartUsecase {
|
||||
.clone()
|
||||
.complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, &opts)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
.map_err(map_get_object_reader_error)?;
|
||||
let _ = invalidate_object_data_cache_after_complete_multipart_success(&cache_adapter, &bucket, &key).await;
|
||||
// Staging reclaim on a replay can free bytes; the dirty mark is
|
||||
// not a usage counter. Quota, replication, lifecycle, and the
|
||||
@@ -880,24 +891,11 @@ impl DefaultMultipartUsecase {
|
||||
)
|
||||
})??;
|
||||
|
||||
// The upload record is gone on a lost-response retry, so SSE-C is
|
||||
// checked against the committed object instead of the staging metadata.
|
||||
if upload_user_defined.is_none() && !contains_key_str(&obj_info.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT) {
|
||||
EncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption: None,
|
||||
ssekms_key_id: None,
|
||||
ssekms_context: None,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
content_size: 0,
|
||||
principal: None,
|
||||
}
|
||||
.validate_complete_multipart_ssec(obj_info.user_defined.as_ref())?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::storage_api::multipart_usecase::helper::pause_committed_mutation_for_test(&bucket).await;
|
||||
|
||||
// Storage has authorized the actual source or committed replay candidate
|
||||
// before comparing checksums/parts, recovery publication or replay effects.
|
||||
let encryption_metadata = upload_user_defined.as_ref().unwrap_or(obj_info.user_defined.as_ref());
|
||||
let server_side_encryption = encryption_metadata
|
||||
.get("x-amz-server-side-encryption")
|
||||
@@ -925,24 +923,55 @@ impl DefaultMultipartUsecase {
|
||||
// checksum: stored (decrypted) values take precedence over the request input;
|
||||
// additional algorithms (XXHash3/64/128, SHA-512, MD5), which have no typed
|
||||
// CompleteMultipartUploadOutput field, are echoed as raw response headers (#1261).
|
||||
let (checksums, is_multipart) = obj_info
|
||||
.decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers)
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let classified = crate::app::object_usecase::classify_response_checksums(checksums, is_multipart);
|
||||
let checksum_crc32 = classified.crc32.or(input.checksum_crc32);
|
||||
let checksum_crc32c = classified.crc32c.or(input.checksum_crc32c);
|
||||
let checksum_sha1 = classified.sha1.or(input.checksum_sha1);
|
||||
let checksum_sha256 = classified.sha256.or(input.checksum_sha256);
|
||||
let checksum_crc64nvme = classified.crc64nvme.or(input.checksum_crc64nvme);
|
||||
let checksum_type = classified.checksum_type.or(input.checksum_type);
|
||||
// Keyless completion remains supported, but plaintext checksums and the
|
||||
// customer-key fingerprint are returned only after verifying that key.
|
||||
// Apply this equally to a new commit and a lost-response replay.
|
||||
let disclose_checksums = (ssec_algorithm.is_none() && ssec_key_md5.is_none()) || customer_key_provided;
|
||||
let classified = if disclose_checksums {
|
||||
let (checksums, is_multipart) = obj_info
|
||||
.decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers)
|
||||
.map_err(ApiError::from)?;
|
||||
crate::app::object_usecase::classify_response_checksums(checksums, is_multipart)
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
let checksum_crc32 = if disclose_checksums {
|
||||
classified.crc32.or(input.checksum_crc32)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let checksum_crc32c = if disclose_checksums {
|
||||
classified.crc32c.or(input.checksum_crc32c)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let checksum_sha1 = if disclose_checksums {
|
||||
classified.sha1.or(input.checksum_sha1)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let checksum_sha256 = if disclose_checksums {
|
||||
classified.sha256.or(input.checksum_sha256)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let checksum_crc64nvme = if disclose_checksums {
|
||||
classified.crc64nvme.or(input.checksum_crc64nvme)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let checksum_type = if disclose_checksums {
|
||||
classified.checksum_type.or(input.checksum_type)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let complete_extra_checksum_headers = classified.extra;
|
||||
|
||||
let location = build_complete_multipart_location(&req.headers, &req.uri, &bucket, &key);
|
||||
let output = CompleteMultipartUploadOutput {
|
||||
bucket: Some(bucket.clone()),
|
||||
key: Some(key.clone()),
|
||||
e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
e_tag: obj_info.etag.clone().map(|etag| to_s3s_xml_etag(&etag)),
|
||||
location: Some(location),
|
||||
server_side_encryption,
|
||||
ssekms_key_id,
|
||||
@@ -970,7 +999,7 @@ impl DefaultMultipartUsecase {
|
||||
.headers
|
||||
.insert("x-amz-server-side-encryption-customer-algorithm", value);
|
||||
}
|
||||
if let Some(key_md5) = ssec_key_md5.as_deref() {
|
||||
if disclose_checksums && let Some(key_md5) = ssec_key_md5.as_deref() {
|
||||
let value =
|
||||
HeaderValue::from_str(key_md5).map_err(|_| s3_error!(InternalError, "Invalid stored SSE-C key metadata"))?;
|
||||
response
|
||||
@@ -993,12 +1022,13 @@ impl DefaultMultipartUsecase {
|
||||
&self,
|
||||
req: S3Request<CreateMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
|
||||
validate_object_key(&req.input.key, "CreateMultipartUpload")?;
|
||||
validate_new_object_key(&req.input.key, "CreateMultipartUpload")?;
|
||||
let multipart_max_total_object_size = parse_presigned_multipart_max_total_object_size(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
)?;
|
||||
validate_new_object_key(&req.input.key, "CreateMultipartUpload")?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
@@ -1085,7 +1115,7 @@ impl DefaultMultipartUsecase {
|
||||
let sse_customer_key_md5 = sse_customer_key_md5.or(header_sse_customer_key_md5);
|
||||
|
||||
// The session data key is generated here, so this is where a multipart upload is held
|
||||
// to the KMS key it names. Parts and the completion reuse the resulting envelope.
|
||||
// to the KMS key it names. Parts and completion also authorize their own caller.
|
||||
let session_principal = SseKmsPrincipal::from_request(&req);
|
||||
let encryption_request = PrepareEncryptionRequest {
|
||||
bucket: &bucket,
|
||||
@@ -1252,6 +1282,7 @@ impl DefaultMultipartUsecase {
|
||||
normalize_presigned_part_checksums(&mut req)?;
|
||||
let mut opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
|
||||
let part_principal = SseKmsPrincipal::from_request(&req);
|
||||
let input = req.input;
|
||||
let UploadPartInput {
|
||||
body,
|
||||
@@ -1276,7 +1307,7 @@ impl DefaultMultipartUsecase {
|
||||
if let Some(size) = size {
|
||||
reject_oversize_single_upload(size)?;
|
||||
}
|
||||
let mut body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
|
||||
let body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
@@ -1284,6 +1315,7 @@ impl DefaultMultipartUsecase {
|
||||
.get_multipart_info(&bucket, &key, &upload_id, &opts)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
authorize_sse_kms_multipart_write(part_principal.as_ref(), &fi.user_defined).await?;
|
||||
let max_total_object_size = multipart_max_total_object_size(&fi.user_defined)?;
|
||||
let mut size = require_upload_part_size(size, max_total_object_size.is_some())?;
|
||||
if let Some(limit) = max_total_object_size
|
||||
@@ -1316,23 +1348,20 @@ impl DefaultMultipartUsecase {
|
||||
.map(|ctx| ctx.request_id.as_str())
|
||||
.unwrap_or_default();
|
||||
let timeout = upload_part_body_read_timeout(put_object_body_read_timeout(), max_total_object_size.is_some());
|
||||
let raw_control = req.extensions.get::<super::object::request_body::BodyReadControl>().cloned();
|
||||
let observe_read_demand = if let Some(control) = &raw_control {
|
||||
control.activate(
|
||||
timeout,
|
||||
&bucket,
|
||||
&key,
|
||||
request_id,
|
||||
u64::try_from(size).map_err(|_| s3_error!(UnexpectedContent))?,
|
||||
)
|
||||
} else {
|
||||
// Direct protocol callers have no raw HTTP body. Retain their
|
||||
// existing capped-session guard without inventing a client cause.
|
||||
if max_total_object_size.is_some() {
|
||||
body_stream = guard_put_object_body_read_timeout(body_stream, &bucket, &key, request_id, Some(size), timeout);
|
||||
}
|
||||
false
|
||||
};
|
||||
let (body_stream, progress_control) =
|
||||
if let Some(control) = req.extensions.get::<super::object::request_body::BodyReadControl>() {
|
||||
control.activate_upload(timeout, &bucket, &key, request_id, size.max(0) as u64);
|
||||
(body_stream, control.clone())
|
||||
} else {
|
||||
super::object::request_body::guard_upload_progress(
|
||||
body_stream,
|
||||
timeout,
|
||||
&bucket,
|
||||
&key,
|
||||
request_id,
|
||||
size.max(0) as u64,
|
||||
)
|
||||
};
|
||||
|
||||
let ingress_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
|
||||
@@ -1453,8 +1482,8 @@ impl DefaultMultipartUsecase {
|
||||
write_plan = write_plan.with_encryption(ssec_write);
|
||||
(Some(ssec_material.server_side_encryption), ssec_material.kms_key_id)
|
||||
} else if let Some(server_side_encryption) = server_side_encryption {
|
||||
// Reuses the envelope the create-multipart-upload call was authorized for; the
|
||||
// KMS key was pinned into the session metadata then and cannot change here.
|
||||
// Reuse the stored envelope after authorizing this caller against its
|
||||
// pinned destination key above.
|
||||
let managed_material = sse_decryption(DecryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
@@ -1478,10 +1507,11 @@ impl DefaultMultipartUsecase {
|
||||
};
|
||||
|
||||
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
|
||||
if observe_read_demand && let Some(control) = raw_control {
|
||||
|
||||
{
|
||||
use rustfs_rio::HashReaderMut;
|
||||
let inner = reader.take_inner();
|
||||
reader.inner = rustfs_rio::boxed_reader(super::object::request_body::DemandReader::new(inner, control));
|
||||
reader.inner = rustfs_rio::boxed_reader(super::object::request_body::DemandReader::new(inner, progress_control));
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::new(reader);
|
||||
@@ -1796,6 +1826,7 @@ impl DefaultMultipartUsecase {
|
||||
.get_multipart_info(&bucket, &key, &upload_id, &dst_opts)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
authorize_sse_kms_multipart_write(copy_principal.as_ref(), &mp_info.user_defined).await?;
|
||||
let destination_size_limit = multipart_max_total_object_size(&mp_info.user_defined)?;
|
||||
EncryptionRequest {
|
||||
bucket: &bucket,
|
||||
@@ -1983,8 +2014,8 @@ impl DefaultMultipartUsecase {
|
||||
mp_info.user_defined.clone(),
|
||||
)
|
||||
} else if let Some(server_side_encryption) = server_side_encryption {
|
||||
// Destination side of the part copy: reuses the session envelope authorized at
|
||||
// create-multipart-upload time. The source side is authorized above.
|
||||
// Reuse the destination envelope after authorizing this caller against
|
||||
// its pinned key above. The source read is separately authorized.
|
||||
let managed_material = sse_decryption(DecryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
@@ -2049,7 +2080,7 @@ impl DefaultMultipartUsecase {
|
||||
checksum_sha1: checksum_value(rustfs_rio::ChecksumType::SHA1),
|
||||
checksum_sha256: checksum_value(rustfs_rio::ChecksumType::SHA256),
|
||||
checksum_crc64nvme: checksum_value(rustfs_rio::ChecksumType::CRC64_NVME),
|
||||
e_tag: part_info.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
e_tag: part_info.etag.map(|etag| to_s3s_xml_etag(&etag)),
|
||||
last_modified: part_info.last_mod.map(Timestamp::from),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -2712,7 +2743,15 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_upload_rejects_reserved_directory_markers() {
|
||||
for key in ["__XLDIR__", "prefix__XLDIR__/object", "object__XLDIR__"] {
|
||||
for key in [
|
||||
"__XLDIR__",
|
||||
"prefix__XLDIR__/object",
|
||||
"object__XLDIR__",
|
||||
"parent/xl.meta",
|
||||
"parent/uuid/xl.meta.bkp",
|
||||
"parent/uuid/delete-prepared",
|
||||
"parent/uuid/delete-quorum-committed",
|
||||
] {
|
||||
let create = CreateMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_owned())
|
||||
.key(key.to_owned())
|
||||
@@ -2976,6 +3015,117 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn ssec_completion_and_replay_require_key_before_disclosing_plaintext_checksums() {
|
||||
crate::app::gating_test_env::run_large_stack_test("ssec-completion-checksums", || async {
|
||||
let (_store, bucket) =
|
||||
crate::app::gating_test_env::durable_quota_test_bucket("ssec-completion-checksums", 16_384).await;
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
let key = "customer-encrypted";
|
||||
let customer_key = "KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKio=";
|
||||
let customer_md5 = "SaaDheEjzuynJH8eW6AEpQ==";
|
||||
let plaintext_sha256 = "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=";
|
||||
let mut create_input = CreateMultipartUploadInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(key.to_string())
|
||||
.build()
|
||||
.expect("create");
|
||||
create_input.sse_customer_algorithm = Some("AES256".to_string());
|
||||
create_input.sse_customer_key = Some(customer_key.to_string());
|
||||
create_input.sse_customer_key_md5 = Some(customer_md5.to_string());
|
||||
create_input.checksum_algorithm = Some(ChecksumAlgorithm::from_static("SHA256"));
|
||||
let mut create = build_request(create_input, Method::POST);
|
||||
create
|
||||
.headers
|
||||
.insert("x-amz-checksum-algorithm", HeaderValue::from_static("SHA256"));
|
||||
create
|
||||
.headers
|
||||
.insert("x-amz-checksum-type", HeaderValue::from_static("COMPOSITE"));
|
||||
let upload_id = usecase
|
||||
.execute_create_multipart_upload(create)
|
||||
.await
|
||||
.expect("SSE-C create")
|
||||
.output
|
||||
.upload_id
|
||||
.expect("upload id");
|
||||
let mut part_input = UploadPartInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(key.to_string())
|
||||
.upload_id(upload_id.clone())
|
||||
.part_number(1)
|
||||
.build()
|
||||
.expect("part");
|
||||
part_input.body = Some(StreamingBlob::from(Bytes::from_static(b"abc")));
|
||||
part_input.content_length = Some(3);
|
||||
part_input.sse_customer_algorithm = Some("AES256".to_string());
|
||||
part_input.sse_customer_key = Some(customer_key.to_string());
|
||||
part_input.sse_customer_key_md5 = Some(customer_md5.to_string());
|
||||
part_input.checksum_algorithm = Some(ChecksumAlgorithm::from_static("SHA256"));
|
||||
part_input.checksum_sha256 = Some(plaintext_sha256.to_string());
|
||||
let mut part = build_request(part_input, Method::PUT);
|
||||
part.headers
|
||||
.insert("x-amz-checksum-sha256", HeaderValue::from_static(plaintext_sha256));
|
||||
let part = usecase.execute_upload_part(part).await.expect("SSE-C part").output;
|
||||
let complete = CompleteMultipartUploadInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(key.to_string())
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(Some(CompletedMultipartUpload {
|
||||
parts: Some(vec![CompletedPart {
|
||||
part_number: Some(1),
|
||||
e_tag: part.e_tag,
|
||||
checksum_sha256: Some(plaintext_sha256.to_string()),
|
||||
..Default::default()
|
||||
}]),
|
||||
}))
|
||||
.build()
|
||||
.expect("complete");
|
||||
// Reject keyless completion before checksum comparison, then enforce
|
||||
// the same key boundary on a successful commit and its replay.
|
||||
for (attempt, keyed) in [false, true, false, true].into_iter().enumerate() {
|
||||
let mut input = complete.clone();
|
||||
if keyed {
|
||||
input.sse_customer_algorithm = Some("AES256".to_string());
|
||||
input.sse_customer_key = Some(customer_key.to_string());
|
||||
input.sse_customer_key_md5 = Some(customer_md5.to_string());
|
||||
}
|
||||
if attempt == 2 {
|
||||
let part = &mut input
|
||||
.multipart_upload
|
||||
.as_mut()
|
||||
.expect("parts")
|
||||
.parts
|
||||
.as_mut()
|
||||
.expect("part list")[0];
|
||||
part.e_tag = Some(ETag::Strong("guessed-etag".to_string()));
|
||||
part.checksum_sha256 = Some("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=".to_string());
|
||||
}
|
||||
let result = usecase
|
||||
.execute_complete_multipart_upload(build_request(input, Method::POST))
|
||||
.await;
|
||||
if keyed {
|
||||
let response = result.expect("verified key completes and replays");
|
||||
assert!(response.output.checksum_sha256.is_some(), "verified key retains checksum response");
|
||||
assert_eq!(
|
||||
response
|
||||
.headers
|
||||
.get("x-amz-server-side-encryption-customer-key-md5")
|
||||
.expect("key MD5"),
|
||||
customer_md5
|
||||
);
|
||||
} else {
|
||||
assert_eq!(
|
||||
result
|
||||
.expect_err("keyless checksum completion must fail before disclosure")
|
||||
.code(),
|
||||
&S3ErrorCode::InvalidRequest
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn conditional_multipart_upload_if_none_match_returns_precondition_failed_when_object_exists() {
|
||||
@@ -3480,6 +3630,80 @@ mod tests {
|
||||
assert_eq!(completed.size, 4096);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelled_multipart_completion_emits_once_and_replay_does_not_emit() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||
let bucket = format!("mpu-owned-event-{}", Uuid::new_v4());
|
||||
let object = "object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("upload");
|
||||
let mut reader = PutObjReader::from_vec(b"part".to_vec());
|
||||
let part = store
|
||||
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("part");
|
||||
let input = CompleteMultipartUploadInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_owned())
|
||||
.upload_id(upload.upload_id)
|
||||
.multipart_upload(Some(CompletedMultipartUpload {
|
||||
parts: Some(vec![CompletedPart {
|
||||
part_number: Some(1),
|
||||
e_tag: part.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
..Default::default()
|
||||
}]),
|
||||
}))
|
||||
.build()
|
||||
.expect("complete input");
|
||||
let probe = crate::app::storage_api::multipart_usecase::helper::MutationCompletionProbe::install(&bucket);
|
||||
let (entered, resume) = probe.pause_after_commit();
|
||||
let request = |input| {
|
||||
let mut request = build_request(input, Method::POST);
|
||||
request.uri = format!("/{bucket}/{object}").parse().expect("complete URI");
|
||||
request
|
||||
};
|
||||
let first = request(input.clone());
|
||||
let completion = tokio::spawn(async move {
|
||||
DefaultMultipartUsecase::from_global()
|
||||
.execute_complete_multipart_upload(first)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), entered.wait())
|
||||
.await
|
||||
.expect("committed MPU");
|
||||
completion.abort();
|
||||
assert!(completion.await.expect_err("cancel waiter").is_cancelled());
|
||||
resume.wait().await;
|
||||
probe.wait_for(1, 1).await;
|
||||
assert_eq!(
|
||||
store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed object")
|
||||
.size,
|
||||
4
|
||||
);
|
||||
DefaultMultipartUsecase::from_global()
|
||||
.execute_complete_multipart_upload(request(input))
|
||||
.await
|
||||
.expect("idempotent replay");
|
||||
let audits = probe.audits.lock().expect("audits");
|
||||
assert_eq!(audits.len(), 2);
|
||||
assert!(audits.iter().all(|entry| entry.api.status.as_deref() == Some("success")));
|
||||
let events = probe.events.lock().expect("events");
|
||||
assert_eq!(events.len(), 1, "replay must not emit another creation event");
|
||||
assert_eq!(events[0].event_name, EventName::ObjectCreatedCompleteMultipartUpload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rejected_empty_parts_preserve_existing_object_and_staging() {
|
||||
@@ -4016,6 +4240,63 @@ mod tests {
|
||||
drop(held);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_upload_part_progress_timeout_releases_admission_and_leaves_no_part() {
|
||||
use crate::app::object::request_body::{BodyReadControl, ObservedBody};
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use http_body::Frame;
|
||||
use http_body_util::StreamBody;
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||
let ambient = crate::app::runtime_sources::current_app_context().expect("test app context");
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
|
||||
let bucket = format!("part-progress-{}", Uuid::new_v4().simple());
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
let manager = Arc::new(ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO));
|
||||
let usecase = DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), Arc::clone(&manager));
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<Result<Frame<Bytes>, std::io::Error>>();
|
||||
let control = BodyReadControl::default();
|
||||
let body = StreamingBlob::from(s3s::Body::http_body_unsync(ObservedBody::new(
|
||||
StreamBody::new(UnboundedReceiverStream::new(receiver)),
|
||||
control.clone(),
|
||||
)));
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("object".to_owned())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.part_number(1)
|
||||
.content_length(Some(65536))
|
||||
.body(Some(body))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut request = build_request(input, Method::PUT);
|
||||
request.extensions.insert(control.clone());
|
||||
let owner = tokio::spawn(async move { usecase.execute_upload_part(request).await });
|
||||
control.wait_until_pending_for_test().await;
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(65536).await.unwrap(),
|
||||
ForegroundWriteAdmission::Rejected
|
||||
));
|
||||
control.advance_wait_for_test(Duration::from_secs(60));
|
||||
sender.send(Ok(Frame::data(Bytes::from_static(b"x")))).unwrap();
|
||||
assert_eq!(owner.await.unwrap().unwrap_err().code(), &S3ErrorCode::RequestTimeout);
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(65536).await.unwrap(),
|
||||
ForegroundWriteAdmission::Admitted(_)
|
||||
));
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(parts.parts.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_upload_part_rejects_when_foreground_write_admission_is_full() {
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
//! CopyObject path.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::object_usecase::object_utils::to_s3s_xml_etag;
|
||||
use futures::FutureExt;
|
||||
use rustfs_utils::path::fold_object_name_path_separators;
|
||||
use tracing::Instrument;
|
||||
@@ -142,7 +143,7 @@ impl DefaultObjectUsecase {
|
||||
Box::pin(self.execute_copy_object_body(req))
|
||||
}
|
||||
|
||||
async fn execute_copy_object_body(&self, req: S3Request<CopyObjectInput>) -> S3Result<S3Response<CopyObjectOutput>> {
|
||||
async fn execute_copy_object_body(&self, mut req: S3Request<CopyObjectInput>) -> S3Result<S3Response<CopyObjectOutput>> {
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
@@ -238,7 +239,7 @@ impl DefaultObjectUsecase {
|
||||
|
||||
// Validate both source and destination keys
|
||||
validate_object_key(&src_key, "COPY (source)")?;
|
||||
validate_object_key(&key, "COPY (dest)")?;
|
||||
validate_new_object_key(&key, "COPY (dest)")?;
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
let replaces_metadata = match metadata_directive.as_ref().map(|directive| directive.as_str()) {
|
||||
@@ -487,6 +488,13 @@ impl DefaultObjectUsecase {
|
||||
let _source_cancellation_guard = source_cancellation.clone().drop_guard();
|
||||
|
||||
let mut src_info = gr.object_info.clone();
|
||||
let effective_tags = replacement_tags.unwrap_or_else(|| (*src_info.user_tags).clone());
|
||||
crate::app::storage_api::object_usecase::access::authorize_copy_object_tags(
|
||||
&mut req,
|
||||
&src_info.user_tags,
|
||||
&effective_tags,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// A copy reads the source plaintext, so it needs the source key's decrypt permission
|
||||
// as well as the destination key's generate permission below. The source read resolves
|
||||
@@ -568,7 +576,6 @@ impl DefaultObjectUsecase {
|
||||
|
||||
// Extract user_defined from Arc for mutation; it will be re-wrapped after all edits.
|
||||
let mut user_defined = (*src_info.user_defined).clone();
|
||||
let effective_tags = replacement_tags.unwrap_or_else(|| (*src_info.user_tags).clone());
|
||||
if !replaces_metadata {
|
||||
let source_expires = src_info.expires.map(Timestamp::from);
|
||||
insert_expires_metadata(&mut user_defined, source_expires.as_ref())?;
|
||||
@@ -888,7 +895,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
}
|
||||
let copy_object_result = CopyObjectResult {
|
||||
e_tag: oi.etag.as_ref().map(|etag| to_s3s_etag(etag)),
|
||||
e_tag: oi.etag.as_ref().map(|etag| to_s3s_xml_etag(etag)),
|
||||
last_modified: oi.mod_time.map(Timestamp::from),
|
||||
checksum_crc32: response_checksums.crc32,
|
||||
checksum_crc32c: response_checksums.crc32c,
|
||||
@@ -1516,6 +1523,71 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn metadata_only_copy_escapes_legacy_etags_in_xml() {
|
||||
use crate::app::storage_api::test::ReqInfo;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use s3s::xml::{SerializeContent, Serializer};
|
||||
let (store, context) = real_store_test_context().await;
|
||||
let bucket = format!("copy-legacy-etag-{}", Uuid::new_v4());
|
||||
let key = "owned.txt";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let etag = "</ETag><Injected>victim</Injected><ETag>&";
|
||||
let mut reader = PutObjReader::from_vec(b"legacy object".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
key,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
preserve_etag: Some(etag.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed legacy ETag directly in storage");
|
||||
let before = store
|
||||
.get_object_info(&bucket, key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy object");
|
||||
assert_eq!(before.etag.as_deref(), Some(etag));
|
||||
let input = CopyObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(key.to_string())
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: key.into(),
|
||||
version_id: None,
|
||||
})
|
||||
.metadata_directive(Some(s3s::dto::MetadataDirective::from_static("REPLACE")))
|
||||
.build()
|
||||
.expect("self-copy request");
|
||||
let mut request = build_request(input, Method::PUT);
|
||||
request.extensions.insert(ReqInfo::default());
|
||||
let result = Box::pin(DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(request))
|
||||
.await
|
||||
.expect("metadata-only self-copy");
|
||||
let mut bytes = Vec::new();
|
||||
result
|
||||
.output
|
||||
.copy_object_result
|
||||
.expect("copy result")
|
||||
.serialize_content(&mut Serializer::new(&mut bytes))
|
||||
.expect("serialize copy result");
|
||||
let xml = String::from_utf8(bytes).expect("XML UTF-8");
|
||||
assert!(!xml.contains("<Injected>"), "{xml}");
|
||||
assert!(xml.contains("<Injected>victim</Injected>"), "{xml}");
|
||||
let stored = store
|
||||
.get_object_info(&bucket, key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("copied object");
|
||||
assert_eq!(stored.etag.as_deref(), Some(etag), "escaping belongs only to the wire response");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_copy_object_refuses_a_bucket_whose_encryption_config_is_unreadable() {
|
||||
|
||||
@@ -54,12 +54,10 @@ async fn authorize_recursive_delete<T>(
|
||||
let info = req_info_mut(req)?;
|
||||
info.object = Some(object.name.clone());
|
||||
info.version_id = version_id.clone();
|
||||
let action = if replica {
|
||||
Action::S3Action(S3Action::ReplicateDeleteAction)
|
||||
} else {
|
||||
delete_object_authorize_action(version_id.as_deref())
|
||||
};
|
||||
authorize_request(req, action).await?;
|
||||
authorize_request(req, delete_object_authorize_action(version_id.as_deref())).await?;
|
||||
if replica {
|
||||
authorize_request(req, Action::S3Action(S3Action::ReplicateDeleteAction)).await?;
|
||||
}
|
||||
if has_bypass_governance_header(&req.headers) {
|
||||
authorize_request(req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await?;
|
||||
}
|
||||
@@ -524,7 +522,12 @@ fn reduce_delete_objects_result<'a>(
|
||||
|
||||
impl DefaultObjectUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_objects(
|
||||
pub async fn execute_delete_objects(&self, req: S3Request<DeleteObjectsInput>) -> S3Result<S3Response<DeleteObjectsOutput>> {
|
||||
let owner = self.clone();
|
||||
run_owned_mutation(async move { Box::pin(owner.execute_delete_objects_owned(req)).await }).await
|
||||
}
|
||||
|
||||
async fn execute_delete_objects_owned(
|
||||
&self,
|
||||
mut req: S3Request<DeleteObjectsInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectsOutput>> {
|
||||
@@ -851,6 +854,8 @@ impl DefaultObjectUsecase {
|
||||
.delete_objects_with_tier_delete_journal_and_accounting(&bucket, object_to_delete.clone(), storage_delete_opts)
|
||||
.await;
|
||||
|
||||
#[cfg(test)]
|
||||
super::super::storage_api::object_usecase::helper::pause_committed_mutation_for_test(&bucket).await;
|
||||
let _manager = get_concurrency_manager();
|
||||
if !errs.is_empty() && errs.iter().all(|err| err.as_ref().is_some_and(is_err_bucket_not_found)) {
|
||||
let result = Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "Bucket not found".to_string()));
|
||||
@@ -1031,6 +1036,8 @@ impl DefaultObjectUsecase {
|
||||
.user_agent(get_request_user_agent(&req_headers))
|
||||
.build();
|
||||
|
||||
#[cfg(test)]
|
||||
super::super::storage_api::object_usecase::helper::capture_mutation_event_for_test(&event_args);
|
||||
notify.notify(event_args).await;
|
||||
}
|
||||
}
|
||||
@@ -1050,7 +1057,15 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
#[instrument(level = "info", skip(self, req))]
|
||||
pub async fn execute_delete_object(&self, mut req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||
pub async fn execute_delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||
let owner = self.clone();
|
||||
run_owned_mutation(async move { Box::pin(owner.execute_delete_object_owned(req)).await }).await
|
||||
}
|
||||
|
||||
async fn execute_delete_object_owned(
|
||||
&self,
|
||||
mut req: S3Request<DeleteObjectInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||
if let Some(context) = &self.context {
|
||||
let _ = context.object_store();
|
||||
}
|
||||
@@ -1261,6 +1276,8 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
super::super::storage_api::object_usecase::helper::pause_committed_mutation_for_test(&bucket).await;
|
||||
if force_delete {
|
||||
let _ = invalidate_object_data_cache_prefix_after_delete(&cache_adapter, &bucket, &key).await;
|
||||
} else {
|
||||
@@ -1454,6 +1471,105 @@ impl DefaultObjectUsecase {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn recursive_replica_delete_checks_each_descendant_delete_permission() {
|
||||
crate::app::gating_test_env::run_large_stack_test("recursive-replica-permissions", || async {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let slot = crate::app::runtime_sources::ServerContextSlot::new();
|
||||
assert!(slot.install(Arc::new(AppContext::new(store.clone(), ambient.iam(), ambient.kms()))));
|
||||
for versioned in [false, true] {
|
||||
let bucket = format!("recursive-replica-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: versioned,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create bucket");
|
||||
for key in ["team/public", "team/secret"] {
|
||||
for _ in 0..2 {
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
key,
|
||||
&mut PutObjReader::from_vec(b"protected payload".to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed descendant");
|
||||
}
|
||||
}
|
||||
let before = store
|
||||
.clone()
|
||||
.list_object_versions(&bucket, "team/", None, None, None, 1000)
|
||||
.await
|
||||
.expect("list fixture")
|
||||
.objects
|
||||
.len();
|
||||
for (grant_delete, deny_delete) in [(false, false), (true, true), (true, false)] {
|
||||
let mut statements = vec![
|
||||
serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":"s3:ReplicateDelete","Resource":format!("arn:aws:s3:::{bucket}/*")}),
|
||||
];
|
||||
if grant_delete {
|
||||
statements.push(serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},"Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/*")}));
|
||||
}
|
||||
if deny_delete {
|
||||
statements.push(serde_json::json!({"Effect":"Deny","Principal":{"AWS":"*"},"Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":format!("arn:aws:s3:::{bucket}/team/secret")}));
|
||||
}
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
|
||||
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(),
|
||||
)
|
||||
.await
|
||||
.expect("publish policy");
|
||||
for replica in [false, true] {
|
||||
let mut req = build_request((), Method::DELETE);
|
||||
req.extensions.insert(slot.clone());
|
||||
req.extensions.insert(crate::storage::access::ReqInfo {
|
||||
bucket: Some(bucket.clone()),
|
||||
object: Some("team/".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
let result = authorize_recursive_delete(&mut req, &store, &bucket, "team/", versioned, replica).await;
|
||||
if grant_delete && !deny_delete {
|
||||
result.expect("ordinary and replication deletes allowed");
|
||||
} else {
|
||||
assert_eq!(
|
||||
result.expect_err("descendant delete permission must be enforced").code(),
|
||||
&S3ErrorCode::AccessDenied
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
req_info_ref(&req).unwrap().object.as_deref(),
|
||||
Some("team/"),
|
||||
"restore original authorization context"
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.clone()
|
||||
.list_object_versions(&bucket, "team/", None, None, None, 1000)
|
||||
.await
|
||||
.expect("versions survive preflight")
|
||||
.objects
|
||||
.len(),
|
||||
before
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
use super::*;
|
||||
use http::{HeaderMap, HeaderValue, Method};
|
||||
use s3s::dto::{
|
||||
@@ -3104,6 +3220,92 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn cancelled_delete_requests_complete_audit_and_notifications() {
|
||||
crate::app::gating_test_env::run_large_stack_test("owned-delete-events", || async {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
let (store, context) = real_store_test_context().await;
|
||||
for batch in [false, true] {
|
||||
let bucket = format!("delete-owned-event-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
for key in ["first", "second"] {
|
||||
let mut reader = PutObjReader::from_vec(b"object".to_vec());
|
||||
store
|
||||
.put_object(&bucket, key, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("object");
|
||||
}
|
||||
let policy_json = format!(
|
||||
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:DeleteObject","s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"#
|
||||
);
|
||||
let mut metadata = (*crate::storage::get_bucket_metadata(&bucket).await.expect("metadata")).clone();
|
||||
metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("policy"));
|
||||
metadata.policy_config_json = policy_json.into_bytes();
|
||||
crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata)
|
||||
.await
|
||||
.expect("policy metadata");
|
||||
let probe = crate::app::storage_api::object_usecase::helper::MutationCompletionProbe::install(&bucket);
|
||||
let (entered, resume) = probe.pause_after_commit();
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(context.clone()));
|
||||
let request_bucket = bucket.clone();
|
||||
let deletion = tokio::spawn(async move {
|
||||
if batch {
|
||||
let input = DeleteObjectsInput::builder()
|
||||
.bucket(request_bucket.clone())
|
||||
.delete(Delete {
|
||||
objects: ["first", "second"]
|
||||
.into_iter()
|
||||
.map(|key| ObjectIdentifier {
|
||||
key: key.to_owned(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
quiet: Some(false),
|
||||
})
|
||||
.build()
|
||||
.expect("batch input");
|
||||
let mut request = build_request(input, Method::POST);
|
||||
request.uri = format!("/{request_bucket}").parse().expect("URI");
|
||||
request.extensions.insert(crate::storage::access::ReqInfo::default());
|
||||
usecase.execute_delete_objects(request).await.map(|_| ())
|
||||
} else {
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(request_bucket.clone())
|
||||
.key("first".to_owned())
|
||||
.build()
|
||||
.expect("delete input");
|
||||
let mut request = build_request(input, Method::DELETE);
|
||||
request.uri = format!("/{request_bucket}/first").parse().expect("URI");
|
||||
usecase.execute_delete_object(request).await.map(|_| ())
|
||||
}
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), entered.wait())
|
||||
.await
|
||||
.expect("committed delete");
|
||||
deletion.abort();
|
||||
assert!(deletion.await.expect_err("cancel waiter").is_cancelled());
|
||||
resume.wait().await;
|
||||
probe.wait_for(1, if batch { 2 } else { 1 }).await;
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(&bucket, "first", &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let audits = probe.audits.lock().expect("audits");
|
||||
assert_eq!(audits.len(), 1);
|
||||
assert_eq!(audits[0].api.status.as_deref(), Some("success"));
|
||||
let events = probe.events.lock().expect("events");
|
||||
assert_eq!(events.len(), if batch { 2 } else { 1 });
|
||||
assert!(events.iter().all(|event| event.event_name == EventName::ObjectRemovedDelete));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn recursive_delete_holds_writer_exclusion_after_authorization() {
|
||||
@@ -3791,11 +3993,16 @@ mod tests {
|
||||
.delete_object(&bucket, "object", ObjectOptions::default())
|
||||
.await
|
||||
.expect("delete quota-tracked object");
|
||||
// Deleted bytes are reclaimed by the owned reconciliation job; requests
|
||||
// no longer scan the bucket synchronously under the ledger lock.
|
||||
crate::app::storage_api::test::set_disk::retry_quota_reconciliation_for_test(Arc::clone(&store), &bucket)
|
||||
.await
|
||||
.expect("reconcile the deleted object's usage through the production worker");
|
||||
let mut replacement = PutObjReader::from_vec(vec![0x43; 4096]);
|
||||
store
|
||||
.put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("deleted bytes must be reclaimed before rejecting a replacement");
|
||||
.expect("reconciled deleted bytes must admit a full-size replacement");
|
||||
|
||||
let mut excess = PutObjReader::from_vec(vec![0x44]);
|
||||
let err = store
|
||||
|
||||
@@ -457,7 +457,9 @@ where
|
||||
}
|
||||
|
||||
let pool = get_concurrency_manager().bytes_pool();
|
||||
let mut buffer = pool.acquire_buffer(size).await;
|
||||
let mut buffer = pool.try_acquire_buffer(size).ok_or_else(|| {
|
||||
S3Error::with_message(S3ErrorCode::SlowDown, "upload buffer pool is full, please reduce your request rate")
|
||||
})?;
|
||||
super::put::read_small_put_body_into(body, &mut *buffer, size).await?;
|
||||
Ok(ExtractStagedBody {
|
||||
reader: Some(super::put::PooledBufferReader::new(buffer, size)),
|
||||
@@ -1282,7 +1284,7 @@ fn is_empty_extract_entry_path(path: &str) -> bool {
|
||||
|
||||
fn validate_extract_member_key(path: &str, limits: ArchiveLimits) -> Result<(), ExtractEntryError> {
|
||||
validate_put_object_extract_entry_path(path, limits).map_err(ExtractEntryError::Recoverable)?;
|
||||
validate_object_key(path, "PUT").map_err(ExtractEntryError::Recoverable)
|
||||
validate_new_object_key(path, "PUT").map_err(ExtractEntryError::Recoverable)
|
||||
}
|
||||
|
||||
fn record_extract_pax_metadata_bytes(
|
||||
@@ -1588,6 +1590,26 @@ fn extract_member_iam_requirements(
|
||||
}
|
||||
}
|
||||
|
||||
// Version and replica controls require authorization even when the archive does not set a time.
|
||||
// An mtime-only member can still be uploaded with server time when the caller lacks that permission.
|
||||
async fn authorized_extract_member_mod_time(
|
||||
req: &mut S3Request<PutObjectInput>,
|
||||
replication_required: bool,
|
||||
archive_time: Option<OffsetDateTime>,
|
||||
) -> S3Result<Option<OffsetDateTime>> {
|
||||
if !replication_required {
|
||||
if archive_time.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
req_info_mut(req)?.suppress_denial_log = true;
|
||||
}
|
||||
match authorize_request(req, Action::S3Action(S3Action::ReplicateObjectAction)).await {
|
||||
Ok(()) => Ok(archive_time),
|
||||
Err(err) if !replication_required && err.code() == &S3ErrorCode::AccessDenied => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_extract_pax_overrides(
|
||||
overrides: &ExtractPaxOverrides,
|
||||
bucket: &str,
|
||||
@@ -2067,14 +2089,26 @@ impl DefaultObjectUsecase {
|
||||
true,
|
||||
)?;
|
||||
let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
|
||||
let body = guard_put_object_body_read_timeout(
|
||||
body,
|
||||
&bucket,
|
||||
&key,
|
||||
&request_context.request_id,
|
||||
content_length,
|
||||
put_object_body_read_timeout(),
|
||||
);
|
||||
let (body, progress_control) = if let Some(control) = req.extensions.get::<super::request_body::BodyReadControl>() {
|
||||
control.activate_upload(
|
||||
put_object_body_read_timeout(),
|
||||
&bucket,
|
||||
&key,
|
||||
&request_context.request_id,
|
||||
content_length.unwrap_or_default().max(0) as u64,
|
||||
);
|
||||
(body, control.clone())
|
||||
} else {
|
||||
super::request_body::guard_upload_progress(
|
||||
body,
|
||||
put_object_body_read_timeout(),
|
||||
&bucket,
|
||||
&key,
|
||||
&request_context.request_id,
|
||||
content_length.unwrap_or_default().max(0) as u64,
|
||||
)
|
||||
};
|
||||
let body = super::request_body::demand_upload_body(body, progress_control.clone());
|
||||
|
||||
let size = match content_length {
|
||||
Some(c) => c,
|
||||
@@ -2092,7 +2126,7 @@ impl DefaultObjectUsecase {
|
||||
if size < 0 {
|
||||
return Err(s3_error!(UnexpectedContent));
|
||||
}
|
||||
validate_object_key(&key, "PUT")?;
|
||||
validate_new_object_key(&key, "PUT")?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
let _ = self
|
||||
.check_bucket_quota(
|
||||
@@ -2156,6 +2190,7 @@ impl DefaultObjectUsecase {
|
||||
ExtractArchiveEtagReader::new(archive_reader, expected_archive_length, archive_upload_state.clone());
|
||||
let decoder = build_admitted_extract_archive_decoder(get_concurrency_manager(), &key, tracked_archive).await?;
|
||||
let decoder = ExtractDecodedLimitReader::new(decoder, extract_limits.max_decoded_size);
|
||||
let decoder = super::request_body::DemandReader::new(decoder, progress_control);
|
||||
|
||||
let mut ar = build_put_object_extract_archive(decoder, extract_limits);
|
||||
let mut entries = ar.entries().map_err(|e| {
|
||||
@@ -2357,7 +2392,10 @@ impl DefaultObjectUsecase {
|
||||
put_opts_with_replication_authorization(
|
||||
&bucket,
|
||||
&fpath,
|
||||
outer_version_id.clone(),
|
||||
// The DTO version is applied with any PAX override below, then
|
||||
// authorized against this member before publication. Header
|
||||
// replication controls still require the outer authorization.
|
||||
None,
|
||||
&member_request_headers,
|
||||
metadata.clone(),
|
||||
replication_authorized,
|
||||
@@ -2456,15 +2494,14 @@ impl DefaultObjectUsecase {
|
||||
if iam_requirements.legal_hold {
|
||||
extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await);
|
||||
}
|
||||
if iam_requirements.replication {
|
||||
extract_try!(authorize_request(&mut auth_req, Action::S3Action(S3Action::ReplicateObjectAction)).await);
|
||||
if let Some(mod_time) = extract_try!(
|
||||
authorized_extract_member_mod_time(&mut auth_req, iam_requirements.replication, archive_entry_mod_time,).await
|
||||
) {
|
||||
opts.mod_time = Some(mod_time);
|
||||
}
|
||||
let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&auth_req);
|
||||
drop(auth_req);
|
||||
extract_try!(reject_veeam_sosapi_mutation(|| Some(Arc::clone(&commit_context.store)), &bucket, &fpath).await);
|
||||
if archive_entry_mod_time.is_some() {
|
||||
opts.mod_time = archive_entry_mod_time;
|
||||
}
|
||||
|
||||
debug!("Extracting file: {}, size: {} bytes", fpath, size);
|
||||
|
||||
@@ -2792,6 +2829,49 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn snowball_stalled_archives_release_actual_decoder_slots() {
|
||||
use futures::poll;
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
let manager = ConcurrencyManager::new();
|
||||
let mut senders = Vec::new();
|
||||
let mut owners = Vec::new();
|
||||
for _ in 0..4 {
|
||||
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<Result<Bytes, std::io::Error>>();
|
||||
let (body, _control) = super::super::request_body::guard_upload_progress(
|
||||
StreamingBlob::wrap(UnboundedReceiverStream::new(receiver)),
|
||||
Duration::ZERO,
|
||||
"bucket",
|
||||
"archive.tar",
|
||||
"request",
|
||||
1024,
|
||||
);
|
||||
let body = StreamReader::new(body.map(|item| item.map_err(s3s_body_error_to_io)));
|
||||
let mut owner = Box::pin(build_admitted_extract_archive_decoder(&manager, "archive.tar", body));
|
||||
assert!(poll!(owner.as_mut()).is_pending());
|
||||
senders.push(sender);
|
||||
owners.push(owner);
|
||||
}
|
||||
assert!(manager.try_acquire_snowball_archive_decoder().is_none());
|
||||
for _ in 0..5 {
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
for (sender, owner) in senders.iter().zip(&mut owners) {
|
||||
sender.send(Ok(Bytes::from_static(b"x"))).unwrap();
|
||||
assert!(poll!(owner.as_mut()).is_pending());
|
||||
}
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
for owner in owners {
|
||||
let error = owner.await.err().expect("stalled archive sniff must expire");
|
||||
assert_eq!(error.code(), &S3ErrorCode::RequestTimeout);
|
||||
}
|
||||
let recovered: Vec<_> = (0..4)
|
||||
.map(|_| manager.try_acquire_snowball_archive_decoder().expect("reusable slot"))
|
||||
.collect();
|
||||
assert!(manager.try_acquire_snowball_archive_decoder().is_none());
|
||||
drop(recovered);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_archive_decoder_admission_is_global_and_lifetime_bound() {
|
||||
struct PanicOnRead;
|
||||
@@ -3585,6 +3665,107 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn snowball_member_mtime_requires_member_replication_authorization() {
|
||||
use crate::app::storage_api::test::ReqInfo;
|
||||
use crate::app::storage_api::test::bucket::metadata::{BUCKET_POLICY_CONFIG, BUCKET_VERSIONING_CONFIG};
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use http::Method;
|
||||
|
||||
let (store, context) = real_store_test_context().await;
|
||||
let bucket = format!("extract-time-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(context));
|
||||
for (replicate, explicit_version, mtime) in [
|
||||
(false, false, 2_524_608_000_u64),
|
||||
(false, false, 1_000_000_000),
|
||||
(false, true, 2_524_608_000),
|
||||
(true, false, 1_704_000_123),
|
||||
(true, true, 2_524_608_000),
|
||||
] {
|
||||
let actions = if replicate {
|
||||
vec!["s3:PutObject", "s3:ReplicateObject"]
|
||||
} else {
|
||||
vec!["s3:PutObject"]
|
||||
};
|
||||
let policy = serde_json::json!({"Version":"2012-10-17", "Statement":[{
|
||||
"Effect":"Allow", "Principal":"*", "Action":actions,
|
||||
"Resource":format!("arn:aws:s3:::{bucket}/*")
|
||||
}]});
|
||||
metadata_sys::update(&bucket, BUCKET_POLICY_CONFIG, serde_json::to_vec(&policy).expect("policy JSON"))
|
||||
.await
|
||||
.expect("install member permissions");
|
||||
let member = match (replicate, explicit_version) {
|
||||
(true, false) => "authorized-time-only.txt",
|
||||
(true, true) => "replicated.txt",
|
||||
(false, true) => "denied-version.txt",
|
||||
(false, false) => "ordinary.txt",
|
||||
};
|
||||
let mut builder = Builder::new(Vec::new());
|
||||
let mut header = Header::new_gnu();
|
||||
header.set_size(1);
|
||||
header.set_mode(0o644);
|
||||
header.set_mtime(mtime);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, member, &b"x"[..])
|
||||
.await
|
||||
.expect("append member");
|
||||
let archive = builder.into_inner().await.expect("finish archive");
|
||||
let before = OffsetDateTime::now_utc();
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("archive.tar".to_string())
|
||||
.version_id(explicit_version.then(|| Uuid::new_v4().to_string()))
|
||||
.content_length(Some(i64::try_from(archive.len()).expect("archive size")))
|
||||
.body(Some(StreamingBlob::from_bytes(Bytes::from(archive))))
|
||||
.build()
|
||||
.expect("extract request");
|
||||
let mut request = build_request(input, Method::PUT);
|
||||
request.extensions.insert(ReqInfo::default());
|
||||
let result = Box::pin(usecase.execute_put_object_extract(request)).await;
|
||||
if explicit_version && !replicate {
|
||||
assert_eq!(
|
||||
result
|
||||
.expect_err("version selection requires member replication permission")
|
||||
.code(),
|
||||
&S3ErrorCode::AccessDenied
|
||||
);
|
||||
let missing = store
|
||||
.get_object_info(&bucket, member, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("unauthorized version cannot publish a member");
|
||||
assert!(is_err_object_not_found(&missing));
|
||||
continue;
|
||||
}
|
||||
result.expect("extract member");
|
||||
let info = store
|
||||
.get_object_info(&bucket, member, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("current member");
|
||||
let actual = info.mod_time.expect("stored timestamp");
|
||||
if replicate {
|
||||
assert_eq!(actual.unix_timestamp(), i64::try_from(mtime).expect("timestamp"));
|
||||
} else {
|
||||
assert!(
|
||||
actual >= before && actual <= OffsetDateTime::now_utc(),
|
||||
"ordinary upload used archive time: {actual}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn archive_etag_reader_validates_sha256_before_completion() {
|
||||
let payload = b"archive-with-wrong-sha256".to_vec();
|
||||
|
||||
+535
-49
@@ -16,6 +16,7 @@
|
||||
|
||||
use super::veeam_sosapi;
|
||||
use super::*;
|
||||
use crate::app::storage_api::object_usecase::access::replication_proxy_allowed;
|
||||
use crate::app::storage_api::object_usecase::object_cache::{
|
||||
GetObjectReadCondition, GetObjectReadMetadata, evaluate_read_preconditions, has_read_preconditions,
|
||||
with_get_object_read_condition,
|
||||
@@ -669,48 +670,186 @@ impl GetObjectBodyLifecycle {
|
||||
}
|
||||
}
|
||||
|
||||
pin_project! {
|
||||
// Keep the disk-read admission permit tied to the response body. This is
|
||||
// intentionally conservative backpressure: a streaming GET should occupy a
|
||||
// read slot until the client drains or drops the body.
|
||||
struct DiskReadPermitReader<R> {
|
||||
#[pin]
|
||||
inner: R,
|
||||
disk_permit: Option<GetObjectDiskPermit>,
|
||||
}
|
||||
// Bound downstream inactivity without imposing a fixed limit on large downloads.
|
||||
const GET_DISK_READ_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const GET_DISK_READ_MIN_WINDOW_BYTES: usize = 64 * 1024;
|
||||
|
||||
struct DiskReadLease<R> {
|
||||
reader: Option<Pin<Box<R>>>,
|
||||
permit: Option<GetObjectDiskPermit>,
|
||||
pending: bool,
|
||||
pending_capacity: usize,
|
||||
closed: bool,
|
||||
expired: bool,
|
||||
last_progress: tokio::time::Instant,
|
||||
window_start: tokio::time::Instant,
|
||||
window_bytes: usize,
|
||||
idle_timeout: Duration,
|
||||
progress_window: Duration,
|
||||
waker: Option<std::task::Waker>,
|
||||
}
|
||||
|
||||
impl<R> DiskReadPermitReader<R> {
|
||||
fn new(inner: R, disk_permit: GetObjectDiskPermit) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
disk_permit: Some(disk_permit),
|
||||
impl<R> DiskReadLease<R> {
|
||||
fn check_deadline(&mut self, now: tokio::time::Instant) {
|
||||
if now.saturating_duration_since(self.last_progress) >= self.idle_timeout {
|
||||
self.expired = true;
|
||||
}
|
||||
if now.saturating_duration_since(self.window_start) >= self.progress_window {
|
||||
if self.window_bytes < GET_DISK_READ_MIN_WINDOW_BYTES {
|
||||
self.expired = true;
|
||||
} else {
|
||||
self.window_start = now;
|
||||
self.window_bytes = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn next_check_delay(&self, now: tokio::time::Instant) -> Duration {
|
||||
self.idle_timeout
|
||||
.saturating_sub(now.saturating_duration_since(self.last_progress))
|
||||
.min(
|
||||
self.progress_window
|
||||
.saturating_sub(now.saturating_duration_since(self.window_start)),
|
||||
)
|
||||
// Keep timer arithmetic representable even for extreme operator overrides.
|
||||
.min(GET_DISK_READ_IDLE_TIMEOUT)
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> AsyncRead for DiskReadPermitReader<R>
|
||||
where
|
||||
R: AsyncRead,
|
||||
{
|
||||
struct DiskReadPermitReader<R> {
|
||||
lease: Arc<Mutex<DiskReadLease<R>>>,
|
||||
changed: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Send + 'static> DiskReadPermitReader<R> {
|
||||
fn new(inner: R, disk_permit: GetObjectDiskPermit) -> Self {
|
||||
// Leave the configured upstream stall/reopen window ahead of this
|
||||
// downstream lease, including the high-latency drive profile.
|
||||
let idle_timeout = get_object_disk_read_timeout()
|
||||
.saturating_mul(2)
|
||||
.max(GET_DISK_READ_IDLE_TIMEOUT);
|
||||
Self::with_idle_timeout(inner, disk_permit, idle_timeout)
|
||||
}
|
||||
|
||||
fn with_idle_timeout(inner: R, disk_permit: GetObjectDiskPermit, idle_timeout: Duration) -> Self {
|
||||
let now = tokio::time::Instant::now();
|
||||
let lease = Arc::new(Mutex::new(DiskReadLease {
|
||||
reader: Some(Box::pin(inner)),
|
||||
permit: Some(disk_permit),
|
||||
pending: false,
|
||||
pending_capacity: 0,
|
||||
closed: false,
|
||||
expired: false,
|
||||
last_progress: now,
|
||||
window_start: now,
|
||||
window_bytes: 0,
|
||||
idle_timeout,
|
||||
progress_window: idle_timeout.saturating_mul(2),
|
||||
waker: None,
|
||||
}));
|
||||
let changed = Arc::new(tokio::sync::Notify::new());
|
||||
tokio::spawn(Self::watch(lease.clone(), changed.clone()));
|
||||
Self { lease, changed }
|
||||
}
|
||||
|
||||
async fn watch(lease: Arc<Mutex<DiskReadLease<R>>>, changed: Arc<tokio::sync::Notify>) {
|
||||
let (reader, permit, pending_capacity, waker) = loop {
|
||||
let next_check = {
|
||||
let mut state = lease.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if !state.closed {
|
||||
state.check_deadline(tokio::time::Instant::now());
|
||||
}
|
||||
if state.closed || state.expired {
|
||||
state.closed = true;
|
||||
break (
|
||||
state.reader.take(),
|
||||
state.permit.take(),
|
||||
state.pending.then_some(state.pending_capacity),
|
||||
state.waker.take(),
|
||||
);
|
||||
}
|
||||
state.next_check_delay(tokio::time::Instant::now())
|
||||
};
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(next_check) => {},
|
||||
_ = changed.notified() => {},
|
||||
}
|
||||
};
|
||||
if let Some(waker) = waker {
|
||||
waker.wake();
|
||||
}
|
||||
if let Some(mut reader) = reader {
|
||||
if let Some(capacity) = pending_capacity {
|
||||
// A cancelled response must not release disk admission while a
|
||||
// submitted read is still running. Complete just that pending
|
||||
// read, discard its result, and never start another read.
|
||||
let mut scratch = vec![0_u8; capacity.min(64 * 1024)];
|
||||
let mut buffer = ReadBuf::new(&mut scratch);
|
||||
let _ = std::future::poll_fn(|cx| reader.as_mut().poll_read(cx, &mut buffer)).await;
|
||||
}
|
||||
drop(reader);
|
||||
}
|
||||
drop(permit);
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead> AsyncRead for DiskReadPermitReader<R> {
|
||||
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
let this = self.project();
|
||||
let had_capacity = buf.remaining() > 0;
|
||||
if buf.remaining() == 0 {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
let mut state = self.lease.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if !state.closed {
|
||||
state.check_deadline(tokio::time::Instant::now());
|
||||
}
|
||||
if state.expired {
|
||||
self.changed.notify_one();
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"GetObject disk-read admission expired",
|
||||
)));
|
||||
}
|
||||
if state.closed {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
state.waker = Some(cx.waker().clone());
|
||||
let filled_before = buf.filled().len();
|
||||
let poll = this.inner.poll_read(cx, buf);
|
||||
// EOF: no more disk reads can happen through this stream, so release
|
||||
// the permit instead of holding it until the client drops the body.
|
||||
if had_capacity
|
||||
&& matches!(poll, Poll::Ready(Ok(())))
|
||||
&& buf.filled().len() == filled_before
|
||||
&& let Some(mut disk_permit) = this.disk_permit.take()
|
||||
{
|
||||
disk_permit.release();
|
||||
state.pending_capacity = buf.remaining();
|
||||
let poll = match state.reader.as_mut() {
|
||||
Some(reader) => reader.as_mut().poll_read(cx, buf),
|
||||
None => return Poll::Ready(Ok(())),
|
||||
};
|
||||
state.pending = poll.is_pending();
|
||||
if let Poll::Ready(result) = &poll {
|
||||
let progress = buf.filled().len() - filled_before;
|
||||
if result.is_err() || progress == 0 {
|
||||
state.closed = true;
|
||||
// The read completed, so normal EOF/error can release inline.
|
||||
state.reader.take();
|
||||
state.permit.take();
|
||||
self.changed.notify_one();
|
||||
} else {
|
||||
state.last_progress = tokio::time::Instant::now();
|
||||
state.window_bytes = state.window_bytes.saturating_add(progress);
|
||||
}
|
||||
}
|
||||
poll
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> Drop for DiskReadPermitReader<R> {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.lease.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
state.closed = true;
|
||||
if !state.pending {
|
||||
state.reader.take();
|
||||
state.permit.take();
|
||||
}
|
||||
// The watchdog takes ownership of an outstanding read on cancellation.
|
||||
self.changed.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
pin_project! {
|
||||
struct GetObjectReaderStream<R> {
|
||||
#[pin]
|
||||
@@ -2506,9 +2645,9 @@ impl DefaultObjectUsecase {
|
||||
) -> S3Result<Option<GetObjectDiskPermit>> {
|
||||
let permit_wait_start = std::time::Instant::now();
|
||||
let permit_wait_timeout = Self::disk_permit_wait_timeout();
|
||||
// Permits are held for the whole body transfer, so slow clients can pin
|
||||
// all of them while disks are idle. Bound the wait on the primary pool
|
||||
// and, on timeout, admit from a bounded degraded overflow lane. Total
|
||||
// Active readers own admission until EOF, drop, or watchdog retirement.
|
||||
// Bound the wait on the primary pool and, on timeout, admit from a
|
||||
// bounded degraded overflow lane. Total
|
||||
// concurrent disk-active GETs are hard-capped at
|
||||
// `primary_cap + degraded_cap`; once that cap is reached we reject with
|
||||
// `SlowDown` instead of reading without any admission token. Never
|
||||
@@ -4328,6 +4467,7 @@ impl DefaultObjectUsecase {
|
||||
// their semantics) may still be served by a remote copy.
|
||||
if let Err(err) = &prepared_read
|
||||
&& matches!(*err.code(), S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion)
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
{
|
||||
// Active-active replication lag window: proxy the GET to a
|
||||
// replication target (backlog#1675 P1-5).
|
||||
@@ -4905,20 +5045,29 @@ fn odm_get_output(head: &SourceHead, content_length: i64, content_range: Option<
|
||||
}
|
||||
}
|
||||
|
||||
/// Feeds the tee primary to the client through a channel: the primary owns a
|
||||
/// `!Sync` boxed source while the response body must be `Sync`. Dropping the
|
||||
/// body ends the pump, which drops the primary and hands the remaining
|
||||
/// source bytes to the tee's drain task (`drain_on_primary_drop`).
|
||||
fn odm_inline_client_body(primary: TeePrimary) -> StreamingBlob {
|
||||
// These downstream budgets are independent of source idle progress. In
|
||||
// particular, reading a little before each idle timeout cannot retain a pull
|
||||
// slot indefinitely. Timeout drops the primary and lets the tee drain to disk.
|
||||
const ODM_INLINE_CLIENT_IDLE_BUDGET: Duration = Duration::from_secs(30);
|
||||
const ODM_INLINE_CLIENT_TOTAL_BUDGET: Duration = Duration::from_secs(120);
|
||||
|
||||
/// Feeds the tee primary through a channel because its source is `!Sync`.
|
||||
/// Dropping the body or exhausting a downstream budget drops the primary and
|
||||
/// hands remaining bytes to the tee's drain task (`drain_on_primary_drop`).
|
||||
fn odm_inline_client_body(primary: TeePrimary, idle_budget: Duration, total_budget: Duration) -> StreamingBlob {
|
||||
let (tx, rx) = tokio::sync::mpsc::channel::<io::Result<Bytes>>(ODM_INLINE_CLIENT_CHANNEL_CHUNKS);
|
||||
spawn_traced(async move {
|
||||
let mut chunks = tokio_util::io::ReaderStream::with_capacity(primary, ODM_SOURCE_BODY_CHUNK_BYTES);
|
||||
while let Some(chunk) = chunks.next().await {
|
||||
let failed = chunk.is_err();
|
||||
if tx.send(chunk).await.is_err() || failed {
|
||||
break;
|
||||
let pump = async move {
|
||||
let mut chunks = tokio_util::io::ReaderStream::with_capacity(primary, ODM_SOURCE_BODY_CHUNK_BYTES);
|
||||
while let Some(chunk) = chunks.next().await {
|
||||
let failed = chunk.is_err();
|
||||
if !matches!(tokio::time::timeout(idle_budget, tx.send(chunk)).await, Ok(Ok(()))) || failed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
// This task is scheduled independently of response body polling.
|
||||
let _ = tokio::time::timeout(total_budget, pump).await;
|
||||
});
|
||||
StreamingBlob::wrap(ReceiverStream::new(rx))
|
||||
}
|
||||
@@ -5039,7 +5188,12 @@ async fn odm_get_inline<S: OdmGetSource>(
|
||||
let (guarded, idle) = idle_guarded_body(source_body, Duration::from_millis(policy.source_timeout.idle_ms));
|
||||
let (primary, secondary) =
|
||||
tee_reader_with_options(Box::pin(tokio_util::io::StreamReader::new(guarded)), ODM_INLINE_TEE_BUFFER_BYTES, options);
|
||||
let output = Box::new(odm_get_output(&head, content_length, content_range, odm_inline_client_body(primary)));
|
||||
let output = Box::new(odm_get_output(
|
||||
&head,
|
||||
content_length,
|
||||
content_range,
|
||||
odm_inline_client_body(primary, ODM_INLINE_CLIENT_IDLE_BUDGET, ODM_INLINE_CLIENT_TOTAL_BUDGET),
|
||||
));
|
||||
let commit_state = Arc::clone(state);
|
||||
let commit_key = key.to_string();
|
||||
spawn_background_with_context(request_context, async move {
|
||||
@@ -5087,11 +5241,12 @@ pub(super) async fn odm_get_from_source<S: OdmGetSource>(
|
||||
if head.size > policy.inline_max_bytes {
|
||||
return odm_get_passthrough(state, source, headers, key, None, Some(PullReason::LargeObject)).await;
|
||||
}
|
||||
let slot = match state.acquire_pull_slot(key).await {
|
||||
Ok(slot) => slot,
|
||||
// The bucket state was torn down under this request: serve it
|
||||
// without queueing anything on the old state.
|
||||
Err(_) => return odm_get_passthrough(state, source, headers, key, None, None).await,
|
||||
let permit_budget = Duration::from_millis(policy.source_timeout.first_byte_ms).min(Duration::from_secs(1));
|
||||
let slot = match tokio::time::timeout(permit_budget, state.acquire_pull_slot(key)).await {
|
||||
Ok(Ok(slot)) => slot,
|
||||
// A saturated or retired bucket state must not hold the response
|
||||
// indefinitely. Dropping acquisition removes its singleflight entry.
|
||||
Ok(Err(_)) | Err(_) => return odm_get_passthrough(state, source, headers, key, None, None).await,
|
||||
};
|
||||
match slot {
|
||||
PullSlot::Leader(leader) => odm_get_inline(state, source, headers, key, leader, request_context).await,
|
||||
@@ -5411,6 +5566,85 @@ mod on_demand_migration_tests {
|
||||
assert!(!odm_get_may_consult_source(&proxy_marked, None));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn odm_get_permit_wait_is_bounded_and_removes_the_abandoned_slot() {
|
||||
let mut policy = PolicyConfig {
|
||||
max_concurrent_pulls: 1,
|
||||
..Default::default()
|
||||
};
|
||||
policy.source_timeout.first_byte_ms = 100;
|
||||
let rt = runtime("permit-budget", policy).await;
|
||||
let state = rt.state("permit-budget");
|
||||
let held = state.acquire_pull_slot("held").await.expect("occupy the only permit");
|
||||
let data = payload(1024);
|
||||
let source = ScriptedSource::for_object(&data);
|
||||
let reply = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
odm_get_from_source(&state, &source, &HeaderMap::new(), KEY, None, None),
|
||||
)
|
||||
.await
|
||||
.expect("saturated inline permits must fall back to passthrough");
|
||||
let (mut output, backfill) = served(reply);
|
||||
assert_eq!(backfill, None);
|
||||
assert_eq!(collect_body(&mut output).await, data);
|
||||
assert_eq!(state.inflight_keys(), 1, "timed-out waiter must not strand a singleflight entry");
|
||||
assert!(rt.write_back.puts().is_empty(), "fallback must not start another inline pull");
|
||||
drop(held);
|
||||
assert_eq!(state.inflight_keys(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn odm_client_backpressure_and_trickle_cannot_retain_a_pull_slot() {
|
||||
for trickle in [false, true] {
|
||||
let rt = runtime("client-budget", PolicyConfig::default()).await;
|
||||
let state = rt.state("client-budget");
|
||||
let leader = match state.acquire_pull_slot(KEY).await.expect("leader") {
|
||||
PullSlot::Leader(leader) => leader,
|
||||
PullSlot::Follower(_) => panic!("fresh key must lead"),
|
||||
};
|
||||
let data = payload(4 * 1024 * 1024);
|
||||
let head = source_head(&data);
|
||||
let source: SourceBody = Box::pin(futures::stream::iter([Ok(Bytes::from(data.clone()))]));
|
||||
let (guarded, idle) = idle_guarded_body(source, Duration::from_secs(5));
|
||||
let (primary, secondary) = tee_reader_with_options(
|
||||
Box::pin(tokio_util::io::StreamReader::new(guarded)),
|
||||
ODM_INLINE_TEE_BUFFER_BYTES,
|
||||
TeeOptions {
|
||||
drain_on_primary_drop: true,
|
||||
max_drain_bytes: data.len(),
|
||||
},
|
||||
);
|
||||
let mut client = odm_inline_client_body(primary, Duration::from_millis(100), Duration::from_millis(250));
|
||||
let commit_state = Arc::clone(&state);
|
||||
let commit = tokio::spawn(async move {
|
||||
let result = commit_inline(&commit_state, KEY, head, None, Box::pin(secondary.into_stream()), &idle).await;
|
||||
leader.complete(result.map(|outcome| PullOutcome {
|
||||
etag: outcome.etag,
|
||||
size: outcome.size,
|
||||
}));
|
||||
});
|
||||
let received = if trickle {
|
||||
let mut received = 0;
|
||||
// Progress is faster than the idle budget but cannot reset the absolute budget.
|
||||
while let Some(chunk) = client.next().await {
|
||||
received += chunk.expect("client chunk").len();
|
||||
tokio::time::sleep(Duration::from_millis(40)).await;
|
||||
}
|
||||
received
|
||||
} else {
|
||||
0
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(3), commit)
|
||||
.await
|
||||
.expect("client pacing cannot stall write-back")
|
||||
.expect("write-back task");
|
||||
assert_eq!(state.inflight_keys(), 0, "permit released while the unread response is still alive");
|
||||
assert_eq!(rt.write_back.puts()[0].1, data, "drain must preserve the entire source object");
|
||||
assert!(received < data.len(), "timed-out response must not pretend to deliver the full object");
|
||||
drop(client);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn odm_get_inline_streams_to_client_and_commits_the_same_bytes() {
|
||||
let rt = runtime("b", PolicyConfig::default()).await;
|
||||
@@ -9861,6 +10095,258 @@ mod tests {
|
||||
assert_eq!(GetObjectGuard::concurrent_count(), initial);
|
||||
}
|
||||
|
||||
struct AdmissionReadProbe {
|
||||
reads: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl AsyncRead for AdmissionReadProbe {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, buffer: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
self.reads.fetch_add(1, AtomicOrdering::SeqCst);
|
||||
buffer.initialize_unfilled().fill(42);
|
||||
buffer.advance(buffer.remaining());
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
struct PendingAdmissionRead {
|
||||
finish: tokio::sync::oneshot::Receiver<()>,
|
||||
started: Arc<AtomicUsize>,
|
||||
completed: Arc<AtomicUsize>,
|
||||
submitted: bool,
|
||||
}
|
||||
|
||||
impl AsyncRead for PendingAdmissionRead {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buffer: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if !self.submitted {
|
||||
self.submitted = true;
|
||||
self.started.fetch_add(1, AtomicOrdering::SeqCst);
|
||||
}
|
||||
match std::future::Future::poll(Pin::new(&mut self.finish), cx) {
|
||||
Poll::Ready(_) => {
|
||||
self.completed.fetch_add(1, AtomicOrdering::SeqCst);
|
||||
buffer.put_slice(b"x");
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn disk_read_admission_expires_unpolled_and_cannot_resume_without_permit() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
|
||||
let permit = semaphore.clone().acquire_owned().await.unwrap();
|
||||
let reads = Arc::new(AtomicUsize::new(0));
|
||||
let mut reader = DiskReadPermitReader::with_idle_timeout(
|
||||
AdmissionReadProbe { reads: reads.clone() },
|
||||
permit.into(),
|
||||
GET_DISK_READ_IDLE_TIMEOUT,
|
||||
);
|
||||
tokio::time::advance(GET_DISK_READ_IDLE_TIMEOUT + Duration::from_secs(1)).await;
|
||||
let replacement = tokio::time::timeout(Duration::from_secs(1), semaphore.clone().acquire_owned())
|
||||
.await
|
||||
.expect("unpolled body must release admission independently")
|
||||
.unwrap();
|
||||
let mut buffer = [0_u8; 1];
|
||||
for _ in 0..2 {
|
||||
assert_eq!(reader.read(&mut buffer).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut);
|
||||
}
|
||||
assert_eq!(reads.load(AtomicOrdering::SeqCst), 0, "expired readers must never touch disk");
|
||||
assert_eq!(semaphore.available_permits(), 0, "replacement still owns the only slot");
|
||||
let reopens = Arc::new(AtomicUsize::new(0));
|
||||
let count = reopens.clone();
|
||||
let reopen: GetObjectReopen<DiskReadPermitReader<AdmissionReadProbe>> = Box::new(move |_| {
|
||||
count.fetch_add(1, AtomicOrdering::SeqCst);
|
||||
Box::pin(async { Err(GetObjectResumeFailure::Fatal) })
|
||||
});
|
||||
let resume = GetObjectResumeControl::new(
|
||||
reopen,
|
||||
RetryTimer::new(
|
||||
GET_OBJECT_RESUME_MAX_ATTEMPTS,
|
||||
Duration::from_millis(1),
|
||||
Duration::from_millis(2),
|
||||
rustfs_utils::retry::NO_JITTER,
|
||||
0,
|
||||
),
|
||||
);
|
||||
let mut streaming = GetObjectStreamingReader::new(
|
||||
reader,
|
||||
"bucket",
|
||||
"object",
|
||||
"expired-admission",
|
||||
None,
|
||||
1024,
|
||||
Duration::from_secs(1),
|
||||
GetObjectBodyLifecycle::disabled(),
|
||||
Some(resume),
|
||||
);
|
||||
assert_eq!(streaming.read(&mut buffer).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut);
|
||||
assert_eq!(
|
||||
reopens.load(AtomicOrdering::SeqCst),
|
||||
0,
|
||||
"expired admission is terminal for mid-stream resume"
|
||||
);
|
||||
drop(streaming);
|
||||
drop(replacement);
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn disk_read_admission_rejects_trickle_but_allows_long_progressing_downloads() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
|
||||
let permit = semaphore.clone().acquire_owned().await.unwrap();
|
||||
let reads = Arc::new(AtomicUsize::new(0));
|
||||
let mut reader = DiskReadPermitReader::with_idle_timeout(
|
||||
AdmissionReadProbe { reads: reads.clone() },
|
||||
permit.into(),
|
||||
GET_DISK_READ_IDLE_TIMEOUT,
|
||||
);
|
||||
let mut byte = [0_u8; 1];
|
||||
for _ in 0..6 {
|
||||
reader.read_exact(&mut byte).await.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
}
|
||||
let permit = tokio::time::timeout(Duration::from_secs(1), semaphore.clone().acquire_owned())
|
||||
.await
|
||||
.expect("small progress cannot renew admission forever")
|
||||
.unwrap();
|
||||
assert_eq!(reader.read(&mut byte).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut);
|
||||
assert_eq!(reads.load(AtomicOrdering::SeqCst), 6);
|
||||
drop(reader);
|
||||
let mut reader = DiskReadPermitReader::with_idle_timeout(
|
||||
AdmissionReadProbe { reads: reads.clone() },
|
||||
permit.into(),
|
||||
GET_DISK_READ_IDLE_TIMEOUT,
|
||||
);
|
||||
let mut buffer = [0_u8; 16 * 1024];
|
||||
// More than 30 minutes is valid: deadlines constrain inactivity and
|
||||
// rolling progress, not the total size or lifetime of a download.
|
||||
for _ in 0..128 {
|
||||
reader.read_exact(&mut buffer).await.unwrap();
|
||||
tokio::time::advance(Duration::from_secs(15)).await;
|
||||
}
|
||||
assert_eq!(semaphore.available_permits(), 0);
|
||||
drop(reader);
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn disk_read_admission_retains_pending_io_until_completion_after_expiry_or_drop() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
for cancel in [false, true] {
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
|
||||
let permit = semaphore.clone().acquire_owned().await.unwrap();
|
||||
let (finish, receiver) = tokio::sync::oneshot::channel();
|
||||
let started = Arc::new(AtomicUsize::new(0));
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
let mut reader = DiskReadPermitReader::with_idle_timeout(
|
||||
PendingAdmissionRead {
|
||||
finish: receiver,
|
||||
started: started.clone(),
|
||||
completed: completed.clone(),
|
||||
submitted: false,
|
||||
},
|
||||
permit.into(),
|
||||
GET_DISK_READ_IDLE_TIMEOUT,
|
||||
);
|
||||
let mut buffer = [0_u8; 1];
|
||||
let mut reading = Box::pin(reader.read(&mut buffer));
|
||||
assert!(futures::poll!(&mut reading).is_pending());
|
||||
drop(reading);
|
||||
assert_eq!(started.load(AtomicOrdering::SeqCst), 1);
|
||||
if cancel {
|
||||
drop(reader);
|
||||
} else {
|
||||
tokio::time::advance(GET_DISK_READ_IDLE_TIMEOUT + Duration::from_secs(1)).await;
|
||||
assert_eq!(reader.read(&mut buffer).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut);
|
||||
drop(reader);
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(semaphore.available_permits(), 0, "pending disk IO still consumes admission");
|
||||
assert_eq!(completed.load(AtomicOrdering::SeqCst), 0);
|
||||
finish.send(()).unwrap();
|
||||
let _replacement = tokio::time::timeout(Duration::from_secs(1), semaphore.clone().acquire_owned())
|
||||
.await
|
||||
.expect("the outstanding read finishes without the response being polled")
|
||||
.unwrap();
|
||||
assert_eq!(completed.load(AtomicOrdering::SeqCst), 1);
|
||||
assert_eq!(started.load(AtomicOrdering::SeqCst), 1, "retirement must never submit a second read");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn disk_read_admission_preserves_configured_slow_io_window() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
|
||||
let permit = semaphore.clone().acquire_owned().await.unwrap();
|
||||
let (finish, receiver) = tokio::sync::oneshot::channel();
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
let mut reader = DiskReadPermitReader::with_idle_timeout(
|
||||
PendingAdmissionRead {
|
||||
finish: receiver,
|
||||
started: Arc::default(),
|
||||
completed: completed.clone(),
|
||||
submitted: false,
|
||||
},
|
||||
permit.into(),
|
||||
Duration::from_secs(120),
|
||||
);
|
||||
let mut buffer = [0_u8; 1];
|
||||
let mut reading = Box::pin(reader.read(&mut buffer));
|
||||
assert!(futures::poll!(&mut reading).is_pending());
|
||||
tokio::time::advance(Duration::from_secs(45)).await;
|
||||
assert!(
|
||||
futures::poll!(&mut reading).is_pending(),
|
||||
"a high-latency read must not inherit the default 30-second cutoff"
|
||||
);
|
||||
assert_eq!(semaphore.available_permits(), 0);
|
||||
finish.send(()).unwrap();
|
||||
assert_eq!(reading.await.unwrap(), 1);
|
||||
assert_eq!(completed.load(AtomicOrdering::SeqCst), 1);
|
||||
drop(reader);
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_read_admission_unread_file_bodies_cannot_exhaust_both_lanes() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("download");
|
||||
let payload = vec![0x5a_u8; 256 * 1024];
|
||||
tokio::fs::write(&path, &payload).await.unwrap();
|
||||
let manager = ConcurrencyManager::with_disk_read_caps_for_test(1, 1);
|
||||
let mut bodies = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let file = tokio::fs::File::open(&path).await.unwrap();
|
||||
let permit = match manager.admit_disk_read(Duration::from_millis(1)).await.unwrap() {
|
||||
DiskReadAdmission::Primary(permit) | DiskReadAdmission::Degraded(permit) => permit,
|
||||
other => panic!("expected an admitted read, got {other:?}"),
|
||||
};
|
||||
bodies.push(DiskReadPermitReader::with_idle_timeout(file, permit.into(), GET_DISK_READ_IDLE_TIMEOUT));
|
||||
}
|
||||
assert!(matches!(
|
||||
manager.admit_disk_read(Duration::from_millis(1)).await.unwrap(),
|
||||
DiskReadAdmission::Rejected
|
||||
));
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(GET_DISK_READ_IDLE_TIMEOUT + Duration::from_secs(1)).await;
|
||||
let permit = match manager.admit_disk_read(Duration::from_secs(1)).await.unwrap() {
|
||||
DiskReadAdmission::Primary(permit) => permit,
|
||||
other => panic!("unread bodies must return the primary slot, got {other:?}"),
|
||||
};
|
||||
tokio::time::resume();
|
||||
for body in &mut bodies {
|
||||
assert_eq!(body.read(&mut [0_u8; 1]).await.unwrap_err().kind(), std::io::ErrorKind::TimedOut);
|
||||
}
|
||||
let file = tokio::fs::File::open(&path).await.unwrap();
|
||||
let mut reader = DiskReadPermitReader::with_idle_timeout(file, permit.into(), GET_DISK_READ_IDLE_TIMEOUT);
|
||||
let mut actual = Vec::new();
|
||||
reader.read_to_end(&mut actual).await.unwrap();
|
||||
assert_eq!(actual, payload, "a fresh reader must read the real file after recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_read_permit_reader_holds_permit_until_reader_is_dropped() {
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
|
||||
|
||||
@@ -16,10 +16,36 @@
|
||||
|
||||
use super::veeam_sosapi;
|
||||
use super::*;
|
||||
use crate::app::storage_api::object_usecase::access::replication_proxy_allowed;
|
||||
use crate::on_demand_migration::{
|
||||
BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, SourceClient, SourceError, SourceHead,
|
||||
};
|
||||
|
||||
async fn head_metadata_action_allowed(req: &S3Request<HeadObjectInput>, action: S3Action) -> S3Result<bool> {
|
||||
let Ok(mut info) = req_info_ref(req).cloned() else {
|
||||
return Ok(false);
|
||||
};
|
||||
info.suppress_denial_log = true;
|
||||
info.suppress_denial_audit = true;
|
||||
let mut probe = S3Request {
|
||||
input: (),
|
||||
method: req.method.clone(),
|
||||
uri: req.uri.clone(),
|
||||
headers: req.headers.clone(),
|
||||
extensions: req.extensions.clone(),
|
||||
credentials: req.credentials.clone(),
|
||||
region: req.region.clone(),
|
||||
service: req.service.clone(),
|
||||
trailing_headers: req.trailing_headers.clone(),
|
||||
};
|
||||
probe.extensions.insert(info);
|
||||
match authorize_request(&mut probe, Action::S3Action(action)).await {
|
||||
Ok(()) => Ok(true),
|
||||
Err(err) if err.code() == &S3ErrorCode::AccessDenied => Ok(false),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
/// Source HEAD seam for the on-demand migration passthrough: production goes
|
||||
/// through [`SourceClient`], tests script the answers.
|
||||
pub(super) trait OdmHeadSource {
|
||||
@@ -413,7 +439,9 @@ impl DefaultObjectUsecase {
|
||||
// Active-active replication lag window: an object missing
|
||||
// locally may still be served by proxying the HEAD to a
|
||||
// replication target (backlog#1675 P1-5).
|
||||
if let Some(output) = Self::proxy_head_object_to_replication_targets(&req, &bucket, &key, &opts).await {
|
||||
if replication_proxy_allowed(&req.extensions)
|
||||
&& let Some(output) = Self::proxy_head_object_to_replication_targets(&req, &bucket, &key, &opts).await
|
||||
{
|
||||
let response = wrap_response_with_cors(&bucket, &req.method, &req.headers, output).await;
|
||||
let result = Ok(response);
|
||||
let _ = helper
|
||||
@@ -615,7 +643,12 @@ impl DefaultObjectUsecase {
|
||||
|
||||
// Add x-amz-tagging-count header if object has tags
|
||||
// Per S3 API spec, this header should be present in HEAD object response when tags exist
|
||||
if tag_count > 0 {
|
||||
let tagging_action = if req.input.version_id.is_some() {
|
||||
S3Action::GetObjectVersionTaggingAction
|
||||
} else {
|
||||
S3Action::GetObjectTaggingAction
|
||||
};
|
||||
if tag_count > 0 && (replication_check || head_metadata_action_allowed(&req, tagging_action).await?) {
|
||||
let header_name = http::HeaderName::from_static(AMZ_TAG_COUNT);
|
||||
if let Ok(header_value) = tag_count.to_string().parse::<HeaderValue>() {
|
||||
response.headers.insert(header_name, header_value);
|
||||
@@ -623,9 +656,20 @@ impl DefaultObjectUsecase {
|
||||
warn!("Failed to parse x-amz-tagging-count header; skipping");
|
||||
}
|
||||
}
|
||||
let retention_allowed = replication_check
|
||||
|| ([
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
|
||||
AMZ_OBJECT_LOCK_MODE_LOWER,
|
||||
AMZ_OBJECT_LOCK_MODE,
|
||||
]
|
||||
.iter()
|
||||
.any(|key| metadata_map.contains_key(*key))
|
||||
&& head_metadata_action_allowed(&req, S3Action::GetObjectRetentionAction).await?);
|
||||
if let Some(retain_date) = metadata_map
|
||||
.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)
|
||||
.or_else(|| metadata_map.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE))
|
||||
&& retention_allowed
|
||||
&& let Ok(header_name) = http::HeaderName::from_bytes(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.as_bytes())
|
||||
&& let Ok(header_value) = HeaderValue::from_str(retain_date)
|
||||
{
|
||||
@@ -634,6 +678,7 @@ impl DefaultObjectUsecase {
|
||||
if let Some(mode) = metadata_map
|
||||
.get(AMZ_OBJECT_LOCK_MODE_LOWER)
|
||||
.or_else(|| metadata_map.get(AMZ_OBJECT_LOCK_MODE))
|
||||
&& retention_allowed
|
||||
&& let Ok(header_name) = http::HeaderName::from_bytes(AMZ_OBJECT_LOCK_MODE_LOWER.as_bytes())
|
||||
&& let Ok(header_value) = HeaderValue::from_str(mode)
|
||||
{
|
||||
@@ -642,6 +687,7 @@ impl DefaultObjectUsecase {
|
||||
if let Some(legal_hold) = metadata_map
|
||||
.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER)
|
||||
.or_else(|| metadata_map.get(AMZ_OBJECT_LOCK_LEGAL_HOLD))
|
||||
&& (replication_check || head_metadata_action_allowed(&req, S3Action::GetObjectLegalHoldAction).await?)
|
||||
&& let Ok(header_name) = http::HeaderName::from_bytes(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.as_bytes())
|
||||
&& let Ok(header_value) = HeaderValue::from_str(legal_hold)
|
||||
{
|
||||
@@ -1061,6 +1107,102 @@ mod tests {
|
||||
build_request(input, Method::HEAD)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn head_response_filters_lock_and_tag_metadata_by_permission() {
|
||||
crate::app::gating_test_env::run_large_stack_test("head-metadata-permissions", || async {
|
||||
use crate::app::runtime_sources::{AppContext, ServerContextSlot};
|
||||
use crate::app::storage_api::test::ReqInfo;
|
||||
use crate::app::storage_api::test::S3Access as _;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::app::storage_api::test::ecfs::FS;
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let context = Arc::new(AppContext::new(store.clone(), ambient.iam(), ambient.kms()));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context.clone()));
|
||||
let fs = FS::with_server_ctx(slot.clone());
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(context));
|
||||
let bucket = format!("head-metadata-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let object = store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"protected",
|
||||
&mut PutObjReader::from_vec(b"body".to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
user_defined: HashMap::from([
|
||||
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_owned(), "2099-01-01T00:00:00Z".to_owned()),
|
||||
(AMZ_OBJECT_LOCK_MODE_LOWER.to_owned(), "GOVERNANCE".to_owned()),
|
||||
(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_owned(), "ON".to_owned()),
|
||||
(AMZ_OBJECT_TAGGING.to_owned(), "classification=secret".to_owned()),
|
||||
]),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
for (grant, explicit_deny, replica) in [
|
||||
(false, false, false),
|
||||
(true, true, false),
|
||||
(true, false, false),
|
||||
(false, false, true),
|
||||
] {
|
||||
let mut statements = vec![serde_json::json!({"Effect":"Allow","Principal":{"AWS":"*"},
|
||||
"Action":["s3:GetObject","s3:GetObjectVersion","s3:ReplicateObject"],"Resource":format!("arn:aws:s3:::{bucket}/*")})];
|
||||
for (enabled, effect) in [(grant, "Allow"), (explicit_deny, "Deny")] {
|
||||
if enabled {
|
||||
statements.push(serde_json::json!({"Effect":effect,"Principal":{"AWS":"*"},
|
||||
"Action":["s3:GetObjectRetention","s3:GetObjectLegalHold","s3:GetObjectTagging","s3:GetObjectVersionTagging"],
|
||||
"Resource":format!("arn:aws:s3:::{bucket}/*")}));
|
||||
}
|
||||
}
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
crate::app::storage_api::object_usecase::bucket::metadata::BUCKET_POLICY_CONFIG,
|
||||
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
for version in [None, object.version_id.map(|version| version.to_string())] {
|
||||
let mut req = head_input(&bucket, "protected", version);
|
||||
req.extensions.insert(ReqInfo::default());
|
||||
req.extensions.insert(slot.clone());
|
||||
if replica {
|
||||
req.headers
|
||||
.insert("x-rustfs-source-replication-check", HeaderValue::from_static("true"));
|
||||
}
|
||||
fs.head_object(&mut req).await.expect("ordinary HEAD read is permitted");
|
||||
let response = Box::pin(usecase.execute_head_object(req)).await.expect("HEAD succeeds");
|
||||
assert_eq!(response.output.content_length, Some(4));
|
||||
for header in [
|
||||
AMZ_TAG_COUNT,
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
AMZ_OBJECT_LOCK_MODE_LOWER,
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER,
|
||||
] {
|
||||
assert_eq!(
|
||||
response.headers.contains_key(header),
|
||||
replica || (grant && !explicit_deny),
|
||||
"header {header}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Drives `execute_head_object` against a real store with the global
|
||||
/// runtime configured as `head = local_only`: any HEAD that wrongly
|
||||
/// enters the runtime shows up as a `filtered` count, so a zero counter
|
||||
|
||||
@@ -129,7 +129,7 @@ fn internal_put_content_input(headers: &HeaderMap, tags: Option<String>) -> PutO
|
||||
}
|
||||
|
||||
async fn validate_internal_write_target(key: &str, bucket: &str, headers: &HeaderMap) -> Result<(), ApiError> {
|
||||
validate_object_key(key, INTERNAL_PUT_METHOD_NAME).map_err(api_error_from_s3)?;
|
||||
validate_new_object_key(key, INTERNAL_PUT_METHOD_NAME).map_err(api_error_from_s3)?;
|
||||
validate_table_catalog_object_mutation(bucket, key)
|
||||
.await
|
||||
.map_err(api_error_from_s3)?;
|
||||
|
||||
@@ -81,7 +81,9 @@ use super::storage_api::object_usecase::error::{
|
||||
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
|
||||
use super::storage_api::object_usecase::helper::{OperationHelper, build_event_resp_elements, spawn_background_with_context};
|
||||
use super::storage_api::object_usecase::helper::{
|
||||
OperationHelper, build_event_resp_elements, run_owned_mutation, spawn_background_with_context,
|
||||
};
|
||||
use super::storage_api::object_usecase::io::{DynReader, HashReader, WritePlan, compression_metadata_value, wrap_reader};
|
||||
#[cfg(test)]
|
||||
use super::storage_api::object_usecase::object_cache::GetObjectBodySource;
|
||||
@@ -112,8 +114,9 @@ use super::storage_api::object_usecase::timeout_wrapper::{GetObjectTimeoutPolicy
|
||||
use super::storage_api::object_usecase::{ECStore, OldCurrentSize};
|
||||
use super::storage_api::object_usecase::{
|
||||
RFC1123, check_preconditions, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize,
|
||||
remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_object_key,
|
||||
validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors,
|
||||
remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_new_object_key,
|
||||
validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
||||
wrap_response_with_cors,
|
||||
};
|
||||
use crate::app::runtime_sources::{
|
||||
AppContext, current_app_context, current_notify_interface_for_context, current_object_data_cache_for_context,
|
||||
@@ -179,7 +182,7 @@ use s3s::dto::{
|
||||
WebsiteRedirectLocation,
|
||||
};
|
||||
use s3s::header::X_AMZ_RESTORE;
|
||||
use s3s::stream::{ByteStream, DynByteStream, RemainingLength};
|
||||
use s3s::stream::{ByteStream, RemainingLength};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
|
||||
fn object_s3_error(code: S3ErrorCode, message: impl Into<std::borrow::Cow<'static, str>>) -> S3Error {
|
||||
@@ -232,10 +235,7 @@ pub(crate) use self::on_demand_migration_put::*;
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::put::PollCountingBody;
|
||||
use self::put::*;
|
||||
pub(crate) use self::put::{
|
||||
guard_put_object_body_read_timeout, previous_current_size_from_backfill, put_object_body_read_timeout,
|
||||
reject_oversize_single_upload,
|
||||
};
|
||||
pub(crate) use self::put::{previous_current_size_from_backfill, put_object_body_read_timeout, reject_oversize_single_upload};
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::restore::RestoreStatusCommitBarrier;
|
||||
pub(crate) use self::shared::*;
|
||||
|
||||
+338
-228
@@ -15,10 +15,11 @@
|
||||
//! PutObject write path: body admission, eager commit, zero-copy tuning.
|
||||
|
||||
use super::*;
|
||||
use futures::FutureExt;
|
||||
|
||||
use crate::app::trailer_adapter::trailer_source;
|
||||
use crate::auth::{RUSTFS_MAX_CONTENT_LENGTH_QUERY, VerifiedPresignedRequest, parse_presigned_put_max_content_length};
|
||||
use crate::error::{ClientBodyReadTimeout, UploadLimitExceeded};
|
||||
use crate::error::UploadLimitExceeded;
|
||||
static PUT_FAILURE_LOGS: rustfs_utils::LogThrottle = rustfs_utils::LogThrottle::new(5_000);
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -92,11 +93,10 @@ static CACHED_SMALL_EAGER_PUT_MAX_SIZE_BYTES: std::sync::OnceLock<usize> = std::
|
||||
const EVENT_PUT_OBJECT_STORE_INFLIGHT_SLOW: &str = "put_object_store_inflight_slow";
|
||||
|
||||
const EVENT_PUT_OBJECT_STORE_RETURNED: &str = "put_object_store_returned";
|
||||
const EVENT_PUT_OBJECT_REQUEST_COMPLETED: &str = "put_object_request_completed";
|
||||
|
||||
const EVENT_PUT_OBJECT_COMMIT_OWNER_DEADLINE: &str = "put_object_commit_owner_deadline";
|
||||
|
||||
const EVENT_PUT_OBJECT_BODY_READ_STALLED: &str = "put_object_body_read_stalled";
|
||||
|
||||
const PUT_OBJECT_STORE_WARN_THRESHOLD: Duration = Duration::from_secs(5);
|
||||
|
||||
// Eager PUT bodies are fully materialized before the storage owner starts. On
|
||||
@@ -148,9 +148,8 @@ pub(crate) fn reject_oversize_single_upload(size: i64) -> S3Result<()> {
|
||||
|
||||
/// Resolve the S3 request-body inter-chunk read timeout from the environment.
|
||||
///
|
||||
/// Returns `Duration::ZERO` when disabled (`RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT=0`),
|
||||
/// in which case [`guard_put_object_body_read_timeout`] passes the body through
|
||||
/// untouched.
|
||||
/// Zero disables inactivity detection; the independent minimum-progress
|
||||
/// policy still bounds uploads that retain scarce resources.
|
||||
pub(crate) fn put_object_body_read_timeout() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_HTTP_REQUEST_BODY_READ_TIMEOUT,
|
||||
@@ -158,36 +157,6 @@ pub(crate) fn put_object_body_read_timeout() -> Duration {
|
||||
))
|
||||
}
|
||||
|
||||
/// A [`ByteStream`] decorator that aborts a request body whose peer stops
|
||||
/// sending bytes without closing the connection.
|
||||
///
|
||||
/// A well-behaved short body ends with EOF and is rejected promptly by the
|
||||
/// eager/streaming readers. The failure this guards against is different: a
|
||||
/// reverse proxy or CDN forwards a *partial* body and then goes silent while
|
||||
/// holding the connection open, so the inner stream neither yields more bytes
|
||||
/// nor reports EOF. Without a bound, RustFS would wait forever for bytes that
|
||||
/// never arrive and the client eventually sees a hang/abort with no server-side
|
||||
/// explanation (issue #3076).
|
||||
///
|
||||
/// The timer resets on every chunk, so slow-but-progressing uploads are not
|
||||
/// penalized; it only fires after `timeout` of complete silence. On timeout the
|
||||
/// stall is logged with the received/expected byte counts and the read fails
|
||||
/// with an `ErrorKind::TimedOut` error instead of hanging.
|
||||
///
|
||||
/// `remaining_length` and `size_hint` are forwarded from the inner stream so
|
||||
/// wrapping is transparent to length/content handling downstream.
|
||||
struct RequestBodyReadTimeout {
|
||||
inner: DynByteStream,
|
||||
timeout: Duration,
|
||||
timer: Option<Pin<Box<tokio::time::Sleep>>>,
|
||||
received: u64,
|
||||
expected: Option<u64>,
|
||||
bucket: String,
|
||||
key: String,
|
||||
request_id: String,
|
||||
timed_out: bool,
|
||||
}
|
||||
|
||||
/// Enforces a maximum size on the decoded request entity while preserving the
|
||||
/// streaming behavior of the underlying S3 body.
|
||||
struct MaxContentLengthStream {
|
||||
@@ -240,86 +209,8 @@ impl ByteStream for MaxContentLengthStream {
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for RequestBodyReadTimeout {
|
||||
type Item = Result<Bytes, StdError>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.get_mut();
|
||||
|
||||
// Once we have surfaced a stall error, treat the stream as terminated so
|
||||
// we never poll the abandoned inner stream again.
|
||||
if this.timed_out {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
|
||||
match Pin::new(&mut this.inner).poll_next(cx) {
|
||||
Poll::Ready(Some(Ok(chunk))) => {
|
||||
this.timer = None;
|
||||
this.received = this.received.saturating_add(chunk.len() as u64);
|
||||
Poll::Ready(Some(Ok(chunk)))
|
||||
}
|
||||
Poll::Ready(other) => {
|
||||
this.timer = None;
|
||||
Poll::Ready(other)
|
||||
}
|
||||
Poll::Pending => {
|
||||
if this.timeout.is_zero() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
|
||||
if this.timer.is_none() {
|
||||
this.timer = Some(Box::pin(tokio::time::sleep(this.timeout)));
|
||||
}
|
||||
|
||||
if let Some(timer) = this.timer.as_mut()
|
||||
&& std::future::Future::poll(timer.as_mut(), cx).is_ready()
|
||||
{
|
||||
this.timer = None;
|
||||
this.timed_out = true;
|
||||
let expected_display = this.expected.map(|v| v.to_string()).unwrap_or_else(|| "unknown".to_string());
|
||||
warn!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
event = EVENT_PUT_OBJECT_BODY_READ_STALLED,
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
request_id = %this.request_id,
|
||||
bucket = %this.bucket,
|
||||
key = %this.key,
|
||||
received_bytes = this.received,
|
||||
expected_bytes = %expected_display,
|
||||
timeout_secs = this.timeout.as_secs(),
|
||||
state = "stall_timeout",
|
||||
"PutObject request body read stalled; aborting. A proxy/CDN likely forwarded a partial body without closing the connection."
|
||||
);
|
||||
return Poll::Ready(Some(Err(Box::new(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
ClientBodyReadTimeout {
|
||||
timeout: this.timeout,
|
||||
raw_bytes_received: this.received,
|
||||
},
|
||||
)) as StdError)));
|
||||
}
|
||||
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
|
||||
impl ByteStream for RequestBodyReadTimeout {
|
||||
fn remaining_length(&self) -> RemainingLength {
|
||||
self.inner.remaining_length()
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrap an incoming request body with [`RequestBodyReadTimeout`] unless the
|
||||
/// feature is disabled (`timeout == 0`), in which case the body is returned
|
||||
/// untouched. `remaining_length` is preserved via [`StreamingBlob::new`].
|
||||
pub(crate) fn guard_put_object_body_read_timeout(
|
||||
#[cfg(test)]
|
||||
pub(super) fn guard_put_object_body_read_timeout(
|
||||
body: StreamingBlob,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
@@ -327,21 +218,8 @@ pub(crate) fn guard_put_object_body_read_timeout(
|
||||
expected: Option<i64>,
|
||||
timeout: Duration,
|
||||
) -> StreamingBlob {
|
||||
if timeout.is_zero() {
|
||||
return body;
|
||||
}
|
||||
|
||||
StreamingBlob::new(RequestBodyReadTimeout {
|
||||
inner: body.into(),
|
||||
timeout,
|
||||
timer: None,
|
||||
received: 0,
|
||||
expected: expected.and_then(|v| u64::try_from(v).ok()),
|
||||
bucket: bucket.to_string(),
|
||||
key: key.to_string(),
|
||||
request_id: request_id.to_string(),
|
||||
timed_out: false,
|
||||
})
|
||||
super::request_body::guard_upload_progress(body, timeout, bucket, key, request_id, expected.unwrap_or_default().max(0) as u64)
|
||||
.0
|
||||
}
|
||||
|
||||
pub(super) struct PooledBufferReader {
|
||||
@@ -700,7 +578,9 @@ async fn read_small_put_body_exact_pooled<R>(mut body: R, size: usize, pool: &By
|
||||
where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
let mut buf = pool.acquire_buffer(size).await;
|
||||
let mut buf = pool.try_acquire_buffer(size).ok_or_else(|| {
|
||||
S3Error::with_message(S3ErrorCode::SlowDown, "upload buffer pool is full, please reduce your request rate")
|
||||
})?;
|
||||
read_small_put_body_into(&mut body, &mut *buf, size).await?;
|
||||
Ok(buf)
|
||||
}
|
||||
@@ -1134,57 +1014,53 @@ pub(super) struct PutObjectCommitted {
|
||||
pub(super) sse_customer_algorithm: Option<SSECustomerAlgorithm>,
|
||||
pub(super) sse_customer_key_md5: Option<SSECustomerKeyMD5>,
|
||||
pub(super) put_extra_checksum_headers: Vec<(&'static str, String)>,
|
||||
completion: PutObjectCompletion,
|
||||
completion: Option<PutObjectCompletion>,
|
||||
put_request_guard: PutObjectGuard,
|
||||
bucket: String,
|
||||
key: String,
|
||||
concurrent_put_requests: usize,
|
||||
buffer_size: usize,
|
||||
start_time: Instant,
|
||||
size: i64,
|
||||
use_zero_copy_eager_put_path: bool,
|
||||
concurrent_put_requests: usize,
|
||||
buffer_size: usize,
|
||||
}
|
||||
|
||||
impl PutObjectCommitted {
|
||||
/// Publish the audit entry / creation event for `result`, record the
|
||||
/// request-level PutObject metrics and release the request guard.
|
||||
pub(super) fn finish<T>(self, result: &S3Result<S3Response<T>>) {
|
||||
let Self {
|
||||
completion,
|
||||
mut put_request_guard,
|
||||
bucket,
|
||||
key,
|
||||
start_time,
|
||||
size,
|
||||
use_zero_copy_eager_put_path,
|
||||
concurrent_put_requests,
|
||||
buffer_size,
|
||||
..
|
||||
} = self;
|
||||
let _ = completion.complete(result);
|
||||
|
||||
// Record PutObject metrics via zero-copy-metrics
|
||||
{
|
||||
let duration_ms = start_time.elapsed().as_millis() as f64;
|
||||
rustfs_io_metrics::record_put_object(
|
||||
duration_ms,
|
||||
size,
|
||||
use_zero_copy_eager_put_path, // Track if zero-copy was enabled
|
||||
);
|
||||
pub(super) fn finish<T>(mut self, result: &S3Result<S3Response<T>>) {
|
||||
if let Some(completion) = self.completion.take() {
|
||||
let _ = completion.complete(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PutObjectCommitted {
|
||||
fn drop(&mut self) {
|
||||
// This value exists only after the owned storage operation succeeds.
|
||||
// A lost HTTP receiver must not discard the committed write's event.
|
||||
if let Some(completion) = self.completion.take() {
|
||||
let result = Ok(S3Response::new(()));
|
||||
let _ = completion.complete(&result);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object(
|
||||
self.start_time.elapsed().as_millis() as f64,
|
||||
self.size,
|
||||
self.use_zero_copy_eager_put_path,
|
||||
);
|
||||
debug!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
component = "app",
|
||||
subsystem = "object",
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
concurrent_put_requests,
|
||||
buffer_size,
|
||||
event = EVENT_PUT_OBJECT_REQUEST_COMPLETED,
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
result = "success",
|
||||
bucket = %self.bucket,
|
||||
key = %self.key,
|
||||
concurrent_put_requests = self.concurrent_put_requests,
|
||||
buffer_size = self.buffer_size,
|
||||
"PutObject request completed"
|
||||
);
|
||||
|
||||
put_request_guard.finish_ok();
|
||||
self.put_request_guard.finish_ok();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1315,7 +1191,7 @@ impl DefaultObjectUsecase {
|
||||
let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?);
|
||||
|
||||
// Validate object key
|
||||
validate_object_key(&key, request_method_name)?;
|
||||
validate_new_object_key(&key, request_method_name)?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
// Validate archive content encoding (reject when strict mode is enabled)
|
||||
@@ -1328,18 +1204,6 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
|
||||
|
||||
// Guard against a proxy/CDN that forwards a partial body then goes silent
|
||||
// without closing the connection: bound the inter-chunk wait so the read
|
||||
// fails (with a diagnostic log) instead of hanging forever (issue #3076).
|
||||
let body = {
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.clone())
|
||||
.unwrap_or_default();
|
||||
guard_put_object_body_read_timeout(body, &bucket, &key, &request_id, content_length, put_object_body_read_timeout())
|
||||
};
|
||||
|
||||
let body = match max_content_length {
|
||||
Some(limit) => StreamingBlob::new(MaxContentLengthStream {
|
||||
inner: body,
|
||||
@@ -1502,6 +1366,33 @@ impl DefaultObjectUsecase {
|
||||
sse_customer_key_md5,
|
||||
} = sse;
|
||||
|
||||
// The same network budget covers eager and streaming reads. Internal
|
||||
// server-generated bodies do not represent a client upload.
|
||||
let (body, progress_control) = match &origin {
|
||||
PutObjectOrigin::S3 { req, .. } => {
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<request_context::RequestContext>()
|
||||
.map(|context| context.request_id.as_str())
|
||||
.unwrap_or_default();
|
||||
let (body, control) = if let Some(control) = req.extensions.get::<super::request_body::BodyReadControl>() {
|
||||
control.activate_upload(put_object_body_read_timeout(), &bucket, &key, request_id, size.max(0) as u64);
|
||||
(body, control.clone())
|
||||
} else {
|
||||
super::request_body::guard_upload_progress(
|
||||
body,
|
||||
put_object_body_read_timeout(),
|
||||
&bucket,
|
||||
&key,
|
||||
request_id,
|
||||
size.max(0) as u64,
|
||||
)
|
||||
};
|
||||
(super::request_body::demand_upload_body(body, control.clone()), Some(control))
|
||||
}
|
||||
PutObjectOrigin::Internal { .. } => (body, None),
|
||||
};
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), bucket.as_str(), key.as_str()).await?;
|
||||
|
||||
// The app check preserves the existing S3 error contract; the storage
|
||||
@@ -1999,6 +1890,11 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
|
||||
if let Some(control) = progress_control {
|
||||
use rustfs_rio::HashReaderMut;
|
||||
let inner = reader.take_inner();
|
||||
reader.inner = rustfs_rio::boxed_reader(super::request_body::DemandReader::new(inner, control));
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_encryption_prepare", encryption_stage_start);
|
||||
|
||||
let reader = PutObjReader::new(reader);
|
||||
@@ -2262,54 +2158,55 @@ impl DefaultObjectUsecase {
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_context(),
|
||||
put_commit_future,
|
||||
);
|
||||
let put_commit = spawn_traced_join(put_commit_future);
|
||||
let commit_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let put_commit_result = if let Some(cancellation) = eager_put_commit_cancellation {
|
||||
let put_commit = spawn_traced_join(async move {
|
||||
let commit_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let result = std::panic::AssertUnwindSafe(put_commit_future).catch_unwind().await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("store_commit", commit_stage_start);
|
||||
let PutObjectCommitResult { obj_info, put_versioned } = match result {
|
||||
Ok(Ok(result)) => result,
|
||||
Ok(Err(err)) => {
|
||||
put_request_guard.finish_err();
|
||||
return Err(fail_put_object(completion, err));
|
||||
}
|
||||
Err(_) => {
|
||||
put_request_guard.finish_err();
|
||||
return Err(fail_put_object(
|
||||
completion,
|
||||
s3_error!(InternalError, "put object commit owner task panicked"),
|
||||
));
|
||||
}
|
||||
};
|
||||
completion = completion.object(obj_info.clone());
|
||||
if let Some(version_id) = obj_info.version_id {
|
||||
completion = completion.version_id(version_id.to_string());
|
||||
}
|
||||
Ok(PutObjectCommitted {
|
||||
obj_info,
|
||||
put_versioned,
|
||||
effective_sse,
|
||||
effective_kms_key_id,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
put_extra_checksum_headers,
|
||||
completion: Some(completion),
|
||||
put_request_guard,
|
||||
bucket,
|
||||
key,
|
||||
concurrent_put_requests,
|
||||
buffer_size,
|
||||
start_time,
|
||||
size,
|
||||
use_zero_copy_eager_put_path,
|
||||
})
|
||||
});
|
||||
let result = if let Some(cancellation) = eager_put_commit_cancellation {
|
||||
EagerPutCommitOwner::new(put_commit, cancellation, EAGER_PUT_COMMIT_CANCELLATION_GRACE)
|
||||
.join()
|
||||
.await
|
||||
} else {
|
||||
put_commit.await
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("store_commit", commit_stage_start);
|
||||
let PutObjectCommitResult { obj_info, put_versioned } = match put_commit_result {
|
||||
Ok(Ok(result)) => result,
|
||||
Ok(Err(err)) => {
|
||||
put_request_guard.finish_err();
|
||||
return Err(fail_put_object(completion, err));
|
||||
}
|
||||
Err(err) => {
|
||||
put_request_guard.finish_err();
|
||||
return Err(fail_put_object(
|
||||
completion,
|
||||
S3Error::with_message(S3ErrorCode::InternalError, format!("put object commit owner task failed: {err}")),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
completion = completion.object(obj_info.clone());
|
||||
if let Some(version_id) = obj_info.version_id {
|
||||
completion = completion.version_id(version_id.to_string());
|
||||
}
|
||||
|
||||
Ok(PutObjectCommitted {
|
||||
obj_info,
|
||||
put_versioned,
|
||||
effective_sse,
|
||||
effective_kms_key_id,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
put_extra_checksum_headers,
|
||||
completion,
|
||||
put_request_guard,
|
||||
bucket,
|
||||
key,
|
||||
start_time,
|
||||
size,
|
||||
use_zero_copy_eager_put_path,
|
||||
concurrent_put_requests,
|
||||
buffer_size,
|
||||
})
|
||||
result.map_err(|_| s3_error!(InternalError, "put object commit owner task failed"))?
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2434,7 +2331,9 @@ mod tests {
|
||||
.content_length(Some(12))
|
||||
.build()
|
||||
.expect("quorum probe input must build");
|
||||
let probe = crate::app::storage_api::object_usecase::helper::MutationCompletionProbe::install(&bucket);
|
||||
let mut req = build_request(input, method.clone());
|
||||
req.uri = format!("/{bucket}/{key}").parse().expect("PUT URI");
|
||||
if method == Method::POST {
|
||||
req.extensions.insert(PostObjectRequestMarker);
|
||||
}
|
||||
@@ -2445,6 +2344,14 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect_err("read quorum failure must refuse the upload");
|
||||
if stage == PutStorageTestStage::Commit {
|
||||
probe.wait_for(1, 0).await;
|
||||
let audits = probe.audits.lock().expect("audits");
|
||||
assert_eq!(audits.len(), 1);
|
||||
assert_eq!(audits[0].api.status.as_deref(), Some("failure"));
|
||||
assert_eq!(audits[0].api.status_code, Some(503));
|
||||
assert!(probe.events.lock().expect("events").is_empty());
|
||||
}
|
||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable, "stage {index}, method {method}");
|
||||
assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(err.message(), Some("The service is unavailable. Please retry."));
|
||||
@@ -2836,6 +2743,7 @@ mod tests {
|
||||
let (store, context) = real_cold_fill_test_context().await;
|
||||
let bucket = format!("put-owner-tail-{}", Uuid::new_v4());
|
||||
let object = "object.bin";
|
||||
let probe = crate::app::storage_api::object_usecase::helper::MutationCompletionProbe::install(&bucket);
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
@@ -2859,11 +2767,9 @@ mod tests {
|
||||
.build()
|
||||
.expect("PUT input must build");
|
||||
let put_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
let put = tokio::spawn(async move {
|
||||
put_usecase
|
||||
.execute_put_object(&FS::new(), build_request(put_input, Method::PUT))
|
||||
.await
|
||||
});
|
||||
let mut request = build_request(put_input, Method::PUT);
|
||||
request.uri = format!("/{bucket}/{object}").parse().expect("PUT URI");
|
||||
let put = tokio::spawn(async move { put_usecase.execute_put_object(&FS::new(), request).await });
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(10), post_store_entered.wait())
|
||||
.await
|
||||
@@ -2896,6 +2802,27 @@ mod tests {
|
||||
.await
|
||||
.expect("cancelled request's owned commit must still publish the object");
|
||||
assert_eq!(recovered.size, i64::try_from(payload.len()).expect("test payload length must fit i64"));
|
||||
probe.wait_for(1, 1).await;
|
||||
let normal_input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("normal.bin".to_owned())
|
||||
.body(Some(StreamingBlob::from(s3s::Body::from(Bytes::from_static(b"normal")))))
|
||||
.content_length(Some(6))
|
||||
.build()
|
||||
.expect("normal PUT");
|
||||
let mut normal_request = build_request(normal_input, Method::PUT);
|
||||
normal_request.uri = format!("/{bucket}/normal.bin").parse().expect("URI");
|
||||
DefaultObjectUsecase::with_context(Some(context))
|
||||
.execute_put_object(&FS::new(), normal_request)
|
||||
.await
|
||||
.expect("normal response completion");
|
||||
let audits = probe.audits.lock().expect("audits");
|
||||
assert_eq!(audits.len(), 2, "both cancelled and returned commits audit exactly once");
|
||||
assert_eq!(audits[0].api.status.as_deref(), Some("success"));
|
||||
let events = probe.events.lock().expect("events");
|
||||
assert_eq!(events.len(), 2, "explicit completion and Drop must not double publish");
|
||||
assert_eq!(events[0].event_name, EventName::ObjectCreatedPut);
|
||||
assert_eq!(events[0].object.name, object);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3393,6 +3320,160 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_put_object_expires_stalled_body_without_creating_object() {
|
||||
use super::super::request_body::{BodyReadControl, ObservedBody};
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use http_body::Frame;
|
||||
use http_body_util::StreamBody;
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
let (store, context) = real_store_test_context().await;
|
||||
let bucket = format!("put-progress-{}", Uuid::new_v4().simple());
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
for size in [8192, 2 * 1024 * 1024, 32 * 1024 * 1024] {
|
||||
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<Result<Frame<Bytes>, std::io::Error>>();
|
||||
let control = BodyReadControl::default();
|
||||
let body = StreamingBlob::from(s3s::Body::http_body_unsync(ObservedBody::new(
|
||||
StreamBody::new(UnboundedReceiverStream::new(receiver)),
|
||||
control.clone(),
|
||||
)));
|
||||
let key = format!("object-{size}.bin");
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(key.clone())
|
||||
.body(Some(body))
|
||||
.content_length(Some(size))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut request = build_request(input, Method::PUT);
|
||||
request.extensions.insert(control.clone());
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
let owner = tokio::spawn(async move { usecase.execute_put_object(&FS::new(), request).await });
|
||||
control.wait_until_pending_for_test().await;
|
||||
control.advance_wait_for_test(Duration::from_secs(60));
|
||||
sender.send(Ok(Frame::data(Bytes::new()))).unwrap();
|
||||
let error = owner.await.unwrap().expect_err("stalled eager/zero-copy/streaming body");
|
||||
assert_eq!(error.code(), &S3ErrorCode::RequestTimeout);
|
||||
assert!(is_err_object_not_found(
|
||||
&store
|
||||
.get_object_info(&bucket, &key, &ObjectOptions::default())
|
||||
.await
|
||||
.unwrap_err()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pooled_upload_keeps_admission_and_buffer_during_real_commit_after_caller_cancel() {
|
||||
use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
use rustfs_io_core::BytesPoolConfig;
|
||||
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("upload-commit-owner", 16384).await;
|
||||
let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO);
|
||||
let admission = manager.admit_put_object(8192).await.unwrap();
|
||||
let permit = match admission {
|
||||
ForegroundWriteAdmission::Admitted(permit) => permit,
|
||||
_ => panic!("initial permit"),
|
||||
};
|
||||
let pool = Arc::new(BytesPool::with_config(BytesPoolConfig {
|
||||
small_max: 1,
|
||||
..Default::default()
|
||||
}));
|
||||
let (body, control) = super::super::request_body::guard_upload_progress(
|
||||
StreamingBlob::from(s3s::Body::from(Bytes::from(vec![7; 8192]))),
|
||||
Duration::ZERO,
|
||||
&bucket,
|
||||
"object",
|
||||
"request",
|
||||
8192,
|
||||
);
|
||||
let buffer =
|
||||
read_small_put_body_exact_pooled(StreamReader::new(body.map(|item| item.map_err(s3s_body_error_to_io))), 8192, &pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let reader = super::super::request_body::DemandReader::new(PooledBufferReader::new(buffer, 8192), control);
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(reader, 8192, 8192, None, None, false).unwrap());
|
||||
let barrier = PutObjectCommitBarrier::install(&bucket, "object", PutObjectCommitPause::BeforeQuotaRename);
|
||||
let write_store = Arc::clone(&store);
|
||||
let write_bucket = bucket.clone();
|
||||
let (finished, completion) = tokio::sync::oneshot::channel();
|
||||
let owner = tokio::spawn(async move {
|
||||
let permit_guard = permit;
|
||||
let result = write_store
|
||||
.put_object(&write_bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await;
|
||||
drop(reader);
|
||||
drop(permit_guard);
|
||||
finished.send(result).unwrap();
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
drop(owner);
|
||||
assert!(pool.try_acquire_buffer(8192).is_none());
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(8192).await.unwrap(),
|
||||
ForegroundWriteAdmission::Rejected
|
||||
));
|
||||
barrier.release();
|
||||
completion.await.unwrap().unwrap();
|
||||
assert!(pool.try_acquire_buffer(8192).is_some());
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(8192).await.unwrap(),
|
||||
ForegroundWriteAdmission::Admitted(_)
|
||||
));
|
||||
assert_eq!(
|
||||
store
|
||||
.get_object_info(&bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.unwrap()
|
||||
.size,
|
||||
8192
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn pooled_upload_exhaustion_rejects_without_allocating_and_recovers_after_stall() {
|
||||
use futures::poll;
|
||||
use rustfs_io_core::BytesPoolConfig;
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
let pool = BytesPool::with_config(BytesPoolConfig {
|
||||
small_max: 1,
|
||||
..Default::default()
|
||||
});
|
||||
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<Result<Bytes, std::io::Error>>();
|
||||
let (body, _control) = super::super::request_body::guard_upload_progress(
|
||||
StreamingBlob::wrap(UnboundedReceiverStream::new(receiver)),
|
||||
Duration::ZERO,
|
||||
"bucket",
|
||||
"object",
|
||||
"request",
|
||||
8192,
|
||||
);
|
||||
let mut owner = Box::pin(read_small_put_body_exact_pooled(
|
||||
StreamReader::new(body.map(|item| item.map_err(s3s_body_error_to_io))),
|
||||
8192,
|
||||
&pool,
|
||||
));
|
||||
assert!(poll!(owner.as_mut()).is_pending());
|
||||
let denied = read_small_put_body_exact_pooled(std::io::Cursor::new(vec![0; 8192]), 8192, &pool).await;
|
||||
assert_eq!(denied.err().expect("full pool must reject immediately").code(), &S3ErrorCode::SlowDown);
|
||||
for _ in 0..5 {
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
sender.send(Ok(Bytes::from_static(b"x"))).unwrap();
|
||||
assert!(poll!(owner.as_mut()).is_pending());
|
||||
assert!(pool.try_acquire_buffer(8192).is_none());
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
assert_eq!(owner.await.err().expect("trickle must expire").code(), &S3ErrorCode::RequestTimeout);
|
||||
let healthy = read_small_put_body_exact_pooled(std::io::Cursor::new(vec![1; 8192]), 8192, &pool)
|
||||
.await
|
||||
.expect("timed-out buffer must be reusable");
|
||||
assert_eq!(&healthy[..], &[1; 8192]);
|
||||
assert!(pool.try_acquire_buffer(8192).is_none(), "commit reader still owns capacity");
|
||||
drop(healthy);
|
||||
assert!(pool.try_acquire_buffer(8192).is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_small_put_body_exact_pooled_reads_exact_bytes_without_prefill() {
|
||||
let pool = get_concurrency_manager().bytes_pool();
|
||||
@@ -4273,6 +4354,9 @@ mod tests {
|
||||
.set_quota_config(&bucket, BucketQuota::new(Some(LIMIT)))
|
||||
.await
|
||||
.expect("configure durable quota");
|
||||
crate::app::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative quota usage");
|
||||
|
||||
let payload = Bytes::from(vec![0x42; 1024]);
|
||||
let input = PutObjectInput::builder()
|
||||
@@ -4456,7 +4540,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelled_put_after_commit_marker_is_reconciled() {
|
||||
async fn cancelled_put_before_rename_releases_marked_quota_reservation() {
|
||||
use crate::app::storage_api::test::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("cancelled-spawned-put-quota", 4096).await;
|
||||
@@ -4470,6 +4554,12 @@ mod tests {
|
||||
.await
|
||||
});
|
||||
commit_barrier.wait_until_paused().await;
|
||||
let mut denied = PutObjReader::from_vec(vec![0x54; 4096]);
|
||||
let error = store
|
||||
.put_object(&bucket, "replacement", &mut denied, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the live commit marker must still reserve the entire quota");
|
||||
assert!(matches!(error, StorageError::QuotaExceeded { .. }));
|
||||
first.abort();
|
||||
assert!(first.await.is_err(), "the outer request task must be cancelled");
|
||||
drop(commit_barrier);
|
||||
@@ -4478,11 +4568,25 @@ mod tests {
|
||||
.get_object_info(&bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("cancelling before rename must not commit the object");
|
||||
// The owned pre-rename cancellation path proves no publication occurred,
|
||||
// releases the mutation fences, and aborts the reservation before unlock.
|
||||
let mut replacement = PutObjReader::from_vec(vec![0x54; 4096]);
|
||||
store
|
||||
.put_object(&bucket, "replacement", &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the next admission must reap the abandoned commit marker");
|
||||
.expect("safe pre-rename rollback must release the marked reservation without a scan");
|
||||
let mut excess = PutObjReader::from_vec(vec![0x55]);
|
||||
let error = store
|
||||
.put_object(&bucket, "excess", &mut excess, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the replacement must consume the reclaimed quota exactly once");
|
||||
assert!(matches!(
|
||||
error,
|
||||
StorageError::QuotaExceeded {
|
||||
current: 4096,
|
||||
limit: 4096
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -4580,6 +4684,9 @@ mod tests {
|
||||
.set_quota_config(&bucket, BucketQuota::new(Some(4096)))
|
||||
.await
|
||||
.expect("lower bucket quota");
|
||||
crate::app::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative quota usage");
|
||||
let mut excess = PutObjReader::from_vec(vec![0x72]);
|
||||
let err = store
|
||||
.put_object(&bucket, "excess", &mut excess, &ObjectOptions::default())
|
||||
@@ -4640,6 +4747,9 @@ mod tests {
|
||||
.await
|
||||
.expect("quota update task should not panic")
|
||||
.expect("quota update should proceed after commit");
|
||||
crate::app::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative quota usage");
|
||||
|
||||
let mut excess = PutObjReader::from_vec(vec![0x74]);
|
||||
let err = store
|
||||
|
||||
@@ -17,8 +17,10 @@
|
||||
//! after its input returned Pending, so the transport cannot infer read demand.
|
||||
|
||||
use super::{LOG_COMPONENT_APP, LOG_SUBSYSTEM_OBJECT};
|
||||
use crate::app::storage_api::request_body::{ByteStream, DynByteStream, RemainingLength, StdError, StreamingBlob};
|
||||
use crate::error::ClientBodyReadTimeout;
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
use http_body::{Body, Frame, SizeHint};
|
||||
use parking_lot::Mutex;
|
||||
use rustfs_rio::{DynReader, EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex};
|
||||
@@ -33,8 +35,12 @@ use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::time::{Instant, Sleep};
|
||||
|
||||
const EVENT_UPLOAD_BODY_PROGRESS_STALLED: &str = "upload_body_progress_stalled";
|
||||
const EVENT_UPLOAD_PART_BODY_READ_STALLED: &str = "upload_part_body_read_stalled";
|
||||
|
||||
pub(crate) const UPLOAD_PROGRESS_WINDOW: Duration = Duration::from_secs(60);
|
||||
pub(crate) const UPLOAD_PROGRESS_BYTES: u64 = 64 * 1024;
|
||||
|
||||
struct ReadPolicy {
|
||||
timeout: Duration,
|
||||
bucket: String,
|
||||
@@ -50,13 +56,18 @@ struct ReadBudget {
|
||||
waiting_since: Option<Instant>,
|
||||
waited: Duration,
|
||||
raw_bytes_received: u64,
|
||||
progress_floor: bool,
|
||||
progress_bytes: u64,
|
||||
progress_waited: Duration,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl ReadBudget {
|
||||
fn pause(&mut self) {
|
||||
if let Some(start) = self.waiting_since.take() {
|
||||
self.waited = self.waited.saturating_add(start.elapsed());
|
||||
let elapsed = start.elapsed();
|
||||
self.waited = self.waited.saturating_add(elapsed);
|
||||
self.progress_waited = self.progress_waited.saturating_add(elapsed);
|
||||
}
|
||||
self.demand = false;
|
||||
}
|
||||
@@ -73,6 +84,7 @@ struct SharedBudget {
|
||||
}
|
||||
|
||||
impl BodyReadControl {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn activate(
|
||||
&self,
|
||||
timeout: Duration,
|
||||
@@ -99,6 +111,22 @@ impl BodyReadControl {
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn activate_upload(&self, timeout: Duration, bucket: &str, key: &str, request_id: &str, expected: u64) {
|
||||
let mut state = self.0.budget.lock();
|
||||
if state.finished {
|
||||
return;
|
||||
}
|
||||
state.policy = Some(ReadPolicy {
|
||||
timeout,
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
request_id: request_id.to_owned(),
|
||||
expected_decoded_bytes: expected,
|
||||
});
|
||||
state.progress_floor = true;
|
||||
self.0.active.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
fn begin_read(&self) {
|
||||
let mut state = self.0.budget.lock();
|
||||
if !state.finished {
|
||||
@@ -111,7 +139,7 @@ impl BodyReadControl {
|
||||
}
|
||||
|
||||
fn progress(&self, bytes: usize) {
|
||||
// Other HTTP operations never activate this UploadPart policy.
|
||||
// Other HTTP operations never activate this upload policy.
|
||||
if !self.0.active.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
@@ -119,8 +147,17 @@ impl BodyReadControl {
|
||||
state.raw_bytes_received = state
|
||||
.raw_bytes_received
|
||||
.saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX));
|
||||
// Charge the pending interval before resetting inactivity. Small
|
||||
// frames do not reset the minimum-progress window or bank credit.
|
||||
if let Some(start) = state.waiting_since.take() {
|
||||
state.progress_waited = state.progress_waited.saturating_add(start.elapsed());
|
||||
}
|
||||
state.progress_bytes = state.progress_bytes.saturating_add(bytes as u64);
|
||||
if state.progress_bytes >= UPLOAD_PROGRESS_BYTES {
|
||||
state.progress_bytes = 0;
|
||||
state.progress_waited = Duration::ZERO;
|
||||
}
|
||||
state.waited = Duration::ZERO;
|
||||
state.waiting_since = None;
|
||||
}
|
||||
|
||||
fn finish(&self) {
|
||||
@@ -141,38 +178,54 @@ impl BodyReadControl {
|
||||
if !state.demand || state.finished {
|
||||
return None;
|
||||
}
|
||||
let remaining = timeout.saturating_sub(state.waited);
|
||||
let inactivity_remaining = if timeout.is_zero() {
|
||||
Duration::MAX
|
||||
} else {
|
||||
timeout.saturating_sub(state.waited)
|
||||
};
|
||||
let remaining = if state.progress_floor {
|
||||
inactivity_remaining.min(UPLOAD_PROGRESS_WINDOW.saturating_sub(state.progress_waited))
|
||||
} else {
|
||||
inactivity_remaining
|
||||
};
|
||||
let start = *state.waiting_since.get_or_insert_with(Instant::now);
|
||||
// A timeout beyond the clock's representable range cannot elapse.
|
||||
start.checked_add(remaining)
|
||||
}
|
||||
|
||||
fn expire(&self) -> Option<ClientBodyReadTimeout> {
|
||||
let (policy, raw_bytes_received) = {
|
||||
let (policy, raw_bytes_received, progress_floor) = {
|
||||
let mut state = self.0.budget.lock();
|
||||
let timeout = state.policy.as_ref()?.timeout;
|
||||
if !state.demand || state.waited.saturating_add(state.waiting_since?.elapsed()) < timeout {
|
||||
let elapsed = state.waiting_since?.elapsed();
|
||||
let inactivity_expired = !timeout.is_zero() && state.waited.saturating_add(elapsed) >= timeout;
|
||||
let progress_expired =
|
||||
state.progress_floor && state.progress_waited.saturating_add(elapsed) >= UPLOAD_PROGRESS_WINDOW;
|
||||
if !state.demand || (!inactivity_expired && !progress_expired) {
|
||||
return None;
|
||||
}
|
||||
if progress_expired {
|
||||
state.policy.as_mut()?.timeout = UPLOAD_PROGRESS_WINDOW;
|
||||
}
|
||||
state.finished = true;
|
||||
self.0.active.store(false, Ordering::Release);
|
||||
state.demand = false;
|
||||
state.waiting_since = None;
|
||||
(state.policy.take()?, state.raw_bytes_received)
|
||||
(state.policy.take()?, state.raw_bytes_received, state.progress_floor)
|
||||
};
|
||||
tracing::error!(
|
||||
event = EVENT_UPLOAD_PART_BODY_READ_STALLED,
|
||||
event = if progress_floor { EVENT_UPLOAD_BODY_PROGRESS_STALLED } else { EVENT_UPLOAD_PART_BODY_READ_STALLED },
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
state = "stall_timeout",
|
||||
operation = "UploadPart",
|
||||
operation = if progress_floor { "Upload" } else { "UploadPart" },
|
||||
request_id = %policy.request_id,
|
||||
bucket = %policy.bucket,
|
||||
key = %policy.key,
|
||||
raw_bytes_received,
|
||||
expected_decoded_bytes = policy.expected_decoded_bytes,
|
||||
timeout_secs = policy.timeout.as_secs(),
|
||||
"UploadPart request body read stalled"
|
||||
"Upload request body read budget exhausted"
|
||||
);
|
||||
Some(ClientBodyReadTimeout {
|
||||
timeout: policy.timeout,
|
||||
@@ -245,6 +298,11 @@ impl<B: Body<Data = Bytes> + Unpin> Body for ObservedBody<B> {
|
||||
if self.ended {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
let already_waiting = self.control.0.budget.lock().waiting_since.is_some();
|
||||
if already_waiting && let Some(error) = self.poll_timeout(cx) {
|
||||
self.ended = true;
|
||||
return Poll::Ready(Some(Err(ObservedBodyError::Inactivity(error))));
|
||||
}
|
||||
// Ignore empty data without resetting the budget. Bound work per poll
|
||||
// even if a body repeatedly returns immediately-ready empty frames.
|
||||
for _ in 0..32 {
|
||||
@@ -308,18 +366,18 @@ impl<B> Drop for ObservedBody<B> {
|
||||
/// Current erasure readers are owned by their read future/producer: canceling
|
||||
/// that owner drops this reader. A future retained-reader cancellation path
|
||||
/// must explicitly pause its read demand before retaining the reader.
|
||||
pub(crate) struct DemandReader {
|
||||
inner: DynReader,
|
||||
pub(crate) struct DemandReader<R = DynReader> {
|
||||
inner: R,
|
||||
control: BodyReadControl,
|
||||
}
|
||||
|
||||
impl DemandReader {
|
||||
pub(crate) fn new(inner: DynReader, control: BodyReadControl) -> Self {
|
||||
impl<R> DemandReader<R> {
|
||||
pub(crate) fn new(inner: R, control: BodyReadControl) -> Self {
|
||||
Self { inner, control }
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for DemandReader {
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for DemandReader<R> {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
self.control.begin_read();
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
@@ -330,13 +388,13 @@ impl AsyncRead for DemandReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DemandReader {
|
||||
impl<R> Drop for DemandReader<R> {
|
||||
fn drop(&mut self) {
|
||||
self.control.finish();
|
||||
}
|
||||
}
|
||||
|
||||
impl EtagResolvable for DemandReader {
|
||||
impl<R: EtagResolvable> EtagResolvable for DemandReader<R> {
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
self.inner.is_etag_reader()
|
||||
}
|
||||
@@ -345,7 +403,7 @@ impl EtagResolvable for DemandReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl HashReaderDetector for DemandReader {
|
||||
impl<R: HashReaderDetector> HashReaderDetector for DemandReader<R> {
|
||||
fn is_hash_reader(&self) -> bool {
|
||||
self.inner.is_hash_reader()
|
||||
}
|
||||
@@ -354,11 +412,145 @@ impl HashReaderDetector for DemandReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryGetIndex for DemandReader {
|
||||
impl<R: TryGetIndex> TryGetIndex for DemandReader<R> {
|
||||
fn try_get_index(&self) -> Option<&Index> {
|
||||
self.inner.try_get_index()
|
||||
}
|
||||
}
|
||||
|
||||
/// Establish demand for eager reads and archive sniffing before the final
|
||||
/// transformed reader exists. Pending demand is subsequently paused by the
|
||||
/// outer DemandReader when a transform returns buffered output.
|
||||
pub(crate) fn demand_upload_body(body: StreamingBlob, control: BodyReadControl) -> StreamingBlob {
|
||||
StreamingBlob::new(DemandStream {
|
||||
inner: body.into(),
|
||||
control,
|
||||
})
|
||||
}
|
||||
struct DemandStream {
|
||||
inner: DynByteStream,
|
||||
control: BodyReadControl,
|
||||
}
|
||||
impl Stream for DemandStream {
|
||||
type Item = Result<Bytes, StdError>;
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
self.control.begin_read();
|
||||
let result = self.inner.poll_next_unpin(cx);
|
||||
if result.is_ready() {
|
||||
self.control.pause_read();
|
||||
}
|
||||
result
|
||||
}
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
impl ByteStream for DemandStream {
|
||||
fn remaining_length(&self) -> RemainingLength {
|
||||
self.inner.remaining_length()
|
||||
}
|
||||
}
|
||||
|
||||
/// Fallback for direct protocol callers without the raw HTTP observation.
|
||||
/// HTTP uses ObservedBody because a signed chunk may not yield decoded output
|
||||
/// until its full payload has arrived and its signature has been verified.
|
||||
/// The final DemandReader pauses pending reads when transforms return output.
|
||||
pub(crate) fn guard_upload_progress(
|
||||
body: StreamingBlob,
|
||||
timeout: Duration,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
expected: u64,
|
||||
) -> (StreamingBlob, BodyReadControl) {
|
||||
let control = BodyReadControl::default();
|
||||
control.activate_upload(timeout, bucket, key, request_id, expected);
|
||||
let stream = UploadProgressStream {
|
||||
inner: body.into(),
|
||||
control: control.clone(),
|
||||
timer: None,
|
||||
ended: false,
|
||||
};
|
||||
(StreamingBlob::new(stream), control)
|
||||
}
|
||||
|
||||
struct UploadProgressStream {
|
||||
inner: DynByteStream,
|
||||
control: BodyReadControl,
|
||||
timer: Option<Pin<Box<Sleep>>>,
|
||||
ended: bool,
|
||||
}
|
||||
|
||||
impl UploadProgressStream {
|
||||
fn poll_timeout(&mut self, cx: &mut Context<'_>) -> Option<ClientBodyReadTimeout> {
|
||||
let deadline = self.control.waiting_deadline()?;
|
||||
let timer = self.timer.get_or_insert_with(|| Box::pin(tokio::time::sleep_until(deadline)));
|
||||
if timer.deadline() != deadline {
|
||||
timer.as_mut().reset(deadline);
|
||||
}
|
||||
timer.as_mut().poll(cx).is_ready().then(|| self.control.expire()).flatten()
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for UploadProgressStream {
|
||||
type Item = Result<Bytes, StdError>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
if self.ended {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
self.control.begin_read();
|
||||
// An overdue read cannot be revived by a late frame (including EOF).
|
||||
// Do not start a wait until the source actually stalls.
|
||||
let already_waiting = self.control.0.budget.lock().waiting_since.is_some();
|
||||
if already_waiting && let Some(error) = self.poll_timeout(cx) {
|
||||
self.ended = true;
|
||||
return Poll::Ready(Some(Err(Box::new(std::io::Error::new(std::io::ErrorKind::TimedOut, error)))));
|
||||
}
|
||||
let mut empty_frames_only = true;
|
||||
for _ in 0..32 {
|
||||
match self.inner.poll_next_unpin(cx) {
|
||||
Poll::Ready(Some(Ok(bytes))) if !bytes.is_empty() => {
|
||||
self.control.progress(bytes.len());
|
||||
self.control.pause_read();
|
||||
return Poll::Ready(Some(Ok(bytes)));
|
||||
}
|
||||
Poll::Ready(Some(Ok(_))) => {}
|
||||
Poll::Ready(other) => {
|
||||
self.ended = true;
|
||||
self.control.finish();
|
||||
return Poll::Ready(other);
|
||||
}
|
||||
Poll::Pending => {
|
||||
empty_frames_only = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(error) = self.poll_timeout(cx) {
|
||||
self.ended = true;
|
||||
return Poll::Ready(Some(Err(Box::new(std::io::Error::new(std::io::ErrorKind::TimedOut, error)))));
|
||||
}
|
||||
// Also bound an always-ready source of empty frames.
|
||||
if empty_frames_only {
|
||||
cx.waker().wake_by_ref();
|
||||
}
|
||||
Poll::Pending
|
||||
}
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
impl ByteStream for UploadProgressStream {
|
||||
fn remaining_length(&self) -> RemainingLength {
|
||||
self.inner.remaining_length()
|
||||
}
|
||||
}
|
||||
impl Drop for UploadProgressStream {
|
||||
fn drop(&mut self) {
|
||||
self.control.finish();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
@@ -28,6 +28,19 @@ mod protocol;
|
||||
type FrameSender = mpsc::UnboundedSender<Result<Frame<Bytes>, io::Error>>;
|
||||
|
||||
impl BodyReadControl {
|
||||
pub(crate) async fn wait_until_pending_for_test(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
if self.0.budget.lock().waiting_since.is_some() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("body must reach a pending network read");
|
||||
}
|
||||
|
||||
/// Elapse this body's active wait without advancing storage timers.
|
||||
/// The fixture must wake the raw body afterward so it polls the deadline.
|
||||
pub(crate) fn advance_wait_for_test(&self, elapsed: Duration) {
|
||||
@@ -37,6 +50,7 @@ impl BodyReadControl {
|
||||
"body must already be waiting for raw input"
|
||||
);
|
||||
state.waited = state.waited.saturating_add(elapsed);
|
||||
state.progress_waited = state.progress_waited.saturating_add(elapsed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -277,3 +291,184 @@ async fn native_body_errors_preserve_their_original_source() {
|
||||
let error = reader.read_to_end(&mut Vec::new()).await.expect_err("native error");
|
||||
assert_eq!(ApiError::from(error).code, S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
fn progress_reader() -> (mpsc::UnboundedSender<Result<Bytes, io::Error>>, impl AsyncRead + Unpin) {
|
||||
let (sender, receiver) = mpsc::unbounded_channel();
|
||||
let (stream, control) = guard_upload_progress(
|
||||
StreamingBlob::wrap(UnboundedReceiverStream::new(receiver)),
|
||||
Duration::ZERO,
|
||||
"bucket",
|
||||
"object",
|
||||
"request",
|
||||
1024 * 1024,
|
||||
);
|
||||
let inner = StreamReader::new(stream.map(|item| item.map_err(io::Error::other)));
|
||||
(sender, DemandReader::new(inner, control))
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn upload_progress_rejects_trickle_empty_frames_and_late_revival() {
|
||||
for empty in [false, true] {
|
||||
let (sender, mut reader) = progress_reader();
|
||||
let mut output = Vec::new();
|
||||
let mut read = Box::pin(reader.read_to_end(&mut output));
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
for _ in 0..5 {
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
sender
|
||||
.send(Ok(if empty { Bytes::new() } else { Bytes::from_static(b"x") }))
|
||||
.unwrap();
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
// Even a full qualifying chunk arriving at the deadline is too late.
|
||||
sender.send(Ok(Bytes::from(vec![0; UPLOAD_PROGRESS_BYTES as usize]))).unwrap();
|
||||
let error = read.await.expect_err("small or empty frames must not renew the window");
|
||||
assert_eq!(ApiError::from(error).code, S3ErrorCode::RequestTimeout);
|
||||
let mut byte = [0];
|
||||
assert_eq!(reader.read(&mut byte).await.unwrap(), 0, "expired reader cannot resume");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn upload_progress_allows_long_uploads_without_banking_burst_credit() {
|
||||
let (sender, mut reader) = progress_reader();
|
||||
let mut output = Vec::new();
|
||||
let mut read = Box::pin(reader.read_to_end(&mut output));
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
for _ in 0..80 {
|
||||
tokio::time::advance(Duration::from_secs(59)).await;
|
||||
sender.send(Ok(Bytes::from(vec![7; UPLOAD_PROGRESS_BYTES as usize]))).unwrap();
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
}
|
||||
drop(sender);
|
||||
assert_eq!(
|
||||
read.await.unwrap(),
|
||||
usize::try_from(80 * UPLOAD_PROGRESS_BYTES).expect("bounded upload fixture size")
|
||||
);
|
||||
|
||||
let (sender, mut reader) = progress_reader();
|
||||
let mut output = Vec::new();
|
||||
let mut read = Box::pin(reader.read_to_end(&mut output));
|
||||
sender
|
||||
.send(Ok(Bytes::from(vec![7; 4 * UPLOAD_PROGRESS_BYTES as usize])))
|
||||
.unwrap();
|
||||
assert!(poll!(read.as_mut()).is_pending());
|
||||
tokio::time::advance(UPLOAD_PROGRESS_WINDOW).await;
|
||||
assert_eq!(ApiError::from(read.await.unwrap_err()).code, S3ErrorCode::RequestTimeout);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn upload_progress_excludes_storage_and_buffered_transform_backpressure() {
|
||||
// A transform may return buffered output after polling its input Pending.
|
||||
struct BufferedOutput<R> {
|
||||
inner: R,
|
||||
first: bool,
|
||||
}
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for BufferedOutput<R> {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
|
||||
if result.is_pending() && self.first {
|
||||
self.first = false;
|
||||
buf.put_slice(b"buffered");
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
let (sender, receiver) = mpsc::unbounded_channel::<Result<Bytes, io::Error>>();
|
||||
let (stream, control) = guard_upload_progress(
|
||||
StreamingBlob::wrap(UnboundedReceiverStream::new(receiver)),
|
||||
Duration::from_secs(300),
|
||||
"bucket",
|
||||
"object",
|
||||
"request",
|
||||
8,
|
||||
);
|
||||
let inner = StreamReader::new(stream.map(|item| item.map_err(io::Error::other)));
|
||||
let mut reader = DemandReader::new(BufferedOutput { inner, first: true }, control);
|
||||
let mut buffer = [0; 8];
|
||||
assert_eq!(reader.read(&mut buffer).await.unwrap(), 8);
|
||||
tokio::time::advance(Duration::from_secs(3600)).await;
|
||||
sender.send(Ok(Bytes::from_static(b"uploaded"))).unwrap();
|
||||
assert_eq!(reader.read(&mut buffer).await.unwrap(), 8);
|
||||
assert_eq!(&buffer, b"uploaded");
|
||||
drop(sender);
|
||||
assert_eq!(reader.read(&mut buffer).await.unwrap(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn upload_progress_reclaims_foreground_and_multipart_admission_after_trickle() {
|
||||
use crate::app::storage_api::object_usecase::concurrency::{ConcurrencyManager, ForegroundWriteAdmission};
|
||||
let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO);
|
||||
for multipart in [false, true] {
|
||||
let admission = if multipart {
|
||||
manager.admit_multipart_part(65536).await.unwrap()
|
||||
} else {
|
||||
manager.admit_put_object(65536).await.unwrap()
|
||||
};
|
||||
let permit = match admission {
|
||||
ForegroundWriteAdmission::Admitted(permit) => permit,
|
||||
_ => panic!("initial admission"),
|
||||
};
|
||||
let (sender, mut reader) = progress_reader();
|
||||
let mut owner = Box::pin(async move {
|
||||
let _permit = permit;
|
||||
let mut output = Vec::new();
|
||||
reader.read_to_end(&mut output).await
|
||||
});
|
||||
assert!(poll!(owner.as_mut()).is_pending());
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(65536).await.unwrap(),
|
||||
ForegroundWriteAdmission::Rejected
|
||||
));
|
||||
for _ in 0..5 {
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
sender.send(Ok(Bytes::from_static(b"x"))).unwrap();
|
||||
assert!(poll!(owner.as_mut()).is_pending());
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
assert!(owner.await.is_err());
|
||||
assert!(matches!(
|
||||
manager.admit_multipart_part(65536).await.unwrap(),
|
||||
ForegroundWriteAdmission::Admitted(_)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn upload_progress_does_not_release_an_owned_commit_during_storage_io_or_cancellation() {
|
||||
use crate::app::storage_api::object_usecase::concurrency::{ConcurrencyManager, ForegroundWriteAdmission};
|
||||
let manager = ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO);
|
||||
let permit = match manager.admit_put_object(65536).await.unwrap() {
|
||||
ForegroundWriteAdmission::Admitted(permit) => permit,
|
||||
_ => panic!("initial admission"),
|
||||
};
|
||||
let (sender, mut reader) = progress_reader();
|
||||
sender.send(Ok(Bytes::from_static(b"x"))).unwrap();
|
||||
let (entered, at_io) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = tokio::sync::oneshot::channel();
|
||||
let owner = tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
let mut byte = [0];
|
||||
reader.read_exact(&mut byte).await.unwrap();
|
||||
entered.send(()).unwrap();
|
||||
released.await.unwrap();
|
||||
reader.read_exact(&mut byte).await
|
||||
});
|
||||
at_io.await.unwrap();
|
||||
// Dropping the request's join handle must leave the commit owner intact.
|
||||
drop(owner);
|
||||
tokio::time::advance(Duration::from_secs(3600)).await;
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(65536).await.unwrap(),
|
||||
ForegroundWriteAdmission::Rejected
|
||||
));
|
||||
sender.send(Ok(Bytes::from_static(b"y"))).unwrap();
|
||||
release.send(()).unwrap();
|
||||
tokio::task::yield_now().await;
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(65536).await.unwrap(),
|
||||
ForegroundWriteAdmission::Admitted(_)
|
||||
));
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ impl S3 for Consumer {
|
||||
.get::<BodyReadControl>()
|
||||
.expect("HTTP control extension")
|
||||
.clone();
|
||||
control.activate(Duration::from_secs(300), "test-bucket", "test-key", "request", expected as u64);
|
||||
control.activate_upload(Duration::from_secs(300), "test-bucket", "test-key", "request", expected as u64);
|
||||
let stream = req.input.body.expect("body");
|
||||
let inner = rustfs_rio::wrap_reader(StreamReader::new(stream.map(|item| item.map_err(io::Error::other))));
|
||||
let mut reader =
|
||||
@@ -207,7 +207,7 @@ fn signed_request(payload: &[u8], unsigned_trailer: bool) -> SignedRequest {
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn signed_chunk_with_raw_progress_survives_eight_minutes_without_decoded_output() {
|
||||
let payload = vec![7; 65536];
|
||||
let payload = vec![7; 1024 * 1024];
|
||||
let SignedRequest {
|
||||
request,
|
||||
sender,
|
||||
@@ -220,13 +220,13 @@ async fn signed_chunk_with_raw_progress_survives_eight_minutes_without_decoded_o
|
||||
sender.send(Ok(Frame::data(prefix))).expect("body receiver");
|
||||
assert!(poll!(call.as_mut()).is_pending());
|
||||
let start = Instant::now();
|
||||
for chunk in payload.chunks(8192) {
|
||||
for chunk in payload.chunks(65536) {
|
||||
assert_eq!(
|
||||
consumer.received.load(Ordering::Relaxed),
|
||||
0,
|
||||
"incomplete chunks must not escape signature validation"
|
||||
);
|
||||
tokio::time::advance(Duration::from_secs(60)).await;
|
||||
tokio::time::advance(Duration::from_secs(30)).await;
|
||||
sender
|
||||
.send(Ok(Frame::data(Bytes::copy_from_slice(chunk))))
|
||||
.expect("body receiver");
|
||||
@@ -356,3 +356,27 @@ async fn trailer_adapter_is_pending_until_the_decoded_body_ends() {
|
||||
assert_eq!(trailer.lookup("x-amz-checksum-sha256"), TrailerValue::Missing);
|
||||
assert_eq!(*consumer.committed.lock(), Some(payload.to_vec()));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn signed_chunk_trickle_cannot_renew_upload_progress_window() {
|
||||
let SignedRequest {
|
||||
request, sender, prefix, ..
|
||||
} = signed_request(&[7; 65536], false);
|
||||
let consumer = Consumer::default();
|
||||
let service = service(consumer.clone());
|
||||
sender.send(Ok(Frame::data(prefix))).unwrap();
|
||||
let mut call = Box::pin(service.call(request));
|
||||
assert!(poll!(call.as_mut()).is_pending());
|
||||
for _ in 0..5 {
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
sender.send(Ok(Frame::data(Bytes::from_static(&[7])))).unwrap();
|
||||
assert!(poll!(call.as_mut()).is_pending());
|
||||
assert_eq!(consumer.received.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(10)).await;
|
||||
let response = call.await.expect("S3 response");
|
||||
assert_eq!(response.status(), http::StatusCode::BAD_REQUEST);
|
||||
let body = BodyExt::collect(response.into_body()).await.unwrap().to_bytes();
|
||||
assert!(String::from_utf8_lossy(&body).contains("<Code>RequestTimeout</Code>"));
|
||||
assert!(consumer.committed.lock().is_none());
|
||||
}
|
||||
|
||||
@@ -701,7 +701,8 @@ fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option<String
|
||||
// HTTP-date, not ISO-8601: the AWS SDKs hand this value to an RFC 822
|
||||
// parser, which rejects `2026-10-02T00:00:00Z` and drops the metadata.
|
||||
let expiry_date = format_http_date(&Timestamp::from(expire_time)).ok()?;
|
||||
Some(format!("expiry-date=\"{}\", rule-id=\"{}\"", expiry_date, event.rule_id))
|
||||
let value = format!("expiry-date=\"{}\", rule-id=\"{}\"", expiry_date, event.rule_id);
|
||||
http::HeaderValue::from_str(&value).is_ok().then_some(value)
|
||||
}
|
||||
|
||||
pub(super) fn internal_object_info_lookup_opts(mut opts: ObjectOptions) -> ObjectOptions {
|
||||
@@ -1956,6 +1957,27 @@ mod tests {
|
||||
assert!(build_put_object_expiration_header(&event).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expiration_header_ignores_unsafe_legacy_rule_ids() {
|
||||
let mut event = lifecycle::Event {
|
||||
action: lifecycle::IlmAction::DeleteAction,
|
||||
rule_id: String::new(),
|
||||
due: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid expiry")),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: String::new(),
|
||||
expiry_policy: None,
|
||||
};
|
||||
for id in ["rule\nname", "rule\rname", "rule\0name", "rule\u{7f}name"] {
|
||||
event.rule_id = id.to_owned();
|
||||
assert!(build_put_object_expiration_header(&event).is_none(), "{id:?}");
|
||||
}
|
||||
event.rule_id = "保留策略".to_owned();
|
||||
let header = build_put_object_expiration_header(&event).expect("valid UTF-8 IDs remain supported");
|
||||
assert!(http::HeaderValue::from_str(&header).is_ok());
|
||||
assert!(header.contains("保留策略"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_put_object_expiration_header_formats_expected_value() {
|
||||
let expire_time = OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap();
|
||||
|
||||
@@ -877,6 +877,7 @@ fn veeam_sosapi_http_contract() {
|
||||
)
|
||||
.await
|
||||
.expect("set quota on fresh bucket");
|
||||
crate::app::storage_api::test::set_disk::initialize_quota_ledger_for_test(Arc::clone(&store), "a_bucket).await.expect("bootstrap quota usage");
|
||||
snapshot.last_update = Some(std::time::SystemTime::now() + Duration::from_secs(2));
|
||||
snapshot.buckets_count = 2;
|
||||
let quota_incarnation = store.bucket_incarnation_id_from_disk("a_bucket).await.expect("quota bucket incarnation");
|
||||
|
||||
@@ -22,6 +22,8 @@ pub(crate) use crate::runtime_sources::{
|
||||
current_notification_system_for_context, current_object_data_cache_handle_for_context,
|
||||
current_object_store_handle_for_context,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::runtime_sources::{IamInterface, KmsInterface};
|
||||
use rustfs_s3select_api::{QueryResult, server::dbms::DatabaseManagerSystem};
|
||||
use s3s::dto::SelectObjectContentInput;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -1781,6 +1781,81 @@ mod tests {
|
||||
assert!(error.source().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn unread_select_response_does_not_block_real_overwrite() {
|
||||
crate::app::gating_test_env::run_large_stack_test("select-unread-writer", || async {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::app::storage_api::test::contract::object::ObjectIO as _;
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||
let bucket = format!("select-unread-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let payload = b"original,123456789012345678901234567890\n".repeat(100_000);
|
||||
let mut initial = crate::app::storage_api::object_usecase::StoragePutObjReader::from_vec(payload);
|
||||
store
|
||||
.put_object(&bucket, "input.csv", &mut initial, &Default::default())
|
||||
.await
|
||||
.expect("CSV fixture");
|
||||
let snapshot = store
|
||||
.prepare_select_object_snapshot(&bucket, "input.csv", &HeaderMap::new(), &Default::default())
|
||||
.await
|
||||
.expect("snapshot for terminal consistency fence");
|
||||
let snapshot = Arc::new(snapshot);
|
||||
let mut input = base_input();
|
||||
input.bucket = bucket.clone();
|
||||
input.key = "input.csv".to_owned();
|
||||
let request = S3Request {
|
||||
input,
|
||||
method: http::Method::POST,
|
||||
uri: format!("/{bucket}/input.csv?select&select-type=2").parse().expect("URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let response = execute_select_object_content(request)
|
||||
.await
|
||||
.expect("start actual SELECT response");
|
||||
// Keep the response alive without polling its event stream. The
|
||||
// producer may fill its bounded channel, but cannot retain object locks.
|
||||
let mut replacement =
|
||||
crate::app::storage_api::object_usecase::StoragePutObjReader::from_vec(b"replacement,2\n".to_vec());
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
store.put_object(&bucket, "input.csv", &mut replacement, &Default::default()),
|
||||
)
|
||||
.await
|
||||
.expect("unread SELECT must not block writer")
|
||||
.expect("normal overwrite");
|
||||
drop(response);
|
||||
assert!(snapshot.open_reader(None).await.is_err(), "first range must reject the real overwrite");
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)]));
|
||||
let output = Box::pin(RecordBatchStreamAdapter::new(
|
||||
schema,
|
||||
futures::stream::empty::<Result<RecordBatch, DataFusionError>>(),
|
||||
));
|
||||
let (mut events, mut rx) = test_event_channel(4);
|
||||
let outcome = send_select_events(
|
||||
output,
|
||||
&mut events,
|
||||
csv_validation(),
|
||||
Arc::new(SelectInputMetrics::default()),
|
||||
Instant::now() + Duration::from_secs(30),
|
||||
30,
|
||||
&snapshot,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, SelectProducerOutcome::Terminal(Err(_))));
|
||||
assert!(rx.try_recv().is_err(), "a real changed-generation fence must never emit Stats or End");
|
||||
});
|
||||
}
|
||||
|
||||
fn base_input() -> SelectObjectContentInput {
|
||||
SelectObjectContentInput {
|
||||
bucket: "bucket".to_string(),
|
||||
|
||||
@@ -24,6 +24,13 @@ pub(crate) fn EndpointServerPools(
|
||||
crate::storage::storage_api::EndpointServerPools::from(pools)
|
||||
}
|
||||
|
||||
/// Streaming body types the request-body adapters implement against.
|
||||
pub(crate) mod request_body {
|
||||
pub(crate) use s3s::StdError;
|
||||
pub(crate) use s3s::dto::StreamingBlob;
|
||||
pub(crate) use s3s::stream::{ByteStream, DynByteStream, RemainingLength};
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub(crate) mod diagnostic {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_diagnostic::{
|
||||
@@ -303,11 +310,11 @@ pub(crate) mod access {
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
PostObjectRequestMarker, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition,
|
||||
TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
|
||||
authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
|
||||
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
authorize_copy_object_tags, authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action,
|
||||
has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
|
||||
replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards,
|
||||
set_requested_object_lock_retain_until,
|
||||
replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref,
|
||||
retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1130,13 +1137,17 @@ pub(crate) mod head_prefix {
|
||||
}
|
||||
|
||||
pub(crate) mod helper {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::helper_consumer::{
|
||||
OperationHelper, build_event_resp_elements, spawn_background_with_context,
|
||||
MutationCompletionProbe, capture_mutation_event_for_test, pause_committed_mutation_for_test,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::helper_consumer::{
|
||||
OperationHelper, build_event_resp_elements, run_owned_mutation, spawn_background_with_context,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod object_utils {
|
||||
pub(crate) use crate::storage::storage_api::to_s3s_etag;
|
||||
pub(crate) use crate::storage::storage_api::{to_s3s_etag, to_s3s_xml_etag};
|
||||
}
|
||||
|
||||
pub(crate) mod io {
|
||||
@@ -1167,14 +1178,15 @@ pub(crate) mod request_context {
|
||||
pub(crate) mod sse {
|
||||
pub(crate) use crate::storage::storage_api::sse_consumer::{
|
||||
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention,
|
||||
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
|
||||
get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption,
|
||||
sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state,
|
||||
authorize_sse_kms_multipart_write, authorize_sse_kms_object_read, classify_sse_read_response,
|
||||
extract_server_side_encryption_from_headers, get_buffer_size_opt_in, load_bucket_object_lock_config_state,
|
||||
project_sse_read_response_headers, sse_decryption, sse_encryption, sse_prepare_encryption,
|
||||
validate_bucket_object_lock_enabled_state,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::sse_consumer::{
|
||||
EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, map_get_object_reader_error,
|
||||
mark_encrypted_multipart_metadata,
|
||||
mark_encrypted_multipart_metadata, multipart_completion_authorization, stored_ssec_metadata,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1213,7 +1225,7 @@ pub(crate) mod s3_api {
|
||||
|
||||
pub(crate) mod tagging {
|
||||
pub(crate) use crate::storage::storage_api::s3_api_consumer::tagging::{
|
||||
parse_copy_object_tags, resolve_copy_object_tags,
|
||||
parse_copy_object_tags, resolve_copy_object_tags, validate_bucket_tag_set,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1248,7 +1260,7 @@ pub(crate) mod bucket_usecase {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3, s3_api};
|
||||
pub(crate) use super::{access, bucket, error, helper, object_utils, request_context, s3, s3_api, sse};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, StorageObjectInfo, StorageObjectOptions, get_validated_store, process_lambda_configurations,
|
||||
process_queue_configurations, process_topic_configurations, validate_list_object_unordered_with_delimiter,
|
||||
@@ -1298,8 +1310,9 @@ pub(crate) mod object_usecase {
|
||||
ECStore, GetObjectReader, OldCurrentSize, RFC1123, StorageDeletedObject, StorageObjectInfo,
|
||||
StorageObjectLockDeleteOptions, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions,
|
||||
parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize,
|
||||
remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_object_key,
|
||||
validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors,
|
||||
remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_new_object_key,
|
||||
validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
||||
wrap_response_with_cors,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1332,7 +1345,7 @@ pub(crate) mod multipart_usecase {
|
||||
access, bucket, compression, data_usage, error, helper, io, object_utils, options, request_context, s3_api, set_disk, sse,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, validate_object_key,
|
||||
ECStore, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, validate_new_object_key,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1359,6 +1372,7 @@ pub(crate) mod context {
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test {
|
||||
pub(crate) use super::EndpointServerPools;
|
||||
pub(crate) use s3s::access::S3Access;
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::storage_contracts::{
|
||||
@@ -1402,7 +1416,8 @@ pub(crate) mod test {
|
||||
pub(crate) mod set_disk {
|
||||
pub(crate) use crate::storage::storage_api::ecstore_set_disk::{
|
||||
ConditionalReadBenchmarkMetadataGuard, MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier,
|
||||
PutObjectCommitPause, fail_next_quota_ledger_save_for_test,
|
||||
PutObjectCommitPause, fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test,
|
||||
retry_quota_reconciliation_for_test,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+34
-13
@@ -197,20 +197,8 @@ impl S3Auth for IAMAuth {
|
||||
return Err(s3_error!(UnauthorizedAccess, "Your account is not signed up"));
|
||||
}
|
||||
|
||||
// Check if this is a Keystone access key (from mixed auth scenario)
|
||||
// Keystone credentials use token authentication, not signature verification
|
||||
if access_key.starts_with("keystone:") {
|
||||
debug!(
|
||||
event = EVENT_KEYSTONE_CREDENTIALS_DETECTED,
|
||||
component = LOG_COMPONENT_AUTH,
|
||||
subsystem = LOG_SUBSYSTEM_KEYSTONE,
|
||||
access_key = %MaskedAccessKey(access_key),
|
||||
result = "token_auth",
|
||||
"Keystone token-auth access key detected"
|
||||
);
|
||||
// Return empty secret key - Keystone uses token validation, not AWS signatures
|
||||
// The actual credentials are stored in task-local storage by KeystoneAuthMiddleware
|
||||
return Ok(SecretKey::from(String::new()));
|
||||
return Err(s3_error!(InvalidAccessKeyId, "Keystone authentication requires X-Auth-Token header"));
|
||||
}
|
||||
|
||||
if access_key == self.access_key {
|
||||
@@ -1737,6 +1725,39 @@ mod tests {
|
||||
assert!(!has_root_access(&sys_cred, &oidc_cred));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn keystone_secret_lookup_requires_validated_token_context() {
|
||||
use rustfs_keystone::KEYSTONE_CREDENTIALS;
|
||||
let auth = IAMAuth::new("root", "root-secret");
|
||||
for key in ["keystone:x", "keystone:admin"] {
|
||||
assert_eq!(auth.get_secret_key(key).await.unwrap_err().code(), &S3ErrorCode::InvalidAccessKeyId);
|
||||
KEYSTONE_CREDENTIALS
|
||||
.scope(None, async {
|
||||
assert_eq!(auth.get_secret_key(key).await.unwrap_err().code(), &S3ErrorCode::InvalidAccessKeyId);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
let credentials = Credentials {
|
||||
access_key: "keystone:validated".to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
KEYSTONE_CREDENTIALS
|
||||
.scope(Some(credentials), async {
|
||||
assert_eq!(
|
||||
auth.get_secret_key("keystone:validated")
|
||||
.await
|
||||
.expect("validated Keystone token")
|
||||
.expose(),
|
||||
""
|
||||
);
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
auth.get_secret_key("root").await.expect("regular root credential").expose(),
|
||||
"root-secret"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_iam_auth_creation() {
|
||||
let access_key = "test-access-key";
|
||||
|
||||
+875
-6
@@ -1429,6 +1429,329 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Validate control-plane digests before s3s discards the serialized body.
|
||||
// Object uploads and multipart/form-data must retain their streaming readers.
|
||||
struct ControlRequestBody(s3s::Body);
|
||||
|
||||
impl http_body::Body for ControlRequestBody {
|
||||
type Data = Bytes;
|
||||
type Error = Error;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<Option<std::result::Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
||||
Pin::new(&mut self.0)
|
||||
.poll_frame(cx)
|
||||
.map(|frame| frame.map(|result| result.map_err(Error::other)))
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
http_body::Body::is_end_stream(&self.0)
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
http_body::Body::size_hint(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
fn control_request_body(body: s3s::Body) -> ControlRequestBody {
|
||||
ControlRequestBody(body)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ControlBodyDigestService<S> {
|
||||
inner: S,
|
||||
idle_timeout: Duration,
|
||||
total_timeout: Duration,
|
||||
max_body_size: usize,
|
||||
virtual_host: Option<Arc<MultiDomain>>,
|
||||
}
|
||||
|
||||
impl<S> ControlBodyDigestService<S> {
|
||||
fn new(inner: S, idle_timeout: Duration) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
idle_timeout,
|
||||
total_timeout: Duration::from_secs(30),
|
||||
max_body_size: rustfs_s3_config().xml_max_body_size,
|
||||
virtual_host: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_virtual_host_domains(mut self, domains: Arc<[String]>) -> Self {
|
||||
if !domains.is_empty() {
|
||||
// Startup already validated this same domain set for S3Service.
|
||||
self.virtual_host = Some(Arc::new(MultiDomain::new(domains.iter()).expect("validated S3 host domains")));
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn control_body_digest_required<B>(request: &HttpRequest<B>, virtual_host: Option<&MultiDomain>) -> Option<bool> {
|
||||
let query = url::form_urlencoded::parse(request.uri().query().unwrap_or_default().as_bytes());
|
||||
// Match s3s' multipart POST precedence, including its unique-header rule.
|
||||
if request.method() == Method::POST
|
||||
&& request.headers().get_all("content-type").iter().count() == 1
|
||||
&& request
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<mime_guess::Mime>().ok())
|
||||
.is_some_and(|mime| mime.type_() == mime_guess::mime::MULTIPART && mime.subtype() == mime_guess::mime::FORM_DATA)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut operation_ids = query.filter(|(key, _)| key == "x-id");
|
||||
let operation_id = operation_ids.next().map(|(_, value)| value);
|
||||
// s3s rejects duplicate declarations and treats a unique x-id as authoritative.
|
||||
if operation_ids.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
if request.method() == Method::POST {
|
||||
let decoded_path = urlencoding::decode(request.uri().path()).ok()?;
|
||||
use s3s::host::S3Host;
|
||||
// Use s3s' host parser, including CNAME fallback, and its Host-first
|
||||
// precedence. A rate-limit host hint is intentionally only best-effort.
|
||||
let mut host_values = request.headers().get_all("host").iter();
|
||||
let host = host_values.next().and_then(|value| value.to_str().ok()).or_else(|| {
|
||||
matches!(request.version(), Version::HTTP_2 | Version::HTTP_3)
|
||||
.then(|| request.uri().authority().map(|value| value.as_str()))
|
||||
.flatten()
|
||||
});
|
||||
if host_values.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let host = match (virtual_host, host) {
|
||||
(Some(parser), Some(host)) if host.parse::<SocketAddr>().is_err() && host.parse::<IpAddr>().is_err() => {
|
||||
Some(parser.parse_host_header(host).ok()?)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let path = s3s::path::parse_virtual_hosted_style(host.as_ref().and_then(|host| host.bucket()), &decoded_path).ok()?;
|
||||
if !matches!(path, s3s::path::S3Path::Bucket { .. }) {
|
||||
return None;
|
||||
}
|
||||
let delete_objects = match operation_id.as_deref() {
|
||||
Some("DeleteObjects") => true,
|
||||
Some(_) => false,
|
||||
None => {
|
||||
// These bucket POST routes precede DeleteObjects in s3s' router.
|
||||
query.clone().any(|(key, _)| key == "delete")
|
||||
&& !query
|
||||
.clone()
|
||||
.any(|(key, _)| matches!(key.as_ref(), "metadataConfiguration" | "metadataTable"))
|
||||
}
|
||||
};
|
||||
return delete_objects.then_some(true);
|
||||
}
|
||||
if request.method() == Method::PUT
|
||||
&& operation_id
|
||||
.as_deref()
|
||||
.is_some_and(|operation| matches!(operation, "PutObject" | "UploadPart"))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
// Other configuration APIs keep their existing optional-MD5 contract, but
|
||||
// an MD5 that is supplied (and signed by SigV2) must actually match.
|
||||
if request.method() == Method::PUT
|
||||
&& request.headers().contains_key("content-md5")
|
||||
&& query.clone().any(|(key, value)| {
|
||||
(key == "x-id" && value.starts_with("Put") && value != "PutObject")
|
||||
|| matches!(
|
||||
key.as_ref(),
|
||||
"acl"
|
||||
| "cors"
|
||||
| "lifecycle"
|
||||
| "logging"
|
||||
| "notification"
|
||||
| "policy"
|
||||
| "replication"
|
||||
| "requestPayment"
|
||||
| "tagging"
|
||||
| "versioning"
|
||||
| "website"
|
||||
| "object-lock"
|
||||
| "retention"
|
||||
| "legal-hold"
|
||||
| "ownershipControls"
|
||||
| "encryption"
|
||||
| "accelerate"
|
||||
| "intelligent-tiering"
|
||||
| "analytics"
|
||||
| "metrics"
|
||||
| "inventory"
|
||||
| "publicAccessBlock"
|
||||
)
|
||||
})
|
||||
{
|
||||
return Some(false);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn control_body_digest_error(code: &'static str) -> Response<s3s::Body> {
|
||||
// Only static protocol error codes reach this function; never reflect headers or bodies.
|
||||
let xml = format!("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>{code}</Code><Message>{code}</Message></Error>");
|
||||
let length = xml.len();
|
||||
let mut response = Response::new(s3s::Body::from(xml));
|
||||
*response.status_mut() = http::StatusCode::BAD_REQUEST;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert("content-type", HeaderValue::from_static("application/xml"));
|
||||
response.headers_mut().insert("content-length", HeaderValue::from(length));
|
||||
response
|
||||
}
|
||||
|
||||
const CONTROL_BODY_CHECKSUM_HEADERS: [&str; 10] = [
|
||||
"x-amz-checksum-crc32",
|
||||
"x-amz-checksum-crc32c",
|
||||
"x-amz-checksum-crc64nvme",
|
||||
"x-amz-checksum-sha1",
|
||||
"x-amz-checksum-sha256",
|
||||
"x-amz-checksum-sha512",
|
||||
"x-amz-checksum-xxhash64",
|
||||
"x-amz-checksum-xxhash3",
|
||||
"x-amz-checksum-xxhash128",
|
||||
"x-amz-checksum-md5",
|
||||
];
|
||||
|
||||
fn verify_control_body_digest(headers: &HeaderMap, bytes: &[u8]) -> std::result::Result<(), &'static str> {
|
||||
use md5::Digest;
|
||||
if let Some(expected) = headers.get("content-md5") {
|
||||
if headers.get_all("content-md5").iter().count() != 1 {
|
||||
return Err("InvalidDigest");
|
||||
}
|
||||
let expected = base64_simd::STANDARD
|
||||
.decode_to_vec(expected.as_bytes())
|
||||
.map_err(|_| "InvalidDigest")?;
|
||||
if expected.len() != 16 {
|
||||
return Err("InvalidDigest");
|
||||
}
|
||||
if md5::Md5::digest(bytes).as_slice() != expected.as_slice() {
|
||||
return Err("BadDigest");
|
||||
}
|
||||
}
|
||||
let mut checksums = s3s::checksum::ChecksumHasher {
|
||||
crc32: headers.contains_key("x-amz-checksum-crc32").then(Default::default),
|
||||
crc32c: headers.contains_key("x-amz-checksum-crc32c").then(Default::default),
|
||||
crc64nvme: headers.contains_key("x-amz-checksum-crc64nvme").then(Default::default),
|
||||
sha1: headers.contains_key("x-amz-checksum-sha1").then(Default::default),
|
||||
sha256: headers.contains_key("x-amz-checksum-sha256").then(Default::default),
|
||||
sha512: headers.contains_key("x-amz-checksum-sha512").then(Default::default),
|
||||
xxhash64: headers.contains_key("x-amz-checksum-xxhash64").then(Default::default),
|
||||
xxhash3: headers.contains_key("x-amz-checksum-xxhash3").then(Default::default),
|
||||
xxhash128: headers.contains_key("x-amz-checksum-xxhash128").then(Default::default),
|
||||
md5: headers.contains_key("x-amz-checksum-md5").then(Default::default),
|
||||
};
|
||||
checksums.update(bytes);
|
||||
let checksums = checksums.finalize();
|
||||
let computed = [
|
||||
checksums.checksum_crc32,
|
||||
checksums.checksum_crc32c,
|
||||
checksums.checksum_crc64nvme,
|
||||
checksums.checksum_sha1,
|
||||
checksums.checksum_sha256,
|
||||
checksums.checksum_sha512,
|
||||
checksums.checksum_xxhash64,
|
||||
checksums.checksum_xxhash3,
|
||||
checksums.checksum_xxhash128,
|
||||
checksums.checksum_md5,
|
||||
];
|
||||
for (name, computed) in CONTROL_BODY_CHECKSUM_HEADERS.into_iter().zip(computed) {
|
||||
if let Some(computed) = computed
|
||||
&& (headers.get_all(name).iter().count() != 1
|
||||
|| headers.get(name).map(HeaderValue::as_bytes) != Some(computed.as_bytes()))
|
||||
{
|
||||
return Err("BadDigest");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl<S, B> Service<HttpRequest<B>> for ControlBodyDigestService<S>
|
||||
where
|
||||
S: Service<HttpRequest<ControlRequestBody>, Response = Response<s3s::Body>> + Clone + Send + 'static,
|
||||
S::Future: Send + 'static,
|
||||
S::Error: Send + 'static,
|
||||
B: http_body::Body<Data = Bytes> + Send + 'static,
|
||||
B::Error: Send + Sync + 'static,
|
||||
Box<dyn std::error::Error + Send + Sync>: From<B::Error>,
|
||||
{
|
||||
type Response = Response<s3s::Body>;
|
||||
type Error = S::Error;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest<B>) -> Self::Future {
|
||||
use http_body_util::BodyExt;
|
||||
let clone = self.inner.clone();
|
||||
let mut inner = std::mem::replace(&mut self.inner, clone);
|
||||
let required = control_body_digest_required(&request, self.virtual_host.as_deref());
|
||||
let idle_timeout = self.idle_timeout;
|
||||
let total_timeout = self.total_timeout;
|
||||
let limit = self.max_body_size;
|
||||
Box::pin(async move {
|
||||
let mut request = request.map(s3s::Body::http_body_unsync);
|
||||
let Some(required) = required else {
|
||||
return inner.call(request.map(control_request_body)).await;
|
||||
};
|
||||
let headers = request.headers();
|
||||
let sig_v2 = headers
|
||||
.get("authorization")
|
||||
.is_some_and(|value| value.as_bytes().starts_with(b"AWS "))
|
||||
|| url::form_urlencoded::parse(request.uri().query().unwrap_or_default().as_bytes())
|
||||
.any(|(key, _)| key == "AWSAccessKeyId");
|
||||
let has_checksum = CONTROL_BODY_CHECKSUM_HEADERS.iter().any(|name| headers.contains_key(*name));
|
||||
if required && !headers.contains_key("content-md5") && (sig_v2 || !has_checksum) {
|
||||
return Ok(control_body_digest_error("MissingContentMD5"));
|
||||
}
|
||||
if headers
|
||||
.get("content-length")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.is_some_and(|length| length > limit as u64)
|
||||
{
|
||||
return Ok(control_body_digest_error("EntityTooLarge"));
|
||||
}
|
||||
let deadline = tokio::time::Instant::now() + total_timeout;
|
||||
let collected = tokio::time::timeout_at(deadline, async {
|
||||
let mut bytes = bytes::BytesMut::new();
|
||||
while let Some(frame) = tokio::time::timeout(idle_timeout, request.body_mut().frame())
|
||||
.await
|
||||
.map_err(|_| "RequestTimeout")?
|
||||
{
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err("RequestTimeout");
|
||||
}
|
||||
let frame = frame.map_err(|_| "IncompleteBody")?;
|
||||
if let Ok(data) = frame.into_data() {
|
||||
if data.len() > limit.saturating_sub(bytes.len()) {
|
||||
return Err("EntityTooLarge");
|
||||
}
|
||||
bytes.extend_from_slice(&data);
|
||||
}
|
||||
}
|
||||
Ok(bytes.freeze())
|
||||
})
|
||||
.await
|
||||
.unwrap_or(Err("RequestTimeout"));
|
||||
let bytes = match collected {
|
||||
Ok(bytes) => bytes,
|
||||
Err(code) => return Ok(control_body_digest_error(code)),
|
||||
};
|
||||
if let Err(code) = verify_control_body_digest(request.headers(), &bytes) {
|
||||
return Ok(control_body_digest_error(code));
|
||||
}
|
||||
*request.body_mut() = s3s::Body::from(bytes);
|
||||
inner.call(request.map(control_request_body)).await
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn trace_on_response<ResBody>(response: &Response<ResBody>, latency: Duration, span: &Span) {
|
||||
span.record("status_code", tracing::field::display(response.status()));
|
||||
let _enter = span.enter();
|
||||
@@ -2139,10 +2462,10 @@ async fn start_http_server_inner(
|
||||
let make_service = move |peer_addr: SocketAddr| -> Http3ExternalService {
|
||||
let remote_addr = RemoteAddr(peer_addr);
|
||||
|
||||
let service = ControlBodyDigestService::new(h3_s3_service.clone(), http_request_body_read_timeout)
|
||||
.with_virtual_host_domains(h3_vh_domain_hosts.clone());
|
||||
#[cfg(feature = "swift")]
|
||||
let service = SwiftService::new(true, None, h3_s3_service.clone());
|
||||
#[cfg(not(feature = "swift"))]
|
||||
let service = h3_s3_service.clone();
|
||||
let service = SwiftService::new(true, None, service);
|
||||
let service = EarlyResponseBodyService::new(service, http_request_body_read_timeout);
|
||||
let service = ServiceBuilder::new()
|
||||
.layer(tower::util::MapResponseLayer::new(|response: Response<s3s::Body>| {
|
||||
@@ -2794,10 +3117,10 @@ fn process_connection(
|
||||
.prepare(),
|
||||
);
|
||||
|
||||
let http_service = ControlBodyDigestService::new(s3_service, request_body_idle_timeout)
|
||||
.with_virtual_host_domains(vh_domain_hosts.clone());
|
||||
#[cfg(feature = "swift")]
|
||||
let http_service = SwiftService::new(true, None, s3_service);
|
||||
#[cfg(not(feature = "swift"))]
|
||||
let http_service = s3_service;
|
||||
let http_service = SwiftService::new(true, None, http_service);
|
||||
let http_service =
|
||||
EarlyResponseBodyService::new(http_service, request_body_idle_timeout).with_task_owner(task_owner.clone());
|
||||
let http_service = InternodeRpcService::new(http_service);
|
||||
@@ -3350,6 +3673,552 @@ mod tests {
|
||||
use tokio::sync::{Notify, mpsc};
|
||||
use tower::{Layer, Service, ServiceBuilder};
|
||||
|
||||
const DIGEST_DELETE_XML: &str = "<Delete><Object><Key>safe-object</Key></Object></Delete>";
|
||||
|
||||
struct DigestCheckedS3(Arc<AtomicUsize>);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl s3s::S3 for DigestCheckedS3 {
|
||||
async fn delete_objects(
|
||||
&self,
|
||||
request: s3s::S3Request<s3s::dto::DeleteObjectsInput>,
|
||||
) -> s3s::S3Result<s3s::S3Response<s3s::dto::DeleteObjectsOutput>> {
|
||||
assert_eq!(request.credentials.expect("authenticated request").access_key, "digest-key");
|
||||
assert_eq!(request.input.delete.objects[0].key, "safe-object");
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(s3s::S3Response::new(s3s::dto::DeleteObjectsOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
struct DigestMultipartS3(Arc<AtomicUsize>);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl s3s::S3 for DigestMultipartS3 {
|
||||
async fn create_multipart_upload(
|
||||
&self,
|
||||
request: s3s::S3Request<s3s::dto::CreateMultipartUploadInput>,
|
||||
) -> s3s::S3Result<s3s::S3Response<s3s::dto::CreateMultipartUploadOutput>> {
|
||||
assert_eq!(request.credentials.expect("authenticated request").access_key, "digest-key");
|
||||
assert_eq!(request.input.bucket, "bucket");
|
||||
assert_eq!(request.input.key, "object");
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(s3s::S3Response::new(s3s::dto::CreateMultipartUploadOutput::default()))
|
||||
}
|
||||
|
||||
async fn complete_multipart_upload(
|
||||
&self,
|
||||
request: s3s::S3Request<s3s::dto::CompleteMultipartUploadInput>,
|
||||
) -> s3s::S3Result<s3s::S3Response<s3s::dto::CompleteMultipartUploadOutput>> {
|
||||
assert_eq!(request.credentials.expect("authenticated request").access_key, "digest-key");
|
||||
assert_eq!(request.input.bucket, "bucket");
|
||||
assert_eq!(request.input.key, "object");
|
||||
self.0.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(s3s::S3Response::new(s3s::dto::CompleteMultipartUploadOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
fn signed_digest_route_request(uri: &str, body: &'static str, with_md5: bool, virtual_host: bool) -> HttpRequest<s3s::Body> {
|
||||
use md5::Digest;
|
||||
let uri: Uri = uri.parse().unwrap();
|
||||
let mut request = HttpRequest::builder()
|
||||
.method("POST")
|
||||
.header("date", chrono::Utc::now().format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("host", uri.authority().unwrap().as_str())
|
||||
.uri(uri)
|
||||
.header("content-length", body.len())
|
||||
.header("content-type", "application/xml")
|
||||
.body(s3s::Body::from(body.to_string()))
|
||||
.unwrap();
|
||||
if with_md5 {
|
||||
request.headers_mut().insert(
|
||||
"content-md5",
|
||||
base64_simd::STANDARD
|
||||
.encode_to_string(md5::Md5::digest(body.as_bytes()))
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
let wire_uri = request.uri().clone();
|
||||
if virtual_host {
|
||||
// The repository V2 signer currently ignores its virtual_host flag.
|
||||
// Sign the canonical bucket-prefixed resource, then restore the wire URI.
|
||||
let bucket = wire_uri.host().unwrap().split('.').next().unwrap();
|
||||
let mut parts = wire_uri.clone().into_parts();
|
||||
parts.path_and_query = Some(format!("/{bucket}{}", wire_uri.path_and_query().unwrap()).parse().unwrap());
|
||||
*request.uri_mut() = Uri::from_parts(parts).unwrap();
|
||||
}
|
||||
let mut request = rustfs_signer::try_sign_v2(request, body.len() as i64, "digest-key", "digest-secret", false).unwrap();
|
||||
*request.uri_mut() = wire_uri;
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_body_digest_follows_s3_multipart_routing_with_delete_query() {
|
||||
use tower::ServiceExt;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let mut builder = S3ServiceBuilder::new(DigestMultipartS3(calls.clone()));
|
||||
builder.set_auth(s3s::auth::SimpleAuth::from_single("digest-key", "digest-secret"));
|
||||
builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(rustfs_s3_config()))));
|
||||
builder.set_host(MultiDomain::new(["example.com"]).unwrap());
|
||||
let service = ControlBodyDigestService::new(builder.build(), Duration::from_secs(2))
|
||||
.with_virtual_host_domains(vec!["example.com".to_owned()].into());
|
||||
for virtual_host in [false, true] {
|
||||
let base = if virtual_host {
|
||||
"http://bucket.example.com/object"
|
||||
} else {
|
||||
"http://example.com/bucket/object"
|
||||
};
|
||||
for (query, body) in [
|
||||
("x-id=CreateMultipartUpload&uploads&delete", ""),
|
||||
("uploads&delete", ""),
|
||||
(
|
||||
"x-id=CompleteMultipartUpload&uploadId=id&delete",
|
||||
"<CompleteMultipartUpload><Part><ETag>etag</ETag><PartNumber>1</PartNumber></Part></CompleteMultipartUpload>",
|
||||
),
|
||||
(
|
||||
"uploadId=id&delete",
|
||||
"<CompleteMultipartUpload><Part><ETag>etag</ETag><PartNumber>1</PartNumber></Part></CompleteMultipartUpload>",
|
||||
),
|
||||
] {
|
||||
let uri = format!("{base}?{query}");
|
||||
let response = service
|
||||
.clone()
|
||||
.oneshot(signed_digest_route_request(&uri, body, false, virtual_host))
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let bytes = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(status, StatusCode::OK, "{uri}: {}", String::from_utf8_lossy(&bytes));
|
||||
}
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_body_digest_keeps_bucket_deletes_guarded_with_multipart_query() {
|
||||
use tower::ServiceExt;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let mut builder = S3ServiceBuilder::new(DigestCheckedS3(calls.clone()));
|
||||
builder.set_auth(s3s::auth::SimpleAuth::from_single("digest-key", "digest-secret"));
|
||||
builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(rustfs_s3_config()))));
|
||||
builder.set_host(MultiDomain::new(["example.com"]).unwrap());
|
||||
let service = ControlBodyDigestService::new(builder.build(), Duration::from_secs(2))
|
||||
.with_virtual_host_domains(vec!["example.com".to_owned()].into());
|
||||
for virtual_host in [false, true] {
|
||||
let base = if virtual_host {
|
||||
"http://bucket.example.com/"
|
||||
} else {
|
||||
"http://example.com/bucket"
|
||||
};
|
||||
for query in ["delete&uploads", "delete&uploadId=id", "x-id=DeleteObjects&uploads"] {
|
||||
let uri = format!("{base}?{query}");
|
||||
digest_error(
|
||||
service
|
||||
.clone()
|
||||
.oneshot(signed_digest_route_request(&uri, DIGEST_DELETE_XML, false, virtual_host))
|
||||
.await
|
||||
.unwrap(),
|
||||
"MissingContentMD5",
|
||||
)
|
||||
.await;
|
||||
let response = service
|
||||
.clone()
|
||||
.oneshot(signed_digest_route_request(&uri, DIGEST_DELETE_XML, true, virtual_host))
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let bytes = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(status, StatusCode::OK, "{uri}: {}", String::from_utf8_lossy(&bytes));
|
||||
let mut tampered = signed_digest_route_request(&uri, DIGEST_DELETE_XML, true, virtual_host);
|
||||
*tampered.body_mut() = s3s::Body::from(DIGEST_DELETE_XML.replace("safe-object", "evil-object"));
|
||||
digest_error(service.clone().oneshot(tampered).await.unwrap(), "BadDigest").await;
|
||||
}
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 6);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_body_digest_guards_cname_and_host_header_precedence() {
|
||||
use tower::ServiceExt;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let mut builder = S3ServiceBuilder::new(DigestCheckedS3(calls.clone()));
|
||||
builder.set_auth(s3s::auth::SimpleAuth::from_single("digest-key", "digest-secret"));
|
||||
builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(rustfs_s3_config()))));
|
||||
builder.set_host(MultiDomain::new(["example.com"]).unwrap());
|
||||
let service = ControlBodyDigestService::new(builder.build(), Duration::from_secs(2))
|
||||
.with_virtual_host_domains(vec!["example.com".to_owned()].into());
|
||||
for different_authority in [false, true] {
|
||||
for md5 in [false, true] {
|
||||
// SigV2 canonicalizes a CNAME request as /<whole-host>/.
|
||||
let mut request = signed_digest_route_request(
|
||||
"http://alias.example.net/alias.example.net/?delete",
|
||||
DIGEST_DELETE_XML,
|
||||
md5,
|
||||
false,
|
||||
);
|
||||
*request.uri_mut() = if different_authority {
|
||||
"http://example.com/?delete"
|
||||
} else {
|
||||
"http://alias.example.net/?delete"
|
||||
}
|
||||
.parse()
|
||||
.unwrap();
|
||||
let response = service.clone().oneshot(request).await.unwrap();
|
||||
if md5 {
|
||||
let status = response.status();
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
} else {
|
||||
digest_error(response, "MissingContentMD5").await;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
fn digest_checked_s3(calls: Arc<AtomicUsize>) -> ControlBodyDigestService<S3Service> {
|
||||
let mut builder = S3ServiceBuilder::new(DigestCheckedS3(calls));
|
||||
builder.set_auth(s3s::auth::SimpleAuth::from_single("digest-key", "digest-secret"));
|
||||
builder.set_config(Arc::new(StaticConfigProvider::new(Arc::new(rustfs_s3_config()))));
|
||||
ControlBodyDigestService::new(builder.build(), Duration::from_secs(2))
|
||||
}
|
||||
|
||||
fn digest_v2_request(presigned: bool, md5: bool, secret: &str) -> HttpRequest<s3s::Body> {
|
||||
digest_v2_request_at(presigned, md5, secret, "http://localhost/bucket?delete")
|
||||
}
|
||||
|
||||
fn digest_v2_request_at(presigned: bool, md5: bool, secret: &str, uri: &str) -> HttpRequest<s3s::Body> {
|
||||
use md5::Digest;
|
||||
let signing_uri = uri.replace("%64elete", "delete");
|
||||
let mut request = HttpRequest::builder()
|
||||
.method("POST")
|
||||
.header("date", chrono::Utc::now().format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.uri(signing_uri)
|
||||
.header("host", "localhost")
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", DIGEST_DELETE_XML.len())
|
||||
.body(s3s::Body::from(DIGEST_DELETE_XML.to_string()))
|
||||
.unwrap();
|
||||
if md5 {
|
||||
request.headers_mut().insert(
|
||||
"content-md5",
|
||||
base64_simd::STANDARD
|
||||
.encode_to_string(md5::Md5::digest(DIGEST_DELETE_XML.as_bytes()))
|
||||
.parse()
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
let mut request = if presigned {
|
||||
let mut request = rustfs_signer::try_pre_sign_v2(request, "digest-key", secret, 60, false).unwrap();
|
||||
// The signer leaves the final base64 query value unescaped.
|
||||
let uri = request.uri().to_string();
|
||||
let (prefix, signature) = uri.rsplit_once("&Signature=").unwrap();
|
||||
let encoded = url::form_urlencoded::Serializer::new(String::new())
|
||||
.append_pair("Signature", signature)
|
||||
.finish();
|
||||
*request.uri_mut() = format!("{prefix}&{encoded}").parse().unwrap();
|
||||
request
|
||||
} else {
|
||||
rustfs_signer::try_sign_v2(request, DIGEST_DELETE_XML.len() as i64, "digest-key", secret, false).unwrap()
|
||||
};
|
||||
if uri.contains("%64elete") {
|
||||
*request.uri_mut() = request
|
||||
.uri()
|
||||
.to_string()
|
||||
.replace("?delete", "?%64elete")
|
||||
.replace("&delete", "&%64elete")
|
||||
.parse()
|
||||
.unwrap();
|
||||
}
|
||||
request
|
||||
}
|
||||
|
||||
async fn digest_error(response: Response<s3s::Body>, code: &str) {
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let bytes = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert!(String::from_utf8_lossy(&bytes).contains(&format!("<Code>{code}</Code>")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_objects_digest_binds_sigv2_header_and_query_bodies() {
|
||||
use tower::ServiceExt;
|
||||
for presigned in [false, true] {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let service = digest_checked_s3(calls.clone());
|
||||
let response = service
|
||||
.clone()
|
||||
.oneshot(digest_v2_request(presigned, true, "digest-secret"))
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
let mut tampered = digest_v2_request(presigned, true, "digest-secret");
|
||||
*tampered.body_mut() = s3s::Body::from(DIGEST_DELETE_XML.replace("safe-object", "evil-object"));
|
||||
digest_error(service.clone().oneshot(tampered).await.unwrap(), "BadDigest").await;
|
||||
digest_error(
|
||||
service
|
||||
.clone()
|
||||
.oneshot(digest_v2_request(presigned, false, "digest-secret"))
|
||||
.await
|
||||
.unwrap(),
|
||||
"MissingContentMD5",
|
||||
)
|
||||
.await;
|
||||
let mut checksum_only = digest_v2_request(presigned, false, "digest-secret");
|
||||
checksum_only
|
||||
.headers_mut()
|
||||
.insert("x-amz-checksum-sha256", HeaderValue::from_static("placeholder"));
|
||||
digest_error(service.clone().oneshot(checksum_only).await.unwrap(), "MissingContentMD5").await;
|
||||
|
||||
let mut malformed = digest_v2_request(presigned, true, "digest-secret");
|
||||
malformed
|
||||
.headers_mut()
|
||||
.insert("content-md5", HeaderValue::from_static("invalid"));
|
||||
digest_error(service.clone().oneshot(malformed).await.unwrap(), "InvalidDigest").await;
|
||||
let response = service
|
||||
.oneshot(digest_v2_request(presigned, true, "wrong-secret"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
StatusCode::FORBIDDEN,
|
||||
"digest verification must not replace authentication"
|
||||
);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1, "only the original authenticated XML may dispatch");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_objects_digest_covers_operation_id_and_encoded_query_routes() {
|
||||
use tower::ServiceExt;
|
||||
for uri in [
|
||||
"http://localhost/bucket?x-id=DeleteObjects",
|
||||
"http://localhost/bucket?%64elete",
|
||||
] {
|
||||
for presigned in [false, true] {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let service = digest_checked_s3(calls.clone());
|
||||
let request = digest_v2_request_at(presigned, true, "digest-secret", uri);
|
||||
let response = service.clone().oneshot(request).await.unwrap();
|
||||
let status = response.status();
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(status, StatusCode::OK, "{uri}: {}", String::from_utf8_lossy(&body));
|
||||
let mut request = digest_v2_request_at(presigned, true, "digest-secret", uri);
|
||||
*request.body_mut() = s3s::Body::from(DIGEST_DELETE_XML.replace("safe-object", "evil-object"));
|
||||
digest_error(service.clone().oneshot(request).await.unwrap(), "BadDigest").await;
|
||||
digest_error(
|
||||
service
|
||||
.oneshot(digest_v2_request_at(presigned, false, "digest-secret", uri))
|
||||
.await
|
||||
.unwrap(),
|
||||
"MissingContentMD5",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn digest_v4_request(checksum: &str) -> HttpRequest<s3s::Body> {
|
||||
use s3s_sigv4::{AmzDate, Payload, calculate_signature, create_canonical_request, create_string_to_sign};
|
||||
let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
|
||||
let date = AmzDate::parse(×tamp).unwrap();
|
||||
let canonical = create_canonical_request(
|
||||
"POST",
|
||||
"/bucket",
|
||||
&[("delete", "")],
|
||||
[
|
||||
("host", "localhost"),
|
||||
("x-amz-checksum-sha256", checksum),
|
||||
("x-amz-content-sha256", "UNSIGNED-PAYLOAD"),
|
||||
("x-amz-date", timestamp.as_str()),
|
||||
],
|
||||
Payload::Unsigned,
|
||||
);
|
||||
let to_sign = create_string_to_sign(&canonical, &date, "us-east-1", "s3");
|
||||
let signature = calculate_signature(&to_sign, "digest-secret", &date, "us-east-1", "s3");
|
||||
HttpRequest::builder().method("POST").uri("http://localhost/bucket?delete")
|
||||
.header("host", "localhost").header("content-length", DIGEST_DELETE_XML.len())
|
||||
.header("x-amz-checksum-sha256", checksum).header("x-amz-content-sha256", "UNSIGNED-PAYLOAD")
|
||||
.header("x-amz-date", ×tamp)
|
||||
.header("authorization", format!("AWS4-HMAC-SHA256 Credential=digest-key/{}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-checksum-sha256;x-amz-content-sha256;x-amz-date, Signature={signature}", date.fmt_date()))
|
||||
.body(s3s::Body::from(DIGEST_DELETE_XML.to_string())).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_objects_digest_preserves_sigv4_checksum_requests() {
|
||||
use sha2::Digest;
|
||||
use tower::ServiceExt;
|
||||
let checksum = base64_simd::STANDARD.encode_to_string(sha2::Sha256::digest(DIGEST_DELETE_XML.as_bytes()));
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let service = digest_checked_s3(calls.clone());
|
||||
let response = service.clone().oneshot(digest_v4_request(&checksum)).await.unwrap();
|
||||
let status = response.status();
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
let mut tampered = digest_v4_request(&checksum);
|
||||
*tampered.body_mut() = s3s::Body::from(DIGEST_DELETE_XML.replace("safe-object", "evil-object"));
|
||||
digest_error(service.oneshot(tampered).await.unwrap(), "BadDigest").await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_body_digest_bounds_declared_and_undeclared_bodies() {
|
||||
use tower::ServiceExt;
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let mut service = digest_checked_s3(calls.clone());
|
||||
service.max_body_size = DIGEST_DELETE_XML.len();
|
||||
for declared in [false, true] {
|
||||
let mut request = digest_v2_request(false, true, "digest-secret");
|
||||
let (sender, receiver) = mpsc::unbounded_channel();
|
||||
let polled = Arc::new(AtomicUsize::new(0));
|
||||
let dropped = Arc::new(AtomicBool::new(false));
|
||||
sender.send(Bytes::from_static(DIGEST_DELETE_XML.as_bytes())).unwrap();
|
||||
sender.send(Bytes::from_static(b" ")).unwrap();
|
||||
if declared {
|
||||
request
|
||||
.headers_mut()
|
||||
.insert(CONTENT_LENGTH, HeaderValue::from(DIGEST_DELETE_XML.len() + 1));
|
||||
} else {
|
||||
request.headers_mut().remove(CONTENT_LENGTH);
|
||||
}
|
||||
let request = request.map(|_| TrackedRequestBody {
|
||||
receiver,
|
||||
bytes_polled: polled.clone(),
|
||||
dropped: dropped.clone(),
|
||||
});
|
||||
digest_error(service.clone().oneshot(request).await.unwrap(), "EntityTooLarge").await;
|
||||
assert!(dropped.load(Ordering::SeqCst));
|
||||
assert_eq!(polled.load(Ordering::SeqCst), if declared { 0 } else { DIGEST_DELETE_XML.len() + 1 });
|
||||
}
|
||||
let mut request = digest_v2_request(false, true, "digest-secret");
|
||||
request.headers_mut().remove(CONTENT_LENGTH);
|
||||
let (sender, receiver) = mpsc::unbounded_channel();
|
||||
sender.send(Bytes::from_static(DIGEST_DELETE_XML.as_bytes())).unwrap();
|
||||
drop(sender);
|
||||
let response = service
|
||||
.oneshot(request.map(|_| TrackedRequestBody {
|
||||
receiver,
|
||||
bytes_polled: Arc::default(),
|
||||
dropped: Arc::default(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let status = response.status();
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(status, StatusCode::LENGTH_REQUIRED, "{}", String::from_utf8_lossy(&body));
|
||||
assert_eq!(
|
||||
calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"preserve s3s' missing-length rejection after bounded collection"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn control_body_digest_idle_total_deadlines_and_cancellation_release_body() {
|
||||
use tower::ServiceExt;
|
||||
for trickle in [false, true] {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let mut service = digest_checked_s3(calls.clone());
|
||||
service.total_timeout = Duration::from_secs(5);
|
||||
let (sender, receiver) = mpsc::unbounded_channel();
|
||||
let dropped = Arc::new(AtomicBool::new(false));
|
||||
let request = digest_v2_request(false, true, "digest-secret").map(|_| TrackedRequestBody {
|
||||
receiver,
|
||||
bytes_polled: Arc::default(),
|
||||
dropped: dropped.clone(),
|
||||
});
|
||||
let task = tokio::spawn(service.oneshot(request));
|
||||
for _ in 0..6 {
|
||||
if trickle {
|
||||
let _ = sender.send(Bytes::from_static(b" "));
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
}
|
||||
digest_error(task.await.unwrap().unwrap(), "RequestTimeout").await;
|
||||
assert!(dropped.load(Ordering::SeqCst));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let (_sender, receiver) = mpsc::unbounded_channel();
|
||||
let dropped = Arc::new(AtomicBool::new(false));
|
||||
let task = tokio::spawn(
|
||||
digest_checked_s3(calls.clone()).oneshot(digest_v2_request(false, true, "digest-secret").map(|_| {
|
||||
TrackedRequestBody {
|
||||
receiver,
|
||||
bytes_polled: Arc::default(),
|
||||
dropped: dropped.clone(),
|
||||
}
|
||||
})),
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
task.abort();
|
||||
assert!(task.await.unwrap_err().is_cancelled());
|
||||
assert!(dropped.load(Ordering::SeqCst));
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_body_digest_preserves_streaming_uploads_and_checks_config_md5() {
|
||||
use tower::ServiceExt;
|
||||
for (method, uri) in [
|
||||
("PUT", "/bucket/key"),
|
||||
("PUT", "/bucket/key?partNumber=1&uploadId=id"),
|
||||
("POST", "/bucket?uploads"),
|
||||
("POST", "/bucket"),
|
||||
("POST", "/bucket/key?uploadId=id"),
|
||||
("POST", "/bucket?delete"),
|
||||
("PUT", "/bucket/key?x-id=PutObject&tagging"),
|
||||
] {
|
||||
let (_sender, receiver) = mpsc::unbounded_channel();
|
||||
let polled = Arc::new(AtomicUsize::new(0));
|
||||
let inner_polled = polled.clone();
|
||||
let inner = tower::service_fn(move |_: HttpRequest<ControlRequestBody>| {
|
||||
assert_eq!(inner_polled.load(Ordering::SeqCst), 0, "uploads must reach dispatch without buffering");
|
||||
async { Ok::<_, Infallible>(Response::new(s3s::Body::empty())) }
|
||||
});
|
||||
let request = HttpRequest::builder()
|
||||
.method(method)
|
||||
.uri(uri)
|
||||
.header("content-md5", "invalid")
|
||||
.header("content-type", "multipart/form-data; boundary=upload")
|
||||
.body(TrackedRequestBody {
|
||||
receiver,
|
||||
bytes_polled: polled,
|
||||
dropped: Arc::default(),
|
||||
})
|
||||
.unwrap();
|
||||
let response = ControlBodyDigestService::new(inner, Duration::from_secs(1))
|
||||
.oneshot(request)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let inner_calls = calls.clone();
|
||||
let inner = tower::service_fn(move |_: HttpRequest<ControlRequestBody>| {
|
||||
inner_calls.fetch_add(1, Ordering::SeqCst);
|
||||
async { Ok::<_, Infallible>(Response::new(s3s::Body::empty())) }
|
||||
});
|
||||
let service = ControlBodyDigestService::new(inner, Duration::from_secs(1));
|
||||
let mut request = digest_v2_request(false, true, "digest-secret");
|
||||
*request.method_mut() = Method::PUT;
|
||||
*request.uri_mut() = "/bucket?versioning".parse().unwrap();
|
||||
assert_eq!(service.clone().oneshot(request).await.unwrap().status(), StatusCode::OK);
|
||||
let request = HttpRequest::builder()
|
||||
.method("PUT")
|
||||
.uri("/bucket?versioning")
|
||||
.header("content-md5", "1B2M2Y8AsgTpgAmY7PhCfg==")
|
||||
.body(s3s::Body::from(String::from("<VersioningConfiguration/>")))
|
||||
.unwrap();
|
||||
digest_error(service.oneshot(request).await.unwrap(), "BadDigest").await;
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
type RateLimitTestCore = tower::util::BoxCloneService<
|
||||
HttpRequest<s3s::Body>,
|
||||
Response<crate::server::hybrid::HybridBody<s3s::Body, Empty<Bytes>>>,
|
||||
|
||||
+195
-5
@@ -90,10 +90,6 @@ const SWIFT_API_PATH_PREFIX: &str = "/v1/";
|
||||
|
||||
pub(crate) fn redact_sensitive_uri_query(uri: &http::Uri) -> String {
|
||||
let path = uri.path();
|
||||
if !is_object_zip_download_path(path) {
|
||||
return uri.to_string();
|
||||
}
|
||||
|
||||
let Some(query) = uri.query() else {
|
||||
return uri.to_string();
|
||||
};
|
||||
@@ -101,7 +97,11 @@ pub(crate) fn redact_sensitive_uri_query(uri: &http::Uri) -> String {
|
||||
let mut redacted_token = false;
|
||||
let mut serializer = form_urlencoded::Serializer::new(String::new());
|
||||
for (key, value) in form_urlencoded::parse(query.as_bytes()) {
|
||||
if key == "token" {
|
||||
if ["X-Amz-Signature", "X-Amz-Security-Token", "Signature"]
|
||||
.iter()
|
||||
.any(|sensitive| key.eq_ignore_ascii_case(sensitive))
|
||||
|| (key == "token" && is_object_zip_download_path(path))
|
||||
{
|
||||
redacted_token = true;
|
||||
serializer.append_pair(&key, REDACTED_QUERY_VALUE);
|
||||
} else {
|
||||
@@ -1045,6 +1045,9 @@ where
|
||||
let (parts, body) = response.into_parts();
|
||||
|
||||
let response = match body {
|
||||
HybridBody::Rest { rest_body } if !is_xml_compat_body(&rest_body) => {
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Rest { rest_body } => {
|
||||
let (rest_body, changed) = fix_s3_error_message_in_xml(rest_body).await.map_err(Into::into)?;
|
||||
let mut parts = parts;
|
||||
@@ -1197,6 +1200,9 @@ where
|
||||
let (parts, body) = response.into_parts();
|
||||
|
||||
let response = match body {
|
||||
HybridBody::Rest { rest_body } if !is_xml_compat_body(&rest_body) => {
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Rest { rest_body } => {
|
||||
let (rest_body, converted_status) = convert_iceberg_error_in_xml(
|
||||
rest_body,
|
||||
@@ -1371,6 +1377,9 @@ where
|
||||
let (parts, body) = response.into_parts();
|
||||
|
||||
let response = match body {
|
||||
HybridBody::Rest { rest_body } if !is_xml_compat_body(&rest_body) => {
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Rest { rest_body } => {
|
||||
let rest_body = fix_object_attributes_etag_in_xml(rest_body).await.map_err(Into::into)?;
|
||||
|
||||
@@ -2010,6 +2019,14 @@ fn is_bodyless_status(status: StatusCode) -> bool {
|
||||
|| status == StatusCode::NOT_MODIFIED
|
||||
}
|
||||
|
||||
const MAX_XML_COMPAT_BODY_BYTES: u64 = 1024 * 1024;
|
||||
|
||||
fn is_xml_compat_body<B: Body>(body: &B) -> bool {
|
||||
body.size_hint()
|
||||
.upper()
|
||||
.is_some_and(|length| length <= MAX_XML_COMPAT_BODY_BYTES)
|
||||
}
|
||||
|
||||
fn is_xml_response(headers: &HeaderMap) -> bool {
|
||||
let is_xml = headers
|
||||
.get(http::header::CONTENT_TYPE)
|
||||
@@ -5483,6 +5500,77 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn xml_compat_layers_do_not_poll_large_or_unbounded_bodies() {
|
||||
struct UnpolledXmlBody(Option<u64>);
|
||||
impl From<Bytes> for UnpolledXmlBody {
|
||||
fn from(_: Bytes) -> Self {
|
||||
panic!("streaming bodies must not be rewritten")
|
||||
}
|
||||
}
|
||||
impl Body for UnpolledXmlBody {
|
||||
type Data = Bytes;
|
||||
type Error = Infallible;
|
||||
fn poll_frame(
|
||||
self: Pin<&mut Self>,
|
||||
_: &mut Context<'_>,
|
||||
) -> Poll<Option<Result<http_body::Frame<Bytes>, Infallible>>> {
|
||||
panic!("compatibility layer must not consume the object stream")
|
||||
}
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
let mut hint = http_body::SizeHint::new();
|
||||
if let Some(length) = self.0 {
|
||||
hint.set_exact(length);
|
||||
}
|
||||
hint
|
||||
}
|
||||
}
|
||||
#[derive(Clone)]
|
||||
struct StreamingXmlResponse {
|
||||
length: Option<u64>,
|
||||
status: StatusCode,
|
||||
}
|
||||
impl Service<Request<()>> for StreamingXmlResponse {
|
||||
type Response = Response<HybridBody<UnpolledXmlBody, Empty<Bytes>>>;
|
||||
type Error = Infallible;
|
||||
type Future = Ready<Result<Self::Response, Self::Error>>;
|
||||
fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
fn call(&mut self, _: Request<()>) -> Self::Future {
|
||||
ready(Ok(Response::builder()
|
||||
.status(self.status)
|
||||
.header(http::header::CONTENT_TYPE, "application/xml")
|
||||
.body(HybridBody::Rest {
|
||||
rest_body: UnpolledXmlBody(self.length),
|
||||
})
|
||||
.expect("streaming response")))
|
||||
}
|
||||
}
|
||||
for length in [None, Some(MAX_XML_COMPAT_BODY_BYTES + 1)] {
|
||||
for (layer, path, status) in [
|
||||
(0, "/bucket/object", StatusCode::FORBIDDEN),
|
||||
(1, "/iceberg/v1/warehouse/namespaces/ns/tables/table", StatusCode::FORBIDDEN),
|
||||
(2, "/bucket/object?attributes", StatusCode::OK),
|
||||
] {
|
||||
let inner = StreamingXmlResponse { length, status };
|
||||
let request = Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(path)
|
||||
.body(())
|
||||
.expect("XML request");
|
||||
let response = match layer {
|
||||
0 => S3ErrorMessageCompatLayer.layer(inner).call(request).await,
|
||||
1 => IcebergRestErrorCompatLayer.layer(inner).call(request).await,
|
||||
_ => ObjectAttributesEtagFixLayer.layer(inner).call(request).await,
|
||||
}
|
||||
.expect("stream passed through");
|
||||
assert_eq!(response.status(), status);
|
||||
assert_eq!(response.headers()[http::header::CONTENT_TYPE], "application/xml");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_error_message_compat_fixes_regular_forbidden_xml() {
|
||||
let body = Bytes::from_static(b"<Error><Code>SignatureDoesNotMatch</Code></Error>");
|
||||
@@ -6132,6 +6220,78 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn conditional_cors_uses_the_host_bucket_with_real_distinct_bucket_configs() {
|
||||
crate::app::gating_test_env::run_large_stack_test("cors-bucket-authority", || async {
|
||||
use crate::app::storage_api::bucket::{metadata::BUCKET_CORS_CONFIG, metadata_sys};
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let owner = format!("cors-owner-{suffix}");
|
||||
let other = format!("cors-other-{suffix}");
|
||||
for (bucket, origin) in [(&owner, "https://owner.example"), (&other, "https://other.example")] {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("CORS bucket");
|
||||
let xml = format!(
|
||||
"<CORSConfiguration><CORSRule><AllowedOrigin>{origin}</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>"
|
||||
);
|
||||
metadata_sys::update(bucket, BUCKET_CORS_CONFIG, xml.into_bytes())
|
||||
.await
|
||||
.expect("persist distinct bucket CORS rules");
|
||||
}
|
||||
let layer = ConditionalCorsLayer {
|
||||
cors_origins: Some("*".to_owned()),
|
||||
vh_domain_hosts: Arc::from(["s3.example".to_owned(), "website.example".to_owned()]),
|
||||
};
|
||||
let mut service = layer.layer(CorsOkService);
|
||||
for (uri, host) in [
|
||||
(format!("/{other}/q3.csv"), Some(format!("{owner}.s3.example"))),
|
||||
(format!("/{other}/q3.csv"), Some(format!("{owner}.website.example"))),
|
||||
(format!("http://{owner}.s3.example/{other}/q3.csv"), None),
|
||||
("/".to_owned(), Some(format!("{owner}.s3.example"))),
|
||||
(format!("/{owner}/{other}/q3.csv"), Some("s3.example".to_owned())),
|
||||
] {
|
||||
for method in [Method::GET, Method::OPTIONS] {
|
||||
for (origin, allowed) in [("https://owner.example", true), ("https://other.example", false)] {
|
||||
let mut request = Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(&uri)
|
||||
.header(cors::standard::ORIGIN, origin);
|
||||
if let Some(host) = &host {
|
||||
request = request.header(http::header::HOST, host);
|
||||
}
|
||||
if method == Method::OPTIONS {
|
||||
request = request.header(cors::request::ACCESS_CONTROL_REQUEST_METHOD, "GET");
|
||||
}
|
||||
let response = service
|
||||
.call(request.body(()).expect("CORS request"))
|
||||
.await
|
||||
.expect("CORS service");
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(cors::response::ACCESS_CONTROL_ALLOW_ORIGIN)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
allowed.then_some(origin),
|
||||
"{method} {uri}: the object owner's CORS rule must be authoritative"
|
||||
);
|
||||
assert_eq!(
|
||||
response.status(),
|
||||
if method == Method::OPTIONS && !allowed {
|
||||
StatusCode::FORBIDDEN
|
||||
} else {
|
||||
StatusCode::OK
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_cors_passthrough_without_origin() {
|
||||
let layer = ConditionalCorsLayer {
|
||||
@@ -6729,6 +6889,36 @@ mod tests {
|
||||
assert_eq!(redact_sensitive_uri_query(&uri), "/rustfs/admin/v3/users?token=not-a-download-token");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_sensitive_uri_query_removes_presigned_credentials() {
|
||||
for path in ["/bucket/object", "/prefix/bucket/object", "/"] {
|
||||
let uri: http::Uri = format!(
|
||||
"{path}?X-Amz-Signature=secret-signature&x-amz-security-token=secret-session&Signature=secret-v2&partNumber=1"
|
||||
)
|
||||
.parse()
|
||||
.expect("presigned URI");
|
||||
let logged = redact_sensitive_uri_query(&uri);
|
||||
assert_eq!(
|
||||
logged,
|
||||
format!("{path}?X-Amz-Signature=redacted&x-amz-security-token=redacted&Signature=redacted&partNumber=1")
|
||||
);
|
||||
assert!(!logged.contains("secret"));
|
||||
}
|
||||
let uri: http::Uri = "/bucket/object?%58-Amz-Signature=secret&X-Amz-Signature=another&signature=v2"
|
||||
.parse()
|
||||
.expect("URI");
|
||||
assert_eq!(
|
||||
redact_sensitive_uri_query(&uri),
|
||||
"/bucket/object?X-Amz-Signature=redacted&X-Amz-Signature=redacted&signature=redacted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_sensitive_uri_query_preserves_ordinary_query_encoding() {
|
||||
let uri: http::Uri = "/bucket?prefix=one%20two&delimiter=%2f".parse().expect("URI");
|
||||
assert_eq!(redact_sensitive_uri_query(&uri), uri.to_string());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn request_logging_watchdog_cancels_dropped_futures() {
|
||||
for poll_request in [false, true] {
|
||||
|
||||
+1277
-49
File diff suppressed because it is too large
Load Diff
+26
-21
@@ -23,7 +23,9 @@ use super::{
|
||||
};
|
||||
use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook;
|
||||
use crate::error::{ApiError, ApiErrorDiagnostic};
|
||||
use crate::storage::access::{apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header};
|
||||
use crate::storage::access::{
|
||||
apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header, replication_proxy_allowed,
|
||||
};
|
||||
use crate::storage::helper::{OperationHelper, audit_only_operation};
|
||||
use crate::storage::options::get_opts;
|
||||
use crate::storage::s3_api::{self, acl};
|
||||
@@ -722,7 +724,7 @@ impl FS {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl S3 for FS {
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn abort_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<AbortMultipartUploadInput>,
|
||||
@@ -775,7 +777,7 @@ impl S3 for FS {
|
||||
usecase.execute_delete_bucket(req).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn delete_bucket_cors(&self, req: S3Request<DeleteBucketCorsInput>) -> S3Result<S3Response<DeleteBucketCorsOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketCors, |req| usecase.execute_delete_bucket_cors(req)).await
|
||||
@@ -792,7 +794,7 @@ impl S3 for FS {
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn delete_bucket_lifecycle(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketLifecycleInput>,
|
||||
@@ -819,7 +821,7 @@ impl S3 for FS {
|
||||
audit_only_operation(req, S3Operation::DeleteBucketReplication, |req| self.delete_bucket_replication_inner(req)).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn delete_bucket_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketTaggingInput>,
|
||||
@@ -849,7 +851,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(DeleteBucketWebsiteOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn delete_public_access_block(
|
||||
&self,
|
||||
req: S3Request<DeletePublicAccessBlockInput>,
|
||||
@@ -877,7 +879,7 @@ impl S3 for FS {
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn delete_object_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectTaggingInput>,
|
||||
@@ -916,6 +918,7 @@ impl S3 for FS {
|
||||
// (backlog#1675 P1-5). No local object exists, so no bucket
|
||||
// notification event is emitted for the proxied write.
|
||||
if (is_err_object_not_found(&e) || is_err_version_not_found(&e))
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
&& Self::proxy_delete_object_tagging(&bucket, &object, version_id.clone(), &req.headers)
|
||||
.await
|
||||
.is_some()
|
||||
@@ -1045,7 +1048,7 @@ impl S3 for FS {
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn get_bucket_cors(&self, req: S3Request<GetBucketCorsInput>) -> S3Result<S3Response<GetBucketCorsOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketCors);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
@@ -1061,7 +1064,7 @@ impl S3 for FS {
|
||||
usecase.execute_get_bucket_encryption(req).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn get_bucket_lifecycle_configuration(
|
||||
&self,
|
||||
req: S3Request<GetBucketLifecycleConfigurationInput>,
|
||||
@@ -1136,14 +1139,14 @@ impl S3 for FS {
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn get_bucket_tagging(&self, req: S3Request<GetBucketTaggingInput>) -> S3Result<S3Response<GetBucketTaggingOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketTagging);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_get_bucket_tagging(req).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn get_public_access_block(
|
||||
&self,
|
||||
req: S3Request<GetPublicAccessBlockInput>,
|
||||
@@ -1153,7 +1156,7 @@ impl S3 for FS {
|
||||
usecase.execute_get_public_access_block(req).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn get_bucket_versioning(
|
||||
&self,
|
||||
req: S3Request<GetBucketVersioningInput>,
|
||||
@@ -1296,7 +1299,7 @@ impl S3 for FS {
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn get_object_lock_configuration(
|
||||
&self,
|
||||
req: S3Request<GetObjectLockConfigurationInput>,
|
||||
@@ -1388,7 +1391,7 @@ impl S3 for FS {
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn get_object_tagging(&self, req: S3Request<GetObjectTaggingInput>) -> S3Result<S3Response<GetObjectTaggingOutput>> {
|
||||
use crate::storage::storage_api::ecstore_bucket::versioning::VersioningApi as _;
|
||||
|
||||
@@ -1436,6 +1439,7 @@ impl S3 for FS {
|
||||
// replication target even though it is missing locally —
|
||||
// proxy the tagging read there (backlog#1675 P1-5).
|
||||
if (is_err_object_not_found(&e) || is_err_version_not_found(&e))
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
&& let Some(output) =
|
||||
Self::proxy_get_object_tagging(bucket, object, req.input.version_id.clone(), &req.headers).await
|
||||
{
|
||||
@@ -1516,7 +1520,7 @@ impl S3 for FS {
|
||||
usecase.execute_head_object(req).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn list_buckets(&self, req: S3Request<ListBucketsInput>) -> S3Result<S3Response<ListBucketsOutput>> {
|
||||
// List buckets not associated with a bucket, give it bucket label "*" to denote "all".
|
||||
record_s3_op(S3Operation::ListBuckets);
|
||||
@@ -1595,7 +1599,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(PutBucketAccelerateConfigurationOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn put_bucket_cors(&self, req: S3Request<PutBucketCorsInput>) -> S3Result<S3Response<PutBucketCorsOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketCors, |req| usecase.execute_put_bucket_cors(req)).await
|
||||
@@ -1632,7 +1636,7 @@ impl S3 for FS {
|
||||
audit_only_operation(req, S3Operation::PutBucketEncryption, |req| usecase.execute_put_bucket_encryption(req)).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn put_bucket_lifecycle_configuration(
|
||||
&self,
|
||||
req: S3Request<PutBucketLifecycleConfigurationInput>,
|
||||
@@ -1694,7 +1698,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(PutBucketRequestPaymentOutput::default()))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn put_public_access_block(
|
||||
&self,
|
||||
req: S3Request<PutPublicAccessBlockInput>,
|
||||
@@ -1703,13 +1707,13 @@ impl S3 for FS {
|
||||
audit_only_operation(req, S3Operation::PutPublicAccessBlock, |req| usecase.execute_put_public_access_block(req)).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn put_bucket_tagging(&self, req: S3Request<PutBucketTaggingInput>) -> S3Result<S3Response<PutBucketTaggingOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketTagging, |req| usecase.execute_put_bucket_tagging(req)).await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn put_bucket_versioning(
|
||||
&self,
|
||||
req: S3Request<PutBucketVersioningInput>,
|
||||
@@ -1885,7 +1889,7 @@ impl S3 for FS {
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn put_object_lock_configuration(
|
||||
&self,
|
||||
req: S3Request<PutObjectLockConfigurationInput>,
|
||||
@@ -2052,6 +2056,7 @@ impl S3 for FS {
|
||||
// (backlog#1675 P1-5). No local object exists, so no bucket
|
||||
// notification event is emitted for the proxied write.
|
||||
if (is_err_object_not_found(&e) || is_err_version_not_found(&e))
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
&& Self::proxy_put_object_tagging(&bucket, &object, version_id.clone(), &req.headers, &tagging.tag_set)
|
||||
.await
|
||||
.is_some()
|
||||
|
||||
@@ -322,11 +322,36 @@ pub(crate) fn get_buffer_size_opt_in(file_size: i64) -> usize {
|
||||
buffer_size
|
||||
}
|
||||
|
||||
/// Validate object key for control characters and log special characters
|
||||
/// Reject XML-incompatible and storage-sidecar keys at new object boundaries.
|
||||
pub(crate) fn validate_new_object_key(key: &str, operation: &str) -> S3Result<()> {
|
||||
validate_object_key(key, operation)?;
|
||||
// These persistent sidecar filenames share directories with client objects.
|
||||
// Case-insensitive checks also protect case-insensitive disk deployments.
|
||||
if key.split('/').any(|segment| {
|
||||
["xl.meta", "xl.meta.bkp", "delete-prepared", "delete-quorum-committed"]
|
||||
.iter()
|
||||
.any(|reserved| segment.eq_ignore_ascii_case(reserved))
|
||||
}) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidArgument,
|
||||
"Object key contains a reserved storage filename",
|
||||
));
|
||||
}
|
||||
if crate::storage::storage_api::ecstore_bucket::utils::has_xml_invalid_control_characters(key) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidArgument,
|
||||
"Object key contains characters unsupported by XML listings",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate client object keys and log special characters
|
||||
///
|
||||
/// This function:
|
||||
/// 1. Rejects keys containing control characters (null bytes, newlines, carriage returns)
|
||||
/// 2. Logs debug information for keys containing spaces, plus signs, or percent signs
|
||||
/// 2. Rejects reserved directory markers and invalid path segments
|
||||
/// 3. Logs debug information for keys containing spaces, plus signs, or percent signs
|
||||
///
|
||||
/// The s3s library handles URL decoding, so keys are already decoded when they reach this function.
|
||||
/// This validation ensures that invalid characters that could cause issues are rejected early.
|
||||
@@ -1155,7 +1180,39 @@ pub(crate) fn parse_part_number_i32_to_usize(part_number: Option<i32>, op: &'sta
|
||||
|
||||
#[cfg(test)]
|
||||
mod object_key_admission_tests {
|
||||
use super::validate_object_key;
|
||||
use super::{validate_new_object_key, validate_object_key};
|
||||
|
||||
#[test]
|
||||
fn storage_sidecar_keys_are_rejected_only_for_new_writes() {
|
||||
for reserved in ["xl.meta", "xl.meta.bkp", "delete-prepared", "delete-quorum-committed"] {
|
||||
for key in [
|
||||
reserved.to_string(),
|
||||
format!("parent/uuid/{reserved}"),
|
||||
format!("parent/{reserved}/child"),
|
||||
reserved.to_uppercase(),
|
||||
] {
|
||||
for operation in ["PutObject", "COPY (dest)", "CreateMultipartUpload", "CompleteMultipartUpload"] {
|
||||
assert!(validate_new_object_key(&key, operation).is_err(), "{operation}: {key}");
|
||||
}
|
||||
assert!(validate_object_key(&key, "GetObject").is_ok(), "legacy read: {key}");
|
||||
assert!(validate_object_key(&key, "DeleteObject").is_ok(), "legacy cleanup: {key}");
|
||||
}
|
||||
assert!(validate_new_object_key(&format!("ordinary/{reserved}-report"), "PutObject").is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xml_invalid_keys_are_rejected_only_for_new_writes() {
|
||||
for byte in (1u8..32).filter(|byte| !matches!(byte, 9 | 10 | 13)) {
|
||||
let key = format!("legacy-{}", char::from(byte));
|
||||
assert!(validate_new_object_key(&key, "PutObject").is_err());
|
||||
assert!(validate_object_key(&key, "GetObject").is_ok());
|
||||
assert!(validate_object_key(&key, "DeleteObject").is_ok());
|
||||
}
|
||||
for key in ["ordinary/key", "tab\tkey", "unicode-中文", "directory/"] {
|
||||
assert!(validate_new_object_key(key, "PutObject").is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reserved_directory_marker_keys_are_rejected() {
|
||||
|
||||
@@ -14,6 +14,26 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn object_key_rejects_reserved_directory_markers() {
|
||||
for key in [
|
||||
"__XLDIR__",
|
||||
"prefix/__XLDIR__",
|
||||
"stash__XLDIR__/blob",
|
||||
"file__XLDIR__",
|
||||
"file__XLDIR__/",
|
||||
"a__XLDIR__b",
|
||||
] {
|
||||
let error = crate::storage::ecfs_extend::validate_object_key(key, "PUT")
|
||||
.expect_err("client keys must not alias internal directory markers");
|
||||
assert_eq!(error.code(), &s3s::S3ErrorCode::InvalidArgument, "{key}");
|
||||
}
|
||||
for key in ["file", "folder/", "prefix/folder/", "__xldir__", "percent%20name"] {
|
||||
crate::storage::ecfs_extend::validate_object_key(key, "PUT")
|
||||
.expect("ordinary objects and directory objects remain valid");
|
||||
}
|
||||
}
|
||||
|
||||
use crate::config::WorkloadProfile;
|
||||
use crate::server::cors;
|
||||
use crate::storage::StorageError;
|
||||
|
||||
@@ -94,6 +94,31 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Own a parsed mutation through its storage and event/audit completion.
|
||||
/// Dropping the HTTP waiter does not cancel the operation. Streaming bodies
|
||||
/// must use their bounded upload owner instead of this entry point.
|
||||
pub(crate) async fn run_owned_mutation<T, F>(operation: F) -> S3Result<S3Response<T>>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: Future<Output = S3Result<S3Response<T>>> + Send + 'static,
|
||||
{
|
||||
let credentials = rustfs_keystone::KEYSTONE_CREDENTIALS.try_with(Clone::clone).ok().flatten();
|
||||
let operation = rustfs_keystone::KEYSTONE_CREDENTIALS.scope(credentials, operation);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let operation = ecstore_diagnostic::with_captured_read_version_context(
|
||||
ecstore_diagnostic::ReadVersionContextBoundary::BackgroundTask,
|
||||
operation,
|
||||
);
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
let operation = rustfs_io_metrics::s3_http_metrics::with_s3_http_operation_context(
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_context(),
|
||||
operation,
|
||||
);
|
||||
super::request_context::spawn_traced_join(operation)
|
||||
.await
|
||||
.map_err(|_| s3s::s3_error!(InternalError, "object mutation owner task failed"))?
|
||||
}
|
||||
|
||||
/// Builds S3-compatible notification response elements with the canonical request ID.
|
||||
pub(crate) fn build_event_resp_elements<T>(response: &S3Response<T>, request_id: &str) -> HashMap<String, String> {
|
||||
let mut resp_elements = extract_resp_elements(response);
|
||||
@@ -137,6 +162,84 @@ fn request_audit_builder<T>(
|
||||
builder
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type MutationCompletionPause = (std::sync::Arc<tokio::sync::Barrier>, std::sync::Arc<tokio::sync::Barrier>);
|
||||
#[cfg(test)]
|
||||
type MutationCompletionProbes = HashMap<String, std::sync::Weak<MutationCompletionProbe>>;
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
pub(crate) struct MutationCompletionProbe {
|
||||
pub(crate) audits: std::sync::Mutex<Vec<AuditEntry>>,
|
||||
pub(crate) events: std::sync::Mutex<Vec<rustfs_notify::EventArgs>>,
|
||||
changed: tokio::sync::Notify,
|
||||
pause: std::sync::Mutex<Option<MutationCompletionPause>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn completion_probes() -> &'static std::sync::Mutex<MutationCompletionProbes> {
|
||||
static PROBES: std::sync::OnceLock<std::sync::Mutex<MutationCompletionProbes>> = std::sync::OnceLock::new();
|
||||
PROBES.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn completion_probe(bucket: &str) -> Option<std::sync::Arc<MutationCompletionProbe>> {
|
||||
completion_probes()
|
||||
.lock()
|
||||
.expect("completion probes")
|
||||
.get(bucket)
|
||||
.and_then(std::sync::Weak::upgrade)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl MutationCompletionProbe {
|
||||
pub(crate) fn install(bucket: &str) -> std::sync::Arc<Self> {
|
||||
let probe = std::sync::Arc::new(Self::default());
|
||||
let mut probes = completion_probes().lock().expect("completion probes");
|
||||
probes.retain(|_, value| value.strong_count() > 0);
|
||||
probes.insert(bucket.to_owned(), std::sync::Arc::downgrade(&probe));
|
||||
probe
|
||||
}
|
||||
|
||||
pub(crate) fn pause_after_commit(&self) -> MutationCompletionPause {
|
||||
let entered = std::sync::Arc::new(tokio::sync::Barrier::new(2));
|
||||
let resume = std::sync::Arc::new(tokio::sync::Barrier::new(2));
|
||||
*self.pause.lock().expect("completion pause") = Some((entered.clone(), resume.clone()));
|
||||
(entered, resume)
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for(&self, audits: usize, events: usize) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
loop {
|
||||
let changed = self.changed.notified();
|
||||
if self.audits.lock().expect("audits").len() >= audits && self.events.lock().expect("events").len() >= events {
|
||||
break;
|
||||
}
|
||||
changed.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("committed mutation must dispatch its audit and events");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn pause_committed_mutation_for_test(bucket: &str) {
|
||||
let pause = completion_probe(bucket).and_then(|probe| probe.pause.lock().expect("completion pause").take());
|
||||
if let Some((entered, resume)) = pause {
|
||||
entered.wait().await;
|
||||
resume.wait().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn capture_mutation_event_for_test(event: &rustfs_notify::EventArgs) {
|
||||
if let Some(probe) = completion_probe(&event.bucket_name) {
|
||||
probe.events.lock().expect("events").push(event.clone());
|
||||
probe.changed.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
/// A unified helper structure for building and distributing audit logs and event notifications via RAII mode at the end of an S3 operation scope.
|
||||
pub enum OperationHelper {
|
||||
Disabled,
|
||||
@@ -191,8 +294,20 @@ impl OperationHelper {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let probe_enabled = completion_probe(
|
||||
req.extensions
|
||||
.get::<ReqInfo>()
|
||||
.and_then(|info| info.bucket.as_deref())
|
||||
.unwrap_or_else(|| req.uri.path().trim_start_matches('/').split('/').next().unwrap_or_default()),
|
||||
)
|
||||
.is_some();
|
||||
let audit_enabled = is_audit_module_enabled();
|
||||
let notify_enabled = !audit_only && should_build_notification_event(is_notify_module_enabled());
|
||||
#[cfg(test)]
|
||||
let audit_enabled = audit_enabled || probe_enabled;
|
||||
#[cfg(test)]
|
||||
let notify_enabled = notify_enabled || (!audit_only && probe_enabled);
|
||||
if audit_only && !audit_enabled {
|
||||
// Nothing to record and no metric to bump: skip the path parsing below.
|
||||
return Self::Disabled;
|
||||
@@ -547,13 +662,39 @@ impl Drop for OperationHelper {
|
||||
return;
|
||||
};
|
||||
|
||||
// An early error or panic must have an outcome, but is never proof
|
||||
// of a successful mutation. Committed owners complete explicitly.
|
||||
if state.api_builder.0.status.is_none() {
|
||||
let elapsed = state.start_time.elapsed();
|
||||
state.api_builder = state
|
||||
.api_builder
|
||||
.clone()
|
||||
.status("failure".to_string())
|
||||
.status_code(StatusCode::INTERNAL_SERVER_ERROR.as_u16() as i32)
|
||||
.time_to_response(format!("{elapsed:.2?}"))
|
||||
.time_to_response_in_ns(elapsed.as_nanos().to_string());
|
||||
if let Some(builder) = state.audit_builder.take() {
|
||||
state.audit_builder = Some(
|
||||
builder
|
||||
.api(state.api_builder.clone().build())
|
||||
.error("Operation ended before reporting its outcome".to_string()),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Distribute audit logs
|
||||
if state.audit_enabled
|
||||
&& let Some(builder) = state.audit_builder.take()
|
||||
{
|
||||
let entry = builder.build();
|
||||
#[cfg(test)]
|
||||
if let Some(probe) = entry.api.bucket.as_deref().and_then(completion_probe) {
|
||||
probe.audits.lock().expect("audits").push(entry.clone());
|
||||
probe.changed.notify_one();
|
||||
}
|
||||
let ctx = Some(state.request_context.clone());
|
||||
spawn_background_with_context(ctx, async move {
|
||||
AuditLogger::log(builder.build()).await;
|
||||
AuditLogger::log(entry).await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -565,6 +706,8 @@ impl Drop for OperationHelper {
|
||||
let event_args = builder.build();
|
||||
// Avoid generating notifications for copy requests
|
||||
if !event_args.is_replication_request() {
|
||||
#[cfg(test)]
|
||||
capture_mutation_event_for_test(&event_args);
|
||||
let ctx = Some(state.request_context.clone());
|
||||
spawn_background_with_context(ctx, async move {
|
||||
runtime_sources::current_notify_interface().notify(event_args).await;
|
||||
@@ -646,6 +789,48 @@ mod tests {
|
||||
assert_eq!(rx.await.expect("unscoped task completes"), (None, None));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_keeps_keystone_context_and_audits_panics_as_failure() {
|
||||
let bucket = "owned-mutation-panic";
|
||||
let probe = super::MutationCompletionProbe::install(bucket);
|
||||
let credentials = Credentials {
|
||||
access_key: "keystone:test".to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
let result: S3Result<S3Response<()>> = rustfs_keystone::KEYSTONE_CREDENTIALS
|
||||
.scope(Some(credentials), async {
|
||||
super::run_owned_mutation(async {
|
||||
assert_eq!(
|
||||
rustfs_keystone::KEYSTONE_CREDENTIALS
|
||||
.with(|credentials| credentials.as_ref().map(|value| value.access_key.clone())),
|
||||
Some("keystone:test".to_owned())
|
||||
);
|
||||
let req = S3Request {
|
||||
input: (),
|
||||
method: Method::DELETE,
|
||||
uri: Uri::from_static("/owned-mutation-panic/object"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let _helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObject);
|
||||
panic!("injected mutation panic before reporting a commit");
|
||||
})
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
assert_eq!(result.expect_err("join failure").code(), &S3ErrorCode::InternalError);
|
||||
probe.wait_for(1, 0).await;
|
||||
let audits = probe.audits.lock().expect("audits");
|
||||
assert_eq!(audits.len(), 1);
|
||||
assert_eq!(audits[0].api.status.as_deref(), Some("failure"));
|
||||
assert_eq!(audits[0].api.status_code, Some(500));
|
||||
assert!(probe.events.lock().expect("events").is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_background_tasks_keep_existing_entry_on_fallback_thread() {
|
||||
|
||||
+183
-24
@@ -360,6 +360,21 @@ pub async fn put_opts(
|
||||
put_opts_with_replication_authorization(bucket, object, vid, headers, metadata, false).await
|
||||
}
|
||||
|
||||
/// Resolve the version addressed by a write before evaluating version-scoped policy.
|
||||
/// Query parameters take precedence over compatible replication headers.
|
||||
pub(crate) fn destination_version_id(vid: Option<&str>, headers: &HeaderMap<HeaderValue>) -> Option<String> {
|
||||
let header = get_header(headers, SUFFIX_SOURCE_VERSION_ID);
|
||||
let value = vid.or(header.as_deref())?.trim();
|
||||
if value.eq_ignore_ascii_case("null") {
|
||||
return Some("null".to_owned());
|
||||
}
|
||||
Some(match Uuid::parse_str(value) {
|
||||
Ok(id) if id.is_nil() => "null".to_owned(),
|
||||
Ok(id) => id.to_string(),
|
||||
Err(_) => value.to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn put_opts_with_replication_authorization(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -379,8 +394,8 @@ pub async fn put_opts_with_replication_authorization(
|
||||
let versioned = versioning_cfg.prefix_enabled(object);
|
||||
let version_suspended = versioning_cfg.prefix_suspended(object);
|
||||
|
||||
let vid = if vid.is_none() && replication_request_authorized {
|
||||
get_header(headers, SUFFIX_SOURCE_VERSION_ID).map(|s| s.into_owned())
|
||||
let vid = if replication_request_authorized {
|
||||
destination_version_id(vid.as_deref(), headers)
|
||||
} else {
|
||||
vid
|
||||
};
|
||||
@@ -439,7 +454,7 @@ pub fn get_complete_multipart_upload_opts_with_replication_authorization(
|
||||
if replication_request_authorized && get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") {
|
||||
replication_request = true;
|
||||
mod_time = replication_source_mtime(headers);
|
||||
preserve_etag = replication_source_etag(headers);
|
||||
preserve_etag = replication_source_etag(headers)?;
|
||||
if let Some(actual_size_str) = get_header(headers, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE) {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut user_defined,
|
||||
@@ -518,25 +533,47 @@ pub fn put_opts_from_headers_with_replication_authorization(
|
||||
if replication_request_authorized && get_header(headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true") {
|
||||
opts.replication_request = true;
|
||||
opts.mod_time = replication_source_mtime(headers);
|
||||
opts.preserve_etag = replication_source_etag(headers);
|
||||
opts.preserve_etag = replication_source_etag(headers)?;
|
||||
// SSE-C ciphertext passthrough: restore the stored encryption metadata
|
||||
// from the transport headers and mark the body as already encrypted so
|
||||
// the write path stores it verbatim.
|
||||
if let Some(restored) = rustfs_utils::http::ssec_transport_to_stored_metadata(headers) {
|
||||
if restored
|
||||
.get(rustfs_utils::http::object_encryption_keys::SSEC_ORIGINAL_SIZE_HEADER)
|
||||
.is_some_and(|value| !value.parse::<i64>().is_ok_and(|size| size >= 0))
|
||||
{
|
||||
return Err(StorageError::InvalidArgument(
|
||||
String::new(),
|
||||
String::new(),
|
||||
"Invalid replicated SSE-C original size".to_string(),
|
||||
));
|
||||
}
|
||||
opts.user_defined.extend(restored);
|
||||
opts.preserve_ciphertext = true;
|
||||
// A compressed passthrough object restores its compression
|
||||
// layout as well, so the replica decompresses after decrypting
|
||||
// (backlog#2363). The value is validated when the object is read.
|
||||
// (backlog#2363). Reject unreadable layouts before persisting them.
|
||||
if let Some(scheme) = get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION) {
|
||||
super::storage_api::options_consumer::validate_compression_scheme(&scheme).map_err(|_| {
|
||||
StorageError::InvalidArgument(
|
||||
String::new(),
|
||||
String::new(),
|
||||
"Unsupported replicated compression scheme".to_string(),
|
||||
)
|
||||
})?;
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut opts.user_defined,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
scheme.into_owned(),
|
||||
);
|
||||
if let Some(actual_size) = get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE)
|
||||
&& actual_size.parse::<i64>().is_ok_and(|size| size >= 0)
|
||||
{
|
||||
if let Some(actual_size) = get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE) {
|
||||
if !actual_size.parse::<i64>().is_ok_and(|size| size >= 0) {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
String::new(),
|
||||
String::new(),
|
||||
"Invalid replicated compression size".to_string(),
|
||||
));
|
||||
}
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut opts.user_defined,
|
||||
rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
|
||||
@@ -556,17 +593,30 @@ pub fn put_opts_from_headers_with_replication_authorization(
|
||||
/// Replicas must keep the source object's ETag: managed-SSE replication
|
||||
/// re-encrypts on the target, so a recomputed ETag would differ from the
|
||||
/// source and every HEAD comparison would re-schedule the object forever.
|
||||
fn replication_source_etag(headers: &HeaderMap<HeaderValue>) -> Option<String> {
|
||||
let value = get_header(headers, SUFFIX_SOURCE_ETAG)?;
|
||||
let value = value.trim().trim_matches('"');
|
||||
(!value.is_empty()).then(|| value.to_string())
|
||||
fn replication_source_etag(headers: &HeaderMap<HeaderValue>) -> std::io::Result<Option<String>> {
|
||||
let Some(value) = get_header(headers, SUFFIX_SOURCE_ETAG) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value = value.trim();
|
||||
let value = value
|
||||
.strip_prefix('"')
|
||||
.and_then(|value| value.strip_suffix('"'))
|
||||
.unwrap_or(value);
|
||||
if value
|
||||
.bytes()
|
||||
.any(|byte| !byte.is_ascii_graphic() || matches!(byte, b'"' | b'<' | b'>' | b'&'))
|
||||
{
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid replication source ETag"));
|
||||
}
|
||||
Ok((!value.is_empty()).then(|| value.to_string()))
|
||||
}
|
||||
|
||||
fn replication_source_mtime(headers: &HeaderMap<HeaderValue>) -> Option<time::OffsetDateTime> {
|
||||
let value = get_header(headers, SUFFIX_SOURCE_MTIME)?;
|
||||
let value = value.trim();
|
||||
match time::OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339) {
|
||||
Ok(mtime) => Some(mtime),
|
||||
Ok(mtime) if mtime.unix_timestamp_nanos() != 0 && i64::try_from(mtime.unix_timestamp_nanos()).is_ok() => Some(mtime),
|
||||
Ok(_) => None,
|
||||
Err(err) => {
|
||||
tracing::warn!("Invalid source-mtime value '{}' (replication request=true): {}", value, err);
|
||||
None
|
||||
@@ -1209,6 +1259,45 @@ mod tests {
|
||||
use std::collections::HashMap;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn replication_etags_reject_xml_and_header_delimiters_before_persistence() {
|
||||
for etag in ["</ETag><Contents><Key>victim</Key></Contents><ETag>", "a&b", "a\"b", "a\tb"] {
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_ETAG, etag);
|
||||
assert!(
|
||||
put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_err(),
|
||||
"{etag:?}"
|
||||
);
|
||||
assert!(
|
||||
get_complete_multipart_upload_opts_with_replication_authorization(&headers, true).is_err(),
|
||||
"{etag:?}"
|
||||
);
|
||||
assert!(
|
||||
put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), false)
|
||||
.expect("untrusted header is inert")
|
||||
.preserve_etag
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
for etag in [
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
"\"0123456789abcdef0123456789abcdef-3\"",
|
||||
"opaque-provider-etag",
|
||||
] {
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_ETAG, etag);
|
||||
assert_eq!(
|
||||
put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true)
|
||||
.expect("valid replica ETag")
|
||||
.preserve_etag
|
||||
.as_deref(),
|
||||
Some(etag.trim_matches('"'))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn create_test_headers() -> HeaderMap<HeaderValue> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-type", HeaderValue::from_static("application/json"));
|
||||
@@ -1564,6 +1653,25 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destination_version_resolution_preserves_query_precedence_and_normalizes_aliases() {
|
||||
for header in ["x-rustfs-source-version-id", "x-minio-source-version-id"] {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(header, HeaderValue::from_static(" ABCDEF12-1234-1234-1234-123456789ABC "));
|
||||
assert_eq!(
|
||||
super::destination_version_id(None, &headers).as_deref(),
|
||||
Some("abcdef12-1234-1234-1234-123456789abc")
|
||||
);
|
||||
for query in ["null", "NULL", " null ", "00000000-0000-0000-0000-000000000000"] {
|
||||
assert_eq!(super::destination_version_id(Some(query), &headers).as_deref(), Some("null"));
|
||||
}
|
||||
headers.insert(header, HeaderValue::from_static("null"));
|
||||
assert_eq!(super::destination_version_id(None, &headers).as_deref(), Some("null"));
|
||||
assert_eq!(super::destination_version_id(Some("invalid"), &headers).as_deref(), Some("invalid"));
|
||||
}
|
||||
assert_eq!(super::destination_version_id(None, &HeaderMap::new()), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_opts_normalizes_null_version_id() {
|
||||
// MinIO-compatible replication senders (including RustFS itself since
|
||||
@@ -1686,9 +1794,10 @@ mod tests {
|
||||
get_str,
|
||||
};
|
||||
|
||||
let scheme = crate::storage::storage_api::compression_metadata_value(rustfs_utils::CompressionAlgorithm::Lz4);
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION, "klauspost/compress/s2");
|
||||
insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION, &scheme);
|
||||
insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE, "6295552");
|
||||
|
||||
// Without SSE-C transport headers the request is not a passthrough:
|
||||
@@ -1716,19 +1825,49 @@ mod tests {
|
||||
let trusted = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true)
|
||||
.expect("authorized replication request should parse");
|
||||
assert!(trusted.preserve_ciphertext);
|
||||
assert_eq!(
|
||||
get_str(&trusted.user_defined, SUFFIX_COMPRESSION).as_deref(),
|
||||
Some("klauspost/compress/s2")
|
||||
);
|
||||
assert_eq!(get_str(&trusted.user_defined, SUFFIX_COMPRESSION).as_deref(), Some(scheme.as_str()));
|
||||
assert_eq!(get_str(&trusted.user_defined, SUFFIX_ACTUAL_SIZE).as_deref(), Some("6295552"));
|
||||
|
||||
// A malformed plaintext size is dropped; the scheme alone still lets
|
||||
// the read path derive the size from the parts.
|
||||
insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE, "-5");
|
||||
let malformed = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true)
|
||||
.expect("authorized replication request should parse");
|
||||
assert!(get_str(&malformed.user_defined, SUFFIX_COMPRESSION).is_some());
|
||||
assert!(get_str(&malformed.user_defined, SUFFIX_ACTUAL_SIZE).is_none());
|
||||
assert!(
|
||||
put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_err(),
|
||||
"invalid replicated sizes must not reach persisted metadata"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replicated_encryption_metadata_rejects_invalid_sizes_and_compression() {
|
||||
use rustfs_utils::http::object_encryption_keys::REPLICATION_SSEC_ORIGINAL_SIZE_HEADER;
|
||||
use rustfs_utils::http::{SUFFIX_REPLICATION_COMPRESSION, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE};
|
||||
for (header, value) in [
|
||||
(REPLICATION_SSEC_ORIGINAL_SIZE_HEADER, "abc"),
|
||||
(REPLICATION_SSEC_ORIGINAL_SIZE_HEADER, "-1"),
|
||||
(REPLICATION_SSEC_ORIGINAL_SIZE_HEADER, "9223372036854775808"),
|
||||
] {
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
headers.insert(
|
||||
http::HeaderName::from_bytes(header.as_bytes()).expect("header name"),
|
||||
HeaderValue::from_static(value),
|
||||
);
|
||||
assert!(put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_err());
|
||||
assert!(
|
||||
put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), false).is_ok(),
|
||||
"untrusted transport headers stay inert"
|
||||
);
|
||||
}
|
||||
for size in ["0", "5", "9223372036854775807"] {
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
headers.insert(
|
||||
http::HeaderName::from_bytes(REPLICATION_SSEC_ORIGINAL_SIZE_HEADER.as_bytes()).expect("size header"),
|
||||
HeaderValue::from_static(size),
|
||||
);
|
||||
assert!(put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_ok());
|
||||
insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION, "not-a-compression-format");
|
||||
insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE, "5");
|
||||
assert!(put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1810,6 +1949,26 @@ mod tests {
|
||||
assert!(opts_invalid.mod_time.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_source_time_must_survive_the_metadata_codec() {
|
||||
for timestamp in ["1970-01-01T00:00:00Z", "9999-12-31T23:59:59Z", "0001-01-01T00:00:00Z"] {
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, timestamp);
|
||||
let opts = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true)
|
||||
.expect("invalid source time falls back to server commit time");
|
||||
assert!(opts.mod_time.is_none(), "{timestamp}");
|
||||
}
|
||||
for timestamp in ["1969-12-31T23:59:59Z", "1970-01-01T00:00:00.000000001Z"] {
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_MTIME, timestamp);
|
||||
assert!(
|
||||
super::replication_source_mtime(&headers).is_some(),
|
||||
"representable time {timestamp} remains readable"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A client PUT must not materialize the replication transport names as
|
||||
/// bare stored user-metadata keys: the outbound replication header
|
||||
/// builder forwards user metadata verbatim on a server-authorized
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::storage::storage_api::s3_api_consumer::bucket::contract::{
|
||||
bucket::BucketInfo,
|
||||
list::{ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info},
|
||||
};
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_etag;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_xml_etag;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use s3s::dto::{
|
||||
Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput,
|
||||
@@ -245,7 +245,7 @@ pub(crate) fn build_list_object_versions_output(
|
||||
.etag
|
||||
.as_ref()
|
||||
.filter(|_| rustfs_utils::http::etag_can_be_listed(&v.user_defined))
|
||||
.map(|etag| to_s3s_etag(etag)),
|
||||
.map(|etag| to_s3s_xml_etag(etag)),
|
||||
storage_class: v.storage_class.clone().map(ObjectVersionStorageClass::from),
|
||||
..Default::default()
|
||||
})
|
||||
@@ -334,7 +334,7 @@ pub(crate) fn build_list_objects_v2_output(
|
||||
.etag
|
||||
.as_ref()
|
||||
.filter(|_| rustfs_utils::http::etag_can_be_listed(&v.user_defined))
|
||||
.map(|etag| to_s3s_etag(etag)),
|
||||
.map(|etag| to_s3s_xml_etag(etag)),
|
||||
storage_class: v.storage_class.clone().map(ObjectStorageClass::from),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -452,7 +452,8 @@ mod tests {
|
||||
use super::{
|
||||
ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_buckets_output,
|
||||
build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output,
|
||||
encode_list_objects_v2_continuation_token, parse_list_object_versions_params, parse_list_objects_v2_params, to_s3s_etag,
|
||||
encode_list_objects_v2_continuation_token, parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
to_s3s_xml_etag,
|
||||
};
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo;
|
||||
@@ -464,6 +465,66 @@ mod tests {
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn listings_escape_legacy_etags_without_creating_xml_entries() {
|
||||
use s3s::xml::{Serialize, Serializer};
|
||||
fn xml<T: Serialize>(value: &T) -> String {
|
||||
let mut bytes = Vec::new();
|
||||
value.serialize(&mut Serializer::new(&mut bytes)).expect("serialize listing");
|
||||
String::from_utf8(bytes).expect("UTF-8 XML")
|
||||
}
|
||||
let malicious = "</ETag><Contents><Key>victim</Key></Contents><ETag>&";
|
||||
let object = ObjectInfo {
|
||||
name: "owned/key".into(),
|
||||
etag: Some(malicious.into()),
|
||||
..Default::default()
|
||||
};
|
||||
let v2 = build_list_objects_v2_output(
|
||||
ListObjectsV2Info {
|
||||
objects: vec![object.clone()],
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
1000,
|
||||
"bucket".into(),
|
||||
String::new(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let v1 = build_list_objects_output(v2.clone(), None);
|
||||
let versions = build_list_object_versions_output(
|
||||
ListObjectVersionsInfo {
|
||||
objects: vec![object],
|
||||
..Default::default()
|
||||
},
|
||||
"bucket".into(),
|
||||
&ListObjectVersionsParams {
|
||||
prefix: String::new(),
|
||||
delimiter: None,
|
||||
key_marker: None,
|
||||
version_id_marker: None,
|
||||
max_keys: 1000,
|
||||
},
|
||||
None,
|
||||
);
|
||||
for output in [xml(&v1), xml(&v2), xml(&versions)] {
|
||||
let mut reader = quick_xml::Reader::from_str(&output);
|
||||
let mut keys = 0;
|
||||
loop {
|
||||
match reader.read_event().expect("legacy ETag must leave well-formed XML") {
|
||||
quick_xml::events::Event::Start(event) if event.name().as_ref() == "Key" => keys += 1,
|
||||
quick_xml::events::Event::Eof => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
assert_eq!(keys, 1, "{output}");
|
||||
assert!(output.contains("</ETag><Contents>"), "{output}");
|
||||
assert!(output.contains("&amp;"), "existing entities must remain literal: {output}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_list_buckets_output_maps_bucket_infos_and_owner() {
|
||||
let bucket_infos = vec![
|
||||
@@ -771,7 +832,7 @@ mod tests {
|
||||
let entries = listing.as_ref().expect("listed objects");
|
||||
assert_eq!(entries.len(), 5);
|
||||
for (index, entry) in entries.iter().enumerate() {
|
||||
assert_eq!(entry.e_tag, (index >= 2).then(|| to_s3s_etag("0123456789abcdef0123456789abcdef")));
|
||||
assert_eq!(entry.e_tag, (index >= 2).then(|| to_s3s_xml_etag("0123456789abcdef0123456789abcdef")));
|
||||
assert_eq!(entry.key.as_deref(), Some(format!("object-{index}").as_str()));
|
||||
assert_eq!(entry.size, Some(42));
|
||||
}
|
||||
@@ -794,7 +855,7 @@ mod tests {
|
||||
let versions = versions.versions.expect("listed versions");
|
||||
assert_eq!(versions.len(), 5);
|
||||
for (index, entry) in versions.iter().enumerate() {
|
||||
assert_eq!(entry.e_tag, (index >= 2).then(|| to_s3s_etag("0123456789abcdef0123456789abcdef")));
|
||||
assert_eq!(entry.e_tag, (index >= 2).then(|| to_s3s_xml_etag("0123456789abcdef0123456789abcdef")));
|
||||
assert_eq!(entry.size, Some(42));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::storage::storage_api::effective_storage_class;
|
||||
use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::{
|
||||
ListMultipartsInfo, ListPartsInfo, MAX_MULTIPART_PART_NUMBER,
|
||||
};
|
||||
use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_etag;
|
||||
use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_xml_etag;
|
||||
use s3s::dto::{CommonPrefix, EncodingType, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
|
||||
@@ -54,7 +54,7 @@ pub(crate) fn build_list_parts_output(res: ListPartsInfo) -> ListPartsOutput {
|
||||
res.parts
|
||||
.into_iter()
|
||||
.map(|p| Part {
|
||||
e_tag: p.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
e_tag: p.etag.map(|etag| to_s3s_xml_etag(&etag)),
|
||||
last_modified: p.last_mod.map(Timestamp::from),
|
||||
part_number: p.part_num.try_into().ok(),
|
||||
// Compressed parts store fewer bytes than the client sent; S3
|
||||
@@ -222,7 +222,7 @@ mod tests {
|
||||
use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::{
|
||||
ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo,
|
||||
};
|
||||
use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_etag;
|
||||
use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_xml_etag;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::Timestamp;
|
||||
use time::OffsetDateTime;
|
||||
@@ -261,7 +261,7 @@ mod tests {
|
||||
assert_eq!(parts.len(), 1);
|
||||
assert_eq!(parts[0].part_number, Some(1));
|
||||
assert_eq!(parts[0].size, Some(11));
|
||||
assert_eq!(parts[0].e_tag, Some(to_s3s_etag("etag-1")));
|
||||
assert_eq!(parts[0].e_tag, Some(to_s3s_xml_etag("etag-1")));
|
||||
assert_eq!(output.owner, Some(rustfs_owner()));
|
||||
assert_eq!(output.initiator, Some(rustfs_initiator()));
|
||||
}
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
use percent_encoding::percent_decode_str;
|
||||
use s3s::dto::{Tag, TaggingDirective};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::OnceLock;
|
||||
use url::form_urlencoded;
|
||||
|
||||
use crate::storage::storage_api::encode_tags;
|
||||
@@ -25,46 +23,18 @@ fn invalid_tag_error(message: &str) -> S3Error {
|
||||
S3Error::with_message(S3ErrorCode::InvalidTag, message.to_string())
|
||||
}
|
||||
|
||||
fn is_valid_tag_text(value: &str) -> bool {
|
||||
static VALID_OBJECT_TAG_TEXT: OnceLock<Result<regex::Regex, regex::Error>> = OnceLock::new();
|
||||
VALID_OBJECT_TAG_TEXT
|
||||
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
|
||||
.as_ref()
|
||||
.is_ok_and(|pattern| pattern.is_match(value))
|
||||
}
|
||||
|
||||
pub(crate) fn validate_object_tag_set(tag_set: &[Tag]) -> S3Result<()> {
|
||||
if tag_set.len() > 10 {
|
||||
return Err(invalid_tag_error("Cannot have more than 10 tags per object"));
|
||||
}
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::validate_tag_set_entries(tag_set).map_err(invalid_tag_error)
|
||||
}
|
||||
|
||||
let mut tag_keys = HashSet::with_capacity(tag_set.len());
|
||||
for tag in tag_set {
|
||||
let key = tag
|
||||
.key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or_else(|| invalid_tag_error("Tag key cannot be empty"))?;
|
||||
|
||||
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
|
||||
return Err(invalid_tag_error("The TagKey you have provided is invalid"));
|
||||
}
|
||||
|
||||
let value = tag
|
||||
.value
|
||||
.as_deref()
|
||||
.ok_or_else(|| invalid_tag_error("Tag value cannot be null"))?;
|
||||
|
||||
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
|
||||
return Err(invalid_tag_error("The TagValue you have provided is invalid"));
|
||||
}
|
||||
|
||||
if !tag_keys.insert(key) {
|
||||
return Err(invalid_tag_error("Cannot provide multiple Tags with the same key"));
|
||||
}
|
||||
pub(crate) fn validate_bucket_tag_set(tag_set: &[Tag]) -> S3Result<()> {
|
||||
if tag_set.len() > crate::storage::storage_api::ecstore_bucket::metadata::MAX_BUCKET_TAGS {
|
||||
return Err(invalid_tag_error("Cannot have more than 50 tags per bucket"));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::validate_tag_set_entries(tag_set).map_err(invalid_tag_error)
|
||||
}
|
||||
|
||||
fn validate_tag_component_encoding(component: &str) -> S3Result<()> {
|
||||
|
||||
+345
-2
@@ -476,6 +476,11 @@ impl EncryptionRequest<'_> {
|
||||
.decode_to_vec(key_md5)
|
||||
.is_ok_and(|decoded| decoded.len() == 16) =>
|
||||
{
|
||||
if rustfs_utils::http::contains_key_str(user_defined, rustfs_utils::http::SUFFIX_PLAINTEXT_CHECKSUM) {
|
||||
return Err(ssec_invalid_request(
|
||||
"SSE-C multipart uploads with additional checksums require the customer key for completion.",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
_ => Err(ssec_invalid_request("The multipart upload contains invalid SSE-C metadata.")),
|
||||
@@ -903,7 +908,7 @@ impl SseKmsPrincipal {
|
||||
conditions: cred.map(|cred| resource_free_condition_values(req, cred)).unwrap_or_default(),
|
||||
request_audit: request_context_from_req(req).and_then(|context| kms_request_audit(&context.request_id)),
|
||||
#[cfg(test)]
|
||||
test_hooks: None,
|
||||
test_hooks: req.extensions.get::<TestAuthorizationHooks>().cloned(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1130,6 +1135,87 @@ pub async fn authorize_sse_kms_object_read(
|
||||
result
|
||||
}
|
||||
|
||||
/// Authorize the current request to write under the upload's stored KMS key.
|
||||
/// Upload IDs do not confer the creator's KMS permissions on another caller.
|
||||
pub async fn authorize_sse_kms_multipart_write(
|
||||
principal: Option<&SseKmsPrincipal>,
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<(), ApiError> {
|
||||
let Some((sse_type, key_id)) = stored_managed_encryption_key(metadata) else {
|
||||
return Ok(());
|
||||
};
|
||||
let result = authorize_sse_kms_key(principal, sse_type, KmsAction::GenerateDataKeyAction, &key_id).await;
|
||||
if let Err(error) = &result {
|
||||
record_managed_kms_outcome(
|
||||
principal,
|
||||
sse_type,
|
||||
Some(&key_id),
|
||||
|| stored_envelope_master_key_version(metadata),
|
||||
Err(error),
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Capture the request's encryption gate without exposing customer keys in
|
||||
/// ObjectOptions Debug output. Storage applies it to the source selected under
|
||||
/// completion locks, including recovered uploads and historical replay versions.
|
||||
pub(crate) fn multipart_completion_authorization(
|
||||
request: EncryptionRequest<'_>,
|
||||
replication_authorized: bool,
|
||||
) -> Arc<dyn super::storage_api::contract::multipart::MultipartCompletionAuthorization<Error = StorageError>> {
|
||||
Arc::new(SseMultipartCompletionAuthorization {
|
||||
bucket: request.bucket.to_string(),
|
||||
key: request.key.to_string(),
|
||||
principal: request.principal.cloned(),
|
||||
algorithm: request.sse_customer_algorithm,
|
||||
customer_key: request.sse_customer_key,
|
||||
customer_md5: request.sse_customer_key_md5,
|
||||
replication_authorized,
|
||||
})
|
||||
}
|
||||
|
||||
struct SseMultipartCompletionAuthorization {
|
||||
bucket: String,
|
||||
key: String,
|
||||
principal: Option<SseKmsPrincipal>,
|
||||
algorithm: Option<String>,
|
||||
customer_key: Option<String>,
|
||||
customer_md5: Option<String>,
|
||||
replication_authorized: bool,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl super::storage_api::contract::multipart::MultipartCompletionAuthorization for SseMultipartCompletionAuthorization {
|
||||
type Error = StorageError;
|
||||
|
||||
async fn authorize(&self, metadata: &HashMap<String, String>) -> Result<(), StorageError> {
|
||||
authorize_sse_kms_multipart_write(self.principal.as_ref(), metadata)
|
||||
.await
|
||||
.map_err(|error| StorageError::other(map_encryption_resolution_error(error)))?;
|
||||
if self.customer_key.is_some()
|
||||
|| !self.replication_authorized
|
||||
|| !rustfs_utils::http::contains_key_str(metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT)
|
||||
{
|
||||
EncryptionRequest {
|
||||
bucket: &self.bucket,
|
||||
key: &self.key,
|
||||
server_side_encryption: None,
|
||||
ssekms_key_id: None,
|
||||
ssekms_context: None,
|
||||
sse_customer_algorithm: self.algorithm.clone(),
|
||||
sse_customer_key: self.customer_key.clone(),
|
||||
sse_customer_key_md5: self.customer_md5.clone(),
|
||||
content_size: 0,
|
||||
principal: self.principal.as_ref(),
|
||||
}
|
||||
.validate_complete_multipart_ssec(metadata)
|
||||
.map_err(|error| StorageError::other(map_encryption_resolution_error(error)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Data-plane KMS audit attachment (SSE-S3 / SSE-KMS)
|
||||
// ============================================================================
|
||||
@@ -4187,7 +4273,7 @@ fn is_legacy_rustfs_managed_metadata(metadata: &HashMap<String, String>) -> bool
|
||||
/// detection only in builds that can actually read it — default builds keep
|
||||
/// the resolver's fail-closed diagnosis instead of demanding customer keys
|
||||
/// they could not use.
|
||||
fn stored_ssec_metadata(metadata: &HashMap<String, String>) -> bool {
|
||||
pub(crate) fn stored_ssec_metadata(metadata: &HashMap<String, String>) -> bool {
|
||||
metadata.contains_key("x-amz-server-side-encryption-customer-algorithm") || minio_sealed_ssec_metadata(metadata)
|
||||
}
|
||||
|
||||
@@ -6877,6 +6963,35 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssec_checksum_completion_requires_key_for_both_metadata_aliases() {
|
||||
let no_key = EncryptionRequest {
|
||||
sse_customer_algorithm: None,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
..multipart_ssec_request(42)
|
||||
};
|
||||
for prefix in ["x-rustfs-internal-", "x-minio-internal-"] {
|
||||
let mut metadata = multipart_ssec_metadata(42);
|
||||
metadata.insert(format!("{prefix}plaintext-checksum"), "true".to_string());
|
||||
assert_eq!(
|
||||
no_key
|
||||
.validate_complete_multipart_ssec(&metadata)
|
||||
.expect_err("no key cannot compare or reveal plaintext checksums")
|
||||
.code,
|
||||
S3ErrorCode::InvalidRequest
|
||||
);
|
||||
multipart_ssec_request(42)
|
||||
.validate_complete_multipart_ssec(&metadata)
|
||||
.expect("correct key");
|
||||
assert!(
|
||||
multipart_ssec_request(43)
|
||||
.validate_complete_multipart_ssec(&metadata)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_complete_multipart_ssec_still_validates_present_parameters() {
|
||||
let partial = EncryptionRequest {
|
||||
@@ -8156,6 +8271,234 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn multipart_handlers_reject_another_callers_denied_kms_upload() {
|
||||
crate::app::gating_test_env::run_large_stack_test("mpu-kms-current-caller", || async {
|
||||
use crate::app::multipart_usecase::DefaultMultipartUsecase;
|
||||
use crate::storage::storage_api::contract::multipart::MultipartOperations as _;
|
||||
use s3s::dto::{
|
||||
CompleteMultipartUploadInput, CompletedMultipartUpload, CompletedPart, CopySource, StreamingBlob,
|
||||
UploadPartCopyInput, UploadPartInput,
|
||||
};
|
||||
let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("mpu-kms-current-caller", 16_384).await;
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
let key = "kms-owned-upload";
|
||||
let metadata = sse_kms_object_metadata();
|
||||
let upload = store
|
||||
.new_multipart_upload(
|
||||
&bucket,
|
||||
key,
|
||||
&crate::storage::ObjectOptions {
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed authorized upload envelope");
|
||||
let authorizer = RecordingKmsKeyAuthorizer::new(false);
|
||||
fn denied_request<T>(input: T, authorizer: Arc<RecordingKmsKeyAuthorizer>) -> s3s::S3Request<T> {
|
||||
let mut req = s3s::S3Request {
|
||||
input,
|
||||
method: http::Method::PUT,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
req.extensions.insert(super::ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials {
|
||||
access_key: "denied-writer".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
req.extensions.insert(super::TestAuthorizationHooks {
|
||||
enforced: true,
|
||||
authorizer,
|
||||
});
|
||||
req
|
||||
}
|
||||
let mut part = UploadPartInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(key.to_string())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.part_number(1)
|
||||
.build()
|
||||
.expect("part");
|
||||
part.body = Some(StreamingBlob::from(bytes::Bytes::from_static(b"abc")));
|
||||
part.content_length = Some(3);
|
||||
// Keep typed request construction separate: each real handler must
|
||||
// consult the destination envelope before encryption or commit.
|
||||
let error = usecase
|
||||
.execute_upload_part(denied_request(part, authorizer.clone()))
|
||||
.await
|
||||
.expect_err("denied part");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
|
||||
let copy = UploadPartCopyInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(key.to_string())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.part_number(1)
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: "source".into(),
|
||||
version_id: None,
|
||||
})
|
||||
.build()
|
||||
.expect("copy");
|
||||
let copy_req = denied_request(copy, authorizer.clone());
|
||||
let error = usecase
|
||||
.execute_upload_part_copy(copy_req)
|
||||
.await
|
||||
.expect_err("denied destination copy");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
|
||||
let complete = CompleteMultipartUploadInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(key.to_string())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.multipart_upload(Some(CompletedMultipartUpload {
|
||||
parts: Some(vec![CompletedPart {
|
||||
part_number: Some(1),
|
||||
e_tag: Some(s3s::dto::ETag::Strong("etag".to_string())),
|
||||
..Default::default()
|
||||
}]),
|
||||
}))
|
||||
.build()
|
||||
.expect("complete");
|
||||
let complete_req = denied_request(complete, authorizer.clone());
|
||||
let error = usecase
|
||||
.execute_complete_multipart_upload(complete_req)
|
||||
.await
|
||||
.expect_err("denied completion");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
|
||||
assert_eq!(authorizer.calls(), vec![(KmsAction::GenerateDataKeyAction, "finance-key".to_string()); 3]);
|
||||
assert!(
|
||||
store
|
||||
.get_multipart_info(&bucket, key, &upload.upload_id, &crate::storage::ObjectOptions::default())
|
||||
.await
|
||||
.is_ok(),
|
||||
"denials must retain the original upload"
|
||||
);
|
||||
let mut reader = crate::storage::PutObjReader::from_vec(b"abc".to_vec());
|
||||
let uploaded_part = store
|
||||
.put_object_part(&bucket, key, &upload.upload_id, 1, &mut reader, &crate::storage::ObjectOptions::default())
|
||||
.await
|
||||
.expect("trusted owner stages part");
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
key,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage::storage_api::contract::multipart::CompletePart {
|
||||
part_num: 1,
|
||||
etag: uploaded_part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&crate::storage::ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("trusted owner completes upload");
|
||||
let replay = CompleteMultipartUploadInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(key.to_string())
|
||||
.upload_id(upload.upload_id)
|
||||
.multipart_upload(Some(CompletedMultipartUpload {
|
||||
parts: Some(vec![CompletedPart {
|
||||
part_number: Some(1),
|
||||
e_tag: Some(s3s::dto::ETag::Strong("guessed-etag".to_string())),
|
||||
..Default::default()
|
||||
}]),
|
||||
}))
|
||||
.build()
|
||||
.expect("replay");
|
||||
let error = usecase
|
||||
.execute_complete_multipart_upload(denied_request(replay, authorizer.clone()))
|
||||
.await
|
||||
.expect_err("replay must authorize before comparing guessed part metadata");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
|
||||
assert_eq!(authorizer.calls().len(), 4);
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_authorization_preserves_typed_denials_and_redacts_customer_keys() {
|
||||
let key = multipart_ssec_request(42).sse_customer_key.expect("test key");
|
||||
let mut metadata = multipart_ssec_metadata(42);
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_PLAINTEXT_CHECKSUM, "true".to_string());
|
||||
let no_key = super::multipart_completion_authorization(
|
||||
EncryptionRequest {
|
||||
sse_customer_algorithm: None,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
..multipart_ssec_request(42)
|
||||
},
|
||||
false,
|
||||
);
|
||||
let error = no_key.authorize(&metadata).await.expect_err("customer key required");
|
||||
assert_eq!(map_get_object_reader_error(error).code, S3ErrorCode::InvalidRequest);
|
||||
let (denied, _) = enforcing_principal(false);
|
||||
let denied = super::multipart_completion_authorization(
|
||||
EncryptionRequest {
|
||||
principal: Some(&denied),
|
||||
..multipart_ssec_request(42)
|
||||
},
|
||||
false,
|
||||
);
|
||||
let error = denied.authorize(&sse_kms_object_metadata()).await.expect_err("KMS denied");
|
||||
assert_eq!(map_get_object_reader_error(error).code, S3ErrorCode::AccessDenied);
|
||||
let valid = super::multipart_completion_authorization(multipart_ssec_request(42), false);
|
||||
valid.authorize(&metadata).await.expect("correct customer key");
|
||||
let opts = crate::storage::ObjectOptions {
|
||||
multipart_completion_authorization: Some(valid),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
!format!("{opts:?}").contains(&key),
|
||||
"ObjectOptions Debug must never expose the captured customer key"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_write_authorization_uses_stored_key_and_current_principal() {
|
||||
let (denied, denied_authorizer) = enforcing_principal(false);
|
||||
let (allowed, allowed_authorizer) = enforcing_principal(true);
|
||||
let metadata = sse_kms_object_metadata();
|
||||
let error = super::authorize_sse_kms_multipart_write(Some(&denied), &metadata)
|
||||
.await
|
||||
.expect_err("upload ID must not grant its creator's KMS permission");
|
||||
assert_eq!(error.code, S3ErrorCode::AccessDenied);
|
||||
assert_eq!(
|
||||
denied_authorizer.calls(),
|
||||
vec![(KmsAction::GenerateDataKeyAction, "finance-key".to_string())]
|
||||
);
|
||||
super::authorize_sse_kms_multipart_write(Some(&allowed), &metadata)
|
||||
.await
|
||||
.expect("authorized writer");
|
||||
assert_eq!(
|
||||
allowed_authorizer.calls(),
|
||||
vec![(KmsAction::GenerateDataKeyAction, "finance-key".to_string())]
|
||||
);
|
||||
for metadata in [HashMap::new(), sse_s3_object_metadata(), multipart_ssec_metadata(42)] {
|
||||
super::authorize_sse_kms_multipart_write(Some(&denied), &metadata)
|
||||
.await
|
||||
.expect("non-KMS upload");
|
||||
}
|
||||
assert_eq!(denied_authorizer.calls().len(), 1);
|
||||
let (off, off_authorizer) = permissive_principal_with_enforcement_off();
|
||||
super::authorize_sse_kms_multipart_write(Some(&off), &metadata)
|
||||
.await
|
||||
.expect("enforcement off");
|
||||
super::authorize_sse_kms_multipart_write(None, &metadata)
|
||||
.await
|
||||
.expect("trusted internal writer");
|
||||
assert!(off_authorizer.calls().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn copy_source_read_authorization_targets_the_source_key() {
|
||||
let (denied, denied_authorizer) = enforcing_principal(false);
|
||||
|
||||
@@ -97,10 +97,11 @@ pub(crate) mod contract {
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use super::super::storage_contracts::MultipartCompletionAuthorization;
|
||||
pub(crate) const MAX_MULTIPART_PART_NUMBER: i32 = 10000;
|
||||
pub(crate) use super::super::storage_contracts::{ListMultipartsInfo, ListPartsInfo};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::super::storage_contracts::{MultipartInfo, MultipartOperations, PartInfo};
|
||||
pub(crate) use super::super::storage_contracts::{CompletePart, MultipartInfo, MultipartOperations, PartInfo};
|
||||
pub(crate) use super::super::storage_contracts::{ListMultipartsInfo, ListPartsInfo};
|
||||
}
|
||||
|
||||
pub(crate) mod object {
|
||||
@@ -133,26 +134,27 @@ pub(crate) use super::ecfs_extend::{
|
||||
get_buffer_size_opt_in, get_validated_store, has_read_preconditions, load_bucket_object_lock_config_state,
|
||||
parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, process_lambda_configurations,
|
||||
process_queue_configurations, process_topic_configurations, remove_object_lock_metadata_for_copy, validate_bucket_exists,
|
||||
validate_bucket_object_lock_enabled_state, validate_list_object_unordered_with_delimiter, validate_object_key,
|
||||
wrap_response_with_cors,
|
||||
validate_bucket_object_lock_enabled_state, validate_list_object_unordered_with_delimiter, validate_new_object_key,
|
||||
validate_object_key, wrap_response_with_cors,
|
||||
};
|
||||
pub(crate) use super::sse::{
|
||||
DecryptionRequest, EncryptionRequest, ObjectDekRewrapOutcome, PrepareEncryptionRequest, SseKmsPrincipal,
|
||||
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
|
||||
project_sse_read_response_headers, rewrap_object_encryption_metadata, sse_decryption, sse_encryption, sse_prepare_encryption,
|
||||
strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
||||
authorize_sse_kms_multipart_write, authorize_sse_kms_object_read, classify_sse_read_response,
|
||||
extract_server_side_encryption_from_headers, project_sse_read_response_headers, rewrap_object_encryption_metadata,
|
||||
sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read,
|
||||
validate_sse_headers_for_write, validate_ssec_for_read,
|
||||
};
|
||||
|
||||
pub(crate) mod access_consumer {
|
||||
pub(crate) use super::super::access::{
|
||||
MultipartStagingOperation, PostObjectRequestMarker, ReqInfo, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX,
|
||||
TableDataPlaneListAccess, TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards,
|
||||
apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_internal_object_request,
|
||||
authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
|
||||
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_copy_object_tags,
|
||||
authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action,
|
||||
has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
|
||||
replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards,
|
||||
set_requested_object_lock_retain_until,
|
||||
replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref,
|
||||
retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -217,7 +219,13 @@ pub(crate) mod head_prefix_consumer {
|
||||
}
|
||||
|
||||
pub(crate) mod helper_consumer {
|
||||
pub(crate) use super::super::helper::{OperationHelper, build_event_resp_elements, spawn_background_with_context};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::super::helper::{
|
||||
MutationCompletionProbe, capture_mutation_event_for_test, pause_committed_mutation_for_test,
|
||||
};
|
||||
pub(crate) use super::super::helper::{
|
||||
OperationHelper, build_event_resp_elements, run_owned_mutation, spawn_background_with_context,
|
||||
};
|
||||
|
||||
pub(crate) type StorageObjectInfo = super::StorageObjectInfo;
|
||||
}
|
||||
@@ -233,6 +241,7 @@ pub(crate) mod options_consumer {
|
||||
parse_copy_source_range, preserve_unclassified_user_metadata, put_opts_with_replication_authorization,
|
||||
validate_archive_content_encoding,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rio::validate_compression_scheme;
|
||||
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod object {
|
||||
@@ -358,9 +367,7 @@ pub(crate) mod s3_api_consumer {
|
||||
|
||||
pub(crate) type StorageObjectInfo = super::super::StorageObjectInfo;
|
||||
|
||||
pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag {
|
||||
super::super::to_s3s_etag(etag)
|
||||
}
|
||||
pub(crate) use super::super::to_s3s_xml_etag;
|
||||
}
|
||||
|
||||
pub(crate) mod common {
|
||||
@@ -384,13 +391,13 @@ pub(crate) mod s3_api_consumer {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag {
|
||||
super::super::to_s3s_etag(etag)
|
||||
}
|
||||
pub(crate) use super::super::to_s3s_xml_etag;
|
||||
}
|
||||
|
||||
pub(crate) mod tagging {
|
||||
pub(crate) use super::super::super::s3_api::tagging::{parse_copy_object_tags, resolve_copy_object_tags};
|
||||
pub(crate) use super::super::super::s3_api::tagging::{
|
||||
parse_copy_object_tags, resolve_copy_object_tags, validate_bucket_tag_set,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -398,13 +405,14 @@ pub(crate) mod sse_consumer {
|
||||
pub(crate) use super::super::sse::{
|
||||
EncryptionKeyKind, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, log_sse_kms_key_policy_mode,
|
||||
map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
||||
map_get_object_reader_error, mark_encrypted_multipart_metadata, multipart_completion_authorization, stored_ssec_metadata,
|
||||
};
|
||||
pub(crate) use super::{
|
||||
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention,
|
||||
authorize_sse_kms_object_read, classify_sse_read_response, extract_server_side_encryption_from_headers,
|
||||
get_buffer_size_opt_in, load_bucket_object_lock_config_state, project_sse_read_response_headers, sse_decryption,
|
||||
sse_encryption, sse_prepare_encryption, validate_bucket_object_lock_enabled_state,
|
||||
authorize_sse_kms_multipart_write, authorize_sse_kms_object_read, classify_sse_read_response,
|
||||
extract_server_side_encryption_from_headers, get_buffer_size_opt_in, load_bucket_object_lock_config_state,
|
||||
project_sse_read_response_headers, sse_decryption, sse_encryption, sse_prepare_encryption,
|
||||
validate_bucket_object_lock_enabled_state,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -649,7 +657,8 @@ pub(crate) mod ecstore_set_disk {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{
|
||||
ConditionalReadBenchmarkMetadataGuard, MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier,
|
||||
PutObjectCommitPause, fail_next_quota_ledger_save_for_test,
|
||||
PutObjectCommitPause, fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test,
|
||||
retry_quota_reconciliation_for_test,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::{
|
||||
DEFAULT_READ_BUFFER_SIZE, file_info_quorum_hash, get_lock_acquire_timeout, is_valid_storage_class,
|
||||
@@ -2447,6 +2456,16 @@ pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag {
|
||||
ecstore_client::object_api_utils::to_s3s_etag(etag)
|
||||
}
|
||||
|
||||
/// The pinned s3s XML serializer writes ETag values as raw text. Escape only
|
||||
/// at XML response boundaries; HTTP headers and conditional comparisons keep
|
||||
/// the original stored value. Existing unsafe ETags must remain listable.
|
||||
pub(crate) fn to_s3s_xml_etag(etag: &str) -> s3s::dto::ETag {
|
||||
let mut etag = to_s3s_etag(etag);
|
||||
let (s3s::dto::ETag::Strong(value) | s3s::dto::ETag::Weak(value)) = &mut etag;
|
||||
*value = quick_xml::escape::escape(value.as_str()).into_owned();
|
||||
etag
|
||||
}
|
||||
|
||||
pub(crate) fn table_catalog_path_hash(value: &str) -> String {
|
||||
ecstore_bucket::metadata::table_catalog_path_hash(value)
|
||||
}
|
||||
|
||||
@@ -22111,10 +22111,10 @@ fn object_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
.expect("delete object entrypoints should remain in app/object/delete.rs");
|
||||
|
||||
for expected in [
|
||||
"validate_object_key(&key, request_method_name)?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;",
|
||||
"validate_new_object_key(&key, request_method_name)?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;",
|
||||
"if let Err(err) = validate_table_catalog_object_mutation(&bucket, &obj_id.key).await",
|
||||
"validate_table_catalog_object_mutation(&bucket, &object).await?;",
|
||||
"validate_object_key(&key, \"PUT\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;",
|
||||
"validate_new_object_key(&key, \"PUT\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;",
|
||||
"extract_try!(validate_table_catalog_object_mutation(&bucket, &fpath).await);",
|
||||
] {
|
||||
assert!(source.contains(expected), "missing object mutation guard: {expected}");
|
||||
@@ -22124,7 +22124,7 @@ fn object_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
.map(|(_, body)| body.split_once("\n#[cfg(test)]").map_or(body, |(entrypoint, _)| entrypoint))
|
||||
.expect("copy object entrypoint should remain in app/object/copy.rs");
|
||||
let copy_key = copy_object
|
||||
.find("validate_object_key(&key, \"COPY (dest)\")?;")
|
||||
.find("validate_new_object_key(&key, \"COPY (dest)\")?;")
|
||||
.expect("copy destination key validation");
|
||||
let copy_guard = copy_object
|
||||
.find("validate_table_catalog_object_mutation(&bucket, &key).await?;")
|
||||
|
||||
Reference in New Issue
Block a user