feat(gateway): serve AbortMultipartUpload

Use the same authorization and audited use case on both S3 stacks. Bind
the gateway claim to the owner selected under the upload write lock
before staged parts can be removed. Preserve the existing generation,
lifecycle and quorum checks.

Verified the fixed legacy baseline, the integrated native wire flow, 212
focused tests and both Clippy modes. Fault checks exercised owner
selection, deletion and orphan handling, scoped IAM, generation
transport, audit, operation counts and dirty usage.

Audit delivery and separate fault injection of the existing final
lock-loss and metadata-incarnation fences were outside this verification.
This commit is contained in:
overtrue
2026-11-15 18:03:01 +08:00
parent 620fdc286a
commit 6f93ac5f47
19 changed files with 1030 additions and 86 deletions
+4
View File
@@ -1388,6 +1388,10 @@ pub struct ObjectOptions {
#[doc(hidden)]
pub multipart_completion_authorization:
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
/// In-process authorization against the selected upload, while Abort holds its write lock.
#[doc(hidden)]
pub multipart_abort_authorization:
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartAbortAuthorization<Error = Error>>>,
pub eval_metadata: Option<HashMap<String, String>>,
/// Internal compare-and-set condition for replication workers publishing
/// terminal status after remote I/O. Storage validates it while holding
@@ -684,6 +684,22 @@ fn fence_commit_on_lock_loss(guard: Option<&ObjectLockDiagGuard>, mode: &'static
Ok(())
}
fn recorded_multipart_owner(
metadata: &HashMap<String, String>,
bucket: &str,
encoded_object: &str,
) -> Option<crate::storage_api_contracts::multipart::MultipartOwner> {
let recorded_bucket = get_consistent_str(metadata, "multipart-bucket")?;
let recorded_object = get_consistent_str(metadata, "multipart-object")?;
if recorded_bucket != bucket || recorded_object != encoded_object {
return None;
}
Some(crate::storage_api_contracts::multipart::MultipartOwner {
bucket: recorded_bucket.to_owned(),
object: decode_dir_object(recorded_object),
})
}
fn multipart_bucket_incarnation_id(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) else {
if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) {
@@ -3612,6 +3628,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
.await?;
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
if let Some(authorization) = &opts.multipart_abort_authorization {
let owner = recorded_multipart_owner(&fi.metadata, bucket, object);
authorization.authorize(owner.as_ref())?;
}
let disks = self.get_disks_internal().await;
let cleanup = prepare_source_cleanup(
&disks,
@@ -1029,3 +1029,229 @@ mod native_abort_closure_tests {
include!("multipart_initial_probe_tests.rs");
}
}
mod abort_owner_tests {
use super::*;
use crate::storage_api_contracts::multipart::{MultipartAbortAuthorization, MultipartOwner};
#[test]
fn recorded_multipart_owner_requires_unambiguous_metadata_for_the_selected_namespace() {
const BUCKET: &str = "abort-owner";
const ENCODED: &str = "directory__XLDIR__";
const RUSTFS: &str = RUSTFS_MULTIPART_BUCKET_KEY;
const MINIO: &str = "x-minio-internal-multipart-bucket";
const RUSTFS_OBJECT: &str = RUSTFS_MULTIPART_OBJECT_KEY;
const MINIO_OBJECT: &str = "x-minio-internal-multipart-object";
let expected = MultipartOwner {
bucket: BUCKET.to_owned(),
object: "directory/".to_owned(),
};
let rustfs = [(RUSTFS, BUCKET), (RUSTFS_OBJECT, ENCODED)];
let minio = [(MINIO, BUCKET), (MINIO_OBJECT, ENCODED)];
for fields in [rustfs.to_vec(), minio.to_vec(), [rustfs, minio].concat()] {
let metadata = fields
.into_iter()
.map(|(key, value)| (key.to_owned(), value.to_owned()))
.collect();
assert_eq!(recorded_multipart_owner(&metadata, BUCKET, ENCODED), Some(expected.clone()));
}
for fields in [
vec![(RUSTFS, BUCKET)],
vec![(RUSTFS_OBJECT, ENCODED)],
vec![(RUSTFS, ""), (RUSTFS_OBJECT, ENCODED)],
vec![(RUSTFS, BUCKET), (RUSTFS_OBJECT, "")],
vec![(RUSTFS, "foreign"), (RUSTFS_OBJECT, ENCODED)],
vec![(RUSTFS, BUCKET), (RUSTFS_OBJECT, "directory/")],
[rustfs, [(MINIO, "foreign"), (MINIO_OBJECT, ENCODED)]].concat(),
[rustfs, [(MINIO, BUCKET), ("X-MINIO-INTERNAL-MULTIPART-OBJECT", "other")]].concat(),
] {
let metadata = fields
.iter()
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect();
assert_eq!(recorded_multipart_owner(&metadata, BUCKET, ENCODED), None, "{fields:?}");
}
}
struct AbortGate {
expected: Option<MultipartOwner>,
accepted: bool,
calls: AtomicUsize,
}
impl MultipartAbortAuthorization for AbortGate {
type Error = StorageError;
fn authorize(&self, owner: Option<&MultipartOwner>) -> Result<()> {
self.calls.fetch_add(1, Ordering::Relaxed);
assert_eq!(owner, self.expected.as_ref(), "gate sees the actual selected upload owner");
if self.accepted {
Ok(())
} else {
Err(StorageError::InvalidUploadID(String::new(), String::new(), String::new()))
}
}
}
#[tokio::test]
async fn abort_owner_callback_controls_physical_deletion() {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "abort-locked-owner";
let key = "staged-object";
make_bucket_on_all(&disks, bucket).await;
let (id, _) = stage_upload_with_create_opts(&set, bucket, key, b"staged part", &ObjectOptions::default()).await;
let path = SetDisks::get_upload_id_dir(bucket, key, &id);
let (fi, _) = set
.check_upload_id_exists(bucket, key, &id, true)
.await
.expect("staged upload");
let part = format!("{path}/{}/part.1", fi.data_dir.expect("staged data directory"));
let expected = MultipartOwner {
bucket: bucket.to_owned(),
object: key.to_owned(),
};
let mut before = Vec::new();
for disk in &disks {
before.push(
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &part)
.await
.expect("staged shard"),
);
}
let rejected = Arc::new(AbortGate {
expected: Some(expected.clone()),
accepted: false,
calls: AtomicUsize::new(0),
});
let opts = ObjectOptions {
multipart_abort_authorization: Some(rejected.clone()),
..Default::default()
};
let result = set.abort_multipart_upload(bucket, key, &id, &opts).await;
for (disk, staged) in disks.iter().zip(before) {
let after = disk
.read_all(RUSTFS_META_MULTIPART_BUCKET, &part)
.await
.expect("denied Abort preserves the shard");
assert_eq!(staged, after);
}
assert!(matches!(result, Err(StorageError::InvalidUploadID(..))));
assert_eq!(rejected.calls.load(Ordering::Relaxed), 1);
let allowed = Arc::new(AbortGate {
expected: Some(expected),
accepted: true,
calls: AtomicUsize::new(0),
});
let opts = ObjectOptions {
multipart_abort_authorization: Some(allowed.clone()),
..Default::default()
};
set.abort_multipart_upload(bucket, key, &id, &opts)
.await
.expect("owner permits Abort");
assert_eq!(allowed.calls.load(Ordering::Relaxed), 1);
for disk in &disks {
assert!(matches!(
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &part).await,
Err(DiskError::FileNotFound)
));
}
}
#[tokio::test]
async fn abort_owner_comes_from_the_selected_metadata_on_all_disks() {
use rustfs_utils::http::{get_str, remove_str};
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "abort-owner-metadata";
make_bucket_on_all(&disks, bucket).await;
for (key, bucket_fields, object_fields, accepted) in [
("rustfs", [Some(bucket), None], [Some("rustfs"), None], true),
("minio", [None, Some(bucket)], [None, Some("minio")], true),
("matching", [Some(bucket), Some(bucket)], [Some("matching"), Some("matching")], true),
("empty", [Some(bucket), Some(bucket)], [Some("empty"), Some("")], false),
("missing", [None, None], [Some("missing"), None], false),
("foreign-bucket", [Some("foreign"), None], [Some("foreign-bucket"), None], false),
("foreign-key", [Some(bucket), None], [Some("unrelated"), None], false),
("conflict", [Some(bucket), Some("foreign")], [Some("conflict"), Some("conflict")], false),
] {
let (id, _) = stage_upload_with_create_opts(&set, bucket, key, b"staged shard data", &ObjectOptions::default()).await;
let path = SetDisks::get_upload_id_dir(bucket, key, &id);
pin_historical_unmarked_upload(&disks, bucket, &path).await;
let mut part_path = None;
let mut staged = Vec::new();
let fields = [
(RUSTFS_MULTIPART_BUCKET_KEY, bucket_fields[0]),
("x-minio-internal-multipart-bucket", bucket_fields[1]),
(RUSTFS_MULTIPART_OBJECT_KEY, object_fields[0]),
("x-minio-internal-multipart-object", object_fields[1]),
];
for disk in &disks {
let mut head = disk
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &path, "", &ReadOptions::default())
.await
.expect("read the staged metadata on each disk");
let part = part_path.get_or_insert_with(|| format!("{path}/{}/part.1", head.data_dir.expect("part directory")));
staged.push(
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, part)
.await
.expect("read staged part before Abort"),
);
assert_eq!(get_str(&head.metadata, "multipart-bucket").as_deref(), Some(bucket));
remove_str(&mut head.metadata, "multipart-bucket");
remove_str(&mut head.metadata, "multipart-object");
for (name, value) in fields {
if let Some(value) = value {
head.metadata.insert(name.to_owned(), value.to_owned());
}
}
disk.write_metadata(bucket, RUSTFS_META_MULTIPART_BUCKET, &path, head)
.await
.expect("write the test owner on each disk");
let persisted = disk
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &path, "", &ReadOptions::default())
.await
.expect("verify the actual persisted metadata");
for (name, value) in fields {
assert_eq!(persisted.metadata.get(name).map(String::as_str), value);
}
}
let gate = Arc::new(AbortGate {
expected: accepted.then(|| MultipartOwner {
bucket: bucket.to_owned(),
object: key.to_owned(),
}),
accepted,
calls: AtomicUsize::new(0),
});
let opts = ObjectOptions {
multipart_abort_authorization: Some(gate.clone()),
..Default::default()
};
let result = set.abort_multipart_upload(bucket, key, &id, &opts).await;
for (disk, previous) in disks.iter().zip(staged) {
let after = disk
.read_all(RUSTFS_META_MULTIPART_BUCKET, part_path.as_deref().expect("part path"))
.await;
if accepted {
assert!(matches!(after, Err(DiskError::FileNotFound)), "{key}: authorized data removed");
} else {
assert_eq!(after.expect("refused Abort preserves the staged part"), previous, "{key}");
}
}
let calls = gate.calls.load(Ordering::Relaxed);
if accepted {
assert_eq!(calls, 1, "{key}");
result.expect("Abort with agreeing persisted owner");
} else {
assert!(result.is_err(), "{key}: invalid metadata must not authorize Abort");
if key == "empty" {
// The existing source inspection refuses this head before selecting an owner.
assert_eq!(calls, 0, "malformed physical owner was refused before the callback");
}
}
}
}
}
@@ -54,8 +54,8 @@ pub(crate) mod list {
pub(crate) mod multipart {
pub(crate) use rustfs_storage_api::multipart::ListPartsOwner;
pub(crate) use rustfs_storage_api::{
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartCompletionAuthorization, MultipartInfo, MultipartOperations,
MultipartUploadInspection, MultipartUploadResult, PartInfo,
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartAbortAuthorization, MultipartCompletionAuthorization,
MultipartInfo, MultipartOperations, MultipartOwner, MultipartUploadInspection, MultipartUploadResult, PartInfo,
};
}
+22
View File
@@ -6825,6 +6825,28 @@ mod tests {
tokio::fs::remove_file(conflicting_metadata)
.await
.expect("remove conflicting fixture");
struct RequiresOwner;
impl crate::storage_api_contracts::multipart::MultipartAbortAuthorization for RequiresOwner {
type Error = StorageError;
fn authorize(&self, _: Option<&crate::storage_api_contracts::multipart::MultipartOwner>) -> Result<()> {
panic!("metadata-absent recovery cannot observe an upload owner");
}
}
let native_opts = ObjectOptions {
multipart_abort_authorization: Some(Arc::new(RequiresOwner)),
..Default::default()
};
let rejected = store
.abort_multipart_upload(&bucket, object, &upload.upload_id, &native_opts)
.await;
for disk_index in 0..3 {
assert!(
staging_dir(0, disk_index).join("part.1").exists(),
"owner-required Abort must preserve late data"
);
}
assert!(matches!(rejected, Err(StorageError::InvalidUploadID(..))));
let (paused, release) = crate::store::multipart::pause_orphan_multipart_cleanup_for_test(&bucket);
let recovery_store = store.clone();
let recovery_bucket = bucket.clone();
+2 -1
View File
@@ -966,7 +966,8 @@ impl ECStore {
bucket_lifecycle_guard: Option<super::BucketIncarnationFenceGuard>,
) -> Result<()> {
let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
if self.ctx.lock_manager().is_disabled() {
// Without upload metadata this path has no recorded owner to authorize.
if opts.multipart_abort_authorization.is_some() || self.ctx.lock_manager().is_disabled() {
return Err(invalid_upload());
}
let Some(bucket_lifecycle_guard) = bucket_lifecycle_guard else {
+1 -1
View File
@@ -109,7 +109,7 @@ pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOption
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
pub use error::{StorageErrorCode, StorageResult};
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
pub use multipart::{MultipartCompletionAuthorization, MultipartUploadInspection};
pub use multipart::{MultipartAbortAuthorization, MultipartCompletionAuthorization, MultipartOwner, MultipartUploadInspection};
pub use object::AuthoritativeListOperations;
pub use object::DeleteAccounting;
pub use object::ObjectLockDeleteOptions;
+18
View File
@@ -15,6 +15,24 @@
use std::collections::HashMap;
use time::OffsetDateTime;
/// Ownership observed in the selected upload metadata.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MultipartOwner {
/// The bucket recorded by the upload.
pub bucket: String,
/// The recorded object, decoded after checking the encoded storage namespace.
pub object: String,
}
/// Authorizes Abort against the upload selected under its existing write lock.
/// An implementation must reject missing ownership and must not acquire storage locks or mutate storage.
pub trait MultipartAbortAuthorization: Send + Sync {
type Error: std::error::Error + Send + Sync + 'static;
/// Returns a refusal before storage prepares or deletes any staged data.
fn authorize(&self, owner: Option<&MultipartOwner>) -> Result<(), Self::Error>;
}
/// Request authorization against the source selected under completion locks.
/// Implementations must not mutate storage or acquire object/upload locks again.
/// Called before comparing client checksums/parts or acknowledging a replay.
+1 -1
View File
@@ -102,7 +102,6 @@ use bytes::Bytes;
use futures::StreamExt;
use http::{HeaderMap, HeaderValue, Uri};
use metrics::counter;
use rustfs_io_metrics::record_s3_op;
use rustfs_s3_ops::S3Operation;
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
@@ -126,6 +125,7 @@ use urlencoding::encode;
use uuid::Uuid;
mod abort;
pub(crate) use abort::{AbortMultipartUploadTarget, reject_abort_capabilities};
mod complete;
mod create;
mod create_values;
+166 -63
View File
@@ -15,76 +15,179 @@
//! AbortMultipartUpload.
use super::*;
use crate::app::storage_api::s3::dto::{AbortMultipartUploadInput, AbortMultipartUploadOutput};
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
use crate::app::RequestEnvelope;
#[cfg(any(feature = "gateway-stack", test))]
use crate::app::storage_api::multipart_usecase::contract::multipart::{MultipartAbortAuthorization, MultipartOwner};
use http::StatusCode;
#[cfg(any(feature = "gateway-stack", test))]
use rustfs_gateway_types::{BucketName, ObjectKey, RecordedUpload, UploadIdClaim, resolve_upload, resolve_upload_async};
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
#[cfg(any(feature = "gateway-stack", test))]
use std::sync::OnceLock;
impl DefaultMultipartUsecase {
#[instrument(level = "debug", skip(self, req))]
pub async fn execute_abort_multipart_upload(
&self,
req: S3Request<AbortMultipartUploadInput>,
) -> S3Result<S3Response<AbortMultipartUploadOutput>> {
reject_presigned_multipart_max_total_object_size_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedSigV4Request>().is_some(),
)?;
reject_presigned_put_max_content_length_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
)?;
record_s3_op(S3Operation::AbortMultipartUpload);
let mut opts = ObjectOptions::default();
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
let AbortMultipartUploadInput {
bucket, key, upload_id, ..
} = req.input;
pub(crate) enum AbortMultipartUploadTarget {
Legacy {
bucket: String,
key: String,
upload_id: String,
},
#[cfg(any(feature = "gateway-stack", test))]
Gateway {
bucket: BucketName,
key: ObjectKey,
upload_id: UploadIdClaim,
},
}
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// Per the AWS S3 API reference, a malformed upload ID is reported as
// NoSuchUpload (404), the same as an upload that was aborted or
// completed. See multipart_upload_api_error.
match store
.abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), &opts)
.await
{
Ok(_) => {
rustfs_scanner::record_dirty_usage_object_from_producer(
&bucket,
&key,
rustfs_scanner::SegmentInvalidationProducerIdentity::AbortMultipartUpload,
);
Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() }))
}
Err(err) => Err(multipart_upload_api_error(err)),
impl AbortMultipartUploadTarget {
fn resource(&self) -> (&str, &str) {
match self {
Self::Legacy { bucket, key, .. } => (bucket, key),
#[cfg(any(feature = "gateway-stack", test))]
Self::Gateway { bucket, key, .. } => (bucket.as_str(), key.as_str()),
}
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
use super::*;
pub(crate) fn reject_abort_capabilities(envelope: &RequestEnvelope) -> S3Result<()> {
reject_presigned_multipart_max_total_object_size_for_other_operation(
envelope.headers(),
envelope.uri().query(),
envelope.verified_sigv4().is_some(),
)?;
reject_presigned_put_max_content_length_for_other_operation(
envelope.headers(),
envelope.uri().query(),
envelope.verified_presigned().is_some(),
)?;
Ok(())
}
#[tokio::test]
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
let input = AbortMultipartUploadInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.build()
.unwrap();
let req = build_request(input, Method::DELETE);
let before = s3_op_total(S3Operation::AbortMultipartUpload);
impl DefaultMultipartUsecase {
#[instrument(level = "debug", skip(self, envelope, target))]
pub(crate) async fn execute_abort_multipart_upload(
&self,
envelope: RequestEnvelope,
target: S3Result<AbortMultipartUploadTarget>,
) -> S3Result<()> {
RequestEnvelope::audit_only_operation(
envelope,
S3Operation::AbortMultipartUpload,
StatusCode::OK,
|envelope| async move {
let target = target?;
let (bucket, key) = target.resource();
let mut opts = ObjectOptions::default();
apply_bucket_generation_guard(&envelope, bucket, &mut opts)?;
#[cfg(any(feature = "gateway-stack", test))]
if matches!(&target, AbortMultipartUploadTarget::Gateway { .. }) && opts.expected_bucket_incarnation_id.is_none()
{
return Err(S3Error::with_message(S3ErrorCode::InternalError, "bucket generation guard is missing"));
}
let err = make_usecase().execute_abort_multipart_upload(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::AbortMultipartUpload), before + 1);
reject_veeam_sosapi_mutation(|| self.object_store(), bucket, key).await?;
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
};
match &target {
AbortMultipartUploadTarget::Legacy { upload_id, .. } => store
.abort_multipart_upload(bucket, key, upload_id, &opts)
.await
.map_err(multipart_upload_api_error)?,
#[cfg(any(feature = "gateway-stack", test))]
AbortMultipartUploadTarget::Gateway { bucket, key, upload_id } => {
abort_gateway(&store, bucket, key, upload_id, opts).await?
}
}
rustfs_scanner::record_dirty_usage_object_from_producer(
bucket,
key,
rustfs_scanner::SegmentInvalidationProducerIdentity::AbortMultipartUpload,
);
Ok(())
},
)
.await
}
}
#[cfg(any(feature = "gateway-stack", test))]
struct AbortOwner(MultipartOwner);
#[cfg(any(feature = "gateway-stack", test))]
impl RecordedUpload for AbortOwner {
fn bucket(&self) -> &str {
&self.0.bucket
}
fn key(&self) -> &str {
&self.0.object
}
}
#[cfg(any(feature = "gateway-stack", test))]
struct AbortAuthorization {
bucket: BucketName,
key: ObjectKey,
upload_id: UploadIdClaim,
observed: OnceLock<MultipartOwner>,
}
#[cfg(any(feature = "gateway-stack", test))]
impl MultipartAbortAuthorization for AbortAuthorization {
type Error = StorageError;
fn authorize(&self, owner: Option<&MultipartOwner>) -> Result<(), Self::Error> {
let (_, owner) =
resolve_upload(&self.upload_id, &self.bucket, &self.key, |_| owner.cloned().map(AbortOwner)).map_err(|_| {
StorageError::InvalidUploadID(self.bucket.as_str().to_owned(), self.key.as_str().to_owned(), String::new())
})?;
self.observed.get_or_init(|| owner.0);
Ok(())
}
}
#[cfg(any(feature = "gateway-stack", test))]
async fn abort_gateway(
store: &ECStore,
bucket: &BucketName,
key: &ObjectKey,
upload_id: &UploadIdClaim,
mut opts: ObjectOptions,
) -> S3Result<()> {
let authorization = Arc::new(AbortAuthorization {
bucket: bucket.clone(),
key: key.clone(),
upload_id: upload_id.clone(),
observed: OnceLock::new(),
});
opts.multipart_abort_authorization = Some(authorization.clone());
// The outer exchange checks the claim's shape. Only the inner exchange, on the
// selected metadata while holding the upload lock, authorizes deletion.
resolve_upload_async(upload_id, bucket, key, |candidate| async move {
store
.abort_multipart_upload(bucket.as_str(), key.as_str(), candidate, &opts)
.await
.map_err(multipart_upload_api_error)?;
let owner = authorization.observed.get().cloned().ok_or_else(|| {
S3Error::with_message(S3ErrorCode::InternalError, "Abort authorization did not observe the upload owner")
})?;
Ok::<_, S3Error>(Some(AbortOwner(owner)))
})
.await?
.map_err(|_| {
// An empty id has the legacy missing-upload response. Other impossible shapes
// have the malformed-upload response. Neither reaches the storage lookup.
let error = if upload_id.is_explicitly_empty() {
StorageError::InvalidUploadID(bucket.as_str().to_owned(), key.as_str().to_owned(), String::new())
} else {
StorageError::MalformedUploadID(String::new())
};
multipart_upload_api_error(error)
})?;
Ok(())
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,205 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::super::test_support::*;
use super::*;
use crate::app::storage_api::multipart_usecase::access::{AuthorizationRequest, load_bucket_generation_from_store};
use crate::app::storage_api::multipart_usecase::helper::MutationCompletionProbe;
use crate::app::storage_api::multipart_usecase::request_context::RequestContext;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
use crate::runtime_sources::ServerContextSlot;
use rustfs_scanner::{SegmentInvalidationProducerIdentity, set_scanner_dirty_usage_mutation_observer};
use std::sync::atomic::{AtomicUsize, Ordering};
#[tokio::test]
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
let request = build_request((), Method::DELETE);
let envelope = RequestEnvelope::from_s3s(&request);
let before = s3_op_total(S3Operation::AbortMultipartUpload);
let target = AbortMultipartUploadTarget::Legacy {
bucket: "bucket".to_owned(),
key: "object".to_owned(),
upload_id: "upload-id".to_owned(),
};
let err = make_usecase()
.execute_abort_multipart_upload(envelope, Ok(target))
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(err.message(), Some("Not init"));
assert_eq!(s3_op_total(S3Operation::AbortMultipartUpload), before);
}
#[test]
#[serial_test::serial]
fn native_abort_requires_current_generation_and_audits_each_result() {
crate::app::gating_test_env::run_large_stack_test("abort-multipart-generation", || async {
let context = crate::app::gating_test_env::shared_gating_ambient().await;
let store = context.object_store();
let usecase = DefaultMultipartUsecase::with_context(Some(context.clone()));
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let bucket = format!("abort-multipart-{}", Uuid::new_v4().simple());
let other = format!("abort-multipart-{}", Uuid::new_v4().simple());
let key = "object";
let opts = ObjectOptions::default();
let request_context = RequestContext::fallback();
let request = || {
let mut req = build_request((), Method::DELETE);
req.uri = format!("/{bucket}/{key}").parse().expect("fixture URI");
req.extensions.insert(slot.clone());
req.extensions.insert(request_context.clone());
req
};
for name in [&bucket, &other] {
store
.make_bucket(name, &MakeBucketOptions::default())
.await
.expect("create bucket");
}
let original = store
.bucket_incarnation_id(&bucket)
.await
.expect("previous bucket incarnation");
let mut stale = request();
let captured = load_bucket_generation_from_store(&store, &stale, &bucket)
.await
.expect("capture prior bucket");
stale.install_bucket_generation(captured);
store
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("delete original bucket");
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("recreate bucket");
assert_ne!(
store
.bucket_incarnation_id(&bucket)
.await
.expect("current bucket incarnation"),
original
);
let upload = store
.new_multipart_upload(&bucket, key, &opts)
.await
.expect("create current upload");
let mut reader = PutObjReader::from_vec(b"current part".to_vec());
store
.put_object_part(&bucket, key, &upload.upload_id, 1, &mut reader, &opts)
.await
.expect("stage current part");
let target = || AbortMultipartUploadTarget::Gateway {
bucket: BucketName::new(bucket.clone()).expect("bucket name"),
key: ObjectKey::new(key).expect("object key"),
upload_id: UploadIdClaim::from_wire(upload.upload_id.as_str()),
};
let dirty = Arc::new(AtomicUsize::new(0));
let observer = {
let dirty = dirty.clone();
let bucket = bucket.clone();
Arc::new(move |observed_bucket: &str, observed_key: &str, producer| {
if observed_bucket == bucket
&& observed_key == key
&& producer == SegmentInvalidationProducerIdentity::AbortMultipartUpload
{
dirty.fetch_add(1, Ordering::Relaxed);
}
}) as rustfs_scanner::ScannerDirtyUsageMutationObserver
};
struct RestoreObserver(Option<rustfs_scanner::ScannerDirtyUsageMutationObserver>);
impl Drop for RestoreObserver {
fn drop(&mut self) {
set_scanner_dirty_usage_mutation_observer(self.0.take());
}
}
let _observer = RestoreObserver(set_scanner_dirty_usage_mutation_observer(Some(observer)));
let probe = MutationCompletionProbe::install(&bucket);
let missing = request();
let mut wrong = request();
let captured = load_bucket_generation_from_store(&store, &wrong, &other)
.await
.expect("capture unrelated bucket");
wrong.install_bucket_generation(captured);
for (req, code) in [
(stale, S3ErrorCode::NoSuchBucket),
(missing, S3ErrorCode::InternalError),
(wrong, S3ErrorCode::InternalError),
] {
let err = usecase
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&req), Ok(target()))
.await
.expect_err("Abort requires current generation");
assert_eq!(err.code(), &code);
let parts = store
.list_object_parts(&bucket, key, &upload.upload_id, None, 10, &opts)
.await
.expect("current upload persists");
assert_eq!(parts.parts.len(), 1, "current staged part persists");
}
assert_eq!(dirty.load(Ordering::Relaxed), 0);
assert_eq!(probe.audits.lock().unwrap().len(), 3);
let mut current = request();
let captured = load_bucket_generation_from_store(&store, &current, &bucket)
.await
.expect("capture current bucket");
current.install_bucket_generation(captured);
usecase
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&current), Ok(target()))
.await
.expect("current generation allows Abort");
assert!(matches!(
store.get_multipart_info(&bucket, key, &upload.upload_id, &opts).await,
Err(StorageError::InvalidUploadID(..))
));
assert_eq!(dirty.load(Ordering::Relaxed), 1);
{
let audits = probe.audits.lock().unwrap();
assert_eq!(audits.len(), 4);
assert_eq!(audits.last().expect("successful audit").api.status_code, Some(200));
}
let legacy = store
.new_multipart_upload(&bucket, key, &opts)
.await
.expect("create legacy-path upload");
usecase
.execute_abort_multipart_upload(
RequestEnvelope::from_s3s(&current),
Ok(AbortMultipartUploadTarget::Legacy {
bucket: bucket.clone(),
key: key.to_owned(),
upload_id: legacy.upload_id,
}),
)
.await
.expect("legacy target with the same authorized generation");
assert_eq!(dirty.load(Ordering::Relaxed), 2);
let audits = probe.audits.lock().unwrap();
assert_eq!(audits.len(), 5, "one audit per owned Abort");
assert_eq!(audits.last().expect("legacy success audit").api.status_code, Some(200));
for audit in audits.iter() {
assert_eq!(audit.request_id.as_deref(), Some(request_context.request_id.as_str()));
assert_eq!(audit.api.name.as_deref(), Some(S3Operation::AbortMultipartUpload.as_str()));
}
assert!(probe.events.lock().unwrap().is_empty());
});
}
+2
View File
@@ -1859,6 +1859,8 @@ pub(crate) mod multipart_usecase {
pub(crate) use super::super::super::storage_contracts::{
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult,
};
#[cfg(any(feature = "gateway-stack", test))]
pub(crate) use super::super::super::storage_contracts::{MultipartAbortAuthorization, MultipartOwner};
pub(crate) use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::MAX_MULTIPART_PART_NUMBER;
}
@@ -24,13 +24,14 @@ use crate::app::RequestEnvelope;
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::dto_values;
use crate::app::multipart_usecase::{
CreateMultipartUploadFields, DefaultMultipartUsecase, ListPartsFields, ListPartsInfo, ListPartsTarget,
AbortMultipartUploadTarget, CreateMultipartUploadFields, DefaultMultipartUsecase, ListPartsFields, ListPartsInfo,
ListPartsTarget, reject_abort_capabilities,
};
use crate::app::storage_api::storage_class::effective_class;
use crate::runtime_sources::{AppContext, ServerContextSlot};
use crate::storage_api::server::http::access::access_owned::listing_multipart::{
CreateMultipartAuthorization, authorize_bucket_listing, authorize_create_multipart_upload, authorize_list_multipart_uploads,
authorize_list_objects_v2, authorize_list_parts,
CreateMultipartAuthorization, authorize_abort_multipart_upload, authorize_bucket_listing, authorize_create_multipart_upload,
authorize_list_multipart_uploads, authorize_list_objects_v2, authorize_list_parts,
};
use rustfs_gateway::{Req, Resp, dto};
use rustfs_policy::policy::action::S3Action;
@@ -45,6 +46,7 @@ pub(super) const ROWS: &[OpRow] = &[
row::<dto::ListMultipartUploads>(),
row::<dto::ListParts>(),
row::<dto::CreateMultipartUpload>(),
row::<dto::AbortMultipartUpload>(),
];
impl Served for dto::ListObjectsV2 {
@@ -245,6 +247,29 @@ impl Served for dto::CreateMultipartUpload {
}
}
impl Served for dto::AbortMultipartUpload {
const METRIC: S3Operation = S3Operation::AbortMultipartUpload;
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
let server = server_context(envelope)?;
let input = request.input();
authorize_abort_multipart_upload(&server, envelope, input.bucket.as_str(), input.key.as_str()).await
}
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
let input = request.into_input();
let target = reject_abort_capabilities(&envelope).map(|()| AbortMultipartUploadTarget::Gateway {
bucket: input.bucket,
key: input.key,
upload_id: input.upload_id,
});
DefaultMultipartUsecase::with_context(context)
.execute_abort_multipart_upload(envelope, target)
.await?;
Ok(Resp::new(dto::AbortMultipartUploadOutput::default()))
}
}
/// The server context a generation guard is read through; the gateway envelope always has one.
fn server_context(envelope: &RequestEnvelope) -> S3Result<Arc<ServerContextSlot>> {
envelope
+4 -8
View File
@@ -3346,14 +3346,10 @@ impl S3Access for FS {
/// This method returns `Ok(())` by default.
async fn abort_multipart_upload(&self, req: &mut S3Request<AbortMultipartUploadInput>) -> S3Result<()> {
let bucket = req.input.bucket.clone();
let bucket_generation = load_bucket_generation(self.server_ctx(), req, &bucket).await;
let req_info = ext_req_info_mut(&mut req.extensions)?;
req_info.bucket = Some(req.input.bucket.clone());
req_info.object = Some(req.input.key.clone());
authorize_request(req, Action::S3Action(S3Action::AbortMultipartUploadAction)).await?;
req.install_bucket_generation(bucket_generation?);
Ok(())
let key = req.input.key.clone();
access_owned::listing_multipart::authorize_abort_multipart_upload(self.server_ctx(), req, &bucket, &key)
.await
.map_err(Into::into)
}
/// Checks whether the CompleteMultipartUpload request has accesses to the resources.
@@ -91,6 +91,23 @@ pub(crate) async fn authorize_list_parts<R: AuthorizationRequest + ?Sized>(
Ok(())
}
/// Authorizes Abort on the object and installs the bucket generation captured before the decision.
pub(crate) async fn authorize_abort_multipart_upload<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
req: &mut R,
bucket: &str,
key: &str,
) -> S3Result<()> {
let bucket_generation = load_bucket_generation(server_ctx, req, bucket).await;
let req_info = req_info_mut(req)?;
req_info.bucket = Some(bucket.to_owned());
req_info.object = Some(key.to_owned());
authorize_request(req, Action::S3Action(S3Action::AbortMultipartUploadAction)).await?;
req.install_bucket_generation(bucket_generation?);
Ok(())
}
/// Decoded request facts used by both multipart initiation access hooks.
pub(crate) struct CreateMultipartAuthorization<'a> {
pub(crate) bucket: &'a str,
@@ -23,11 +23,10 @@
//! operations onto the gateway owns this file and is the only one that edits it.
use crate::storage::ecfs::*;
use crate::storage::helper::audit_only_operation;
use crate::storage::s3_api::bucket::encode_list_output_value;
use crate::storage::s3_api::{
self, CreateMultipartUploadFields, DefaultMultipartUsecase, ListPartsFields, ListPartsTarget,
mark_on_demand_migration_list_local_only, multipart::build_list_parts_output,
self, AbortMultipartUploadTarget, CreateMultipartUploadFields, DefaultMultipartUsecase, ListPartsFields, ListPartsTarget,
mark_on_demand_migration_list_local_only, multipart::build_list_parts_output, reject_abort_capabilities,
};
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::{
list_multipart_uploads, list_object_versions, list_objects, list_objects_v2,
@@ -87,8 +86,18 @@ impl FS {
&self,
req: S3Request<AbortMultipartUploadInput>,
) -> S3Result<S3Response<AbortMultipartUploadOutput>> {
let usecase = s3_api::multipart_usecase_for(self);
audit_only_operation(req, S3Operation::AbortMultipartUpload, |req| usecase.execute_abort_multipart_upload(req)).await
let envelope = RequestEnvelope::from_s3s(&req);
let target = reject_abort_capabilities(&envelope).map(|()| {
record_s3_op(S3Operation::AbortMultipartUpload);
let AbortMultipartUploadInput {
bucket, key, upload_id, ..
} = req.input;
AbortMultipartUploadTarget::Legacy { bucket, key, upload_id }
});
s3_api::multipart_usecase_for(self)
.execute_abort_multipart_upload(envelope, target)
.await?;
Ok(S3Response::new(AbortMultipartUploadOutput::default()))
}
#[instrument(name = "create_multipart_upload", level = "debug", skip(self, req))]
+2 -2
View File
@@ -20,8 +20,8 @@
use crate::app::bucket_usecase::DefaultBucketUsecase;
pub(crate) use crate::app::bucket_usecase::{PreparedLifecycleConfiguration, validate_lifecycle_rule_status};
pub(crate) use crate::app::multipart_usecase::{
CreateMultipartUploadFields, DefaultMultipartUsecase, ListPartsFields, ListPartsTarget, UploadPartTarget,
UploadPartWriteInput, UploadPartWriteOutput,
AbortMultipartUploadTarget, CreateMultipartUploadFields, DefaultMultipartUsecase, ListPartsFields, ListPartsTarget,
UploadPartTarget, UploadPartWriteInput, UploadPartWriteOutput, reject_abort_capabilities,
};
pub(crate) use crate::app::object::reject_veeam_sosapi_mutation;
pub(crate) use crate::app::object::shared::mark_on_demand_migration_list_local_only;
@@ -25,10 +25,17 @@ use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
use rustfs_s3_ops::S3Operation;
use rustfs_storage_api::{MultipartOperations as _, ObjectIO as _};
mod abort;
mod create;
mod parts;
pub(crate) const MATRIX: &[MatrixRow] = &[
MatrixRow {
operation: "AbortMultipartUpload",
metric: S3Operation::AbortMultipartUpload,
has_error_body: true,
call: abort::matrix_abort,
},
MatrixRow {
operation: "CreateMultipartUpload",
metric: S3Operation::CreateMultipartUpload,
@@ -0,0 +1,289 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! AbortMultipartUpload wire responses and staged part removal.
use super::super::{
ENV_S3_STACK, GATEWAY, LEGACY, Outcome, ROOT_ACCESS_KEY, ROOT_SECRET_KEY, USER_SECRET_KEY, common,
create_bucket_in_the_store, create_user_allowed_on_one_bucket, outcome, refusal, s3_client, with_stack,
};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use futures::future::BoxFuture;
use rustfs::app::context::get_global_app_context;
use rustfs::embedded::{RustFSServerBuilder, find_available_port};
use rustfs_ecstore::api::disk::RUSTFS_META_MULTIPART_BUCKET;
use rustfs_s3_ops::S3Operation;
use rustfs_storage_api::MultipartOperations as _;
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};
const BUCKET: &str = "abort-multipart-wire";
const OTHER: &str = "abort-multipart-other";
const KEY: &str = "directory/object";
const DENIED: &str = "other-object";
const USER: &str = "abortpartscoped";
pub(super) fn matrix_abort<'a>(client: &'a Client, bucket: &'a str) -> BoxFuture<'a, Outcome> {
Box::pin(async move {
let store = get_global_app_context().expect("embedded server context").object_store();
let key = "matrix-abort";
let opts = Default::default();
let upload = store
.new_multipart_upload(bucket, key, &opts)
.await
.expect("stage a matrix upload");
let result = outcome(
client
.abort_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(&upload.upload_id)
.send()
.await,
);
if result.is_err() {
store
.abort_multipart_upload(bucket, key, &upload.upload_id, &opts)
.await
.expect("remove matrix fixture");
}
result
})
}
fn operation_count() -> u64 {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
.find(|row| row.op == S3Operation::AbortMultipartUpload.as_str())
.map_or(0, |row| row.total)
}
async fn stage(client: &Client, bucket: &str, key: &str) -> (String, String) {
let initiated = client
.create_multipart_upload()
.bucket(bucket)
.key(key)
.send()
.await
.expect("initiate fixture");
let id = initiated.upload_id().expect("fixture UploadId").to_owned();
let part = client
.upload_part()
.bucket(bucket)
.key(key)
.upload_id(&id)
.part_number(1)
.body(ByteStream::from_static(b"real staged part"))
.send()
.await
.expect("upload fixture part");
(id, part.e_tag().expect("fixture ETag").to_owned())
}
fn staged_part(volume: &Path, bucket: &str, key: &str) -> PathBuf {
let key = rustfs_utils::path::encode_dir_object_ref(key);
let hash = hex_simd::encode_to_string(Sha256::digest(format!("{bucket}/{key}")), hex_simd::AsciiCase::Lower);
let path = volume.join(RUSTFS_META_MULTIPART_BUCKET).join(hash);
let children = |path: &Path| {
std::fs::read_dir(path)
.expect("read staging directory")
.map(|entry| entry.expect("read staging entry").path())
.filter(|path| path.is_dir())
.collect::<Vec<_>>()
};
let parts: Vec<_> = children(&path)
.into_iter()
.flat_map(|session| children(&session))
.map(|data| data.join("part.1"))
.filter(|path| path.is_file())
.collect();
assert_eq!(parts.len(), 1, "{bucket}/{key} must have one physical part");
parts.into_iter().next().expect("physical part path")
}
async fn abort(client: &Client, bucket: &str, key: &str, id: &str, expected: Option<(&str, u16)>) {
let abort = client.abort_multipart_upload().bucket(bucket).key(key);
let result = if id.is_empty() {
// The SDK rejects empty UploadIds before sending a request.
abort
.upload_id("placeholder")
.customize()
.mutate_request(|request| {
let (path, _) = request.uri().split_once('?').expect("serialized Abort query");
request.set_uri(format!("{path}?uploadId=")).expect("empty uploadId URI");
})
.send()
.await
} else {
abort.upload_id(id).send().await
};
if let Some((code, status)) = expected {
let error = result.expect_err("Abort must be refused");
assert_eq!(refusal(&error), (Some(code.to_owned()), Some(status)), "{bucket}/{key}: {error:?}");
let raw = error.raw_response().expect("HTTP response");
assert!(raw.headers().get("x-amz-request-id").is_some());
assert!(raw.headers().get("x-request-id").is_some());
if code == "NoSuchUpload" {
let message = if id == "bad-id" {
"The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
} else {
"The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed."
};
assert_eq!(error.message(), Some(message), "{bucket}/{key}");
}
} else {
result.expect("Abort should remove the current upload");
}
}
fn wire_contract(stack: Option<&'static str>) {
temp_env::with_vars([(ENV_S3_STACK, stack)], || {
common::run_embedded_test(move || async move {
let volume = tempfile::tempdir().expect("isolated data volume");
let server = RustFSServerBuilder::new()
.address(format!("127.0.0.1:{}", find_available_port().expect("local port")))
.access_key(ROOT_ACCESS_KEY)
.secret_key(ROOT_SECRET_KEY)
.volume(volume.path().to_string_lossy())
.build()
.await
.expect("start embedded server");
for bucket in [BUCKET, OTHER] {
create_bucket_in_the_store(bucket).await;
}
let root = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
let (id, _) = stage(&root, BUCKET, KEY).await;
let physical = staged_part(volume.path(), BUCKET, KEY);
let before = operation_count();
abort(&root, BUCKET, KEY, &id, None).await;
assert_eq!(operation_count(), before + 1);
assert!(!physical.exists(), "successful Abort removed the physical staged part");
let store = get_global_app_context().expect("embedded server context").object_store();
assert!(
store.get_multipart_info(BUCKET, KEY, &id, &Default::default()).await.is_err(),
"the store no longer sees the upload"
);
abort(&root, BUCKET, KEY, &id, Some(("NoSuchUpload", 404))).await;
abort(&root, BUCKET, KEY, "bad-id", Some(("NoSuchUpload", 404))).await;
let directory = "directory/marker/";
let initiated = root
.create_multipart_upload()
.bucket(BUCKET)
.key(directory)
.send()
.await
.expect("directory upload");
let id = initiated.upload_id().expect("directory upload ID");
abort(&root, BUCKET, directory, id, None).await;
let completed_key = "completed";
let (completed_id, etag) = stage(&root, BUCKET, completed_key).await;
let part = aws_sdk_s3::types::CompletedPart::builder().part_number(1).e_tag(etag).build();
root.complete_multipart_upload()
.bucket(BUCKET)
.key(completed_key)
.upload_id(&completed_id)
.multipart_upload(aws_sdk_s3::types::CompletedMultipartUpload::builder().parts(part).build())
.send()
.await
.expect("complete fixture");
abort(&root, BUCKET, completed_key, &completed_id, Some(("NoSuchUpload", 404))).await;
let (id, _) = stage(&root, BUCKET, KEY).await;
let physical = staged_part(volume.path(), BUCKET, KEY);
let before = operation_count();
let cap_error = root
.abort_multipart_upload()
.bucket(BUCKET)
.key(KEY)
.upload_id(&id)
.customize()
.mutate_request(|req| {
let uri = format!("{}&x-rustfs-max-total-object-size=17&x-rustfs-max-content-length=12", req.uri());
req.set_uri(uri).expect("capability query URI");
})
.send()
.await
.expect_err("Abort cannot take an upload capability");
assert_eq!(refusal(&cap_error), (Some("InvalidRequest".to_owned()), Some(400)));
assert!(
cap_error
.message()
.is_some_and(|message| message.contains("x-rustfs-max-content-length"))
);
assert_eq!(operation_count(), before);
assert!(physical.exists(), "refused capability preserved the physical part");
let (denied_id, _) = stage(&root, BUCKET, DENIED).await;
let denied_part = staged_part(volume.path(), BUCKET, DENIED);
let (other_id, _) = stage(&root, OTHER, KEY).await;
let other_part = staged_part(volume.path(), OTHER, KEY);
for (bucket, key) in [(BUCKET, DENIED), (OTHER, KEY)] {
abort(&root, bucket, key, &id, Some(("NoSuchUpload", 404))).await;
}
assert!(physical.exists() && denied_part.exists() && other_part.exists());
create_user_allowed_on_one_bucket(USER, "s3:AbortMultipartUpload", &format!("{BUCKET}/{KEY}")).await;
let scoped = s3_client(&server.endpoint(), USER, USER_SECRET_KEY);
for (bucket, key, session, part) in [
(BUCKET, DENIED, &denied_id, &denied_part),
(OTHER, KEY, &other_id, &other_part),
] {
let before = operation_count();
abort(&scoped, bucket, key, session, Some(("AccessDenied", 403))).await;
assert_eq!(operation_count(), before);
assert!(part.exists(), "denied Abort preserved the physical staged part");
}
abort(&scoped, BUCKET, KEY, &id, None).await;
assert!(!physical.exists());
assert!(denied_part.exists() && other_part.exists());
server.shutdown().await;
});
});
}
#[test]
#[serial_test::serial]
fn legacy_abort_multipart_fixed_main_contract() {
wire_contract(LEGACY);
}
#[test]
#[serial_test::serial]
fn gateway_abort_multipart_wire_contract() {
wire_contract(GATEWAY);
}
async fn empty_upload_id(server: rustfs::embedded::RustFSServer) {
create_bucket_in_the_store(BUCKET).await;
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
abort(&client, BUCKET, KEY, "", Some(("NoSuchUpload", 404))).await;
server.shutdown().await;
}
#[test]
#[serial_test::serial]
fn legacy_abort_multipart_empty_upload_id_returns_no_such_upload() {
with_stack(LEGACY, empty_upload_id);
}
#[test]
#[serial_test::serial]
fn gateway_abort_multipart_empty_upload_id_returns_no_such_upload() {
with_stack(GATEWAY, empty_upload_id);
}