mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 03:59:14 +00:00
fix(restore): reject SELECT restore and keep typed S3 errors (#7113)
* fix(restore): reject SELECT restore and keep typed S3 errors RestoreObject accepted `Type=SELECT` requests, but the restore path can only write the retrieved bytes back to the source key: `put_restore_opts` built SELECT output options and `restore_transitioned_object` then PUT them over the source bucket/object. On an unversioned bucket that dropped `x-amz-restore`, user metadata and tags from the live object; on a versioned bucket it published a bogus latest version. Nothing was ever written to `OutputLocation.S3`, yet the response still carried a fabricated `x-amz-restore-output-path`. Reject SELECT at the API boundary with a typed NotImplemented, before any guard or metadata write, and fail closed in `put_restore_opts` as the backstop for any other caller. Every other RestoreObject failure was collapsed into a `Custom` error code, which serializes as a generic retryable 500: a missing key or version, a malformed version-id, an object that was never transitioned, an illegal `Days`, and authorization or storage failures all looked the same to a client. Map them to their S3 identities instead — NoSuchKey, NoSuchVersion, InvalidArgument, InvalidObjectState, InvalidRequest, MalformedXML — by preserving `StorageError` through `post_restore_opts` and letting `ApiError` do the mapping. The intentional 409 RestoreAlreadyInProgress and 503 SlowDown behaviour is unchanged, and request validation now runs before any lock is taken. backlog#1341, backlog#2205 * test(restore): give the typed-error regression the ecstore test stack `execute_restore_object_maps_failures_to_typed_s3_errors` builds a real ECStore fixture, and under nextest each test runs in a spawned thread with libtest's 2 MiB stack. On Linux CI that overflowed: the test aborted with SIGABRT / "fatal runtime error: stack overflow" while every other test in the run passed. Add it to the `ecstore-base-stack` filter in both the default and ci profiles, alongside the other `package(rustfs)` tests that drive the same store fixture. 4 MiB matches what the deeper multipart and access roundtrips already use.
This commit is contained in:
@@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
@@ -210,7 +210,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
|
||||
@@ -87,16 +87,12 @@ use rustfs_filemeta::{
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
|
||||
};
|
||||
use rustfs_utils::{
|
||||
get_env_i64, get_env_usize,
|
||||
path::encode_dir_object,
|
||||
string::{parse_bool, strings_has_prefix_fold},
|
||||
};
|
||||
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
|
||||
Timestamp,
|
||||
};
|
||||
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
|
||||
use s3s::header::X_AMZ_RESTORE;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::any::Any;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
@@ -165,7 +161,6 @@ pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
)]
|
||||
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
|
||||
pub const AMZ_ENCRYPTION_AES: &str = "AES256";
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
|
||||
@@ -4802,24 +4797,24 @@ fn attach_tier_operation_lease(mut reader: GetObjectReader, lease: TierOperation
|
||||
reader
|
||||
}
|
||||
|
||||
pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, std::io::Error> {
|
||||
/// Resolve the RestoreObject request options.
|
||||
///
|
||||
/// Returns the typed [`StorageError`]: flattening these into an opaque
|
||||
/// `io::Error` string erased the identity the S3 layer needs to answer
|
||||
/// InvalidArgument instead of a generic 500 (backlog#2205).
|
||||
pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, Error> {
|
||||
let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
|
||||
let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
|
||||
let vid = version_id.trim();
|
||||
if !vid.is_empty() && vid != NULL_VERSION_ID {
|
||||
if let Err(_err) = Uuid::parse_str(vid) {
|
||||
return Err(std::io::Error::other(
|
||||
StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(),
|
||||
));
|
||||
return Err(StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()));
|
||||
}
|
||||
if !versioned && !version_suspended {
|
||||
return Err(std::io::Error::other(
|
||||
StorageError::InvalidArgument(
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
format!("version-id specified {} but versioning is not enabled on {}", vid, bucket),
|
||||
)
|
||||
.to_string(),
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
format!("version-id specified {vid} but versioning is not enabled on {bucket}"),
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -4872,43 +4867,18 @@ pub async fn put_restore_opts(
|
||||
}
|
||||
meta.insert(X_AMZ_STORAGE_CLASS.as_str().to_lowercase(), sc);*/
|
||||
|
||||
if let Some(type_) = &rreq.type_
|
||||
&& type_.as_str() == RestoreRequestType::SELECT
|
||||
// A SELECT restore must never reach the restore writer: the caller writes
|
||||
// the retrieved bytes back to the source bucket/object, so building
|
||||
// SELECT output options here produced a source overwrite carrying only
|
||||
// the OutputLocation metadata instead of a write to `OutputLocation.S3`
|
||||
// (backlog#1341). RestoreObject rejects SELECT at the API boundary; this
|
||||
// is the fail-closed backstop for any other caller.
|
||||
if rreq
|
||||
.type_
|
||||
.as_ref()
|
||||
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
|
||||
{
|
||||
let Some(s3) = select_restore_s3_location(rreq)? else {
|
||||
return Err(std::io::Error::other("OutputLocation.S3 required for SELECT requests"));
|
||||
};
|
||||
if let Some(user_metadata) = s3.user_metadata.as_ref() {
|
||||
for metadata in user_metadata {
|
||||
let name = metadata
|
||||
.name
|
||||
.as_deref()
|
||||
.ok_or_else(|| std::io::Error::other("SELECT restore metadata name is required"))?;
|
||||
let value = metadata.value.clone().unwrap_or_default();
|
||||
if strings_has_prefix_fold(name, "x-amz-meta") {
|
||||
meta.insert(name.to_string(), value);
|
||||
} else {
|
||||
meta.insert(format!("x-amz-meta-{name}"), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(tags) = &s3.tagging {
|
||||
meta.insert(
|
||||
AMZ_OBJECT_TAGGING.to_string(),
|
||||
serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
|
||||
);
|
||||
}
|
||||
if let Some(encryption) = &s3.encryption
|
||||
&& encryption.encryption_type.as_str() != ""
|
||||
{
|
||||
meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
|
||||
}
|
||||
return Ok(ObjectOptions {
|
||||
versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
|
||||
version_suspended: BucketVersioningSys::prefix_suspended(bucket, object).await,
|
||||
user_defined: meta,
|
||||
..Default::default()
|
||||
});
|
||||
return Err(std::io::Error::other("SELECT restore requests are not supported"));
|
||||
}
|
||||
for (k, v) in oi.user_defined.iter() {
|
||||
meta.insert(k.to_string(), v.clone());
|
||||
|
||||
@@ -170,12 +170,12 @@ use s3s::dto::{
|
||||
DeleteObjectsOutput, DeletedObject, ETag, GetObjectAttributesInput, GetObjectAttributesOutput, GetObjectAttributesParts,
|
||||
GetObjectInput, GetObjectOutput, HeadObjectInput, HeadObjectOutput, MetadataDirective, ObjectAttributes, ObjectLockLegalHold,
|
||||
ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ObjectPart, PutObjectInput,
|
||||
PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreStatus, SSECustomerAlgorithm,
|
||||
SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, ServerSideEncryption,
|
||||
ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat,
|
||||
WebsiteRedirectLocation,
|
||||
PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreRequestType, RestoreStatus,
|
||||
SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput,
|
||||
ServerSideEncryption, ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader,
|
||||
Timestamp, TimestampFormat, WebsiteRedirectLocation,
|
||||
};
|
||||
use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
|
||||
use s3s::header::X_AMZ_RESTORE;
|
||||
use s3s::stream::{ByteStream, DynByteStream, RemainingLength};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
|
||||
|
||||
@@ -165,18 +165,49 @@ impl DefaultObjectUsecase {
|
||||
|
||||
validate_table_catalog_object_mutation(&bucket, &object).await?;
|
||||
|
||||
let rreq = rreq.ok_or_else(|| {
|
||||
S3Error::with_message(S3ErrorCode::Custom("ErrValidRestoreObject".into()), "restore request is required")
|
||||
})?;
|
||||
// Typed S3 errors on every RestoreObject failure (backlog#2205): a
|
||||
// `Custom` code serializes as a generic 500, which makes SDK clients
|
||||
// retry client errors and conflicts alike.
|
||||
let rreq = rreq.ok_or_else(|| S3Error::with_message(S3ErrorCode::MalformedXML, "restore request is required"))?;
|
||||
|
||||
// SELECT-type restore is not supported (backlog#1341). The restore
|
||||
// path can only write the retrieved bytes back to the source key, so
|
||||
// honouring a SELECT request overwrote the source object with
|
||||
// SELECT-only metadata (dropping `x-amz-restore`, user metadata and
|
||||
// tags on an unversioned bucket, or publishing a bogus latest version
|
||||
// on a versioned one) while never writing anything to
|
||||
// `OutputLocation.S3`. Reject before any guard, metadata write or
|
||||
// fabricated `x-amz-restore-output-path` response header.
|
||||
if rreq
|
||||
.type_
|
||||
.as_ref()
|
||||
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
|
||||
{
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::NotImplemented,
|
||||
"SELECT restore requests are not supported.",
|
||||
));
|
||||
}
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
// Validate the request shape before taking any lock or reading the
|
||||
// object: a malformed request or an illegal `Days` value is a client
|
||||
// error, and the validator messages are static — they carry no
|
||||
// backend or credential detail.
|
||||
if let Err(e) = validate_restore_request(&rreq, store.clone()) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("Restore object validation failed: {e}"),
|
||||
));
|
||||
}
|
||||
|
||||
let version_id_str = version_id.clone().unwrap_or_default();
|
||||
let mut opts = post_restore_opts(&version_id_str, &bucket, &object)
|
||||
.await
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrPostRestoreOpts".into()), "restore object failed."))?;
|
||||
.map_err(ApiError::from)?;
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
||||
// `apply_bucket_generation_guard` deliberately tolerates a missing guard
|
||||
// (only the S3 access layer installs one), so this must not hard-require
|
||||
@@ -192,11 +223,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
};
|
||||
|
||||
// SELECT-type restores skip both the ongoing check and the metadata
|
||||
// write below, so the accept guard would protect nothing for them —
|
||||
// they keep the plain (read-locked) accept path.
|
||||
let is_select = rreq.type_.as_ref().is_some_and(|t| t.as_str() == "SELECT");
|
||||
let restore_operation_id = (!is_select).then(Uuid::new_v4);
|
||||
let restore_operation_id = Some(Uuid::new_v4());
|
||||
let mut restore_worker_guard = if let Some(operation_id) = restore_operation_id {
|
||||
Some(
|
||||
store
|
||||
@@ -210,10 +237,10 @@ impl DefaultObjectUsecase {
|
||||
|
||||
// Hold the restore-accept guard across the restore-status read, the
|
||||
// ongoing/already-restored decision, and the metadata write below, so
|
||||
// two concurrent (non-SELECT) POST ?restore cannot both observe
|
||||
// ongoing=false and both start a copy-back (backlog#1304). Reads and
|
||||
// writes inside this scope run with no_lock; the guard is dropped
|
||||
// before the copy-back is spawned so it never blocks readers.
|
||||
// two concurrent POST ?restore cannot both observe ongoing=false and
|
||||
// both start a copy-back (backlog#1304). Reads and writes inside this
|
||||
// scope run with no_lock; the guard is dropped before the copy-back is
|
||||
// spawned so it never blocks readers.
|
||||
// Contention on the accept guard (e.g. a concurrent accept or an
|
||||
// in-flight commit on the same object) is transient — answer 503
|
||||
// SlowDown so SDK clients back off and retry instead of treating it
|
||||
@@ -222,9 +249,7 @@ impl DefaultObjectUsecase {
|
||||
if store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)? != restore_bucket_incarnation_id {
|
||||
return Err(ApiError::from(StorageError::BucketNotFound(bucket.clone())).into());
|
||||
}
|
||||
let mut accept_guard = if is_select {
|
||||
None
|
||||
} else {
|
||||
let mut accept_guard = {
|
||||
let guard = store
|
||||
.acquire_restore_accept_guard(&bucket, &object)
|
||||
.await
|
||||
@@ -233,24 +258,17 @@ impl DefaultObjectUsecase {
|
||||
Some(guard)
|
||||
};
|
||||
|
||||
let mut obj_info = store
|
||||
.get_object_info(&bucket, &object, &opts)
|
||||
.await
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed."))?;
|
||||
// A missing key or version must stay NoSuchKey / NoSuchVersion, and an
|
||||
// authorization or storage failure must keep its own identity, so map
|
||||
// the storage error instead of flattening it (backlog#2205).
|
||||
let mut obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?;
|
||||
|
||||
// Check if object is in a transitioned state
|
||||
// Restoring an object that was never transitioned is the S3
|
||||
// InvalidObjectState case, not an internal error.
|
||||
if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::Custom("ErrInvalidTransitionedState".into()),
|
||||
"restore object failed.",
|
||||
));
|
||||
}
|
||||
|
||||
// Validate restore request
|
||||
if let Err(e) = validate_restore_request(&rreq, store.clone()) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::Custom("ErrValidRestoreObject".into()),
|
||||
format!("Restore object validation failed: {}", e),
|
||||
S3ErrorCode::InvalidObjectState,
|
||||
"The operation is not valid for the object's storage class.",
|
||||
));
|
||||
}
|
||||
|
||||
@@ -260,7 +278,7 @@ impl DefaultObjectUsecase {
|
||||
// would create an ABBA cycle. If the probe succeeds, reacquire and
|
||||
// re-read the object before replacing the exact orphan generation.
|
||||
let mut superseded_worker_guard = None;
|
||||
if obj_info.restore_ongoing && !is_select {
|
||||
if obj_info.restore_ongoing {
|
||||
match classify_ongoing_restore(obj_info.user_defined.as_ref(), OffsetDateTime::now_utc()) {
|
||||
OngoingRestoreRecovery::ActiveOrUnsafe => {
|
||||
return Err(S3Error::with_message(
|
||||
@@ -293,13 +311,11 @@ impl DefaultObjectUsecase {
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::SlowDown, "restore object failed."))?,
|
||||
);
|
||||
opts.no_lock = true;
|
||||
obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(|_| {
|
||||
S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed.")
|
||||
})?;
|
||||
obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?;
|
||||
if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::Custom("ErrInvalidTransitionedState".into()),
|
||||
"restore object failed.",
|
||||
S3ErrorCode::InvalidObjectState,
|
||||
"The operation is not valid for the object's storage class.",
|
||||
));
|
||||
}
|
||||
if obj_info.restore_ongoing {
|
||||
@@ -327,11 +343,11 @@ impl DefaultObjectUsecase {
|
||||
remove_str(&mut metadata, SUFFIX_RESTORE_OPERATION_ID);
|
||||
remove_str(&mut metadata, SUFFIX_RESTORE_WORKER_LOCK);
|
||||
|
||||
let mut header = HeaderMap::new();
|
||||
|
||||
let event_object_info = obj_info.clone();
|
||||
let obj_info_ = obj_info.clone();
|
||||
if !is_select {
|
||||
// Scopes the accept-guarded metadata write: everything below runs
|
||||
// inside the accept critical section, which is released right after.
|
||||
{
|
||||
obj_info.metadata_only = true;
|
||||
metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap_or(1).to_string());
|
||||
let request_date = OffsetDateTime::now_utc().format(&Rfc3339).map_err(|e| {
|
||||
@@ -403,7 +419,7 @@ impl DefaultObjectUsecase {
|
||||
&restore_dst_opts,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrCopyObject".into()), "restore object failed."))?;
|
||||
.map_err(ApiError::from)?;
|
||||
rustfs_scanner::record_dirty_usage_bucket(&bucket);
|
||||
#[cfg(test)]
|
||||
maybe_pause_after_restore_status_commit(&bucket, &object).await;
|
||||
@@ -429,17 +445,6 @@ impl DefaultObjectUsecase {
|
||||
drop(accept_guard);
|
||||
drop(restore_bucket_lifecycle_guard);
|
||||
|
||||
// Handle output location for SELECT requests
|
||||
if let Some(output_location) = &rreq.output_location
|
||||
&& let Some(s3) = &output_location.s3
|
||||
&& !s3.bucket_name.is_empty()
|
||||
{
|
||||
let restore_object = Uuid::new_v4().to_string();
|
||||
if let Ok(header_value) = format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse() {
|
||||
header.insert(X_AMZ_RESTORE_OUTPUT_PATH, header_value);
|
||||
}
|
||||
}
|
||||
|
||||
// Spawn restoration task in the background. Pin the copy-back to the
|
||||
// version the accept resolved and flagged: with a versionless request
|
||||
// on a versioned bucket, a PUT landing between the accept and the
|
||||
@@ -499,7 +504,7 @@ impl DefaultObjectUsecase {
|
||||
restore_output_path: None,
|
||||
};
|
||||
helper = helper.object(event_object_info).version_id(version_id_str);
|
||||
let result = Ok(S3Response::with_headers(output, header));
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
@@ -629,6 +634,28 @@ mod tests {
|
||||
assert_eq!(classify_ongoing_restore(&conflicting_date, now), OngoingRestoreRecovery::ActiveOrUnsafe);
|
||||
}
|
||||
|
||||
fn restore_request(days: Option<i32>) -> RestoreRequest {
|
||||
RestoreRequest {
|
||||
days,
|
||||
description: None,
|
||||
glacier_job_parameters: None,
|
||||
output_location: None,
|
||||
select_parameters: None,
|
||||
tier: None,
|
||||
type_: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_input(bucket: &str, key: &str, rreq: RestoreRequest) -> RestoreObjectInput {
|
||||
RestoreObjectInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.key(key.to_string())
|
||||
.restore_request(Some(rreq))
|
||||
.build()
|
||||
.expect("restore input should build")
|
||||
}
|
||||
|
||||
/// backlog#2205: a missing restore body is a client error, not a 500.
|
||||
#[tokio::test]
|
||||
async fn execute_restore_object_rejects_missing_restore_request() {
|
||||
let input = RestoreObjectInput::builder()
|
||||
@@ -641,10 +668,92 @@ mod tests {
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_restore_object(req).await.unwrap_err();
|
||||
match err.code() {
|
||||
S3ErrorCode::Custom(code) => assert_eq!(code, "ErrValidRestoreObject"),
|
||||
code => panic!("unexpected error code: {:?}", code),
|
||||
}
|
||||
assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
|
||||
}
|
||||
|
||||
/// backlog#1341: a SELECT restore must be rejected outright — the restore
|
||||
/// path can only write back to the source key, never to
|
||||
/// `OutputLocation.S3`. Rejection happens before the store is resolved, so
|
||||
/// an uninitialized usecase still answers NotImplemented rather than the
|
||||
/// InternalError every request that gets past this point returns.
|
||||
#[tokio::test]
|
||||
async fn execute_restore_object_rejects_select_type() {
|
||||
let mut rreq = restore_request(None);
|
||||
rreq.type_ = Some(s3s::dto::RestoreRequestType::from_static(s3s::dto::RestoreRequestType::SELECT));
|
||||
|
||||
let req = build_request(restore_input("test-bucket", "test-key", rreq), Method::POST);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_restore_object(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
|
||||
}
|
||||
|
||||
/// backlog#2205: every RestoreObject failure that reaches storage must
|
||||
/// keep its typed S3 identity. Before this, a missing key, a malformed
|
||||
/// version-id, an illegal `Days` and an object that was never transitioned
|
||||
/// all collapsed into `Custom(...)` codes, which serialize as a retryable
|
||||
/// HTTP 500.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_restore_object_maps_failures_to_typed_s3_errors() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let context = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let bucket = format!("restore-typed-errors-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create restore test bucket");
|
||||
let mut reader = PutObjReader::from_vec(b"never transitioned".to_vec());
|
||||
store
|
||||
.put_object(&bucket, "local-object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("put untransitioned test object");
|
||||
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(context));
|
||||
|
||||
// An illegal `Days` is a client error, rejected before any lock or
|
||||
// object read.
|
||||
let err = usecase
|
||||
.execute_restore_object(build_request(
|
||||
restore_input(&bucket, "local-object", restore_request(Some(0))),
|
||||
Method::POST,
|
||||
))
|
||||
.await
|
||||
.expect_err("days=0 must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
// A malformed version-id keeps InvalidArgument instead of being
|
||||
// flattened inside `post_restore_opts`.
|
||||
let mut input = restore_input(&bucket, "local-object", restore_request(Some(1)));
|
||||
input.version_id = Some("not-a-uuid".to_string());
|
||||
let err = usecase
|
||||
.execute_restore_object(build_request(input, Method::POST))
|
||||
.await
|
||||
.expect_err("malformed version-id must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
|
||||
// A missing key stays NoSuchKey.
|
||||
let err = usecase
|
||||
.execute_restore_object(build_request(
|
||||
restore_input(&bucket, "missing-object", restore_request(Some(1))),
|
||||
Method::POST,
|
||||
))
|
||||
.await
|
||||
.expect_err("missing key must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::NoSuchKey);
|
||||
|
||||
// Restoring an object that was never transitioned is the S3
|
||||
// InvalidObjectState case, not an internal error.
|
||||
let err = usecase
|
||||
.execute_restore_object(build_request(
|
||||
restore_input(&bucket, "local-object", restore_request(Some(1))),
|
||||
Method::POST,
|
||||
))
|
||||
.await
|
||||
.expect_err("untransitioned object must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidObjectState);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -368,7 +368,8 @@ pub(crate) mod bucket {
|
||||
version_id: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<crate::storage::storage_api::StorageObjectOptions, std::io::Error> {
|
||||
) -> Result<crate::storage::storage_api::StorageObjectOptions, crate::storage::storage_api::StorageError>
|
||||
{
|
||||
crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts(
|
||||
version_id, bucket, object,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user