mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
feat(gateway): serve RestoreObject through the owned use case
Share the authorization snapshot and restore admission between both stacks.
Preserve HTTP 200 for both success paths and emit ObjectRestorePost on new
admission only.
Verified on c0763a43 with 114 focused tests, four serialized ILM tests, and
16 signed wire responses per stack. Faults covered shared authorization,
generation capture and admission, SELECT rejection, worker parameters and
liveness, and both audit and event paths. Cluster lock loss and mixed-version
rollout remain unverified.
This commit is contained in:
@@ -15,11 +15,12 @@
|
||||
//! RestoreObject path.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::object_usecase::bucket::restore_request_from_wire;
|
||||
use crate::app::storage_api::s3::dto::{RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreRequestType};
|
||||
use crate::app::storage_api::s3::header::X_AMZ_RESTORE;
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::storage_api::object_usecase::bucket::RestoreRequest;
|
||||
use rustfs_filemeta::RestoreStatus;
|
||||
use rustfs_gateway_types::dto::{RequestCharged, RestoreObjectOutput};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_utils::http::headers::AMZ_RESTORE;
|
||||
|
||||
// RUSTFS_COMPAT_TODO(backlog-1337): legacy restores lack a liveness marker. Remove after the minimum supported release writes v1 on every restore.
|
||||
const LEGACY_RESTORE_ORPHAN_GRACE: time::Duration = time::Duration::hours(24);
|
||||
@@ -153,20 +154,24 @@ async fn maybe_pause_after_restore_status_commit(bucket: &str, object: &str) {
|
||||
}
|
||||
|
||||
impl DefaultObjectUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_restore_object(&self, req: S3Request<RestoreObjectInput>) -> S3Result<S3Response<RestoreObjectOutput>> {
|
||||
#[instrument(
|
||||
name = "execute_restore_object",
|
||||
level = "debug",
|
||||
skip(self, envelope, bucket, object, version_id, restore_request)
|
||||
)]
|
||||
pub(crate) async fn execute_restore_object_owned(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
restore_request: Option<RestoreRequest>,
|
||||
) -> S3Result<RestoreObjectOutput> {
|
||||
if let Some(context) = &self.context {
|
||||
let _ = context.object_store();
|
||||
}
|
||||
|
||||
let mut helper = OperationHelper::new(&req, EventName::ObjectRestorePost, S3Operation::RestoreObject);
|
||||
let RestoreObjectInput {
|
||||
bucket,
|
||||
key: object,
|
||||
restore_request: rreq,
|
||||
version_id,
|
||||
..
|
||||
} = req.input.clone();
|
||||
let helper = OperationHelper::from_view(&envelope.audit_view(), EventName::ObjectRestorePost, S3Operation::RestoreObject);
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &object).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &object).await?;
|
||||
@@ -174,7 +179,8 @@ impl DefaultObjectUsecase {
|
||||
// Typed S3 errors on every RestoreObject failure (backlog#2205): a
|
||||
// `Custom` code serializes as a generic 500, which makes SDK clients
|
||||
// retry client errors and conflicts alike.
|
||||
let rreq = rreq.ok_or_else(|| S3Error::with_message(S3ErrorCode::MalformedXML, "restore request is required"))?;
|
||||
let rreq =
|
||||
restore_request.ok_or_else(|| S3Error::with_message(S3ErrorCode::MalformedXML, "restore request is required"))?;
|
||||
|
||||
// SELECT-type restore is not supported (backlog#1341). The restore
|
||||
// path can only write the retrieved bytes back to the source key, so
|
||||
@@ -184,11 +190,7 @@ impl DefaultObjectUsecase {
|
||||
// on a versioned one) while never writing anything to
|
||||
// `OutputLocation.S3`. Reject before any guard, metadata write or
|
||||
// fabricated `x-amz-restore-output-path` response header.
|
||||
if rreq
|
||||
.type_
|
||||
.as_ref()
|
||||
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
|
||||
{
|
||||
if rreq.type_.as_deref() == Some("SELECT") {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::NotImplemented,
|
||||
"SELECT restore requests are not supported.",
|
||||
@@ -199,8 +201,6 @@ impl DefaultObjectUsecase {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let rreq = restore_request_from_wire(rreq);
|
||||
|
||||
// Validate the request shape before taking any lock or reading the
|
||||
// object: a malformed request or an illegal `Days` value is a client
|
||||
// error, and the validator messages are static — they carry no
|
||||
@@ -212,11 +212,11 @@ impl DefaultObjectUsecase {
|
||||
));
|
||||
}
|
||||
|
||||
let version_id_str = version_id.clone().unwrap_or_default();
|
||||
let version_id_str = version_id.unwrap_or_default();
|
||||
let mut opts = post_restore_opts(&version_id_str, &bucket, &object)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
||||
apply_bucket_generation_guard(&envelope, &bucket, &mut opts)?;
|
||||
// `apply_bucket_generation_guard` deliberately tolerates a missing guard
|
||||
// (only the S3 access layer installs one), so this must not hard-require
|
||||
// it. Resolve the current generation instead, exactly as the copy path
|
||||
@@ -370,7 +370,7 @@ impl DefaultObjectUsecase {
|
||||
metadata.insert(AMZ_RESTORE_REQUEST_DATE.to_string(), request_date);
|
||||
if already_restored {
|
||||
metadata.insert(
|
||||
X_AMZ_RESTORE.as_str().to_string(),
|
||||
AMZ_RESTORE.to_owned(),
|
||||
RestoreStatus {
|
||||
is_restore_in_progress: Some(false),
|
||||
restore_expiry_date: Some(restore_expiry),
|
||||
@@ -379,7 +379,7 @@ impl DefaultObjectUsecase {
|
||||
);
|
||||
} else {
|
||||
metadata.insert(
|
||||
X_AMZ_RESTORE.as_str().to_string(),
|
||||
AMZ_RESTORE.to_owned(),
|
||||
RestoreStatus {
|
||||
is_restore_in_progress: Some(true),
|
||||
restore_expiry_date: Some(OffsetDateTime::now_utc()),
|
||||
@@ -444,16 +444,12 @@ impl DefaultObjectUsecase {
|
||||
drop(superseded_worker_guard.take());
|
||||
|
||||
if already_restored {
|
||||
let output = RestoreObjectOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
restore_output_path: None,
|
||||
};
|
||||
helper = helper
|
||||
let result = Ok(restore_output());
|
||||
let _ = helper
|
||||
.object(event_object_info.clone())
|
||||
.version_id(version_id_str.clone())
|
||||
.suppress_event();
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
.suppress_event()
|
||||
.complete_owned(&result, StatusCode::OK, &HeaderMap::new());
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -485,7 +481,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
let restore_worker_guard = restore_worker_guard.take();
|
||||
let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&req);
|
||||
let table_data_plane_publication_guards = retained_table_data_plane_publication_guards(&envelope);
|
||||
spawn_traced(async move {
|
||||
let _restore_worker_guard = restore_worker_guard;
|
||||
let _table_data_plane_publication_guards = table_data_plane_publication_guards;
|
||||
@@ -526,22 +522,29 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
});
|
||||
|
||||
let output = RestoreObjectOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
restore_output_path: None,
|
||||
};
|
||||
helper = helper.object(event_object_info).version_id(version_id_str);
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
let result = Ok(restore_output());
|
||||
let _ = helper.object(event_object_info).version_id(version_id_str).complete_owned(
|
||||
&result,
|
||||
StatusCode::OK,
|
||||
&HeaderMap::new(),
|
||||
);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_output() -> RestoreObjectOutput {
|
||||
RestoreObjectOutput {
|
||||
request_charged: Some(RequestCharged::custom("requester".to_owned())),
|
||||
restore_output_path: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::Method;
|
||||
use s3s::dto::RestoreRequest;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::{RestoreObjectInput, RestoreRequest};
|
||||
|
||||
fn ongoing_metadata(operation_id: Uuid) -> HashMap<String, String> {
|
||||
let mut metadata = HashMap::new();
|
||||
|
||||
@@ -346,7 +346,7 @@ pub(crate) mod access {
|
||||
}
|
||||
|
||||
pub(crate) mod bucket {
|
||||
use crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
pub(crate) use crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
@@ -393,7 +393,7 @@ pub(crate) mod bucket {
|
||||
crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::Lifecycle::predict_expiration(lifecycle, obj).await
|
||||
}
|
||||
|
||||
// Remove this conversion when RestoreObject receives Gateway inputs.
|
||||
// The legacy edge keeps values that its decoder permits before domain validation.
|
||||
pub(crate) fn restore_request_from_wire(request: s3s::dto::RestoreRequest) -> RestoreRequest {
|
||||
use rustfs_gateway_types::dto;
|
||||
|
||||
|
||||
@@ -13,19 +13,23 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch E2 (rustfs/backlog#2765): DeleteObject, DeleteObjects, RestoreObject and SelectObjectContent.
|
||||
//! DeleteObject is served; the batch appends the others here.
|
||||
//! DeleteObject and RestoreObject are served here.
|
||||
|
||||
use super::{OpRow, Served, row};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
use crate::app::storage_api::object_usecase::bucket::{
|
||||
RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging,
|
||||
};
|
||||
use crate::runtime_sources::AppContext;
|
||||
use crate::storage_api::server::http::access::access_owned::object_misc::authorize_delete_object;
|
||||
use crate::storage_api::server::http::access::access_owned::object_misc::{authorize_delete_object, authorize_restore_object};
|
||||
use http::StatusCode;
|
||||
use rustfs_gateway::{Req, Resp, dto};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::{S3Result, s3_error};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) const ROWS: &[OpRow] = &[row::<dto::DeleteObject>()];
|
||||
pub(super) const ROWS: &[OpRow] = &[row::<dto::DeleteObject>(), row::<dto::RestoreObject>()];
|
||||
|
||||
impl Served for dto::DeleteObject {
|
||||
const METRIC: S3Operation = S3Operation::DeleteObject;
|
||||
@@ -48,3 +52,70 @@ impl Served for dto::DeleteObject {
|
||||
Ok(Resp::new(output))
|
||||
}
|
||||
}
|
||||
|
||||
impl Served for dto::RestoreObject {
|
||||
const METRIC: S3Operation = S3Operation::RestoreObject;
|
||||
|
||||
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
|
||||
let input = request.input();
|
||||
let server_ctx = envelope
|
||||
.server_context()
|
||||
.cloned()
|
||||
.ok_or_else(|| s3_error!(InternalError, "server context is missing"))?;
|
||||
Ok(authorize_restore_object(
|
||||
&server_ctx,
|
||||
envelope,
|
||||
input.bucket.as_str(),
|
||||
input.key.as_str(),
|
||||
input.version_id.as_deref(),
|
||||
)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let input = request.into_input();
|
||||
let output = DefaultObjectUsecase::with_context(context)
|
||||
.execute_restore_object_owned(
|
||||
envelope,
|
||||
input.bucket.as_str().to_owned(),
|
||||
input.key.as_str().to_owned(),
|
||||
input.version_id,
|
||||
Some(restore_request(input.restore_request)),
|
||||
)
|
||||
.await?;
|
||||
// RustFS answers 200 for both a new restore and a renewal.
|
||||
Ok(Resp::with_status(output, StatusCode::OK.as_u16()))
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_request(request: dto::RestoreRequest) -> RestoreRequest {
|
||||
RestoreRequest {
|
||||
days: request.days,
|
||||
description: request.description,
|
||||
glacier_job_parameters: request.glacier_job_parameters,
|
||||
output_location: request.output_location.map(|output| RestoreOutputLocation {
|
||||
s3: output.s3.map(|location| RestoreS3Location {
|
||||
access_control_list: location.access_control_list,
|
||||
bucket_name: location.bucket_name.as_str().to_owned(),
|
||||
canned_acl: location.canned_acl,
|
||||
encryption: location.encryption,
|
||||
prefix: location.prefix,
|
||||
storage_class: location.storage_class,
|
||||
tagging: location.tagging.map(|tagging| RestoreTagging {
|
||||
tag_set: tagging
|
||||
.tag_set
|
||||
.into_iter()
|
||||
.map(|tag| RestoreTag {
|
||||
key: tag.key,
|
||||
value: tag.value,
|
||||
})
|
||||
.collect(),
|
||||
}),
|
||||
user_metadata: location.user_metadata,
|
||||
}),
|
||||
}),
|
||||
select_parameters: request.select_parameters,
|
||||
tier: request.tier,
|
||||
type_: request.r#type.map(|value| value.as_str().to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4270,17 +4270,9 @@ impl S3Access for FS {
|
||||
/// This method returns `Ok(())` by default.
|
||||
async fn restore_object(&self, req: &mut S3Request<RestoreObjectInput>) -> S3Result<()> {
|
||||
let bucket = req.input.bucket.clone();
|
||||
let bucket_generation = load_bucket_generation(self.server_ctx(), req, &bucket).await;
|
||||
#[cfg(test)]
|
||||
wait_for_object_generation_test_hook(&bucket).await;
|
||||
let req_info = ext_req_info_mut(&mut req.extensions)?;
|
||||
req_info.bucket = Some(req.input.bucket.clone());
|
||||
req_info.object = Some(req.input.key.clone());
|
||||
req_info.version_id = req.input.version_id.clone();
|
||||
|
||||
authorize_request(req, Action::S3Action(S3Action::RestoreObjectAction)).await?;
|
||||
req.install_bucket_generation(bucket_generation?);
|
||||
Ok(())
|
||||
let key = req.input.key.clone();
|
||||
let version_id = req.input.version_id.clone();
|
||||
access_owned::object_misc::authorize_restore_object(self.server_ctx(), req, &bucket, &key, version_id.as_deref()).await
|
||||
}
|
||||
|
||||
/// Checks whether the SelectObjectContent request has accesses to the resources.
|
||||
@@ -8459,6 +8451,16 @@ mod tests {
|
||||
.await
|
||||
.expect("read the recreated bucket incarnation");
|
||||
assert_ne!(authorized_incarnation_id, recreated_incarnation_id);
|
||||
let replacement = b"recreated bucket object";
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"object",
|
||||
&mut crate::storage::PutObjReader::from_vec(replacement.to_vec()),
|
||||
&crate::storage::ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("write the recreated bucket's object");
|
||||
resume_tx
|
||||
.send(())
|
||||
.expect("RestoreObject access should still be waiting at the test hook");
|
||||
@@ -8467,20 +8469,46 @@ mod tests {
|
||||
result.expect("the equivalent policy should authorize RestoreObject without refreshing its snapshot");
|
||||
let mut opts = crate::storage::ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut opts).expect("apply the RestoreObject authorization guard");
|
||||
assert_eq!(opts.expected_bucket_incarnation_id, Some(authorized_incarnation_id));
|
||||
|
||||
// The RestoreObject usecase future is large enough that heap-pinning a
|
||||
// single future was not enough on Linux: the whole scenario runs on a
|
||||
// dedicated large stack (see the `#[test]` wrapper above) so unrelated
|
||||
// growth in bucket metadata futures cannot tip the test over.
|
||||
let err = Box::pin(
|
||||
let result = Box::pin(
|
||||
crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(app_context)).execute_restore_object(req),
|
||||
)
|
||||
.await
|
||||
.expect_err("the old RestoreObject authorization must not reach the recreated bucket");
|
||||
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
|
||||
.await;
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, "object", None, HeaderMap::new(), &crate::storage::ObjectOptions::default())
|
||||
.await
|
||||
.expect("read the recreated bucket's object");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read replacement bytes");
|
||||
assert_eq!(
|
||||
(
|
||||
opts.expected_bucket_incarnation_id,
|
||||
result.as_ref().err().map(|error| error.code()),
|
||||
bytes.as_slice(),
|
||||
reader.object_info.restore_ongoing
|
||||
),
|
||||
(
|
||||
Some(authorized_incarnation_id),
|
||||
Some(&S3ErrorCode::NoSuchBucket),
|
||||
replacement.as_slice(),
|
||||
false
|
||||
),
|
||||
"stale RestoreObject must refuse and preserve the recreated bucket's object"
|
||||
);
|
||||
drop(reader);
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.delete_bucket(
|
||||
&bucket,
|
||||
&DeleteBucketOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("clean up the recreated bucket");
|
||||
}
|
||||
|
||||
@@ -31,13 +31,34 @@ use crate::storage::access::{
|
||||
AuthorizationRequest, BucketGenerationGuard, PendingDeleteBucketGenerationGuard, S3Error, S3ErrorCode, S3Result,
|
||||
ServerContextSlot, audit_request_denial, authorize_request, delete_object_authorize_action, has_bypass_governance_header,
|
||||
invalid_force_delete_header, load_bucket_generation, missing_req_info_error, recursive_force_delete_has_authenticated_caller,
|
||||
s3_error,
|
||||
req_info_mut, s3_error,
|
||||
};
|
||||
use crate::storage::denial_audit::DenialAuditor;
|
||||
use crate::storage::options::destination_version_id;
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, force_delete_header};
|
||||
|
||||
/// Both RestoreObject edges retain the bucket generation read before policy evaluation.
|
||||
pub(crate) async fn authorize_restore_object<R: AuthorizationRequest + ?Sized>(
|
||||
server_ctx: &ServerContextSlot,
|
||||
req: &mut R,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
) -> S3Result<()> {
|
||||
let bucket_generation = load_bucket_generation(server_ctx, req, bucket).await;
|
||||
#[cfg(test)]
|
||||
wait_for_object_generation_test_hook(bucket).await;
|
||||
let req_info = req_info_mut(req)?;
|
||||
req_info.bucket = Some(bucket.to_owned());
|
||||
req_info.object = Some(key.to_owned());
|
||||
req_info.version_id = version_id.map(str::to_owned);
|
||||
|
||||
authorize_request(req, Action::S3Action(S3Action::RestoreObjectAction)).await?;
|
||||
req.install_bucket_generation(bucket_generation?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// DeleteObject's authorization on either edge: the bucket generation is loaded before the
|
||||
/// decision, and the guard it becomes is installed for the use case after an allow.
|
||||
pub(crate) async fn authorize_delete_object<R: AuthorizationRequest + ?Sized>(
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::storage_api::object_usecase::bucket::restore_request_from_wire;
|
||||
use crate::storage::access::legacy_copy_source;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::ecfs_extend::validate_new_object_key;
|
||||
@@ -223,6 +224,29 @@ impl FS {
|
||||
}
|
||||
|
||||
impl DefaultObjectUsecase {
|
||||
/// Projects a legacy RestoreObject request onto the shared restore path.
|
||||
pub async fn execute_restore_object(&self, req: S3Request<RestoreObjectInput>) -> S3Result<S3Response<RestoreObjectOutput>> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let input = req.input;
|
||||
self.execute_restore_object_owned(
|
||||
envelope,
|
||||
input.bucket,
|
||||
input.key,
|
||||
input.version_id,
|
||||
input.restore_request.map(restore_request_from_wire),
|
||||
)
|
||||
.await
|
||||
.map(|output| {
|
||||
S3Response::new(RestoreObjectOutput {
|
||||
request_charged: output
|
||||
.request_charged
|
||||
.map(|value| RequestCharged::from(value.as_str().to_owned())),
|
||||
restore_output_path: output.restore_output_path,
|
||||
})
|
||||
})
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Projects a legacy CopyObject request onto the shared copy path.
|
||||
pub async fn execute_copy_object(&self, req: S3Request<CopyObjectInput>) -> S3Result<S3Response<CopyObjectOutput>> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
|
||||
@@ -14489,7 +14489,7 @@ fn assert_object_mutation_reserved_prefix_sources(delete: &str, legacy_delete: &
|
||||
assert_ordered(method(source, signature), &[key_validation, key_guard, mutation_boundary]);
|
||||
}
|
||||
assert_ordered(
|
||||
method(restore, "fn execute_restore_object("),
|
||||
method(restore, "fn execute_restore_object_owned("),
|
||||
&[
|
||||
"validate_table_catalog_object_mutation(&bucket, &object).await?;",
|
||||
store_resolution,
|
||||
|
||||
@@ -360,6 +360,7 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
|
||||
create_bucket_in_the_store(bucket_replication_lifecycle::LOCK_CONFIGURATION_MATRIX_BUCKET).await;
|
||||
create_bucket_in_the_store(object_meta::MATRIX_BUCKET).await;
|
||||
create_bucket_in_the_store(object_misc::MATRIX_BUCKET).await;
|
||||
object_misc::seed_restore_object(object_misc::MATRIX_BUCKET, false).await;
|
||||
create_bucket_in_the_store(object_read::MATRIX_BUCKET).await;
|
||||
create_bucket_in_the_store(object_commit::COPY_MATRIX_BUCKET).await;
|
||||
put_object_in_the_store(object_commit::COPY_MATRIX_BUCKET, object_commit::COPY_MATRIX_SOURCE).await;
|
||||
@@ -409,6 +410,7 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
|
||||
| S3Operation::PutBucketTagging
|
||||
| S3Operation::DeleteBucketEncryption
|
||||
| S3Operation::DeleteBucketTagging
|
||||
| S3Operation::RestoreObject
|
||||
) {
|
||||
"unknown"
|
||||
} else {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows and behaviour of migration batch E2 (rustfs/backlog#2765): DeleteObject.
|
||||
//! Refusal-matrix rows and behaviour of DeleteObject and RestoreObject.
|
||||
|
||||
use super::bucket_configs::StatusCapture;
|
||||
use super::{
|
||||
@@ -21,20 +21,32 @@ use super::{
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata as _;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, RestoreRequest, VersioningConfiguration};
|
||||
use futures::future::BoxFuture;
|
||||
use rustfs::app::context::get_global_app_context;
|
||||
use rustfs::embedded::RustFSServer;
|
||||
use rustfs_ecstore::api::error::is_err_object_not_found;
|
||||
use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
|
||||
use rustfs_ecstore::api::runtime::global_tier_config_mgr;
|
||||
use rustfs_ecstore::api::tier::test_util::{MockWarmBackend, register_mock_tier};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
|
||||
|
||||
pub(crate) const MATRIX: &[MatrixRow] = &[MatrixRow {
|
||||
operation: "DeleteObject",
|
||||
metric: S3Operation::DeleteObject,
|
||||
has_error_body: true,
|
||||
call: delete_object,
|
||||
}];
|
||||
pub(crate) const MATRIX: &[MatrixRow] = &[
|
||||
MatrixRow {
|
||||
operation: "DeleteObject",
|
||||
metric: S3Operation::DeleteObject,
|
||||
has_error_body: true,
|
||||
call: delete_object,
|
||||
},
|
||||
MatrixRow {
|
||||
operation: "RestoreObject",
|
||||
metric: S3Operation::RestoreObject,
|
||||
has_error_body: true,
|
||||
call: restore_object,
|
||||
},
|
||||
];
|
||||
|
||||
pub(super) const MATRIX_BUCKET: &str = "e2-delete-matrix";
|
||||
|
||||
@@ -42,6 +54,228 @@ fn delete_object<'a>(client: &'a Client, _bucket: &'a str) -> BoxFuture<'a, Outc
|
||||
Box::pin(async move { outcome(client.delete_object().bucket(MATRIX_BUCKET).key("matrix-absent").send().await) })
|
||||
}
|
||||
|
||||
const RESTORE_KEY: &str = "archive";
|
||||
const RESTORE_TIER: &str = "RESTORETIER";
|
||||
|
||||
fn restore_object<'a>(client: &'a Client, _bucket: &'a str) -> BoxFuture<'a, Outcome> {
|
||||
Box::pin(async move {
|
||||
outcome(
|
||||
client
|
||||
.restore_object()
|
||||
.bucket(MATRIX_BUCKET)
|
||||
.key(RESTORE_KEY)
|
||||
.restore_request(RestoreRequest::builder().days(1).build())
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn seed_restore_object(bucket: &str, versioned: bool) -> (MockWarmBackend, Option<String>) {
|
||||
let backend = register_mock_tier(&global_tier_config_mgr(), RESTORE_TIER).await;
|
||||
let store = get_global_app_context().expect("embedded server context").object_store();
|
||||
let uploaded = store
|
||||
.put_object(
|
||||
bucket,
|
||||
RESTORE_KEY,
|
||||
&mut PutObjReader::from_vec(b"archived bytes".to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write restore fixture");
|
||||
let mut opts = ObjectOptions {
|
||||
version_id: uploaded.version_id.map(|version| version.to_string()),
|
||||
versioned,
|
||||
mod_time: uploaded.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
opts.transition.status = "pending".to_owned();
|
||||
opts.transition.tier = RESTORE_TIER.to_owned();
|
||||
opts.transition.etag = uploaded.etag.unwrap_or_default();
|
||||
store
|
||||
.transition_object(bucket, RESTORE_KEY, &opts)
|
||||
.await
|
||||
.expect("transition restore fixture");
|
||||
let archived = store
|
||||
.get_object_info(bucket, RESTORE_KEY, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read transitioned fixture");
|
||||
assert_eq!(archived.transitioned_object.status, "complete");
|
||||
(backend, opts.version_id)
|
||||
}
|
||||
|
||||
async fn restore_admits_once_and_renews_in_place(server: RustFSServer) {
|
||||
const BUCKET: &str = "e2-restore";
|
||||
create_bucket_in_the_store(BUCKET).await;
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(BUCKET)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("enable fixture versioning");
|
||||
let (backend, version) = seed_restore_object(BUCKET, true).await;
|
||||
let version = version.expect("archive has a UUID version");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(RESTORE_KEY)
|
||||
.body(ByteStream::from_static(b"current before restore"))
|
||||
.send()
|
||||
.await
|
||||
.expect("create a distinct current version before restore");
|
||||
let restore = |days| {
|
||||
client
|
||||
.restore_object()
|
||||
.bucket(BUCKET)
|
||||
.key(RESTORE_KEY)
|
||||
.version_id(&version)
|
||||
.restore_request(RestoreRequest::builder().days(days).build())
|
||||
};
|
||||
client.put_bucket_policy().bucket(BUCKET).policy(format!(
|
||||
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Deny","Principal":"*","Action":"s3:RestoreObject","Resource":"arn:aws:s3:::{BUCKET}/{RESTORE_KEY}"}}]}}"#
|
||||
)).send().await.expect("set explicit RestoreObject deny");
|
||||
let denied = outcome(restore(1).send().await);
|
||||
let untouched = client
|
||||
.head_object()
|
||||
.bucket(BUCKET)
|
||||
.key(RESTORE_KEY)
|
||||
.version_id(&version)
|
||||
.send()
|
||||
.await
|
||||
.expect("head denied fixture");
|
||||
assert_eq!(
|
||||
(denied.err().map(|error| (error.status, error.code)), untouched.restore()),
|
||||
(Some((Some(403), Some("AccessDenied".to_owned()))), None),
|
||||
"denied restore changed admission"
|
||||
);
|
||||
assert_eq!(backend.get_count().await, 0, "denial triggered a tier read");
|
||||
client
|
||||
.delete_bucket_policy()
|
||||
.bucket(BUCKET)
|
||||
.send()
|
||||
.await
|
||||
.expect("remove explicit deny");
|
||||
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
let status = StatusCapture::default();
|
||||
let admitted = restore(1)
|
||||
.customize()
|
||||
.interceptor(status.clone())
|
||||
.send()
|
||||
.await
|
||||
.expect("restore admission");
|
||||
assert_eq!(status.take(), Some(200));
|
||||
assert_eq!(admitted.request_charged().map(|value| value.as_str()), Some("requester"));
|
||||
assert_eq!(admitted.restore_output_path(), None);
|
||||
get_barrier.wait_until_paused().await;
|
||||
let conflict = restore(1).send().await.expect_err("duplicate ongoing restore");
|
||||
assert_eq!(refusal(&conflict), (Some("RestoreAlreadyInProgress".to_owned()), Some(409)));
|
||||
let current = client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(RESTORE_KEY)
|
||||
.body(ByteStream::from_static(b"intervening PUT"))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT while copy-back is reading the tier");
|
||||
let replacement_version = current.version_id().expect("replacement UUID version");
|
||||
get_barrier.release();
|
||||
|
||||
let before = tokio::time::timeout(std::time::Duration::from_secs(15), async {
|
||||
loop {
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(BUCKET)
|
||||
.key(RESTORE_KEY)
|
||||
.version_id(&version)
|
||||
.send()
|
||||
.await
|
||||
.expect("head restoring object");
|
||||
if let Some(header) = head
|
||||
.restore()
|
||||
.filter(|header| header.starts_with("ongoing-request=\"false\""))
|
||||
{
|
||||
break header.to_owned();
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("wait for restored copy");
|
||||
let renewed = restore(2)
|
||||
.customize()
|
||||
.interceptor(status.clone())
|
||||
.send()
|
||||
.await
|
||||
.expect("restore renewal");
|
||||
assert_eq!(status.take(), Some(200));
|
||||
assert_eq!(renewed.request_charged().map(|value| value.as_str()), Some("requester"));
|
||||
assert_eq!(renewed.restore_output_path(), None);
|
||||
let output = client
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key(RESTORE_KEY)
|
||||
.version_id(&version)
|
||||
.send()
|
||||
.await
|
||||
.expect("read renewed copy");
|
||||
assert!(
|
||||
output
|
||||
.restore()
|
||||
.is_some_and(|header| header.starts_with("ongoing-request=\"false\"") && header != before)
|
||||
);
|
||||
assert_eq!(
|
||||
output
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect renewed copy")
|
||||
.into_bytes()
|
||||
.as_ref(),
|
||||
b"archived bytes"
|
||||
);
|
||||
assert_eq!(backend.get_count().await, 1, "renewal must reuse the local copy");
|
||||
let current = client
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key(RESTORE_KEY)
|
||||
.send()
|
||||
.await
|
||||
.expect("read intervening PUT");
|
||||
assert_eq!(current.version_id(), Some(replacement_version));
|
||||
assert_eq!(current.restore(), None);
|
||||
assert_eq!(
|
||||
current
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect intervening PUT")
|
||||
.into_bytes()
|
||||
.as_ref(),
|
||||
b"intervening PUT"
|
||||
);
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_admits_and_renews_restore_object() {
|
||||
with_stack(GATEWAY, restore_admits_once_and_renews_in_place);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_admits_and_renews_restore_object() {
|
||||
with_stack(LEGACY, restore_admits_once_and_renews_in_place);
|
||||
}
|
||||
|
||||
const DELETE_BUCKET: &str = "e2-delete-object";
|
||||
/// A bucket name RustFS creates and the AWS grammar refuses (`sthree-` is reserved).
|
||||
const RESERVED_NAME_BUCKET: &str = "sthree-e2-delete";
|
||||
|
||||
Reference in New Issue
Block a user