mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f4266a4576 | |||
| 00ecde1250 | |||
| e9da1d3096 |
@@ -57,7 +57,8 @@ use tracing::{Span, debug, error, info, instrument, warn};
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||
|
||||
#[derive(RustEmbed)]
|
||||
#[folder = "$CARGO_MANIFEST_DIR/static"]
|
||||
#[cfg_attr(test, folder = "$CARGO_MANIFEST_DIR/tests/fixtures/native-console")]
|
||||
#[cfg_attr(not(test), folder = "$CARGO_MANIFEST_DIR/static")]
|
||||
struct StaticFiles;
|
||||
|
||||
/// Static file handler
|
||||
|
||||
@@ -922,6 +922,14 @@ pub(crate) fn claims_route(method: &Method, uri: &Uri, headers: &HeaderMap) -> b
|
||||
|| is_control_plane_route(method, uri, headers)
|
||||
}
|
||||
|
||||
/// Console paths whose requests do not belong to an earlier extension route.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) fn claims_native_console_route(method: &Method, uri: &Uri) -> bool {
|
||||
is_console_path(uri.path())
|
||||
&& parse_object_lambda_request(method, uri).is_none()
|
||||
&& s3_extension::route_key(method, uri).is_none()
|
||||
}
|
||||
|
||||
fn is_public_health_path(path: &str) -> bool {
|
||||
path == HEALTH_PREFIX || path == HEALTH_READY_PATH
|
||||
}
|
||||
|
||||
@@ -332,7 +332,7 @@ async fn delete_objects_resolves_bucket_versioning_once_per_request() {
|
||||
let lookups_before = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst);
|
||||
let snapshots_before = DELETE_CONFIG_SNAPSHOT_LOADS.load(Ordering::SeqCst);
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.execute_delete_objects_from_s3s(req)
|
||||
.await
|
||||
.expect("delete objects should succeed");
|
||||
|
||||
@@ -389,7 +389,7 @@ async fn delete_objects_treats_empty_directory_version_as_absent() {
|
||||
});
|
||||
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.execute_delete_objects_from_s3s(req)
|
||||
.await
|
||||
.expect("empty VersionId should be treated as an absent version");
|
||||
let deleted = response
|
||||
@@ -438,7 +438,7 @@ async fn delete_objects_preserves_explicit_null_directory_version() {
|
||||
});
|
||||
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.execute_delete_objects_from_s3s(req)
|
||||
.await
|
||||
.expect("explicit null VersionId should delete the directory's null version");
|
||||
let deleted = response
|
||||
@@ -839,7 +839,7 @@ async fn delete_objects_replication_config_failure_does_not_call_storage_delete(
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_objects(req)
|
||||
.execute_delete_objects_from_s3s(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject DeleteObjects");
|
||||
|
||||
|
||||
+145
-122
@@ -18,10 +18,14 @@ use super::shared::{LOG_COMPONENT_APP, LOG_SUBSYSTEM_OBJECT};
|
||||
use super::*;
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::storage_api::object_usecase::DeleteObjectCondition;
|
||||
use crate::app::storage_api::s3::dto::ObjectIdentifier;
|
||||
use crate::app::storage_api::s3::dto::{DeleteObjectsInput, DeleteObjectsOutput, DeletedObject};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use crate::app::storage_api::object_usecase::access::AuthorizationRequest as _;
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::dto::{DeleteObjectsInput, DeleteObjectsOutput};
|
||||
use crate::app::storage_api::s3::{S3ErrorCode, s3_error};
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::{S3Request, S3Response};
|
||||
use rustfs_gateway_types::dto as gateway_dto;
|
||||
use rustfs_s3_types::S3Result;
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
|
||||
@@ -100,11 +104,10 @@ fn normalize_delete_object_key(key: &str) -> Cow<'_, str> {
|
||||
}
|
||||
|
||||
fn successful_delete_audit_objects(
|
||||
delete: &s3s::dto::Delete,
|
||||
objects: &[DeleteBatchEntry],
|
||||
successful_results: impl IntoIterator<Item = bool>,
|
||||
) -> Vec<AuditObjectVersion> {
|
||||
delete
|
||||
.objects
|
||||
objects
|
||||
.iter()
|
||||
.zip(successful_results)
|
||||
.filter(|(_, successful)| *successful)
|
||||
@@ -131,7 +134,7 @@ fn normalize_delete_objects_version_id(
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_delete_objects_condition(object: &ObjectIdentifier) -> Result<Option<DeleteObjectCondition>, s3s::dto::Error> {
|
||||
fn parse_delete_objects_condition(object: &DeleteBatchEntry) -> Result<Option<DeleteObjectCondition>, DeleteBatchError> {
|
||||
let Some(etag) = object.e_tag.as_ref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -140,15 +143,15 @@ fn parse_delete_objects_condition(object: &ObjectIdentifier) -> Result<Option<De
|
||||
S3ErrorCode::InvalidRequest,
|
||||
"An ETag condition cannot be combined with an explicit VersionId".to_string(),
|
||||
)
|
||||
} else if let Some(etag) = etag.as_strong() {
|
||||
match DeleteObjectCondition::parse(etag) {
|
||||
} else if !etag.weak {
|
||||
match DeleteObjectCondition::parse(&etag.value) {
|
||||
Ok(condition) => return Ok(Some(condition)),
|
||||
Err(error) => (S3ErrorCode::InvalidArgument, error.to_string()),
|
||||
}
|
||||
} else {
|
||||
(S3ErrorCode::InvalidArgument, "A delete ETag condition must use a strong ETag".to_string())
|
||||
};
|
||||
Err(s3s::dto::Error {
|
||||
Err(DeleteBatchError {
|
||||
code: Some(code.as_str().to_string()),
|
||||
key: Some(object.key.clone()),
|
||||
message: Some(message),
|
||||
@@ -341,7 +344,7 @@ async fn replica_delete_fanout_state(
|
||||
replica_delete_fanout_state_from_config(config, replication_source, version_id, &site_peer_deployment_ids)
|
||||
}
|
||||
|
||||
fn validate_undo_delete_version(expected: Option<&str>, requested: Option<&str>) -> S3Result<()> {
|
||||
fn validate_undo_delete_version(expected: Option<&str>, requested: Option<&str>) -> crate::app::storage_api::s3::S3Result<()> {
|
||||
if expected.is_some() && expected != requested {
|
||||
return Err(s3_error!(PreconditionFailed));
|
||||
}
|
||||
@@ -485,12 +488,12 @@ fn project_delete_objects_pre_stat_error(
|
||||
object: ObjectToDelete,
|
||||
synthetic_version_id: bool,
|
||||
error: ApiError,
|
||||
) -> S3Result<s3s::dto::Error> {
|
||||
) -> S3Result<DeleteBatchError> {
|
||||
// Bucket loss invalidates the shared request, not just one object.
|
||||
if error.code == S3ErrorCode::NoSuchBucket {
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(s3s::dto::Error {
|
||||
Ok(DeleteBatchError {
|
||||
code: Some(error.code.as_str().to_string()),
|
||||
key: Some(object.object_name),
|
||||
message: Some(error.message),
|
||||
@@ -524,12 +527,12 @@ fn reduce_delete_objects_result<'a>(
|
||||
deleted: &'a StorageDeletedObject,
|
||||
error: Option<&EcstoreError>,
|
||||
synthetic_version_id: bool,
|
||||
) -> Result<&'a StorageDeletedObject, s3s::dto::Error> {
|
||||
) -> Result<&'a StorageDeletedObject, DeleteBatchError> {
|
||||
match delete_objects_reported_storage_error(object, error) {
|
||||
None => Ok(deleted),
|
||||
Some(error) => {
|
||||
let api_error = ApiError::from(error.clone());
|
||||
Err(s3s::dto::Error {
|
||||
Err(DeleteBatchError {
|
||||
code: Some(api_error.code.as_str().to_string()),
|
||||
key: Some(object.object_name.clone()),
|
||||
message: Some(api_error.message),
|
||||
@@ -582,50 +585,54 @@ fn delete_objects_storage_error_counts(
|
||||
}
|
||||
|
||||
impl DefaultObjectUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_delete_objects(&self, req: S3Request<DeleteObjectsInput>) -> S3Result<S3Response<DeleteObjectsOutput>> {
|
||||
#[instrument(level = "debug", skip(self, envelope, input))]
|
||||
pub(crate) async fn execute_delete_objects(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
input: DeleteBatchInput,
|
||||
) -> S3Result<DeleteBatchOutput> {
|
||||
let owner = self.clone();
|
||||
run_owned_mutation(async move { Box::pin(owner.execute_delete_objects_owned(req)).await }).await
|
||||
run_owned_mutation(async move { Box::pin(owner.execute_delete_objects_owned(envelope, input)).await }).await
|
||||
}
|
||||
|
||||
async fn execute_delete_objects_owned(
|
||||
&self,
|
||||
mut req: S3Request<DeleteObjectsInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectsOutput>> {
|
||||
mut envelope: RequestEnvelope,
|
||||
input: DeleteBatchInput,
|
||||
) -> S3Result<DeleteBatchOutput> {
|
||||
if let Some(context) = &self.context {
|
||||
let _ = context.object_store();
|
||||
}
|
||||
|
||||
let helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObjects).suppress_event();
|
||||
let request_context = helper.request_context_or_from_request(&req);
|
||||
let (bucket, delete) = {
|
||||
let bucket = req.input.bucket.clone();
|
||||
let delete = req.input.delete.clone();
|
||||
(bucket, delete)
|
||||
};
|
||||
let view = envelope.audit_view();
|
||||
let helper =
|
||||
OperationHelper::from_view(&view, EventName::ObjectRemovedDelete, S3Operation::DeleteObjects).suppress_event();
|
||||
let request_context = helper.request_context_or_from_view(&view);
|
||||
// Both edges carry LastModifiedTime and Size. The batch engine only evaluates ETag.
|
||||
let DeleteBatchInput { bucket, objects, quiet } = input;
|
||||
|
||||
if delete.objects.is_empty() || delete.objects.len() > 1000 {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidArgument,
|
||||
"No objects to delete or too many objects to delete".to_string(),
|
||||
if objects.is_empty() || objects.len() > 1000 {
|
||||
return Err(rustfs_s3_types::s3_error!(
|
||||
InvalidArgument,
|
||||
"No objects to delete or too many objects to delete"
|
||||
));
|
||||
}
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
// Capture the bucket generation before per-object authorization, but
|
||||
// do not expose a bucket-state error unless at least one object is authorized.
|
||||
let bucket_generation = load_bucket_generation_from_store(store.as_ref(), &req, &bucket).await;
|
||||
let bucket_generation = load_bucket_generation_from_store(store.as_ref(), &envelope, &bucket).await;
|
||||
#[cfg(test)]
|
||||
wait_for_delete_objects_generation_test_hook(&bucket).await;
|
||||
|
||||
let bypass_governance = has_bypass_governance_header(&req.headers);
|
||||
let bypass_governance = has_bypass_governance_header(envelope.headers());
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
struct DeleteResult {
|
||||
delete_object: Option<StorageDeletedObject>,
|
||||
error: Option<s3s::dto::Error>,
|
||||
error: Option<DeleteBatchError>,
|
||||
emit_side_effects: bool,
|
||||
synthetic_version_id: bool,
|
||||
version_suspended: bool,
|
||||
@@ -636,29 +643,40 @@ impl DefaultObjectUsecase {
|
||||
requested_version_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
let mut delete_results = vec![DeleteResult::default(); delete.objects.len()];
|
||||
let mut delete_results = vec![DeleteResult::default(); objects.len()];
|
||||
|
||||
struct AuthorizedDelete {
|
||||
idx: usize,
|
||||
object: ObjectToDelete,
|
||||
}
|
||||
|
||||
let mut authorized_deletes = Vec::with_capacity(delete.objects.len());
|
||||
let mut authorized_deletes = Vec::with_capacity(objects.len());
|
||||
// Issue #5740: keep the first per-key denial of this bulk request at
|
||||
// warn and demote the rest to debug, so a denied 1000-key DeleteObjects
|
||||
// cannot flood the log.
|
||||
let mut bulk_denial_logged = false;
|
||||
// Per-key denials are audited once per denied action, in aggregate, after the loop.
|
||||
let mut bulk_denials = crate::storage::access::BulkDenialAudit::start(&mut req)?;
|
||||
for (idx, obj_id) in delete.objects.iter().enumerate() {
|
||||
let mut bulk_denials = crate::storage::access::BulkDenialAudit::start_owned(&mut envelope)?;
|
||||
for (idx, obj_id) in objects.iter().enumerate() {
|
||||
// Authorization and every storage side effect must address the same
|
||||
// normalized object; retain the request key only for response entries.
|
||||
let key = normalize_delete_object_key(&obj_id.key);
|
||||
if let Err(err) = validate_object_key(&key, "DeleteObjects") {
|
||||
delete_results[idx].error = Some(s3s::dto::Error {
|
||||
code: Some(err.code().as_str().to_string()),
|
||||
// A rejected wire key has no authorization resource. Its normalized text cannot become a target.
|
||||
let key_error = if obj_id.key_too_long {
|
||||
Some((
|
||||
S3ErrorCode::InvalidArgument,
|
||||
ApiError::error_code_to_message(&S3ErrorCode::InvalidArgument),
|
||||
))
|
||||
} else {
|
||||
validate_object_key(&key, "DeleteObjects")
|
||||
.err()
|
||||
.map(|error| (error.code().clone(), error.to_string()))
|
||||
};
|
||||
if let Some((code, message)) = key_error {
|
||||
delete_results[idx].error = Some(DeleteBatchError {
|
||||
code: Some(code.as_str().to_string()),
|
||||
key: Some(obj_id.key.clone()),
|
||||
message: Some(err.to_string()),
|
||||
message: Some(message),
|
||||
version_id: obj_id.version_id.clone(),
|
||||
});
|
||||
continue;
|
||||
@@ -674,7 +692,7 @@ impl DefaultObjectUsecase {
|
||||
let (version_id, version_uuid) = match normalize_delete_objects_version_id(raw_version_id.clone()) {
|
||||
Ok(parsed) => parsed,
|
||||
Err(err) => {
|
||||
delete_results[idx].error = Some(s3s::dto::Error {
|
||||
delete_results[idx].error = Some(DeleteBatchError {
|
||||
code: Some("NoSuchVersion".to_string()),
|
||||
key: Some(obj_id.key.clone()),
|
||||
message: Some(err),
|
||||
@@ -685,29 +703,29 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
|
||||
{
|
||||
let req_info = req_info_mut(&mut req)?;
|
||||
let req_info = req_info_mut(&mut envelope)?;
|
||||
req_info.bucket = Some(bucket.clone());
|
||||
req_info.object = Some(key.to_string());
|
||||
req_info.version_id = version_id.clone();
|
||||
}
|
||||
|
||||
let mut denied_action = delete_object_authorize_action(version_id.as_deref());
|
||||
let mut auth_res = authorize_request(&mut req, denied_action).await;
|
||||
let mut auth_res = authorize_request(&mut envelope, denied_action).await;
|
||||
if auth_res.is_ok()
|
||||
&& delete_condition
|
||||
.as_ref()
|
||||
.is_some_and(DeleteObjectCondition::requires_read_permission)
|
||||
{
|
||||
denied_action = Action::S3Action(S3Action::GetObjectAction);
|
||||
auth_res = authorize_request(&mut req, denied_action).await;
|
||||
auth_res = authorize_request(&mut envelope, denied_action).await;
|
||||
}
|
||||
if auth_res.is_err() {
|
||||
bulk_denials.denied(denied_action);
|
||||
if !bulk_denial_logged {
|
||||
bulk_denial_logged = true;
|
||||
req_info_mut(&mut req)?.suppress_denial_log = true;
|
||||
req_info_mut(&mut envelope)?.suppress_denial_log = true;
|
||||
}
|
||||
delete_results[idx].error = Some(s3s::dto::Error {
|
||||
delete_results[idx].error = Some(DeleteBatchError {
|
||||
code: Some("AccessDenied".to_string()),
|
||||
key: Some(obj_id.key.clone()),
|
||||
message: Some("Access Denied".to_string()),
|
||||
@@ -717,14 +735,15 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
if bypass_governance {
|
||||
let auth_res = authorize_request(&mut req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await;
|
||||
let auth_res =
|
||||
authorize_request(&mut envelope, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await;
|
||||
if auth_res.is_err() {
|
||||
bulk_denials.denied(Action::S3Action(S3Action::BypassGovernanceRetentionAction));
|
||||
if !bulk_denial_logged {
|
||||
bulk_denial_logged = true;
|
||||
req_info_mut(&mut req)?.suppress_denial_log = true;
|
||||
req_info_mut(&mut envelope)?.suppress_denial_log = true;
|
||||
}
|
||||
delete_results[idx].error = Some(s3s::dto::Error {
|
||||
delete_results[idx].error = Some(DeleteBatchError {
|
||||
code: Some("AccessDenied".to_string()),
|
||||
key: Some(obj_id.key.clone()),
|
||||
message: Some("Access Denied".to_string()),
|
||||
@@ -735,7 +754,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
if let Err(err) = validate_table_catalog_object_mutation(&bucket, &key).await {
|
||||
delete_results[idx].error = Some(s3s::dto::Error {
|
||||
delete_results[idx].error = Some(DeleteBatchError {
|
||||
code: Some("InvalidRequest".to_string()),
|
||||
key: Some(obj_id.key.clone()),
|
||||
message: Some(err.to_string()),
|
||||
@@ -758,19 +777,18 @@ impl DefaultObjectUsecase {
|
||||
authorized_deletes.push(AuthorizedDelete { idx, object });
|
||||
}
|
||||
|
||||
bulk_denials.finish(&mut req)?;
|
||||
bulk_denials.finish_owned(&mut envelope, RequestEnvelope::audit_view)?;
|
||||
|
||||
if authorized_deletes.is_empty() {
|
||||
let output = DeleteObjectsOutput {
|
||||
deleted: (!delete.quiet.unwrap_or_default()).then(Vec::new),
|
||||
let output = DeleteBatchOutput {
|
||||
deleted: (!quiet.unwrap_or_default()).then(Vec::new),
|
||||
errors: Some(delete_results.into_iter().filter_map(|result| result.error).collect()),
|
||||
..Default::default()
|
||||
};
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
let result = Ok(output);
|
||||
let _ = helper.complete_owned(&result, StatusCode::OK, &HeaderMap::new());
|
||||
return result;
|
||||
}
|
||||
req.extensions.insert(bucket_generation?);
|
||||
envelope.install_bucket_generation(bucket_generation?);
|
||||
let bucket_lock_enabled = object_lock_checks_required(&bucket).await;
|
||||
|
||||
let delete_config_snapshot = Arc::new(
|
||||
@@ -795,7 +813,7 @@ impl DefaultObjectUsecase {
|
||||
// DeleteObjects names each key explicitly. The force-delete header is a
|
||||
// DeleteObject/DeleteBucket extension and must not change this batch,
|
||||
// including by rejecting a non-boolean value.
|
||||
let mut batch_headers = req.headers.clone();
|
||||
let mut batch_headers = envelope.headers().clone();
|
||||
batch_headers.remove("x-rustfs-force-delete");
|
||||
batch_headers.remove("x-minio-force-delete");
|
||||
let mut prepared_deletes: Vec<PreparedDelete> = Vec::with_capacity(authorized_deletes.len());
|
||||
@@ -913,7 +931,7 @@ impl DefaultObjectUsecase {
|
||||
object_lock_delete: Some(StorageObjectLockDeleteOptions { bypass_governance }),
|
||||
..Default::default()
|
||||
};
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut storage_delete_opts)?;
|
||||
apply_bucket_generation_guard(&envelope, &bucket, &mut storage_delete_opts)?;
|
||||
let storage_started = std::time::Instant::now();
|
||||
let (dobjs, errs, accounting) = store
|
||||
.delete_objects_with_tier_delete_journal_and_accounting(&bucket, object_to_delete.clone(), storage_delete_opts)
|
||||
@@ -952,8 +970,8 @@ impl DefaultObjectUsecase {
|
||||
super::super::storage_api::object_usecase::helper::pause_committed_mutation_for_test(&bucket).await;
|
||||
let _manager = get_concurrency_manager();
|
||||
if !errs.is_empty() && errs.iter().all(|err| err.as_ref().is_some_and(is_err_bucket_not_found)) {
|
||||
let result = Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "Bucket not found".to_string()));
|
||||
let _ = helper.complete(&result);
|
||||
let result = Err(rustfs_s3_types::s3_error!(NoSuchBucket, "Bucket not found"));
|
||||
let _ = helper.complete_owned(&result, StatusCode::OK, &HeaderMap::new());
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -1001,19 +1019,19 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
let deleted = (!delete.quiet.unwrap_or_default()).then(|| {
|
||||
let deleted = (!quiet.unwrap_or_default()).then(|| {
|
||||
delete_results
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(idx, result)| result.delete_object.as_ref().map(|object| (idx, result, object)))
|
||||
.map(|(idx, result, object)| DeletedObject {
|
||||
.map(|(idx, result, object)| DeleteBatchSuccess {
|
||||
delete_marker: { if object.delete_marker { Some(true) } else { None } },
|
||||
delete_marker_version_id: delete_marker_response_version_id(
|
||||
object.delete_marker_version_id,
|
||||
object.delete_marker && result.version_suspended && result.requested_version_id.is_none(),
|
||||
result.synthetic_version_id,
|
||||
),
|
||||
key: Some(delete.objects[idx].key.clone()),
|
||||
key: Some(objects[idx].key.clone()),
|
||||
version_id: delete_entry_response_version_id(
|
||||
object.version_id,
|
||||
object.delete_marker,
|
||||
@@ -1034,18 +1052,17 @@ impl DefaultObjectUsecase {
|
||||
.enumerate()
|
||||
.filter_map(|(idx, result)| {
|
||||
let mut error = result.error.clone()?;
|
||||
error.key = Some(delete.objects[idx].key.clone());
|
||||
error.key = Some(objects[idx].key.clone());
|
||||
Some(error)
|
||||
})
|
||||
.collect::<Vec<s3s::dto::Error>>();
|
||||
let output = DeleteObjectsOutput {
|
||||
.collect::<Vec<_>>();
|
||||
let output = DeleteBatchOutput {
|
||||
deleted,
|
||||
errors: Some(errors),
|
||||
..Default::default()
|
||||
};
|
||||
let helper = if helper.wants_audit_object_info() {
|
||||
let audit_objects =
|
||||
successful_delete_audit_objects(&delete, delete_results.iter().map(|result| result.delete_object.is_some()));
|
||||
successful_delete_audit_objects(&objects, delete_results.iter().map(|result| result.delete_object.is_some()));
|
||||
helper.audit_objects(audit_objects)
|
||||
} else {
|
||||
helper
|
||||
@@ -1074,11 +1091,10 @@ impl DefaultObjectUsecase {
|
||||
let _ = replication_task.await;
|
||||
}
|
||||
|
||||
let req_headers = req.headers.clone();
|
||||
let req_headers = envelope.headers().clone();
|
||||
let notify = current_notify_interface_for_context(self.context.as_deref());
|
||||
let req_params = rustfs_targets::extract_params_header(&req_headers);
|
||||
let resp_elements =
|
||||
build_event_resp_elements(&S3Response::new(DeleteObjectsOutput::default()), &request_context.request_id);
|
||||
let resp_elements = build_owned_event_resp_elements(&HeaderMap::new(), &request_context.request_id);
|
||||
let deleted_any = delete_results.iter().any(|result| result.delete_object.is_some());
|
||||
let delete_producers = delete_results
|
||||
.iter()
|
||||
@@ -1143,8 +1159,8 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
});
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
let result = Ok(output);
|
||||
let _ = helper.complete_owned(&result, StatusCode::OK, &HeaderMap::new());
|
||||
if deleted_any {
|
||||
rustfs_scanner::record_dirty_usage_bucket_from_producers(&bucket, delete_producers);
|
||||
}
|
||||
@@ -1600,12 +1616,20 @@ impl DefaultObjectUsecase {
|
||||
|
||||
#[cfg(test)]
|
||||
impl DefaultObjectUsecase {
|
||||
pub(crate) async fn execute_delete_objects_from_s3s(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectsInput>,
|
||||
) -> crate::app::storage_api::s3::S3Result<S3Response<DeleteObjectsOutput>> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
Ok(S3Response::new(self.execute_delete_objects(envelope, req.input.into()).await?.into()))
|
||||
}
|
||||
|
||||
/// DeleteObject for a request built as the legacy stack receives it, converted the way its
|
||||
/// edge converts it.
|
||||
pub(crate) async fn execute_delete_object_from_s3s(
|
||||
&self,
|
||||
req: S3Request<crate::app::storage_api::s3::dto::DeleteObjectInput>,
|
||||
) -> S3Result<gateway_dto::DeleteObjectOutput> {
|
||||
) -> crate::app::storage_api::s3::S3Result<gateway_dto::DeleteObjectOutput> {
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let input = req.input;
|
||||
Ok(Box::pin(self.execute_delete_object_fields(envelope, input.bucket, input.key, input.version_id)).await?)
|
||||
@@ -1843,7 +1867,7 @@ mod tests {
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
let output = Box::pin(usecase.execute_delete_objects(req))
|
||||
let output = Box::pin(usecase.execute_delete_objects_from_s3s(req))
|
||||
.await
|
||||
.expect("batch delete")
|
||||
.output;
|
||||
@@ -2371,7 +2395,7 @@ mod tests {
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
let response = Box::pin(usecase.execute_delete_objects(req))
|
||||
let response = Box::pin(usecase.execute_delete_objects_from_s3s(req))
|
||||
.await
|
||||
.expect("batch marker delete must reach the authoritative mutation");
|
||||
assert!(response.output.errors.as_ref().is_none_or(Vec::is_empty), "{:?}", response.output.errors);
|
||||
@@ -2552,7 +2576,7 @@ mod tests {
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
let response = Box::pin(usecase.execute_delete_objects(req))
|
||||
let response = Box::pin(usecase.execute_delete_objects_from_s3s(req))
|
||||
.await
|
||||
.expect("a pre-stat failure must not abort independent batch entries");
|
||||
let deleted = response.output.deleted.expect("per-key successes");
|
||||
@@ -2586,7 +2610,7 @@ mod tests {
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
let response = Box::pin(usecase.execute_delete_objects(req))
|
||||
let response = Box::pin(usecase.execute_delete_objects_from_s3s(req))
|
||||
.await
|
||||
.expect("all pre-stat failures still produce per-key results");
|
||||
assert!(response.output.deleted.as_ref().is_none_or(Vec::is_empty));
|
||||
@@ -2654,7 +2678,7 @@ mod tests {
|
||||
ApiError::from(StorageError::BucketNotFound("missing".to_string())),
|
||||
)
|
||||
.expect_err("shared bucket loss remains a request failure");
|
||||
assert_eq!(error.code(), &S3ErrorCode::NoSuchBucket);
|
||||
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::NoSuchBucket);
|
||||
|
||||
let version = Uuid::new_v4();
|
||||
for (version_id, synthetic, expected) in [
|
||||
@@ -2818,7 +2842,7 @@ mod tests {
|
||||
let req = build_request(input, Method::POST);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_delete_objects(req).await.unwrap_err();
|
||||
let err = usecase.execute_delete_objects_from_s3s(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
@@ -2840,7 +2864,7 @@ mod tests {
|
||||
let req = build_request(input, Method::POST);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_delete_objects(req).await.unwrap_err();
|
||||
let err = usecase.execute_delete_objects_from_s3s(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
@@ -2862,7 +2886,7 @@ mod tests {
|
||||
let req = build_request(input, Method::POST);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_delete_objects(req).await.unwrap_err();
|
||||
let err = usecase.execute_delete_objects_from_s3s(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(err.message(), Some("Not init"));
|
||||
}
|
||||
@@ -2920,7 +2944,7 @@ mod tests {
|
||||
install_delete_objects_generation_test_hook(bucket.clone(), Arc::clone(&loaded), Arc::clone(&resume));
|
||||
|
||||
let usecase = DefaultObjectUsecase::with_context(Some(context));
|
||||
let delete = tokio::spawn(async move { usecase.execute_delete_objects(req).await });
|
||||
let delete = tokio::spawn(async move { usecase.execute_delete_objects_from_s3s(req).await });
|
||||
loaded.wait().await;
|
||||
|
||||
store
|
||||
@@ -3046,13 +3070,8 @@ mod tests {
|
||||
},
|
||||
];
|
||||
|
||||
let objects = successful_delete_audit_objects(
|
||||
&Delete {
|
||||
objects: requested,
|
||||
quiet: Some(true),
|
||||
},
|
||||
[true, false, true],
|
||||
);
|
||||
let requested: Vec<_> = requested.into_iter().map(Into::into).collect();
|
||||
let objects = successful_delete_audit_objects(&requested, [true, false, true]);
|
||||
|
||||
assert_eq!(
|
||||
objects,
|
||||
@@ -3071,16 +3090,8 @@ mod tests {
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
assert!(
|
||||
successful_delete_audit_objects(
|
||||
&Delete {
|
||||
objects: requested,
|
||||
quiet: None,
|
||||
},
|
||||
[false]
|
||||
)
|
||||
.is_empty()
|
||||
);
|
||||
let requested: Vec<_> = requested.into_iter().map(Into::into).collect();
|
||||
assert!(successful_delete_audit_objects(&requested, [false]).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3103,22 +3114,31 @@ mod tests {
|
||||
key: "key".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(parse_delete_objects_condition(&object).expect("missing ETag is unconditional"), None);
|
||||
assert_eq!(
|
||||
parse_delete_objects_condition(&object.clone().into()).expect("missing ETag is unconditional"),
|
||||
None
|
||||
);
|
||||
for etag in ["", "\"\"", " \t "] {
|
||||
let error = parse_delete_objects_condition(&ObjectIdentifier {
|
||||
e_tag: Some(ETag::Strong(etag.to_string())),
|
||||
..object.clone()
|
||||
})
|
||||
let error = parse_delete_objects_condition(
|
||||
&ObjectIdentifier {
|
||||
e_tag: Some(ETag::Strong(etag.to_string())),
|
||||
..object.clone()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.expect_err("an explicit empty ETag must never become an unconditional delete");
|
||||
assert_eq!(error.code.as_deref(), Some("InvalidArgument"));
|
||||
assert_eq!(error.key.as_deref(), Some("key"));
|
||||
}
|
||||
for version in ["null", "", "bad-version"] {
|
||||
let error = parse_delete_objects_condition(&ObjectIdentifier {
|
||||
e_tag: Some(ETag::Strong("*".to_string())),
|
||||
version_id: Some(version.to_string()),
|
||||
..object.clone()
|
||||
})
|
||||
let error = parse_delete_objects_condition(
|
||||
&ObjectIdentifier {
|
||||
e_tag: Some(ETag::Strong("*".to_string())),
|
||||
version_id: Some(version.to_string()),
|
||||
..object.clone()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.expect_err("any explicit VersionId is unsupported with a condition");
|
||||
assert_eq!(error.code.as_deref(), Some("InvalidRequest"));
|
||||
assert_eq!(error.version_id.as_deref(), Some(version));
|
||||
@@ -3159,7 +3179,7 @@ mod tests {
|
||||
e_tag: Some(ETag::Weak(value.to_string())),
|
||||
..Default::default()
|
||||
};
|
||||
let error = parse_delete_objects_condition(&object)
|
||||
let error = parse_delete_objects_condition(&object.into())
|
||||
.expect_err("a weak validator must fail instead of falling back to an unconditional delete");
|
||||
assert_eq!(error.code.as_deref(), Some("InvalidArgument"));
|
||||
assert_eq!(error.key.as_deref(), Some("key"));
|
||||
@@ -3170,7 +3190,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
parse_delete_objects_condition(&object).expect("strong validator retains its condition"),
|
||||
parse_delete_objects_condition(&object.into()).expect("strong validator retains its condition"),
|
||||
Some(DeleteObjectCondition::MatchEtag("current-etag".to_string()))
|
||||
);
|
||||
}
|
||||
@@ -3398,7 +3418,7 @@ mod tests {
|
||||
req.extensions.insert(crate::storage::access::ReqInfo::default());
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.execute_delete_objects_from_s3s(req)
|
||||
.await
|
||||
.expect("mixed batch returns per-entry outcomes");
|
||||
let deleted = response
|
||||
@@ -3470,7 +3490,10 @@ mod tests {
|
||||
.expect("build quiet mixed batch");
|
||||
let mut quiet_req = build_request(quiet_input, Method::POST);
|
||||
quiet_req.extensions.insert(crate::storage::access::ReqInfo::default());
|
||||
let quiet_response = usecase.execute_delete_objects(quiet_req).await.expect("quiet batch succeeds");
|
||||
let quiet_response = usecase
|
||||
.execute_delete_objects_from_s3s(quiet_req)
|
||||
.await
|
||||
.expect("quiet batch succeeds");
|
||||
assert!(quiet_response.output.deleted.is_none(), "Quiet hides only successful entries");
|
||||
let quiet_errors = quiet_response.output.errors.expect("Quiet preserves errors");
|
||||
assert_eq!(quiet_errors.len(), 1);
|
||||
@@ -3562,7 +3585,7 @@ mod tests {
|
||||
let mut request = build_request(input, Method::POST);
|
||||
request.uri = format!("/{request_bucket}").parse().expect("URI");
|
||||
request.extensions.insert(crate::storage::access::ReqInfo::default());
|
||||
usecase.execute_delete_objects(request).await.map(|_| ())
|
||||
usecase.execute_delete_objects_from_s3s(request).await.map(|_| ())
|
||||
} else {
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(request_bucket.clone())
|
||||
@@ -3942,7 +3965,7 @@ mod tests {
|
||||
..Default::default()
|
||||
});
|
||||
DefaultObjectUsecase::from_global()
|
||||
.execute_delete_objects(batch_req)
|
||||
.execute_delete_objects_from_s3s(batch_req)
|
||||
.await
|
||||
.expect("batch compressed delete should succeed");
|
||||
assert_eq!(
|
||||
@@ -4322,10 +4345,10 @@ mod bulk_denial_audit_wiring_tests {
|
||||
.next()
|
||||
.unwrap();
|
||||
let started = source
|
||||
.find("BulkDenialAudit::start(&mut req)")
|
||||
.find("BulkDenialAudit::start_owned(&mut envelope)")
|
||||
.expect("per-key denial audit is not suppressed");
|
||||
let finished = source
|
||||
.find("bulk_denials.finish(&mut req)")
|
||||
.find("bulk_denials.finish_owned(&mut envelope, RequestEnvelope::audit_view)")
|
||||
.expect("the aggregate entry is not emitted");
|
||||
assert!(started < finished, "the aggregate must be emitted after the per-key checks");
|
||||
assert_eq!(
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Owned input and result fields for the batch deletion engine.
|
||||
//!
|
||||
//! Keys and versions stay raw so legacy refusals can echo entries that did not pass validation.
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct DeleteBatchInput {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) objects: Vec<DeleteBatchEntry>,
|
||||
pub(crate) quiet: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct DeleteBatchEntry {
|
||||
pub(crate) key: String,
|
||||
pub(crate) key_too_long: bool,
|
||||
pub(crate) version_id: Option<String>,
|
||||
pub(crate) e_tag: Option<DeleteBatchETag>,
|
||||
/// Seconds since the Unix epoch and nanoseconds within that second.
|
||||
pub(crate) _last_modified_time: Option<(i64, u32)>,
|
||||
pub(crate) _size: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct DeleteBatchETag {
|
||||
pub(crate) value: String,
|
||||
pub(crate) weak: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct DeleteBatchOutput {
|
||||
pub(crate) deleted: Option<Vec<DeleteBatchSuccess>>,
|
||||
pub(crate) errors: Option<Vec<DeleteBatchError>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct DeleteBatchSuccess {
|
||||
pub(crate) delete_marker: Option<bool>,
|
||||
pub(crate) delete_marker_version_id: Option<String>,
|
||||
pub(crate) key: Option<String>,
|
||||
pub(crate) version_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub(crate) struct DeleteBatchError {
|
||||
pub(crate) code: Option<String>,
|
||||
pub(crate) key: Option<String>,
|
||||
pub(crate) message: Option<String>,
|
||||
pub(crate) version_id: Option<String>,
|
||||
}
|
||||
@@ -86,8 +86,7 @@ use super::storage_api::object_usecase::error::{
|
||||
};
|
||||
use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
|
||||
use super::storage_api::object_usecase::helper::{
|
||||
OperationHelper, build_event_resp_elements, build_owned_event_resp_elements, run_owned_mutation,
|
||||
spawn_background_with_context,
|
||||
OperationHelper, build_owned_event_resp_elements, run_owned_mutation, spawn_background_with_context,
|
||||
};
|
||||
use super::storage_api::object_usecase::io::{
|
||||
DynReader, EtagAlgorithm, HashReader, WritePlan, compression_metadata_value, wrap_reader,
|
||||
@@ -220,6 +219,8 @@ pub(crate) use copy_input::{CopyObjectWriteInput, CopyObjectWriteOutput, CopySou
|
||||
#[cfg(all(any(feature = "cuobject", feature = "nixl"), target_os = "linux"))]
|
||||
pub(crate) mod cuobject;
|
||||
mod delete;
|
||||
mod delete_batch_types;
|
||||
pub(crate) use delete_batch_types::*;
|
||||
mod extract;
|
||||
mod get;
|
||||
mod get_response;
|
||||
|
||||
@@ -1638,8 +1638,7 @@ pub(crate) mod helper {
|
||||
MutationCompletionProbe, capture_mutation_event_for_test, pause_committed_mutation_for_test,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::helper_consumer::{
|
||||
OperationHelper, build_event_resp_elements, build_owned_event_resp_elements, run_owned_mutation,
|
||||
spawn_background_with_context,
|
||||
OperationHelper, build_owned_event_resp_elements, run_owned_mutation, spawn_background_with_context,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+275
-111
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod owned;
|
||||
|
||||
use crate::runtime_sources::current_action_credentials;
|
||||
#[cfg(feature = "webdav")]
|
||||
use crate::runtime_sources::current_object_store_handle;
|
||||
@@ -494,38 +496,13 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
"Protocol storage client request"
|
||||
);
|
||||
|
||||
let mut builder = GetObjectInput::builder().bucket(bucket.to_string()).key(key.to_string());
|
||||
|
||||
// Add range header if start_pos is specified
|
||||
if let Some(start) = start_pos {
|
||||
let range = s3s::dto::Range::Int {
|
||||
first: start,
|
||||
last: None, // Read to end of file
|
||||
};
|
||||
builder = builder.range(Some(range));
|
||||
}
|
||||
|
||||
let input = builder.build().map_err(|e| {
|
||||
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build GetObjectInput: {}", e))
|
||||
})?;
|
||||
|
||||
let uri = build_object_uri(bucket, key, &[])?;
|
||||
let req = Self::create_request(
|
||||
input,
|
||||
Method::GET,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_string()),
|
||||
object: Some(key.to_string()),
|
||||
credentials,
|
||||
},
|
||||
)?;
|
||||
|
||||
match self.fs.get_object(req).await {
|
||||
Ok(response) => gateway_s3s_compat::generated::ops::get_object::output_from_s3s(response.output)
|
||||
.map_err(|e| conversion_error("GetObject output", e)),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
self.get_object_owned(
|
||||
bucket,
|
||||
key,
|
||||
credentials,
|
||||
start_pos.map(|first| rustfs_gateway_types::ByteRange::From { first }),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
@@ -588,31 +565,20 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
) -> Result<gateway_dto::DeleteObjectOutput, Self::Error> {
|
||||
trace_protocol_request("delete_object", Some(bucket), Some(key));
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.key(key.to_string())
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build DeleteObjectInput: {}", e))
|
||||
})?;
|
||||
|
||||
let uri = build_object_uri(bucket, key, &[])?;
|
||||
let req = Self::create_request(
|
||||
input,
|
||||
let envelope = Self::create_envelope(
|
||||
Method::DELETE,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_string()),
|
||||
object: Some(key.to_string()),
|
||||
bucket: Some(bucket.to_owned()),
|
||||
object: Some(key.to_owned()),
|
||||
credentials,
|
||||
},
|
||||
)?;
|
||||
|
||||
match self.fs.delete_object(req).await {
|
||||
Ok(response) => gateway_s3s_compat::generated::ops::delete_object::output_from_s3s(response.output)
|
||||
.map_err(|e| conversion_error("DeleteObject output", e)),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
);
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::DeleteObject);
|
||||
self.owned_usecase()
|
||||
.execute_delete_object_fields(envelope, bucket.to_owned(), key.to_owned(), None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn head_object(
|
||||
@@ -623,31 +589,32 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
) -> Result<gateway_dto::HeadObjectOutput, Self::Error> {
|
||||
trace_protocol_request("head_object", Some(bucket), Some(key));
|
||||
|
||||
let input = HeadObjectInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.key(key.to_string())
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build HeadObjectInput: {}", e))
|
||||
})?;
|
||||
|
||||
let uri = build_object_uri(bucket, key, &[])?;
|
||||
let req = Self::create_request(
|
||||
input,
|
||||
let envelope = Self::create_envelope(
|
||||
Method::HEAD,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_string()),
|
||||
object: Some(key.to_string()),
|
||||
bucket: Some(bucket.to_owned()),
|
||||
object: Some(key.to_owned()),
|
||||
credentials,
|
||||
},
|
||||
)?;
|
||||
|
||||
match self.fs.head_object(req).await {
|
||||
Ok(response) => gateway_s3s_compat::generated::ops::head_object::output_from_s3s(response.output)
|
||||
.map_err(|e| conversion_error("HeadObject output", e)),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
);
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::HeadObject);
|
||||
self.owned_usecase()
|
||||
.execute_head_object_fields(
|
||||
envelope,
|
||||
crate::app::object_usecase::HeadObjectFields {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
version_id: None,
|
||||
part_number: None,
|
||||
range: None,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.and_then(|answered| owned::head_output(answered.output))
|
||||
}
|
||||
|
||||
async fn head_bucket(
|
||||
@@ -891,37 +858,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
"Protocol storage client request"
|
||||
);
|
||||
|
||||
let range = s3s::dto::Range::Int {
|
||||
first: start_pos,
|
||||
last: Some(start_pos + length - 1),
|
||||
};
|
||||
|
||||
let input = GetObjectInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.key(key.to_string())
|
||||
.range(Some(range))
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build GetObjectInput: {}", e))
|
||||
})?;
|
||||
|
||||
let uri = build_object_uri(bucket, key, &[])?;
|
||||
let req = Self::create_request(
|
||||
input,
|
||||
Method::GET,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_string()),
|
||||
object: Some(key.to_string()),
|
||||
credentials,
|
||||
},
|
||||
)?;
|
||||
|
||||
match self.fs.get_object(req).await {
|
||||
Ok(response) => gateway_s3s_compat::generated::ops::get_object::output_from_s3s(response.output)
|
||||
.map_err(|e| conversion_error("GetObject output", e)),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
self.get_object_owned(
|
||||
bucket,
|
||||
key,
|
||||
credentials,
|
||||
Some(rustfs_gateway_types::ByteRange::FromTo {
|
||||
first: start_pos,
|
||||
last: start_pos + length - 1,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn copy_object(
|
||||
@@ -1403,16 +1349,30 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.expect("request should build");
|
||||
let request_info = request.extensions.get::<ReqInfo>().expect("request info should be present");
|
||||
let copied = request_info.cred.as_ref().expect("credentials should be present");
|
||||
|
||||
assert_eq!(copied.access_key, credentials.access_key);
|
||||
assert_eq!(copied.secret_key, credentials.secret_key);
|
||||
assert_eq!(copied.session_token, credentials.session_token);
|
||||
assert_eq!(copied.parent_user, credentials.parent_user);
|
||||
assert_eq!(copied.groups, credentials.groups);
|
||||
assert_eq!(copied.claims, Some(claims));
|
||||
assert!(copied.is_service_account());
|
||||
let envelope = ProtocolStorageClient::create_envelope(
|
||||
Method::GET,
|
||||
http::Uri::from_static("/bucket?list-type=2"),
|
||||
RequestParams {
|
||||
bucket: Some("bucket".to_owned()),
|
||||
object: None,
|
||||
credentials: &credentials,
|
||||
},
|
||||
);
|
||||
assert_eq!(envelope.uri(), &request.uri);
|
||||
assert_eq!(envelope.headers(), &request.headers);
|
||||
for request_info in [
|
||||
request.extensions.get::<ReqInfo>().expect("request info should be present"),
|
||||
envelope.req_info().expect("owned request info"),
|
||||
] {
|
||||
let copied = request_info.cred.as_ref().expect("credentials should be present");
|
||||
assert_eq!(copied.access_key, credentials.access_key);
|
||||
assert_eq!(copied.secret_key, credentials.secret_key);
|
||||
assert_eq!(copied.session_token, credentials.session_token);
|
||||
assert_eq!(copied.parent_user, credentials.parent_user);
|
||||
assert_eq!(copied.groups, credentials.groups);
|
||||
assert_eq!(copied.claims, Some(claims.clone()));
|
||||
assert!(copied.is_service_account());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "webdav", feature = "sftp"))]
|
||||
@@ -1523,4 +1483,208 @@ mod tests {
|
||||
|
||||
assert_eq!(uri.to_string(), "/bucket?list-type=2");
|
||||
}
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_markers_versions_and_errors_match_legacy() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-oracle", || async {
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::protocols::client::{
|
||||
BucketOperations, MakeBucketOptions, ObjectIO, StorageObjectOptions, StoragePutObjReader,
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_protocols::common::client::s3::StorageBackend;
|
||||
let root = rustfs_credentials::Credentials {
|
||||
access_key: "protocol-oracle-root".into(),
|
||||
secret_key: "protocol-oracle-secret".into(),
|
||||
status: "on".into(),
|
||||
..Default::default()
|
||||
};
|
||||
let (_temp, context, _iam) = crate::protocols::policy_test::protocol_test_context(root.clone()).await;
|
||||
let store = context.object_store();
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let fs = FS::with_server_ctx(slot);
|
||||
let backend = ProtocolStorageClient::new(fs.clone());
|
||||
for (bucket, key, data, versioned) in [
|
||||
("protocol-version-oracle", "versioned-key", b"versioned data".as_slice(), true),
|
||||
("protocol-marker-oracle", "folder/", b"".as_slice(), false),
|
||||
] {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: versioned,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let mut reader = StoragePutObjReader::from_vec(data.to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
key,
|
||||
&mut reader,
|
||||
&StorageObjectOptions {
|
||||
versioned,
|
||||
mod_time: Some(
|
||||
time::OffsetDateTime::from_unix_timestamp(1_700_000_000)
|
||||
.expect("valid test time")
|
||||
.replace_nanosecond(123_456_789)
|
||||
.expect("valid nanoseconds"),
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed object");
|
||||
let params = || RequestParams {
|
||||
bucket: Some(bucket.to_owned()),
|
||||
object: Some(key.to_owned()),
|
||||
credentials: &root,
|
||||
};
|
||||
let uri = build_object_uri(bucket, key, &[]).expect("object URI");
|
||||
let old = fs
|
||||
.get_object(
|
||||
ProtocolStorageClient::create_request(
|
||||
GetObjectInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.key(key.to_owned())
|
||||
.build()
|
||||
.expect("old GET input"),
|
||||
Method::GET,
|
||||
uri.clone(),
|
||||
params(),
|
||||
)
|
||||
.expect("old GET request"),
|
||||
)
|
||||
.await
|
||||
.expect("old GET");
|
||||
let old = gateway_s3s_compat::generated::ops::get_object::output_from_s3s(old.output).expect("old GET output");
|
||||
if versioned {
|
||||
assert_eq!(old.last_modified.expect("old LastModified").subsec_nanos(), 123_000_000);
|
||||
}
|
||||
let current = backend.get_object(bucket, key, &root, None).await.expect("protocol GET");
|
||||
assert_eq!(current.last_modified, old.last_modified);
|
||||
assert_eq!(current.version_id, old.version_id);
|
||||
assert_eq!(current.version_id.is_some(), versioned);
|
||||
assert_eq!(current.e_tag, old.e_tag);
|
||||
assert_eq!(current.content_length, old.content_length);
|
||||
let collect = |output: gateway_dto::GetObjectOutput| async move {
|
||||
output
|
||||
.body
|
||||
.expect("GET body")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("read body")
|
||||
.to_bytes()
|
||||
};
|
||||
assert_eq!(collect(current).await, collect(old).await);
|
||||
let old = fs
|
||||
.head_object(
|
||||
ProtocolStorageClient::create_request(
|
||||
HeadObjectInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.key(key.to_owned())
|
||||
.build()
|
||||
.expect("old HEAD input"),
|
||||
Method::HEAD,
|
||||
uri,
|
||||
params(),
|
||||
)
|
||||
.expect("old HEAD request"),
|
||||
)
|
||||
.await
|
||||
.expect("old HEAD");
|
||||
let old = gateway_s3s_compat::generated::ops::head_object::output_from_s3s(old.output).expect("old HEAD output");
|
||||
if versioned {
|
||||
assert_eq!(old.last_modified.expect("old LastModified").subsec_nanos(), 123_000_000);
|
||||
}
|
||||
let current = backend.head_object(bucket, key, &root).await.expect("protocol HEAD");
|
||||
assert_eq!(current.last_modified, old.last_modified);
|
||||
assert_eq!(current.version_id, old.version_id);
|
||||
assert_eq!(current.e_tag, old.e_tag);
|
||||
assert_eq!(current.content_length, old.content_length);
|
||||
let deleted = backend.delete_object(bucket, key, &root).await.expect("protocol DELETE");
|
||||
if versioned {
|
||||
assert_eq!(deleted.delete_marker, Some(true));
|
||||
}
|
||||
assert_eq!(deleted.version_id.is_some(), versioned);
|
||||
let old_error: rustfs_s3_types::S3Error = fs
|
||||
.get_object(
|
||||
ProtocolStorageClient::create_request(
|
||||
GetObjectInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.key(key.to_owned())
|
||||
.build()
|
||||
.expect("old GET input"),
|
||||
Method::GET,
|
||||
build_object_uri(bucket, key, &[]).expect("object URI"),
|
||||
params(),
|
||||
)
|
||||
.expect("old GET request"),
|
||||
)
|
||||
.await
|
||||
.expect_err("old GET deleted object")
|
||||
.into();
|
||||
let current = backend
|
||||
.get_object(bucket, key, &root, None)
|
||||
.await
|
||||
.expect_err("protocol GET deleted object");
|
||||
assert_eq!(current.code(), old_error.code());
|
||||
assert_eq!(current.status_code(), old_error.status_code());
|
||||
assert_eq!(current.headers(), old_error.headers());
|
||||
if versioned {
|
||||
let headers = current.headers().expect("delete marker error headers");
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("x-amz-version-id")
|
||||
.expect("marker version")
|
||||
.to_str()
|
||||
.expect("version text"),
|
||||
deleted.version_id.as_deref().expect("delete version")
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
#[test]
|
||||
fn protocol_mainflow_retention_timestamp_matches_legacy_precision() {
|
||||
use crate::app::object_usecase::{GetObjectOutputFacts, HeadObjectOutputFacts};
|
||||
let instant = rustfs_gateway_types::Timestamp::from_secs_nanos(1_700_000_000, 123_456_789).expect("test timestamp");
|
||||
let get = || GetObjectOutputFacts {
|
||||
dto: gateway_dto::GetObjectOutput {
|
||||
object_lock_retain_until_date: Some(instant),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let head = || HeadObjectOutputFacts {
|
||||
dto: gateway_dto::HeadObjectOutput {
|
||||
object_lock_retain_until_date: Some(instant),
|
||||
..Default::default()
|
||||
},
|
||||
etag: None,
|
||||
};
|
||||
let old_get =
|
||||
gateway_s3s_compat::generated::ops::get_object::output_from_s3s(get().into_s3s().expect("old GET projection"))
|
||||
.expect("old GET output");
|
||||
let old_head =
|
||||
gateway_s3s_compat::generated::ops::head_object::output_from_s3s(head().into_s3s().expect("old HEAD projection"))
|
||||
.expect("old HEAD output");
|
||||
let old = (old_get.object_lock_retain_until_date, old_head.object_lock_retain_until_date);
|
||||
let expected =
|
||||
Some(rustfs_gateway_types::Timestamp::from_secs_nanos(1_700_000_000, 123_000_000).expect("millisecond timestamp"));
|
||||
assert_eq!(old, (expected, expected));
|
||||
let current = (
|
||||
owned::get_output(get())
|
||||
.expect("protocol GET output")
|
||||
.object_lock_retain_until_date,
|
||||
owned::head_output(head())
|
||||
.expect("protocol HEAD output")
|
||||
.object_lock_retain_until_date,
|
||||
);
|
||||
assert_eq!(current, old);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Request and response projections for protocol reads and single-object deletion.
|
||||
//! Authorization stays on the protocol session edge; storage runs in the client's server context.
|
||||
|
||||
use super::{ProtocolStorageClient, RequestParams};
|
||||
use crate::app::object_usecase::{DefaultObjectUsecase, GetObjectBody, GetObjectOutputFacts, HeadObjectOutputFacts};
|
||||
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
|
||||
use crate::runtime_sources::current_action_credentials;
|
||||
use crate::storage_api::protocols::client::{ReqInfo, RequestContext};
|
||||
use http::{HeaderMap, Method, Uri};
|
||||
use rustfs_gateway_types::{ETag, Timestamp, dto};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
|
||||
impl ProtocolStorageClient {
|
||||
pub(super) async fn get_object_owned(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
credentials: &rustfs_credentials::Credentials,
|
||||
range: Option<rustfs_gateway_types::ByteRange>,
|
||||
) -> S3Result<dto::GetObjectOutput> {
|
||||
let uri = super::build_object_uri(bucket, key, &[])?;
|
||||
let envelope = Self::create_envelope(
|
||||
Method::GET,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_owned()),
|
||||
object: Some(key.to_owned()),
|
||||
credentials,
|
||||
},
|
||||
);
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
|
||||
self.owned_usecase()
|
||||
.execute_get_object_fields(
|
||||
envelope,
|
||||
crate::app::object_usecase::GetObjectFields {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
version_id: None,
|
||||
part_number: None,
|
||||
range,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.and_then(|answered| get_output(answered.output))
|
||||
}
|
||||
|
||||
pub(super) fn owned_usecase(&self) -> DefaultObjectUsecase {
|
||||
DefaultObjectUsecase::with_context(self.fs.server_ctx().app_context())
|
||||
}
|
||||
|
||||
pub(super) fn create_envelope(method: Method, uri: Uri, params: RequestParams<'_>) -> RequestEnvelope {
|
||||
let is_owner = current_action_credentials().is_some_and(|root| params.credentials.access_key == root.access_key);
|
||||
let credentials =
|
||||
RequestCredentials::new(params.credentials.access_key.clone(), params.credentials.secret_key.clone(), None)
|
||||
.unwrap_or_else(|_| RequestCredentials::without_secret_key(params.credentials.access_key.clone(), None));
|
||||
let mut extensions = http::Extensions::new();
|
||||
extensions.insert(ReqInfo {
|
||||
cred: Some(params.credentials.clone()),
|
||||
is_owner,
|
||||
bucket: params.bucket,
|
||||
object: params.object,
|
||||
version_id: None,
|
||||
replication_request_authorized: false,
|
||||
region: None,
|
||||
request_context: Some(RequestContext::fallback()),
|
||||
suppress_denial_log: false,
|
||||
suppress_denial_audit: false,
|
||||
});
|
||||
RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method,
|
||||
uri,
|
||||
headers: HeaderMap::new(),
|
||||
credentials: Some(credentials),
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
extensions,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn etag(opaque: String, weak: bool) -> S3Result<ETag> {
|
||||
if weak { ETag::new_weak(opaque) } else { ETag::new(opaque) }
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Object response contains an invalid ETag"))
|
||||
}
|
||||
|
||||
// Protocol clients received millisecond timestamps through the legacy response codec.
|
||||
fn millisecond_timestamp(value: Timestamp) -> Timestamp {
|
||||
Timestamp::from_secs_nanos(value.secs(), value.subsec_millis() * 1_000_000)
|
||||
.expect("a millisecond remainder is below one second") // subsec_millis is at most 999.
|
||||
}
|
||||
|
||||
pub(super) fn get_output(mut output: GetObjectOutputFacts) -> S3Result<dto::GetObjectOutput> {
|
||||
output.dto.last_modified = output.dto.last_modified.map(millisecond_timestamp);
|
||||
output.dto.object_lock_retain_until_date = output.dto.object_lock_retain_until_date.map(millisecond_timestamp);
|
||||
output.dto.e_tag = output.etag.map(|tag| etag(tag.opaque, tag.weak)).transpose()?;
|
||||
output.dto.body = output
|
||||
.body
|
||||
.map(|body| match body {
|
||||
GetObjectBody::Bytes(bytes) => Ok(rustfs_gateway_stream::ByteStream::from_bytes(bytes)),
|
||||
GetObjectBody::Stream(stream) => rustfs_gateway_stream::ByteStream::new(stream)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Invalid GetObject body capabilities")),
|
||||
})
|
||||
.transpose()?;
|
||||
Ok(output.dto)
|
||||
}
|
||||
|
||||
pub(super) fn head_output(mut output: HeadObjectOutputFacts) -> S3Result<dto::HeadObjectOutput> {
|
||||
output.dto.last_modified = output.dto.last_modified.map(millisecond_timestamp);
|
||||
output.dto.object_lock_retain_until_date = output.dto.object_lock_retain_until_date.map(millisecond_timestamp);
|
||||
output.dto.e_tag = output.etag.map(|tag| etag(tag.opaque, tag.weak)).transpose()?;
|
||||
Ok(output.dto)
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Protocol mainflow behavior through the installed server's object use case.
|
||||
//! These probes exercise the protocol client, not the HTTP transport or authorization hook.
|
||||
|
||||
use super::ProtocolStorageClient;
|
||||
use super::policy_test::protocol_test_context;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::protocols::client::{
|
||||
BucketOperations, MakeBucketOptions, ObjectIO, StorageObjectOptions, StoragePutObjReader,
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_protocols::common::client::s3::StorageBackend;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
const KEY: &str = "directory/a ?#+%.txt";
|
||||
|
||||
async fn object_client(bucket: &str) -> (tempfile::TempDir, Credentials, ProtocolStorageClient, String) {
|
||||
let root = Credentials {
|
||||
access_key: "protocol-mainflow-root".into(),
|
||||
secret_key: "protocol-mainflow-secret".into(),
|
||||
status: "on".into(),
|
||||
..Default::default()
|
||||
};
|
||||
let (temp, context, _iam) = protocol_test_context(root.clone()).await;
|
||||
let store = context.object_store();
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let mut reader = StoragePutObjReader::from_vec(b"protocol-data".to_vec());
|
||||
let stored = store
|
||||
.put_object(bucket, KEY, &mut reader, &StorageObjectOptions::default())
|
||||
.await
|
||||
.expect("seed object");
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
(
|
||||
temp,
|
||||
root,
|
||||
crate::init::protocol_storage_client_for_server(slot),
|
||||
stored.etag.expect("stored ETag"),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_reads_ranges_and_data() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-read", || async {
|
||||
let bucket = "protocol-mainflow-read";
|
||||
let (_temp, root, backend, stored_etag) = object_client(bucket).await;
|
||||
for (start, expected) in [(None, b"protocol-data".as_slice()), (Some(9), b"data".as_slice())] {
|
||||
let output = backend.get_object(bucket, KEY, &root, start).await.expect("protocol GET");
|
||||
assert_eq!(output.content_length, Some(expected.len() as i64));
|
||||
assert_eq!(output.e_tag.as_ref().expect("GET ETag").opaque_tag(), stored_etag);
|
||||
let body = output
|
||||
.body
|
||||
.expect("GET body")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("consume GET")
|
||||
.to_bytes();
|
||||
assert_eq!(body.as_ref(), expected);
|
||||
assert_eq!(output.version_id, None);
|
||||
}
|
||||
let ranged = backend
|
||||
.get_object_range(bucket, KEY, &root, 2, 4)
|
||||
.await
|
||||
.expect("bounded range");
|
||||
assert_eq!(ranged.content_range.as_deref(), Some("bytes 2-5/13"));
|
||||
assert_eq!(
|
||||
ranged
|
||||
.body
|
||||
.expect("range body")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("consume range")
|
||||
.to_bytes()
|
||||
.as_ref(),
|
||||
b"otoc"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_heads_object_metadata() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-head", || async {
|
||||
let bucket = "protocol-mainflow-head";
|
||||
let (_temp, root, backend, stored_etag) = object_client(bucket).await;
|
||||
let head = backend.head_object(bucket, KEY, &root).await.expect("protocol HEAD");
|
||||
assert_eq!(head.content_length, Some(13));
|
||||
assert_eq!(head.e_tag.as_ref().expect("HEAD ETag").opaque_tag(), stored_etag);
|
||||
assert_eq!(head.version_id, None);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_deletes_object() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-delete", || async {
|
||||
let bucket = "protocol-mainflow-delete";
|
||||
let (_temp, root, backend, _stored_etag) = object_client(bucket).await;
|
||||
let deleted = backend.delete_object(bucket, KEY, &root).await.expect("protocol DELETE");
|
||||
assert_eq!(deleted.version_id, None);
|
||||
assert_eq!(deleted.delete_marker, Some(false));
|
||||
let missing = backend
|
||||
.get_object(bucket, KEY, &root, None)
|
||||
.await
|
||||
.expect_err("deleted object is absent");
|
||||
assert_eq!(missing.code(), &S3ErrorCode::NoSuchKey);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_refuses_missing_objects_buckets_and_unsatisfied_ranges() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-errors", || async {
|
||||
let bucket = "protocol-mainflow-errors";
|
||||
let (_temp, root, backend, _stored_etag) = object_client(bucket).await;
|
||||
for target in [bucket, "protocol-absent-bucket"] {
|
||||
let expected = if target == bucket {
|
||||
S3ErrorCode::NoSuchKey
|
||||
} else {
|
||||
S3ErrorCode::NoSuchBucket
|
||||
};
|
||||
assert_eq!(
|
||||
backend
|
||||
.get_object(target, "absent", &root, None)
|
||||
.await
|
||||
.expect_err("missing GET")
|
||||
.code(),
|
||||
&expected
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.get_object(target, "absent", &root, Some(1))
|
||||
.await
|
||||
.expect_err("missing open-range GET")
|
||||
.code(),
|
||||
&expected
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.get_object_range(target, "absent", &root, 1, 1)
|
||||
.await
|
||||
.expect_err("missing bounded-range GET")
|
||||
.code(),
|
||||
&expected
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.head_object(target, "absent", &root)
|
||||
.await
|
||||
.expect_err("missing HEAD")
|
||||
.code(),
|
||||
&expected
|
||||
);
|
||||
}
|
||||
for start in [13, 14, 1024] {
|
||||
for error in [
|
||||
backend
|
||||
.get_object(bucket, KEY, &root, Some(start))
|
||||
.await
|
||||
.expect_err("open range past end"),
|
||||
backend
|
||||
.get_object_range(bucket, KEY, &root, start, 1)
|
||||
.await
|
||||
.expect_err("bounded range past end"),
|
||||
] {
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidRange);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
backend
|
||||
.delete_object("protocol-absent-bucket", KEY, &root)
|
||||
.await
|
||||
.expect_err("missing DELETE bucket")
|
||||
.code(),
|
||||
&S3ErrorCode::NoSuchBucket
|
||||
);
|
||||
});
|
||||
}
|
||||
@@ -18,3 +18,6 @@ pub use client::ProtocolStorageClient;
|
||||
|
||||
#[cfg(test)]
|
||||
mod policy_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod mainflow_test;
|
||||
|
||||
@@ -54,7 +54,7 @@ fn decode_policy(value: serde_json::Value) -> Result<rustfs_policy::policy::Poli
|
||||
serde_json::from_str(&value.to_string())
|
||||
}
|
||||
|
||||
async fn protocol_test_context(root: Credentials) -> (tempfile::TempDir, Arc<AppContext>, Arc<IamSys<ObjectStore>>) {
|
||||
pub(super) async fn protocol_test_context(root: Credentials) -> (tempfile::TempDir, Arc<AppContext>, Arc<IamSys<ObjectStore>>) {
|
||||
let (temp, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
ObjectStore::new(store.clone())
|
||||
.save_iam_config(json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
|
||||
|
||||
@@ -13,17 +13,17 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch E2 (rustfs/backlog#2765): DeleteObject, DeleteObjects, RestoreObject and SelectObjectContent.
|
||||
//! DeleteObject, RestoreObject and SelectObjectContent are served here.
|
||||
|
||||
use super::{OpRow, Served, resp, row};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
use crate::app::dto_values::stored_object_key;
|
||||
use crate::app::object_usecase::{DefaultObjectUsecase, DeleteBatchETag, DeleteBatchEntry, DeleteBatchInput, DeleteBatchOutput};
|
||||
use crate::app::storage_api::object_usecase::bucket::{
|
||||
RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging,
|
||||
};
|
||||
use crate::runtime_sources::AppContext;
|
||||
use crate::storage_api::server::http::access::access_owned::object_misc::{
|
||||
authorize_delete_object, authorize_restore_object, authorize_select_object_content,
|
||||
authorize_delete_object, authorize_delete_objects, authorize_restore_object, authorize_select_object_content,
|
||||
};
|
||||
use http::StatusCode;
|
||||
use rustfs_gateway::{Req, Resp, dto};
|
||||
@@ -33,6 +33,7 @@ use std::sync::Arc;
|
||||
|
||||
pub(super) const ROWS: &[OpRow] = &[
|
||||
row::<dto::DeleteObject>(),
|
||||
row::<dto::DeleteObjects>(),
|
||||
row::<dto::RestoreObject>(),
|
||||
row::<dto::SelectObjectContent>(),
|
||||
];
|
||||
@@ -142,3 +143,197 @@ impl Served for dto::SelectObjectContent {
|
||||
Ok(resp::<Self>(answered))
|
||||
}
|
||||
}
|
||||
|
||||
impl Served for dto::DeleteObjects {
|
||||
const METRIC: S3Operation = S3Operation::DeleteObjects;
|
||||
|
||||
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
|
||||
Ok(authorize_delete_objects(envelope, request.input().bucket.as_str())?)
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let input = delete_batch_input(&request)?;
|
||||
let output = DefaultObjectUsecase::with_context(context)
|
||||
.execute_delete_objects(envelope, input)
|
||||
.await?;
|
||||
Ok(Resp::new(output.try_into()?))
|
||||
}
|
||||
}
|
||||
|
||||
fn delete_batch_input(request: &Req<dto::DeleteObjects>) -> S3Result<DeleteBatchInput> {
|
||||
Ok(DeleteBatchInput {
|
||||
bucket: request.input().bucket.as_str().to_owned(),
|
||||
quiet: request.input().delete.quiet,
|
||||
objects: request
|
||||
.resources()
|
||||
.resolve_objects(request.read_proof())
|
||||
.ok_or_else(|| s3_error!(InternalError, "DeleteObjects resources could not be resolved"))?
|
||||
.map(Into::into)
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
impl From<&dto::ObjectIdentifier> for DeleteBatchEntry {
|
||||
fn from(object: &dto::ObjectIdentifier) -> Self {
|
||||
Self {
|
||||
key: object.key.as_str().to_owned(),
|
||||
key_too_long: object.key.rejection() == Some(dto::DeleteObjectKeyRejection::TooLong),
|
||||
version_id: object.version_id.clone(),
|
||||
e_tag: object.e_tag.as_ref().map(|condition| match condition.etag() {
|
||||
Some(tag) => DeleteBatchETag {
|
||||
value: tag.opaque_tag().to_owned(),
|
||||
weak: tag.is_weak(),
|
||||
},
|
||||
None => DeleteBatchETag {
|
||||
value: String::new(),
|
||||
weak: false,
|
||||
},
|
||||
}),
|
||||
_last_modified_time: object
|
||||
.last_modified_time
|
||||
.as_ref()
|
||||
.map(|value| (value.secs(), value.subsec_nanos())),
|
||||
_size: object.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<DeleteBatchOutput> for dto::DeleteObjectsOutput {
|
||||
type Error = rustfs_s3_types::S3Error;
|
||||
|
||||
fn try_from(output: DeleteBatchOutput) -> S3Result<Self> {
|
||||
// Keep the original decoded key on both successes and per-key errors.
|
||||
Ok(Self {
|
||||
deleted: output
|
||||
.deleted
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
Ok(dto::DeletedObject {
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
key: entry.key.as_deref().map(stored_object_key).transpose()?,
|
||||
version_id: entry.version_id,
|
||||
})
|
||||
})
|
||||
.collect::<S3Result<_>>()?,
|
||||
errors: output
|
||||
.errors
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
Ok(dto::Error {
|
||||
code: entry.code,
|
||||
key: entry
|
||||
.key
|
||||
.map(dto::DeleteKeyEcho::new)
|
||||
.transpose()
|
||||
.map_err(|error| s3_error!(InternalError, "DeleteObjects error key cannot be encoded: {error}"))?,
|
||||
message: entry.message,
|
||||
version_id: entry.version_id,
|
||||
})
|
||||
})
|
||||
.collect::<S3Result<_>>()?,
|
||||
request_charged: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::RequestHead;
|
||||
use crate::server::gateway_handlers::test_support::{Signed, signed_request};
|
||||
use crate::storage_api::server::http::access::test_support::ReqInfo;
|
||||
use bytes::Bytes;
|
||||
use http::{Extensions, HeaderMap, Method, StatusCode};
|
||||
use http_body_util::{BodyExt as _, Full};
|
||||
use rustfs_gateway::{
|
||||
Credentials, Handler, HandlerResult, RegionSet, ServiceBuilder, SigV4Authenticator, StaticCredentials, allow_when,
|
||||
};
|
||||
|
||||
struct BatchProbe;
|
||||
|
||||
impl Handler<dto::DeleteObjects> for BatchProbe {
|
||||
async fn call(&self, request: Req<dto::DeleteObjects>) -> HandlerResult<dto::DeleteObjects> {
|
||||
let mut envelope = RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: Method::POST,
|
||||
uri: "/fixture-bucket?delete".parse().unwrap(),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
);
|
||||
envelope.install_req_info(ReqInfo {
|
||||
bucket: Some("previous-bucket".to_owned()),
|
||||
object: Some("previous-key".to_owned()),
|
||||
version_id: Some("previous-version".to_owned()),
|
||||
..ReqInfo::default()
|
||||
});
|
||||
|
||||
<dto::DeleteObjects as Served>::authorize(&mut envelope, &request)
|
||||
.await
|
||||
.unwrap();
|
||||
let info = envelope.req_info().unwrap();
|
||||
assert_eq!(info.bucket.as_deref(), Some("fixture-bucket"));
|
||||
assert_eq!((info.object.as_deref(), info.version_id.as_deref()), (None, None));
|
||||
|
||||
let input = delete_batch_input(&request).unwrap();
|
||||
assert_eq!(input.bucket, "fixture-bucket");
|
||||
assert_eq!(input.quiet, Some(true));
|
||||
let first = DeleteBatchEntry {
|
||||
key: "first".to_owned(),
|
||||
key_too_long: false,
|
||||
version_id: Some("null".to_owned()),
|
||||
e_tag: Some(DeleteBatchETag {
|
||||
value: "weak-tag".to_owned(),
|
||||
weak: true,
|
||||
}),
|
||||
_last_modified_time: Some((946_684_800, 0)),
|
||||
_size: Some(-19),
|
||||
};
|
||||
let second = DeleteBatchEntry {
|
||||
key: "second".to_owned(),
|
||||
e_tag: Some(DeleteBatchETag {
|
||||
value: "*".to_owned(),
|
||||
weak: false,
|
||||
}),
|
||||
_size: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(input.objects, [first.clone(), second, first]);
|
||||
Ok(Resp::new(dto::DeleteObjectsOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_objects_sets_the_batch_target_and_carries_frozen_entries() {
|
||||
let access = "batch-fixture";
|
||||
let secret = "fixture-delete-batch-secret";
|
||||
let credentials = StaticCredentials::new().with(Credentials::new(access, secret.as_bytes()).unwrap());
|
||||
let regions = RegionSet::new(["us-east-1"]).unwrap();
|
||||
let service = ServiceBuilder::new()
|
||||
.rustfs_profile()
|
||||
.authenticator(SigV4Authenticator::new(Arc::new(credentials), regions).rustfs_profile())
|
||||
.authorizer(allow_when(|_| true))
|
||||
.register::<dto::DeleteObjects, _>(Arc::new(BatchProbe))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let first = concat!(
|
||||
"<Object><Key>first</Key><VersionId>null</VersionId><ETag>W/"weak-tag"</ETag>",
|
||||
"<LastModifiedTime>Sat, 01 Jan 2000 00:00:00 GMT</LastModifiedTime><Size>-19</Size></Object>"
|
||||
);
|
||||
let second = "<Object><Key>second</Key><ETag>*</ETag><Size>0</Size></Object>";
|
||||
let body = format!("<Delete>{first}{second}{first}<Quiet>true</Quiet></Delete>");
|
||||
let request = signed_request(Signed::Header, Method::POST, "/fixture-bucket", "delete", access, secret)
|
||||
.map(|_| Full::new(Bytes::from(body)));
|
||||
let response = service.call(request).await;
|
||||
let status = response.status();
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
|
||||
}
|
||||
}
|
||||
|
||||
+360
-2
@@ -1538,7 +1538,7 @@ type S3EdgeService = S3Service;
|
||||
/// The S3 service mounted under the middleware: the legacy stack or the gateway stack, selected at
|
||||
/// startup (`server/stack.rs`).
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService>;
|
||||
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService, NativeConsoleEdgeService>;
|
||||
|
||||
/// The gateway stack where the legacy service is mounted: the same response and error types, so
|
||||
/// the layers around it do not change with the stack.
|
||||
@@ -1566,6 +1566,44 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
#[derive(Clone)]
|
||||
struct NativeConsoleEdgeService(axum::Router);
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
impl NativeConsoleEdgeService {
|
||||
fn for_config(config: &config::Config) -> Option<Self> {
|
||||
(matches!(config.s3_stack, config::S3Stack::Gateway) && config.console_enable)
|
||||
.then(|| Self(admin::console::make_console_server()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
impl<B> Service<HttpRequest<B>> for NativeConsoleEdgeService
|
||||
where
|
||||
B: http_body::Body<Data = Bytes> + Send + 'static,
|
||||
B::Error: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
type Response = Response<s3s::Body>;
|
||||
type Error = s3s::HttpError;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest<B>) -> Self::Future {
|
||||
let answer = self.0.call(request.map(axum::body::Body::new));
|
||||
Box::pin(async move {
|
||||
let response = match answer.await {
|
||||
Ok(response) => response,
|
||||
Err(never) => match never {},
|
||||
};
|
||||
Ok(response.map(s3s::Body::http_body_unsync))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Selects the S3 stack of this server, once. A gateway stack that cannot be assembled fails
|
||||
/// startup; the legacy stack is never started in its place.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
@@ -1577,7 +1615,8 @@ fn select_s3_stack(
|
||||
let service = select_stack(config.s3_stack, legacy, || -> Result<GatewayEdgeService> {
|
||||
let stack = GatewayStack::assemble(assembly()).map_err(Error::other)?;
|
||||
Ok(GatewayEdgeService(stack))
|
||||
})?;
|
||||
})?
|
||||
.with_native_console(NativeConsoleEdgeService::for_config(config));
|
||||
info!(
|
||||
event = EVENT_S3_STACK_ASSEMBLED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
@@ -4382,6 +4421,325 @@ mod tests {
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
mod native_console_tests {
|
||||
use super::*;
|
||||
use crate::config::S3Stack;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tower::ServiceExt;
|
||||
|
||||
async fn assert_console_response_equivalent(
|
||||
expected: Response<s3s::Body>,
|
||||
actual: Response<s3s::Body>,
|
||||
method: &Method,
|
||||
target: &str,
|
||||
) {
|
||||
use http_body_util::BodyExt;
|
||||
assert_eq!(actual.status(), expected.status(), "{method} {target}");
|
||||
for header in [http::header::CONTENT_TYPE, http::header::CONTENT_LENGTH, http::header::ALLOW] {
|
||||
assert_eq!(
|
||||
actual.headers().get(&header),
|
||||
expected.headers().get(&header),
|
||||
"{method} {target}: {header}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
actual.into_body().collect().await.unwrap().to_bytes(),
|
||||
expected.into_body().collect().await.unwrap().to_bytes(),
|
||||
"{method} {target}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_console_route_contract() {
|
||||
use http_body_util::BodyExt;
|
||||
legacy_console_characterizes_public_static_and_malformed_authorization().await;
|
||||
let target = format!("{}/route-fixture.txt", crate::server::console_prefix());
|
||||
let (service, calls) = edge(S3Stack::Gateway, true);
|
||||
let response = service.oneshot(console_request(Method::GET, &target, false)).await.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers().get(http::header::CONTENT_TYPE).unwrap(), "text/plain");
|
||||
assert_eq!(response.headers().get(http::header::CONTENT_LENGTH).unwrap(), "28");
|
||||
assert_eq!(
|
||||
response.into_body().collect().await.unwrap().to_bytes().as_ref(),
|
||||
b"console-route-static-content"
|
||||
);
|
||||
assert_eq!(counts(&calls), [0, 0, 1]);
|
||||
public_static_uses_native_console_without_legacy_or_gateway_s3_calls().await;
|
||||
neighboring_prefixes_admin_sts_s3_and_console_extension_queries_are_excluded().await;
|
||||
disabled_console_and_legacy_mode_keep_the_existing_entrypoint().await;
|
||||
}
|
||||
|
||||
struct ConsoleTestS3;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl s3s::S3 for ConsoleTestS3 {}
|
||||
|
||||
fn legacy_console(enabled: bool) -> s3s::service::S3Service {
|
||||
let mut builder = s3s::service::S3ServiceBuilder::new(ConsoleTestS3);
|
||||
builder.set_auth(crate::auth::IAMAuth::new("console-test-access", "console-test-secret"));
|
||||
builder.set_route(crate::admin::router::S3Router::<crate::admin::router::AdminOperation>::new(enabled));
|
||||
builder.build()
|
||||
}
|
||||
|
||||
fn console_request(method: Method, target: &str, malformed_auth: bool) -> HttpRequest<Empty<Bytes>> {
|
||||
let mut builder = HttpRequest::builder().method(method).uri(target).header("host", "localhost");
|
||||
if malformed_auth {
|
||||
builder = builder.header(http::header::AUTHORIZATION, "invalid");
|
||||
}
|
||||
builder.body(Empty::new()).expect("static request")
|
||||
}
|
||||
|
||||
fn static_console_cases() -> [(Method, String); 5] {
|
||||
let prefix = crate::server::console_prefix();
|
||||
[
|
||||
(Method::GET, format!("{prefix}/index.html")),
|
||||
(Method::HEAD, format!("{prefix}/index.html")),
|
||||
(Method::OPTIONS, format!("{prefix}/index.html")),
|
||||
(Method::GET, "/favicon.ico".to_owned()),
|
||||
(Method::GET, format!("{prefix}/missing-static-file.js")),
|
||||
]
|
||||
}
|
||||
|
||||
async fn static_response_body(response: Response<s3s::Body>, method: &Method, target: &str) -> Bytes {
|
||||
use http_body_util::BodyExt;
|
||||
let (fixture, content_type): (&[u8], &str) = if target == "/favicon.ico" {
|
||||
(include_bytes!("../../tests/fixtures/native-console/favicon.ico"), "image/x-icon")
|
||||
} else {
|
||||
(include_bytes!("../../tests/fixtures/native-console/index.html"), "text/html")
|
||||
};
|
||||
assert_eq!(response.status(), StatusCode::OK, "{method} {target}");
|
||||
if method == Method::OPTIONS {
|
||||
assert_eq!(
|
||||
response.headers().get("access-control-allow-methods").unwrap(),
|
||||
"GET,POST,PUT,DELETE,OPTIONS"
|
||||
);
|
||||
assert_eq!(response.headers().get("access-control-allow-headers").unwrap(), "*");
|
||||
assert!(response.headers().get(http::header::CONTENT_TYPE).is_none());
|
||||
assert!(response.headers().get(http::header::CONTENT_LENGTH).is_none());
|
||||
} else {
|
||||
assert_eq!(response.headers().get(http::header::CONTENT_TYPE).unwrap(), content_type);
|
||||
assert_eq!(
|
||||
response
|
||||
.headers()
|
||||
.get(http::header::CONTENT_LENGTH)
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap(),
|
||||
fixture.len().to_string()
|
||||
);
|
||||
}
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
let expected = if method == Method::HEAD || method == Method::OPTIONS {
|
||||
b"".as_slice()
|
||||
} else {
|
||||
fixture
|
||||
};
|
||||
assert_eq!(body.as_ref(), expected, "{method} {target}");
|
||||
body
|
||||
}
|
||||
|
||||
async fn legacy_console_characterizes_public_static_and_malformed_authorization() {
|
||||
use http_body_util::BodyExt;
|
||||
for (method, target) in static_console_cases() {
|
||||
let legacy = legacy_console(true)
|
||||
.oneshot(console_request(method.clone(), &target, false))
|
||||
.await
|
||||
.unwrap();
|
||||
let native = crate::admin::console::make_console_server()
|
||||
.oneshot(console_request(method.clone(), &target, false))
|
||||
.await
|
||||
.unwrap()
|
||||
.map(s3s::Body::http_body_unsync);
|
||||
let malformed_native = crate::admin::console::make_console_server()
|
||||
.oneshot(console_request(method.clone(), &target, true))
|
||||
.await
|
||||
.unwrap()
|
||||
.map(s3s::Body::http_body_unsync);
|
||||
assert_eq!(legacy.status(), native.status(), "{method} {target}");
|
||||
assert_eq!(malformed_native.status(), native.status(), "native {method} {target}");
|
||||
for header in [http::header::CONTENT_TYPE, http::header::CONTENT_LENGTH, http::header::ALLOW] {
|
||||
assert_eq!(
|
||||
legacy.headers().get(&header),
|
||||
native.headers().get(&header),
|
||||
"{method} {target}: {header}"
|
||||
);
|
||||
assert_eq!(
|
||||
malformed_native.headers().get(&header),
|
||||
native.headers().get(&header),
|
||||
"native malformed auth {method} {target}: {header}"
|
||||
);
|
||||
}
|
||||
let legacy_body = static_response_body(legacy, &method, &target).await;
|
||||
let native_body = static_response_body(native, &method, &target).await;
|
||||
let malformed_native_body = static_response_body(malformed_native, &method, &target).await;
|
||||
assert_eq!(legacy_body, native_body, "{method} {target}");
|
||||
assert_eq!(malformed_native_body, native_body, "native malformed auth {method} {target}");
|
||||
let malformed = legacy_console(true)
|
||||
.oneshot(console_request(method.clone(), &target, true))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(malformed.status(), StatusCode::BAD_REQUEST, "legacy {method} {target}");
|
||||
let malformed_body = malformed.into_body().collect().await.unwrap().to_bytes();
|
||||
if method == Method::HEAD {
|
||||
assert!(malformed_body.is_empty());
|
||||
} else {
|
||||
assert!(String::from_utf8_lossy(&malformed_body).contains("<Code>InvalidRequest</Code>"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type Probe = tower::util::BoxCloneService<HttpRequest<Empty<Bytes>>, Response<s3s::Body>, s3s::HttpError>;
|
||||
|
||||
fn counted<S>(inner: S, calls: Arc<AtomicUsize>) -> Probe
|
||||
where
|
||||
S: Service<HttpRequest<Empty<Bytes>>, Response = Response<s3s::Body>, Error = s3s::HttpError>
|
||||
+ Clone
|
||||
+ Send
|
||||
+ 'static,
|
||||
S::Future: Send + 'static,
|
||||
{
|
||||
Probe::new(tower::service_fn(move |request| {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
inner.clone().oneshot(request)
|
||||
}))
|
||||
}
|
||||
|
||||
fn edge(stack: S3Stack, enabled: bool) -> (S3StackService<Probe, Probe>, [Arc<AtomicUsize>; 3]) {
|
||||
let calls = std::array::from_fn(|_| Arc::new(AtomicUsize::new(0)));
|
||||
let mut config = config::Config::new("127.0.0.1:9001", Vec::new());
|
||||
config.s3_stack = stack;
|
||||
config.console_enable = enabled;
|
||||
let legacy = counted(legacy_console(enabled), calls[0].clone());
|
||||
let gateway = counted(
|
||||
tower::service_fn(|_| async {
|
||||
Ok::<_, s3s::HttpError>(
|
||||
Response::builder()
|
||||
.status(StatusCode::IM_A_TEAPOT)
|
||||
.body(s3s::Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
}),
|
||||
calls[1].clone(),
|
||||
);
|
||||
let console = NativeConsoleEdgeService::for_config(&config).map(|service| counted(service, calls[2].clone()));
|
||||
let edge = select_stack(stack, legacy, || Ok::<_, s3s::HttpError>(gateway))
|
||||
.unwrap()
|
||||
.with_native_console(console);
|
||||
(edge, calls)
|
||||
}
|
||||
|
||||
fn counts(calls: &[Arc<AtomicUsize>; 3]) -> [usize; 3] {
|
||||
std::array::from_fn(|i| calls[i].load(Ordering::SeqCst))
|
||||
}
|
||||
|
||||
async fn public_static_uses_native_console_without_legacy_or_gateway_s3_calls() {
|
||||
let (service, calls) = edge(S3Stack::Gateway, true);
|
||||
let prefix = crate::server::console_prefix();
|
||||
let cases: Vec<_> = static_console_cases()
|
||||
.into_iter()
|
||||
.chain([
|
||||
(Method::GET, "/apple-touch-icon.png".to_owned()),
|
||||
(Method::GET, format!("{prefix}/index.html?events")),
|
||||
(Method::HEAD, format!("{prefix}/index.html?lambdaArn=bad")),
|
||||
(Method::GET, "/favicon.ico?lambdaArn=bad".to_owned()),
|
||||
])
|
||||
.collect();
|
||||
for (method, target) in &cases {
|
||||
let actual = service
|
||||
.clone()
|
||||
.oneshot(console_request(method.clone(), target, false))
|
||||
.await
|
||||
.unwrap();
|
||||
let expected = legacy_console(true)
|
||||
.oneshot(console_request(method.clone(), target, false))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_console_response_equivalent(expected, actual, method, target).await;
|
||||
}
|
||||
assert_eq!(counts(&calls), [0, 0, cases.len()]);
|
||||
}
|
||||
|
||||
async fn neighboring_prefixes_admin_sts_s3_and_console_extension_queries_are_excluded() {
|
||||
let (service, calls) = edge(S3Stack::Gateway, true);
|
||||
let prefix = crate::server::console_prefix();
|
||||
let cases = [
|
||||
(Method::GET, format!("{prefix}x/index.html"), false, false),
|
||||
(Method::GET, "/favicon.icox".to_owned(), false, false),
|
||||
(Method::GET, "/apple-touch-icon.pngx".to_owned(), false, false),
|
||||
(Method::GET, "/rustfs/administrator".to_owned(), false, false),
|
||||
(Method::GET, "/bucket/key".to_owned(), false, false),
|
||||
(Method::PUT, "/bucket/key".to_owned(), false, false),
|
||||
(Method::POST, "/".to_owned(), false, false),
|
||||
(Method::GET, "/rustfs/admin/v3/info".to_owned(), true, false),
|
||||
(Method::GET, "/minio/admin/v3/info".to_owned(), true, false),
|
||||
(Method::GET, "/iceberg/v1/config".to_owned(), true, false),
|
||||
(Method::POST, "/".to_owned(), true, true),
|
||||
(Method::GET, "/favicon.ico?events".to_owned(), true, false),
|
||||
(Method::GET, "/favicon.ico?replication-metrics".to_owned(), true, false),
|
||||
(Method::GET, format!("{prefix}/index.html?lambdaArn="), true, false),
|
||||
(Method::GET, format!("{prefix}/index.html?lambdaArn=bad"), true, false),
|
||||
(Method::GET, format!("{prefix}/index.html?lambdaArn=bad&lambdaArn=bad"), true, false),
|
||||
];
|
||||
for (method, target, legacy, sts) in &cases {
|
||||
for malformed_auth in [false, true] {
|
||||
let make_request = || {
|
||||
let mut request = console_request(method.clone(), target, malformed_auth);
|
||||
if *sts {
|
||||
request.headers_mut().insert(
|
||||
http::header::CONTENT_TYPE,
|
||||
http::HeaderValue::from_static("application/x-www-form-urlencoded"),
|
||||
);
|
||||
}
|
||||
request
|
||||
};
|
||||
let actual = service.clone().oneshot(make_request()).await.unwrap();
|
||||
if *legacy {
|
||||
let expected = legacy_console(true).oneshot(make_request()).await.unwrap();
|
||||
if target == "/rustfs/admin/v3/info" || target == "/minio/admin/v3/info" {
|
||||
assert_eq!(
|
||||
actual.status(),
|
||||
if malformed_auth {
|
||||
StatusCode::BAD_REQUEST
|
||||
} else {
|
||||
StatusCode::FORBIDDEN
|
||||
},
|
||||
"admin on console listener"
|
||||
);
|
||||
}
|
||||
assert_console_response_equivalent(expected, actual, method, target).await;
|
||||
} else {
|
||||
assert_eq!(actual.status(), StatusCode::IM_A_TEAPOT, "{method} {target}");
|
||||
}
|
||||
}
|
||||
}
|
||||
let legacy_cases = cases.iter().filter(|case| case.2).count();
|
||||
assert_eq!(counts(&calls), [legacy_cases * 2, (cases.len() - legacy_cases) * 2, 0]);
|
||||
}
|
||||
|
||||
async fn disabled_console_and_legacy_mode_keep_the_existing_entrypoint() {
|
||||
for (stack, enabled) in [(S3Stack::Gateway, false), (S3Stack::Legacy, true)] {
|
||||
let (service, calls) = edge(stack, enabled);
|
||||
let cases = static_console_cases();
|
||||
for (method, target) in &cases {
|
||||
for malformed_auth in [false, true] {
|
||||
let actual = service
|
||||
.clone()
|
||||
.oneshot(console_request(method.clone(), target, malformed_auth))
|
||||
.await
|
||||
.unwrap();
|
||||
let expected = legacy_console(enabled)
|
||||
.oneshot(console_request(method.clone(), target, malformed_auth))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_console_response_equivalent(expected, actual, method, target).await;
|
||||
}
|
||||
}
|
||||
assert_eq!(counts(&calls), [cases.len() * 2, 0, 0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type RateLimitTestCore = tower::util::BoxCloneService<
|
||||
HttpRequest<s3s::Body>,
|
||||
Response<crate::server::hybrid::HybridBody<s3s::Body, Empty<Bytes>>>,
|
||||
|
||||
+38
-13
@@ -18,16 +18,16 @@
|
||||
//! be, and [`select_stack`], the only place a stack is chosen. The whole stack is one or the
|
||||
//! other: no request is routed between stacks by its operation, and a request the gateway stack
|
||||
//! refuses is answered by that refusal and never retried on the legacy stack. With the gateway
|
||||
//! selected, the legacy service receives only what the legacy router claims (the RustFS extension
|
||||
//! routes and the control plane), so no S3 request reaches the legacy S3 implementation.
|
||||
//! selected, console static requests use the native console service; the legacy service receives
|
||||
//! the remaining claimed extension and control-plane routes, never S3 operations.
|
||||
//! NOT responsible for: assembling either stack (`server/http.rs` builds the legacy service,
|
||||
//! `server/gateway_stack.rs` the gateway one), or the middleware around them.
|
||||
//! Upstream: `server/http.rs`. Downstream: the legacy S3 service and the gateway stack.
|
||||
//! Upstream: `server/http.rs`. Downstream: the native console, legacy S3 service and gateway stack.
|
||||
//!
|
||||
//! This file names neither stack's types: it is generic over two services with one response
|
||||
//! This file names no stack's types: it is generic over the services with one response
|
||||
//! type, so `server/http.rs` decides what they are and how their bodies meet.
|
||||
|
||||
use crate::admin::router::claims_route;
|
||||
use crate::admin::router::{claims_native_console_route, claims_route};
|
||||
use crate::config::S3Stack;
|
||||
use http::Request;
|
||||
use std::future::Future;
|
||||
@@ -38,12 +38,22 @@ use tower::Service;
|
||||
/// The S3 service of one server: the legacy stack, or the gateway stack with the legacy router
|
||||
/// kept for the routes that are not S3 operations.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct S3StackService<L, G> {
|
||||
pub(crate) struct S3StackService<L, G, C = G> {
|
||||
legacy: L,
|
||||
gateway: Option<G>,
|
||||
native_console: Option<C>,
|
||||
}
|
||||
|
||||
impl<L, G> S3StackService<L, G> {
|
||||
impl<L, G, C> S3StackService<L, G, C> {
|
||||
/// Attaches the console service used only by the gateway selection.
|
||||
pub(crate) fn with_native_console<N>(self, native_console: Option<N>) -> S3StackService<L, G, N> {
|
||||
S3StackService {
|
||||
legacy: self.legacy,
|
||||
gateway: self.gateway,
|
||||
native_console,
|
||||
}
|
||||
}
|
||||
|
||||
/// The stack this service was built with, as the startup line names it: read from the service
|
||||
/// that answers, never from the configuration that asked for it.
|
||||
pub(crate) fn assembled(&self) -> &'static str {
|
||||
@@ -71,15 +81,21 @@ pub(crate) fn select_stack<L, G, E>(
|
||||
S3Stack::Legacy => None,
|
||||
S3Stack::Gateway => Some(assemble_gateway()?),
|
||||
};
|
||||
Ok(S3StackService { legacy, gateway })
|
||||
Ok(S3StackService {
|
||||
legacy,
|
||||
gateway,
|
||||
native_console: None,
|
||||
})
|
||||
}
|
||||
|
||||
impl<L, G, B, R, E> Service<Request<B>> for S3StackService<L, G>
|
||||
impl<L, G, C, B, R, E> Service<Request<B>> for S3StackService<L, G, C>
|
||||
where
|
||||
L: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
G: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
C: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
L::Future: Send + 'static,
|
||||
G::Future: Send + 'static,
|
||||
C::Future: Send + 'static,
|
||||
{
|
||||
type Response = http::Response<R>;
|
||||
type Error = E;
|
||||
@@ -88,16 +104,25 @@ where
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
if let Some(gateway) = &mut self.gateway {
|
||||
ready!(gateway.poll_ready(cx))?;
|
||||
if let Some(console) = &mut self.native_console {
|
||||
ready!(console.poll_ready(cx))?;
|
||||
}
|
||||
}
|
||||
self.legacy.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, request: Request<B>) -> Self::Future {
|
||||
match &mut self.gateway {
|
||||
// REMOVED BY T2.11: the gateway serves these routes itself once they are migrated.
|
||||
Some(gateway) if !claims_route(request.method(), request.uri(), request.headers()) => Box::pin(gateway.call(request)),
|
||||
_ => Box::pin(self.legacy.call(request)),
|
||||
if let Some(gateway) = &mut self.gateway {
|
||||
if !claims_route(request.method(), request.uri(), request.headers()) {
|
||||
return Box::pin(gateway.call(request));
|
||||
}
|
||||
if let Some(console) = &mut self.native_console
|
||||
&& claims_native_console_route(request.method(), request.uri())
|
||||
{
|
||||
return Box::pin(console.call(request));
|
||||
}
|
||||
}
|
||||
Box::pin(self.legacy.call(request))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use self::access_owned::listing_multipart::{
|
||||
authorize_bucket_listing, authorize_list_multipart_uploads, authorize_list_objects_v2, authorize_list_parts,
|
||||
};
|
||||
use self::access_owned::object_misc::authorize_delete_object;
|
||||
use self::access_owned::object_misc::{authorize_delete_object, authorize_delete_objects};
|
||||
use super::ObjectOptions;
|
||||
use super::ecfs::FS;
|
||||
use super::{ECStore, PolicySys, StorageError, get_lock_acquire_timeout, is_err_bucket_not_found};
|
||||
@@ -33,7 +33,7 @@ use crate::error::ApiError;
|
||||
use crate::license::license_check;
|
||||
use crate::shared_types::RemoteAddr;
|
||||
use crate::storage::denial_audit::DenialAuditor;
|
||||
use crate::storage::helper::DenialScope;
|
||||
use crate::storage::helper::{AuditRequestView, DenialScope};
|
||||
use crate::storage::request_context::RequestContext;
|
||||
use crate::storage::storage_api::contract::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, BucketOperations, BucketOptions};
|
||||
use crate::storage::storage_api::contract::namespace::NamespaceLocking as _;
|
||||
@@ -2172,15 +2172,15 @@ fn audit_request_denial<R: AuthorizationRequest + ?Sized>(
|
||||
|
||||
/// Audits the refused keys of a bulk request (DeleteObjects) in aggregate instead of once per key.
|
||||
///
|
||||
/// `start` marks the request so per-key denials are logged but not audited; `finish` clears the
|
||||
/// `start_owned` marks the request so per-key denials are logged but not audited; `finish_owned` clears the
|
||||
/// mark and emits one entry per denied action with the number of keys refused for it.
|
||||
pub(crate) struct BulkDenialAudit {
|
||||
denied: Vec<(Action, u64)>,
|
||||
}
|
||||
|
||||
impl BulkDenialAudit {
|
||||
pub(crate) fn start<T>(req: &mut S3Request<T>) -> S3Result<Self> {
|
||||
ext_req_info_mut(&mut req.extensions)?.suppress_denial_audit = true;
|
||||
pub(crate) fn start_owned<R: AuthorizationRequest + ?Sized>(request: &mut R) -> S3Result<Self> {
|
||||
req_info_mut(request)?.suppress_denial_audit = true;
|
||||
Ok(Self { denied: Vec::new() })
|
||||
}
|
||||
|
||||
@@ -2192,14 +2192,24 @@ impl BulkDenialAudit {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn finish<T>(self, req: &mut S3Request<T>) -> S3Result<()> {
|
||||
self.finish_with(req, DenialAuditor::global())
|
||||
pub(crate) fn finish_owned<R: AuthorizationRequest + ?Sized>(
|
||||
self,
|
||||
request: &mut R,
|
||||
audit_view: impl FnOnce(&R) -> AuditRequestView<'_>,
|
||||
) -> S3Result<()> {
|
||||
self.finish_owned_with(request, audit_view, DenialAuditor::global())
|
||||
}
|
||||
|
||||
fn finish_with<T>(self, req: &mut S3Request<T>, auditor: &DenialAuditor) -> S3Result<()> {
|
||||
ext_req_info_mut(&mut req.extensions)?.suppress_denial_audit = false;
|
||||
fn finish_owned_with<R: AuthorizationRequest + ?Sized>(
|
||||
self,
|
||||
request: &mut R,
|
||||
audit_view: impl FnOnce(&R) -> AuditRequestView<'_>,
|
||||
auditor: &DenialAuditor,
|
||||
) -> S3Result<()> {
|
||||
req_info_mut(request)?.suppress_denial_audit = false;
|
||||
let view = audit_view(request);
|
||||
for (action, keys) in self.denied {
|
||||
auditor.audit(req, &action, "bulk_request_keys_denied", DenialScope::Keys(keys));
|
||||
auditor.audit_owned(&view, &action, "bulk_request_keys_denied", DenialScope::Keys(keys));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -3573,12 +3583,8 @@ impl S3Access for FS {
|
||||
///
|
||||
/// This method returns `Ok(())` by default.
|
||||
async fn delete_objects(&self, req: &mut S3Request<DeleteObjectsInput>) -> S3Result<()> {
|
||||
let req_info = ext_req_info_mut(&mut req.extensions)?;
|
||||
req_info.bucket = Some(req.input.bucket.clone());
|
||||
req_info.object = None;
|
||||
req_info.version_id = None;
|
||||
|
||||
Ok(())
|
||||
let bucket = req.input.bucket.clone();
|
||||
authorize_delete_objects(req, &bucket)
|
||||
}
|
||||
|
||||
/// Checks whether the DeletePublicAccessBlock request has accesses to the resources.
|
||||
@@ -6884,21 +6890,25 @@ mod tests {
|
||||
#[test]
|
||||
fn bulk_denial_is_one_entry_per_denied_action_with_the_key_count() {
|
||||
let (auditor, captured) = test_auditor(true, 10, 10);
|
||||
let mut req = denial_request("denial-bulk", true);
|
||||
let req = denial_request("denial-bulk", true);
|
||||
let mut envelope = crate::app::RequestEnvelope::from_s3s(&req);
|
||||
|
||||
let nothing_denied = BulkDenialAudit::start(&mut req).unwrap();
|
||||
assert!(req.extensions.get::<ReqInfo>().unwrap().suppress_denial_audit);
|
||||
nothing_denied.finish_with(&mut req, &auditor).unwrap();
|
||||
assert!(!req.extensions.get::<ReqInfo>().unwrap().suppress_denial_audit);
|
||||
let nothing_denied = BulkDenialAudit::start_owned(&mut envelope).unwrap();
|
||||
assert!(envelope.req_info().unwrap().suppress_denial_audit);
|
||||
nothing_denied
|
||||
.finish_owned_with(&mut envelope, crate::app::RequestEnvelope::audit_view, &auditor)
|
||||
.unwrap();
|
||||
assert!(!envelope.req_info().unwrap().suppress_denial_audit);
|
||||
assert!(captured.lock().unwrap().is_empty());
|
||||
|
||||
let mut bulk = BulkDenialAudit::start(&mut req).unwrap();
|
||||
let mut bulk = BulkDenialAudit::start_owned(&mut envelope).unwrap();
|
||||
let delete = Action::S3Action(S3Action::DeleteObjectAction);
|
||||
let bypass = Action::S3Action(S3Action::BypassGovernanceRetentionAction);
|
||||
for action in [delete, delete, bypass, delete] {
|
||||
bulk.denied(action);
|
||||
}
|
||||
bulk.finish_with(&mut req, &auditor).unwrap();
|
||||
bulk.finish_owned_with(&mut envelope, crate::app::RequestEnvelope::audit_view, &auditor)
|
||||
.unwrap();
|
||||
|
||||
let entries = captured.lock().unwrap();
|
||||
assert_eq!(entries.len(), 2);
|
||||
@@ -8596,6 +8606,9 @@ mod tests {
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
bucket: Some("previous-bucket".to_string()),
|
||||
object: Some("previous-key".to_string()),
|
||||
version_id: Some("previous-version".to_string()),
|
||||
..ReqInfo::default()
|
||||
});
|
||||
|
||||
|
||||
@@ -38,6 +38,15 @@ use crate::storage::options::destination_version_id;
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, force_delete_header};
|
||||
|
||||
/// The bulk engine authorizes each entry after this request-level target setup.
|
||||
pub(crate) fn authorize_delete_objects<R: AuthorizationRequest + ?Sized>(req: &mut R, bucket: &str) -> S3Result<()> {
|
||||
let info = req_info_mut(req)?;
|
||||
info.bucket = Some(bucket.to_owned());
|
||||
info.object = None;
|
||||
info.version_id = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Both RestoreObject edges retain the bucket generation read before policy evaluation.
|
||||
pub(crate) async fn authorize_restore_object<R: AuthorizationRequest + ?Sized>(
|
||||
server_ctx: &ServerContextSlot,
|
||||
|
||||
@@ -27,6 +27,7 @@ use rustfs_audit::{
|
||||
global::AuditLogger,
|
||||
};
|
||||
use rustfs_iam::sys::{OIDC_ISSUER_CLAIM, is_rustfs_oidc_claims};
|
||||
#[cfg(test)]
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_notify::EventArgsBuilder;
|
||||
use rustfs_policy::policy::action::Action;
|
||||
@@ -173,10 +174,6 @@ where
|
||||
}
|
||||
|
||||
/// Builds S3-compatible notification response elements with the canonical request ID.
|
||||
pub(crate) fn build_event_resp_elements<T>(response: &S3Response<T>, request_id: &str) -> HashMap<String, String> {
|
||||
build_owned_event_resp_elements(&response.headers, request_id)
|
||||
}
|
||||
|
||||
pub(crate) fn build_owned_event_resp_elements(response_headers: &HeaderMap, request_id: &str) -> HashMap<String, String> {
|
||||
let mut resp_elements = extract_params_header(response_headers);
|
||||
resp_elements.insert(AMZ_REQUEST_ID.to_string(), request_id.to_string());
|
||||
@@ -370,15 +367,15 @@ pub struct EnabledOperationHelper {
|
||||
}
|
||||
|
||||
impl OperationHelper {
|
||||
/// Create a new OperationHelper for S3 requests.
|
||||
/// Builds the legacy request helper for audit and notification fixtures.
|
||||
#[cfg(test)]
|
||||
pub fn new(req: &S3Request<impl Send + Sync>, event: EventName, op: S3Operation) -> Self {
|
||||
record_s3_op(op);
|
||||
Self::from_view(&AuditRequestView::legacy(req), event, op)
|
||||
}
|
||||
|
||||
/// [`Self::new`] for a request whose audit facts an owned edge decoded
|
||||
/// (`RequestEnvelope::audit_view`). It does not count the operation: the owned edge already
|
||||
/// did, once, before calling the use case.
|
||||
/// Builds the helper from the audit facts decoded by the operation edge.
|
||||
/// The edge counts the operation before calling the use case.
|
||||
pub(crate) fn from_view(req: &AuditRequestView<'_>, event: EventName, op: S3Operation) -> Self {
|
||||
Self::build(req, event, op, false)
|
||||
}
|
||||
@@ -387,8 +384,7 @@ impl OperationHelper {
|
||||
///
|
||||
/// For operations that have no notification event of their own (`UploadPart`, the
|
||||
/// bucket configuration writes) or whose event is internal (`AbortMultipartUpload`).
|
||||
/// Unlike `new` it does not call `record_s3_op`, so per-operation metrics stay as the
|
||||
/// handlers already report them.
|
||||
/// The handler records the per-operation metric.
|
||||
pub fn audit_only(req: &S3Request<impl Send + Sync>, op: S3Operation) -> Self {
|
||||
Self::audit_only_from_view(&AuditRequestView::legacy(req), op)
|
||||
}
|
||||
@@ -691,11 +687,6 @@ impl OperationHelper {
|
||||
|
||||
/// Returns the operation's canonical context, reading the ingress extension
|
||||
/// when the disabled fast path holds no request state.
|
||||
pub(crate) fn request_context_or_from_request<T>(&self, req: &S3Request<T>) -> RequestContext {
|
||||
self.request_context_or_from_view(&AuditRequestView::legacy(req))
|
||||
}
|
||||
|
||||
/// [`Self::request_context_or_from_request`] for a request whose audit facts an owned edge decoded.
|
||||
pub(crate) fn request_context_or_from_view(&self, req: &AuditRequestView<'_>) -> RequestContext {
|
||||
match self {
|
||||
Self::Enabled(state) => state.request_context.clone(),
|
||||
@@ -1845,7 +1836,7 @@ mod tests {
|
||||
});
|
||||
|
||||
let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
|
||||
let request_context = helper.request_context_or_from_request(&req);
|
||||
let request_context = helper.request_context_or_from_view(&super::AuditRequestView::legacy(&req));
|
||||
let request_id = request_context.request_id;
|
||||
assert!(uuid::Uuid::parse_str(&request_id).is_ok());
|
||||
let result = Ok(S3Response::new(DeleteObjectTaggingOutput::default()));
|
||||
@@ -1893,7 +1884,12 @@ mod tests {
|
||||
let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
|
||||
|
||||
assert!(matches!(&helper, OperationHelper::Disabled));
|
||||
assert_eq!(helper.request_context_or_from_request(&req).request_id, "server-request-id");
|
||||
assert_eq!(
|
||||
helper
|
||||
.request_context_or_from_view(&super::AuditRequestView::legacy(&req))
|
||||
.request_id,
|
||||
"server-request-id"
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -26,8 +26,10 @@ use crate::storage::access::legacy_copy_source;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::ecfs_extend::validate_new_object_key;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use crate::storage::s3_api::{CopyObjectWriteInput, CopyObjectWriteOutput, CopySourceCondition, DefaultObjectUsecase};
|
||||
use crate::storage::s3_api::{
|
||||
self, CopyObjectWriteInput, CopyObjectWriteOutput, CopySourceCondition, DefaultObjectUsecase, DeleteBatchETag,
|
||||
DeleteBatchEntry, DeleteBatchInput, DeleteBatchOutput,
|
||||
};
|
||||
use crate::storage::storage_api::to_s3s_xml_etag;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
@@ -192,8 +194,11 @@ impl FS {
|
||||
&self,
|
||||
req: S3Request<DeleteObjectsInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectsOutput>> {
|
||||
record_s3_op(S3Operation::DeleteObjects);
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_delete_objects(req).await
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let output = usecase.execute_delete_objects(envelope, req.input.into()).await?;
|
||||
Ok(S3Response::new(output.into()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn restore_object_edge(
|
||||
@@ -344,3 +349,62 @@ fn copy_output_to_legacy(output: CopyObjectWriteOutput) -> S3Response<CopyObject
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
impl From<DeleteObjectsInput> for DeleteBatchInput {
|
||||
fn from(input: DeleteObjectsInput) -> Self {
|
||||
Self {
|
||||
bucket: input.bucket,
|
||||
objects: input.delete.objects.into_iter().map(Into::into).collect(),
|
||||
quiet: input.delete.quiet,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ObjectIdentifier> for DeleteBatchEntry {
|
||||
fn from(object: ObjectIdentifier) -> Self {
|
||||
Self {
|
||||
key: object.key,
|
||||
key_too_long: false,
|
||||
version_id: object.version_id,
|
||||
e_tag: object.e_tag.map(|tag| match tag {
|
||||
ETag::Strong(value) => DeleteBatchETag { value, weak: false },
|
||||
ETag::Weak(value) => DeleteBatchETag { value, weak: true },
|
||||
}),
|
||||
_last_modified_time: object.last_modified_time.map(|value| {
|
||||
let instant = time::OffsetDateTime::from(value);
|
||||
(instant.unix_timestamp(), instant.nanosecond())
|
||||
}),
|
||||
_size: object.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<DeleteBatchOutput> for DeleteObjectsOutput {
|
||||
fn from(output: DeleteBatchOutput) -> Self {
|
||||
Self {
|
||||
deleted: output.deleted.map(|entries| {
|
||||
entries
|
||||
.into_iter()
|
||||
.map(|entry| DeletedObject {
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
key: entry.key,
|
||||
version_id: entry.version_id,
|
||||
})
|
||||
.collect()
|
||||
}),
|
||||
errors: output.errors.map(|entries| {
|
||||
entries
|
||||
.into_iter()
|
||||
.map(|entry| Error {
|
||||
code: entry.code,
|
||||
key: entry.key,
|
||||
message: entry.message,
|
||||
version_id: entry.version_id,
|
||||
})
|
||||
.collect()
|
||||
}),
|
||||
request_charged: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,8 +31,9 @@ pub(crate) use crate::app::object::{
|
||||
};
|
||||
pub(crate) use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
pub(crate) use crate::app::object_usecase::{
|
||||
GetObjectAttributesFields, GetObjectAttributesOutputFacts, GetObjectBody, GetObjectFields, GetObjectOutputFacts,
|
||||
HeadObjectFields, HeadObjectOutputFacts,
|
||||
DeleteBatchETag, DeleteBatchEntry, DeleteBatchInput, DeleteBatchOutput, GetObjectAttributesFields,
|
||||
GetObjectAttributesOutputFacts, GetObjectBody, GetObjectFields, GetObjectOutputFacts, HeadObjectFields,
|
||||
HeadObjectOutputFacts,
|
||||
};
|
||||
|
||||
pub(crate) mod bucket;
|
||||
|
||||
@@ -257,8 +257,7 @@ pub(crate) mod helper_consumer {
|
||||
MutationCompletionProbe, capture_mutation_event_for_test, pause_committed_mutation_for_test,
|
||||
};
|
||||
pub(crate) use super::super::helper::{
|
||||
OperationHelper, build_event_resp_elements, build_owned_event_resp_elements, run_owned_mutation,
|
||||
spawn_background_with_context,
|
||||
OperationHelper, build_owned_event_resp_elements, run_owned_mutation, spawn_background_with_context,
|
||||
};
|
||||
|
||||
pub(crate) type StorageObjectInfo = super::StorageObjectInfo;
|
||||
|
||||
@@ -14440,7 +14440,16 @@ fn assert_object_mutation_reserved_prefix_sources(delete: &str, legacy_delete: &
|
||||
"fn execute_delete_object_fields(",
|
||||
"owner.execute_delete_object_owned(envelope, bucket, key, version_id)",
|
||||
),
|
||||
(delete, "fn execute_delete_objects(", "owner.execute_delete_objects_owned(req)"),
|
||||
(
|
||||
legacy_delete,
|
||||
"fn delete_objects_edge(",
|
||||
"usecase.execute_delete_objects(envelope, req.input.into())",
|
||||
),
|
||||
(
|
||||
delete,
|
||||
"fn execute_delete_objects(",
|
||||
"owner.execute_delete_objects_owned(envelope, input)",
|
||||
),
|
||||
(
|
||||
legacy_put,
|
||||
"fn execute_put_object_extract(",
|
||||
@@ -14551,6 +14560,18 @@ fn object_mutation_reserved_prefix_guard_rejects_broken_delete_chains() {
|
||||
"owner.execute_delete_object_owned(envelope, bucket, key, version_id)",
|
||||
"owner.execute_delete_object_unchecked(envelope, bucket, key, version_id)",
|
||||
),
|
||||
(
|
||||
"legacy batch uses a different use case",
|
||||
true,
|
||||
"usecase.execute_delete_objects(envelope, req.input.into())",
|
||||
"usecase.execute_other_batch(envelope, req.input.into())",
|
||||
),
|
||||
(
|
||||
"batch delegate uses different input",
|
||||
false,
|
||||
"owner.execute_delete_objects_owned(envelope, input)",
|
||||
"owner.execute_delete_objects_owned(envelope, changed_input)",
|
||||
),
|
||||
("typed method missing", false, "fn execute_delete_object(", "fn missing_delete_object("),
|
||||
("legacy method missing", true, "fn delete_object_edge(", "fn missing_delete_object_edge("),
|
||||
(
|
||||
@@ -14597,6 +14618,12 @@ fn object_mutation_reserved_prefix_guard_rejects_broken_delete_chains() {
|
||||
"owner.execute_delete_object_owned(envelope, bucket, key, version_id)" => {
|
||||
"fn execute_delete_object_fields( must call its guarded implementation"
|
||||
}
|
||||
"usecase.execute_delete_objects(envelope, req.input.into())" => {
|
||||
"fn delete_objects_edge( must call its guarded implementation"
|
||||
}
|
||||
"owner.execute_delete_objects_owned(envelope, input)" => {
|
||||
"fn execute_delete_objects( must call its guarded implementation"
|
||||
}
|
||||
_ => from,
|
||||
};
|
||||
(
|
||||
|
||||
@@ -405,12 +405,22 @@ fn single_rule(method: &str, origin: &str) -> CorsConfiguration {
|
||||
.expect("valid configuration")
|
||||
}
|
||||
|
||||
/// Records the status of the one response a request receives.
|
||||
/// Records the status and request ID headers of the response.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(super) struct StatusCapture(Arc<Mutex<Option<u16>>>);
|
||||
pub(super) struct StatusCapture(Arc<Mutex<Option<CapturedResponse>>>);
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct CapturedResponse {
|
||||
pub(super) status: u16,
|
||||
pub(super) request_ids: [Option<String>; 2],
|
||||
}
|
||||
|
||||
impl StatusCapture {
|
||||
pub(super) fn take(&self) -> Option<u16> {
|
||||
self.take_response().map(|response| response.status)
|
||||
}
|
||||
|
||||
pub(super) fn take_response(&self) -> Option<CapturedResponse> {
|
||||
self.0.lock().ok().and_then(|mut slot| slot.take())
|
||||
}
|
||||
}
|
||||
@@ -426,11 +436,15 @@ impl Intercept for StatusCapture {
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let status = context.response().status().as_u16();
|
||||
let response = context.response();
|
||||
let header = |name| response.headers().get(name).map(str::to_owned);
|
||||
*self
|
||||
.0
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("status capture mutex was poisoned"))? = Some(status);
|
||||
.map_err(|_| std::io::Error::other("status capture mutex was poisoned"))? = Some(CapturedResponse {
|
||||
status: response.status().as_u16(),
|
||||
request_ids: [header("x-amz-request-id"), header("x-request-id")],
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -437,7 +437,8 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
|
||||
Err(refusal) => refusal,
|
||||
};
|
||||
let context = format!("{} as the {caller} caller: {refusal:?}", row.operation);
|
||||
assert_eq!(refusal.status, Some(403), "{context}");
|
||||
let per_key = row.metric == S3Operation::DeleteObjects && !by_authentication;
|
||||
assert_eq!(refusal.status, Some(if per_key { 200 } else { 403 }), "{context}");
|
||||
if row.has_error_body {
|
||||
assert_eq!(refusal.code.as_deref(), Some(*code), "{context}");
|
||||
}
|
||||
@@ -448,7 +449,17 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
|
||||
);
|
||||
let counted = counted_since(&before);
|
||||
assert_eq!(counted.values().sum::<u64>(), 1, "counted once: {context} {counted:?}");
|
||||
assert!(counted.keys().all(|(_, _, outcome)| *outcome == "4xx"), "{context} {counted:?}");
|
||||
let expected_outcome = if per_key { "2xx" } else { "4xx" };
|
||||
assert!(
|
||||
counted.keys().all(|(_, _, outcome)| *outcome == expected_outcome),
|
||||
"{context} {counted:?}"
|
||||
);
|
||||
if per_key {
|
||||
assert!(
|
||||
counted.keys().all(|(_, label, _)| *label == row.metric.as_str()),
|
||||
"under the batch operation: {context} {counted:?}"
|
||||
);
|
||||
}
|
||||
if *by_authentication {
|
||||
assert!(
|
||||
counted.keys().all(|(_, label, _)| *label == "unknown"),
|
||||
|
||||
@@ -23,8 +23,8 @@ use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata as _;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, OutputSerialization,
|
||||
RestoreRequest, VersioningConfiguration,
|
||||
BucketVersioningStatus, CsvInput, CsvOutput, Delete, ExpressionType, FileHeaderInfo, InputSerialization, ObjectIdentifier,
|
||||
OutputSerialization, RestoreRequest, VersioningConfiguration,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
@@ -46,6 +46,12 @@ pub(crate) const MATRIX: &[MatrixRow] = &[
|
||||
has_error_body: true,
|
||||
call: delete_object,
|
||||
},
|
||||
MatrixRow {
|
||||
operation: "DeleteObjects",
|
||||
metric: S3Operation::DeleteObjects,
|
||||
has_error_body: true,
|
||||
call: delete_objects,
|
||||
},
|
||||
MatrixRow {
|
||||
operation: "RestoreObject",
|
||||
metric: S3Operation::RestoreObject,
|
||||
@@ -288,6 +294,46 @@ fn the_legacy_stack_admits_and_renews_restore_object() {
|
||||
with_stack(LEGACY, restore_admits_once_and_renews_in_place);
|
||||
}
|
||||
|
||||
fn batch(keys: &[&str]) -> aws_sdk_s3::types::builders::DeleteBuilder {
|
||||
Delete::builder().set_objects(Some(
|
||||
keys.iter()
|
||||
.map(|key| ObjectIdentifier::builder().key(*key).build().expect("object key"))
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
fn delete_objects<'a>(client: &'a Client, _bucket: &'a str) -> BoxFuture<'a, Outcome> {
|
||||
Box::pin(async move {
|
||||
let status = StatusCapture::default();
|
||||
let result = client
|
||||
.delete_objects()
|
||||
.bucket(MATRIX_BUCKET)
|
||||
.delete(batch(&["matrix-absent"]).build().unwrap())
|
||||
.customize()
|
||||
.interceptor(status.clone())
|
||||
.send()
|
||||
.await;
|
||||
let Ok(output) = result else {
|
||||
return outcome(result);
|
||||
};
|
||||
let response = status.take_response().expect("receive DeleteObjects response");
|
||||
assert_eq!(response.status, 200);
|
||||
if let [error] = output.errors() {
|
||||
assert_eq!(error.key(), Some("matrix-absent"));
|
||||
assert!(output.deleted().is_empty());
|
||||
return Err(super::Refusal {
|
||||
code: error.code().map(str::to_owned),
|
||||
status: Some(response.status),
|
||||
request_ids: response.request_ids,
|
||||
});
|
||||
}
|
||||
assert!(output.errors().is_empty(), "unexpected response: {output:?}");
|
||||
assert_eq!(output.deleted().len(), 1);
|
||||
assert_eq!(output.deleted()[0].key(), Some("matrix-absent"));
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
const DELETE_BUCKET: &str = "e2-delete-object";
|
||||
/// A bucket name RustFS creates and the AWS grammar refuses (`sthree-` is reserved).
|
||||
const RESERVED_NAME_BUCKET: &str = "sthree-e2-delete";
|
||||
@@ -692,3 +738,132 @@ fn gateway_select_preserves_the_event_sequence_and_denies_unauthorized_reads() {
|
||||
fn legacy_select_preserves_the_event_sequence_and_denies_unauthorized_reads() {
|
||||
with_stack(LEGACY, select_parity);
|
||||
}
|
||||
|
||||
async fn delete_objects_checks_each_key_and_preserves_quiet_errors(server: RustFSServer) {
|
||||
const BUCKET: &str = "e2-delete-batch-scoped";
|
||||
const USER: &str = "e2deletebatchscoped";
|
||||
create_bucket_in_the_store(BUCKET).await;
|
||||
create_user_without_a_policy(USER).await;
|
||||
let policy = rustfs_policy::policy::Policy::parse_config(format!(
|
||||
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::{BUCKET}/allowed*"}}]}}"#
|
||||
).as_bytes()).expect("parse batch policy");
|
||||
let iam = get_global_app_context().expect("embedded server context").iam().handle();
|
||||
iam.set_policy(USER, policy).await.expect("store batch policy");
|
||||
iam.policy_db_set(USER, rustfs_iam::store::UserType::Reg, false, USER)
|
||||
.await
|
||||
.expect("attach batch policy");
|
||||
|
||||
let client = s3_client(&server.endpoint(), USER, USER_SECRET_KEY);
|
||||
let root = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
for quiet in [false, true] {
|
||||
put_in_the_store(BUCKET, "allowed").await;
|
||||
put_in_the_store(BUCKET, "allowed-etag").await;
|
||||
put_in_the_store(BUCKET, "denied").await;
|
||||
let head = root
|
||||
.head_object()
|
||||
.bucket(BUCKET)
|
||||
.key("allowed-etag")
|
||||
.send()
|
||||
.await
|
||||
.expect("read fixture ETag");
|
||||
let objects = [
|
||||
ObjectIdentifier::builder().key("denied").build().unwrap(),
|
||||
ObjectIdentifier::builder().key("allowed").e_tag("*").build().unwrap(),
|
||||
ObjectIdentifier::builder().key("allowed-absent").build().unwrap(),
|
||||
ObjectIdentifier::builder().key("allowed-absent").build().unwrap(),
|
||||
ObjectIdentifier::builder()
|
||||
.key("allowed-etag")
|
||||
.e_tag(head.e_tag().expect("stored ETag"))
|
||||
.build()
|
||||
.unwrap(),
|
||||
];
|
||||
let output = client
|
||||
.delete_objects()
|
||||
.bucket(BUCKET)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.set_objects(Some(objects.into()))
|
||||
.quiet(quiet)
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect("mixed DeleteObjects returns per-key outcomes");
|
||||
assert!(is_in_the_store(BUCKET, "denied").await, "key-level denial must preserve the object");
|
||||
assert!(
|
||||
is_in_the_store(BUCKET, "allowed-etag").await,
|
||||
"concrete ETag without GetObject must preserve the object"
|
||||
);
|
||||
assert!(
|
||||
!is_in_the_store(BUCKET, "allowed").await,
|
||||
"the wildcard needs DeleteObject but not GetObject"
|
||||
);
|
||||
let deleted: Vec<_> = output.deleted().iter().map(|object| object.key().unwrap()).collect();
|
||||
assert_eq!(
|
||||
deleted,
|
||||
if quiet {
|
||||
vec![]
|
||||
} else {
|
||||
vec!["allowed", "allowed-absent", "allowed-absent"]
|
||||
}
|
||||
);
|
||||
let errors: Vec<_> = output
|
||||
.errors()
|
||||
.iter()
|
||||
.map(|error| (error.key().unwrap(), error.code().unwrap()))
|
||||
.collect();
|
||||
assert_eq!(errors, [("denied", "AccessDenied"), ("allowed-etag", "AccessDenied")]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_deletes_only_authorized_batch_entries() {
|
||||
with_stack(GATEWAY, delete_objects_checks_each_key_and_preserves_quiet_errors);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_deletes_only_authorized_batch_entries() {
|
||||
with_stack(LEGACY, delete_objects_checks_each_key_and_preserves_quiet_errors);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_echoes_rejected_entries_without_executing_them() {
|
||||
with_stack(GATEWAY, |server| async move {
|
||||
const BUCKET: &str = "e2-delete-batch-refusals";
|
||||
create_bucket_in_the_store(BUCKET).await;
|
||||
for key in ["kept", "conditional", "deleted"] {
|
||||
put_in_the_store(BUCKET, key).await;
|
||||
}
|
||||
let oversized_key = format!("{}kept", "/".repeat(1025));
|
||||
let objects = [
|
||||
ObjectIdentifier::builder().key("").build().unwrap(),
|
||||
ObjectIdentifier::builder().key(&oversized_key).build().unwrap(),
|
||||
ObjectIdentifier::builder().key("conditional").e_tag("\"\"").build().unwrap(),
|
||||
ObjectIdentifier::builder().key("deleted").build().unwrap(),
|
||||
];
|
||||
let root = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
let output = root
|
||||
.delete_objects()
|
||||
.bucket(BUCKET)
|
||||
.delete(Delete::builder().set_objects(Some(objects.into())).build().unwrap())
|
||||
.send()
|
||||
.await
|
||||
.expect("the independent entry still succeeds");
|
||||
|
||||
assert_eq!(output.deleted().len(), 1);
|
||||
assert_eq!(output.deleted()[0].key(), Some("deleted"));
|
||||
let errors: Vec<_> = output.errors().iter().map(|error| (error.key(), error.code())).collect();
|
||||
assert_eq!(
|
||||
errors,
|
||||
[
|
||||
(Some(""), Some("InvalidArgument")),
|
||||
(Some(oversized_key.as_str()), Some("InvalidArgument")),
|
||||
(Some("conditional"), Some("InvalidArgument")),
|
||||
],
|
||||
);
|
||||
assert!(is_in_the_store(BUCKET, "kept").await);
|
||||
assert!(is_in_the_store(BUCKET, "conditional").await);
|
||||
assert!(!is_in_the_store(BUCKET, "deleted").await);
|
||||
});
|
||||
}
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 70 B |
@@ -0,0 +1 @@
|
||||
<!doctype html><html lang="en"><title>Console fixture</title><body>Console static fixture</body></html>
|
||||
@@ -0,0 +1 @@
|
||||
console-route-static-content
|
||||
@@ -22,6 +22,8 @@ classify_source_layer() {
|
||||
printf 'composition'
|
||||
elif [[ "$file" == rustfs/src/app/* ]]; then
|
||||
printf 'app'
|
||||
elif [[ "$file" == rustfs/src/protocols/client.rs ]] || [[ "$file" == rustfs/src/protocols/client/* ]]; then
|
||||
printf 'interface'
|
||||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] ||
|
||||
[[ "$file" == rustfs/src/storage/legacy_edge/* ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
|
||||
[[ "$file" == rustfs/src/storage/s3_compat/* ]]; then
|
||||
@@ -47,6 +49,12 @@ classify_target_layer() {
|
||||
# their exported HTTP contracts belong to the interface boundary.
|
||||
printf 'interface'
|
||||
;;
|
||||
protocols)
|
||||
case "${path#protocols::}" in
|
||||
client | client::* | ProtocolStorageClient | ProtocolStorageClient::*) printf 'interface' ;;
|
||||
*) printf 'infra' ;;
|
||||
esac
|
||||
;;
|
||||
app)
|
||||
printf 'app'
|
||||
;;
|
||||
@@ -147,6 +155,16 @@ run_layer_model_self_tests() {
|
||||
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/tests/object_meta.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge_helper.rs
|
||||
assert_source_layer 'interface' rustfs/src/protocols/client.rs
|
||||
assert_source_layer 'interface' rustfs/src/protocols/client/owned.rs
|
||||
assert_source_layer 'infra' rustfs/src/protocols/client_helper.rs
|
||||
assert_target_layer 'interface' protocols::client
|
||||
assert_target_layer 'interface' protocols::client::ProtocolStorageClient
|
||||
assert_target_layer 'interface' protocols::ProtocolStorageClient
|
||||
assert_target_layer 'infra' protocols::client_helper
|
||||
assert_dependency_direction 'allowed' "$(classify_source_layer rustfs/src/protocols/client/owned.rs)" "$app_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer protocols::client::ProtocolStorageClient)"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer protocols::ProtocolStorageClient)"
|
||||
assert_source_layer 'app' rustfs/src/app/bucket_usecase/cors.rs
|
||||
assert_source_layer 'app' rustfs/src/app/multipart_usecase/complete.rs
|
||||
assert_target_layer 'interface' storage::legacy_edge
|
||||
|
||||
@@ -286,6 +286,18 @@ for silent in ${REAL_SPLIT_FILES[@]+"${REAL_SPLIT_FILES[@]}"}; do
|
||||
done
|
||||
|
||||
# --- layer dependencies ------------------------------------------------------
|
||||
# Protocol edges may call the app; app code cannot call either public client path.
|
||||
mkdir -p "$FX/rustfs/src/protocols/client"
|
||||
cat >"$FX/rustfs/src/protocols/client/layer_probe.rs" <<'PROTOCOL_EDGE'
|
||||
use crate::app::RequestEnvelope;
|
||||
PROTOCOL_EDGE
|
||||
cat >"$FX/rustfs/src/protocols/client_helper.rs" <<'PROTOCOL_NEIGHBOR'
|
||||
use crate::app::RequestEnvelope;
|
||||
PROTOCOL_NEIGHBOR
|
||||
cat >"$FX/rustfs/src/app/protocol_client_layer_probe.rs" <<'PROTOCOL_REVERSE'
|
||||
use crate::protocols::client::ProtocolStorageClient;
|
||||
use crate::protocols::ProtocolStorageClient;
|
||||
PROTOCOL_REVERSE
|
||||
LAYER_OUT="$FX/.layer.out"
|
||||
if (cd "$FX" && ./scripts/check_layer_dependencies.sh) >"$LAYER_OUT" 2>&1; then
|
||||
fail "layer guard passed over a planted reverse dependency"
|
||||
@@ -294,6 +306,10 @@ expect_reported "$LAYER_OUT" "dep|$E|interface->composition|" "crate::startup_se
|
||||
expect_silent "$LAYER_OUT" "dep|$EDGE_IMPORTS|" "moved ecfs imports stay legal"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/app/lifecycle_transition_api_test/" "moved lifecycle transition tests stay excluded"
|
||||
expect_silent "$LAYER_OUT" "dep|$LISTING|" "moved bucket usecase code keeps the app layer"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/protocols/client/layer_probe.rs|" "protocol client edge may use app contracts"
|
||||
expect_reported "$LAYER_OUT" "dep|rustfs/src/protocols/client_helper.rs|infra->app|" "crate::app::RequestEnvelope" "neighboring protocol helper remains infrastructure"
|
||||
expect_reported "$LAYER_OUT" "dep|rustfs/src/app/protocol_client_layer_probe.rs|app->interface|" "crate::protocols::client::ProtocolStorageClient" "app cannot import the protocol client module"
|
||||
expect_reported "$LAYER_OUT" "dep|rustfs/src/app/protocol_client_layer_probe.rs|app->interface|" "crate::protocols::ProtocolStorageClient" "app cannot import the protocol client re-export"
|
||||
|
||||
# --- logging hot span --------------------------------------------------------
|
||||
LOG_OUT="$FX/.logging.out"
|
||||
|
||||
Reference in New Issue
Block a user