Compare commits

..

9 Commits

35 changed files with 561 additions and 1832 deletions
+1 -1
View File
@@ -309,7 +309,7 @@ hostname = "0.4.2"
ipnetwork = { version = "0.21.1" }
lazy_static = "1.5.1"
libc = "0.2.191"
cuobject = "0.1.0"
cuobject = { version = "0.1.0", default-features = false }
libsystemd = "0.7.2"
local-ip-address = "0.6.13"
log = "0.4"
+9
View File
@@ -3657,6 +3657,11 @@ fn complete_pending_rebuild_on_cache_hit(dial_addr: &str, cache_key: &str) -> bo
// Limits transport response futures; client RPC metrics separately cover unary body decoding.
const INTERNODE_RPC_CONCURRENCY_LIMIT: usize = 64;
// Quorum reads cancel surplus metadata RPCs in bursts. Keep bounded reset state
// for their in-flight replies without spending the connection's protocol-error
// budget (h2 remembers only 50 resets by default).
const INTERNODE_RPC_MAX_CONCURRENT_RESET_STREAMS: usize = 4096;
/// Whether control/bulk channel isolation is enabled (env-gated, default off for safe rollout).
fn channel_isolation_enabled() -> bool {
rustfs_utils::get_env_bool(
@@ -4110,6 +4115,7 @@ async fn dial_channel(
let conn_window = checked_rpc_window(conn_window)?;
let mut endpoint = Endpoint::from_shared(dial_addr.to_string())?
.concurrency_limit(concurrency_limit)
.http2_max_concurrent_reset_streams(INTERNODE_RPC_MAX_CONCURRENT_RESET_STREAMS)
// HTTP/2 PING frames for application-layer health check
.http2_keep_alive_interval(http2_keepalive_interval)
// How long to wait for PING ACK before considering connection dead
@@ -4266,6 +4272,9 @@ pub async fn evict_failed_connection_with_reason(
}
}
#[cfg(test)]
mod reset_stream_tests;
#[cfg(test)]
mod tests {
use super::*;
+186
View File
@@ -0,0 +1,186 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::io;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::oneshot;
use tonic::codegen::{Service, http};
const CANCELLED_REQUESTS: usize = 64;
async fn write_frame(socket: &mut TcpStream, kind: u8, flags: u8, stream: u32, payload: &[u8]) -> io::Result<()> {
let length = u32::try_from(payload.len()).expect("fixture frame length");
let mut frame = Vec::with_capacity(9 + payload.len());
frame.extend_from_slice(&length.to_be_bytes()[1..]);
frame.extend_from_slice(&[kind, flags]);
frame.extend_from_slice(&stream.to_be_bytes());
frame.extend_from_slice(payload);
socket.write_all(&frame).await
}
async fn read_frame(socket: &mut TcpStream) -> io::Result<(u8, u8, u32, Vec<u8>)> {
let mut header = [0; 9];
socket.read_exact(&mut header).await?;
let length = u32::from_be_bytes([0, header[0], header[1], header[2]]);
let stream = u32::from_be_bytes(header[5..9].try_into().expect("stream ID bytes"));
let mut payload = vec![0; usize::try_from(length).expect("fixture frame length")];
socket.read_exact(&mut payload).await?;
Ok((header[3], header[4], stream, payload))
}
// A raw peer can release replies already in transit after observing CANCEL.
// An h2 server API would discard those replies upon receiving the reset.
async fn delayed_peer(
listener: TcpListener,
requests_seen: oneshot::Sender<()>,
resets_seen: oneshot::Sender<usize>,
) -> io::Result<usize> {
let (mut socket, _) = listener.accept().await?;
socket.set_nodelay(true)?;
let mut preface = [0; 24];
socket.read_exact(&mut preface).await?;
assert_eq!(&preface, b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n");
write_frame(&mut socket, 4, 0, 0, &[]).await?;
let mut requests = Vec::new();
while requests.len() < CANCELLED_REQUESTS {
let (kind, flags, stream, _) = read_frame(&mut socket).await?;
match (kind, flags) {
(4, 0) => write_frame(&mut socket, 4, 1, 0, &[]).await?,
(1, _) => requests.push(stream),
_ => {}
}
}
requests_seen.send(()).expect("caller waits for dispatched requests");
let mut cancellations = 0;
while cancellations < CANCELLED_REQUESTS {
let (kind, _, stream, payload) = read_frame(&mut socket).await?;
if kind == 3 {
assert!(requests.contains(&stream), "reset belongs to a dispatched request");
assert_eq!(payload, 8_u32.to_be_bytes(), "caller cancellation sends CANCEL");
cancellations += 1;
}
}
for stream in requests {
// HPACK static index 8 is ':status: 200'; END_HEADERS | END_STREAM.
write_frame(&mut socket, 1, 5, stream, &[0x88]).await?;
}
let ping = b"rst-test";
write_frame(&mut socket, 6, 0, 0, ping).await?;
let mut error_resets = 0;
loop {
let (kind, flags, _, payload) = read_frame(&mut socket).await?;
match (kind, flags) {
(3, _) => {
assert_eq!(payload, 5_u32.to_be_bytes(), "late reply rejected as STREAM_CLOSED");
error_resets += 1;
}
(6, 1) if payload == ping => break,
(7, _) => panic!("late replies closed the shared connection: {payload:?}"),
_ => {}
}
}
// PING acknowledgment establishes that all preceding replies were processed.
resets_seen.send(error_resets).expect("caller waits for reset accounting");
loop {
let (kind, _, stream, payload) = read_frame(&mut socket).await?;
if kind == 3 {
assert_eq!(payload, 5_u32.to_be_bytes());
error_resets += 1;
}
if kind == 1 {
write_frame(&mut socket, 1, 5, stream, &[0x88]).await?;
return Ok(error_resets);
}
}
}
async fn cancelled_metadata_replies_keep_channel_usable(bulk: bool) {
tokio::time::timeout(Duration::from_secs(10), async {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fixture peer");
let address = format!("http://{}", listener.local_addr().expect("fixture address"));
let key = if bulk {
super::bulk_cache_key(&address, 0)
} else {
address.clone()
};
let (requests_tx, requests_rx) = oneshot::channel();
let (resets_tx, resets_rx) = oneshot::channel();
let server = tokio::spawn(delayed_peer(listener, requests_tx, resets_tx));
let (channel, _) = super::dial_channel(&address, &key, Duration::from_secs(10))
.await
.expect("production channel connects");
let mut calls = Vec::new();
for _ in 0..CANCELLED_REQUESTS {
let mut channel = channel.clone();
let uri = format!("{address}/node_service.NodeService/ReadVersion");
calls.push(tokio::spawn(async move {
std::future::poll_fn(|cx| channel.poll_ready(cx))
.await
.expect("channel ready");
channel
.call(
http::Request::builder()
.uri(uri)
.body(tonic::body::Body::empty())
.expect("read request"),
)
.await
}));
}
requests_rx.await.expect("all metadata requests dispatched");
for call in &calls {
call.abort();
}
for call in calls {
assert!(call.await.expect_err("request cancelled before response").is_cancelled());
}
let _ = resets_rx.await.expect("peer accounts for late response frames");
let mut channel = channel;
std::future::poll_fn(|cx| channel.poll_ready(cx))
.await
.expect("channel remains ready");
let response = channel
.call(
http::Request::builder()
.uri(format!("{address}/node_service.NodeService/Ping"))
.body(tonic::body::Body::empty())
.expect("probe request"),
)
.await
.expect("subsequent RPC shares the connection");
assert_eq!(response.status(), http::StatusCode::OK);
let error_resets = server
.await
.expect("fixture task completes")
.expect("fixture transport succeeds");
assert_eq!(
error_resets, 0,
"cancelled metadata replies must not spend the connection's protocol-error budget"
);
})
.await
.expect("bounded cancellation exchange");
}
#[tokio::test]
async fn control_channel_tolerates_late_cancelled_metadata_replies() {
cancelled_metadata_replies_keep_channel_usable(false).await;
}
#[tokio::test]
async fn bulk_channel_tolerates_late_cancelled_metadata_replies() {
cancelled_metadata_replies_keep_channel_usable(true).await;
}
+1 -2
View File
@@ -57,8 +57,7 @@ use tracing::{Span, debug, error, info, instrument, warn};
use tracing_opentelemetry::OpenTelemetrySpanExt;
#[derive(RustEmbed)]
#[cfg_attr(test, folder = "$CARGO_MANIFEST_DIR/tests/fixtures/native-console")]
#[cfg_attr(not(test), folder = "$CARGO_MANIFEST_DIR/static")]
#[folder = "$CARGO_MANIFEST_DIR/static"]
struct StaticFiles;
/// Static file handler
-8
View File
@@ -922,14 +922,6 @@ pub(crate) fn claims_route(method: &Method, uri: &Uri, headers: &HeaderMap) -> b
|| is_control_plane_route(method, uri, headers)
}
/// Console paths whose requests do not belong to an earlier extension route.
#[cfg(feature = "gateway-stack")]
pub(crate) fn claims_native_console_route(method: &Method, uri: &Uri) -> bool {
is_console_path(uri.path())
&& parse_object_lambda_request(method, uri).is_none()
&& s3_extension::route_key(method, uri).is_none()
}
fn is_public_health_path(path: &str) -> bool {
path == HEALTH_PREFIX || path == HEALTH_READY_PATH
}
@@ -332,7 +332,7 @@ async fn delete_objects_resolves_bucket_versioning_once_per_request() {
let lookups_before = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst);
let snapshots_before = DELETE_CONFIG_SNAPSHOT_LOADS.load(Ordering::SeqCst);
let response = usecase
.execute_delete_objects_from_s3s(req)
.execute_delete_objects(req)
.await
.expect("delete objects should succeed");
@@ -389,7 +389,7 @@ async fn delete_objects_treats_empty_directory_version_as_absent() {
});
let response = usecase
.execute_delete_objects_from_s3s(req)
.execute_delete_objects(req)
.await
.expect("empty VersionId should be treated as an absent version");
let deleted = response
@@ -438,7 +438,7 @@ async fn delete_objects_preserves_explicit_null_directory_version() {
});
let response = usecase
.execute_delete_objects_from_s3s(req)
.execute_delete_objects(req)
.await
.expect("explicit null VersionId should delete the directory's null version");
let deleted = response
@@ -839,7 +839,7 @@ async fn delete_objects_replication_config_failure_does_not_call_storage_delete(
});
let err = usecase
.execute_delete_objects_from_s3s(req)
.execute_delete_objects(req)
.await
.expect_err("replication config failure must reject DeleteObjects");
+122 -145
View File
@@ -18,14 +18,10 @@ use super::shared::{LOG_COMPONENT_APP, LOG_SUBSYSTEM_OBJECT};
use super::*;
use crate::app::RequestEnvelope;
use crate::app::storage_api::object_usecase::DeleteObjectCondition;
use crate::app::storage_api::object_usecase::access::AuthorizationRequest as _;
#[cfg(test)]
use crate::app::storage_api::s3::dto::{DeleteObjectsInput, DeleteObjectsOutput};
use crate::app::storage_api::s3::{S3ErrorCode, s3_error};
#[cfg(test)]
use crate::app::storage_api::s3::{S3Request, S3Response};
use crate::app::storage_api::s3::dto::ObjectIdentifier;
use crate::app::storage_api::s3::dto::{DeleteObjectsInput, DeleteObjectsOutput, DeletedObject};
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use rustfs_gateway_types::dto as gateway_dto;
use rustfs_s3_types::S3Result;
use std::borrow::Cow;
use std::collections::{BTreeMap, HashSet};
@@ -104,10 +100,11 @@ fn normalize_delete_object_key(key: &str) -> Cow<'_, str> {
}
fn successful_delete_audit_objects(
objects: &[DeleteBatchEntry],
delete: &s3s::dto::Delete,
successful_results: impl IntoIterator<Item = bool>,
) -> Vec<AuditObjectVersion> {
objects
delete
.objects
.iter()
.zip(successful_results)
.filter(|(_, successful)| *successful)
@@ -134,7 +131,7 @@ fn normalize_delete_objects_version_id(
}
}
fn parse_delete_objects_condition(object: &DeleteBatchEntry) -> Result<Option<DeleteObjectCondition>, DeleteBatchError> {
fn parse_delete_objects_condition(object: &ObjectIdentifier) -> Result<Option<DeleteObjectCondition>, s3s::dto::Error> {
let Some(etag) = object.e_tag.as_ref() else {
return Ok(None);
};
@@ -143,15 +140,15 @@ fn parse_delete_objects_condition(object: &DeleteBatchEntry) -> Result<Option<De
S3ErrorCode::InvalidRequest,
"An ETag condition cannot be combined with an explicit VersionId".to_string(),
)
} else if !etag.weak {
match DeleteObjectCondition::parse(&etag.value) {
} else if let Some(etag) = etag.as_strong() {
match DeleteObjectCondition::parse(etag) {
Ok(condition) => return Ok(Some(condition)),
Err(error) => (S3ErrorCode::InvalidArgument, error.to_string()),
}
} else {
(S3ErrorCode::InvalidArgument, "A delete ETag condition must use a strong ETag".to_string())
};
Err(DeleteBatchError {
Err(s3s::dto::Error {
code: Some(code.as_str().to_string()),
key: Some(object.key.clone()),
message: Some(message),
@@ -344,7 +341,7 @@ async fn replica_delete_fanout_state(
replica_delete_fanout_state_from_config(config, replication_source, version_id, &site_peer_deployment_ids)
}
fn validate_undo_delete_version(expected: Option<&str>, requested: Option<&str>) -> crate::app::storage_api::s3::S3Result<()> {
fn validate_undo_delete_version(expected: Option<&str>, requested: Option<&str>) -> S3Result<()> {
if expected.is_some() && expected != requested {
return Err(s3_error!(PreconditionFailed));
}
@@ -488,12 +485,12 @@ fn project_delete_objects_pre_stat_error(
object: ObjectToDelete,
synthetic_version_id: bool,
error: ApiError,
) -> S3Result<DeleteBatchError> {
) -> S3Result<s3s::dto::Error> {
// Bucket loss invalidates the shared request, not just one object.
if error.code == S3ErrorCode::NoSuchBucket {
return Err(error.into());
}
Ok(DeleteBatchError {
Ok(s3s::dto::Error {
code: Some(error.code.as_str().to_string()),
key: Some(object.object_name),
message: Some(error.message),
@@ -527,12 +524,12 @@ fn reduce_delete_objects_result<'a>(
deleted: &'a StorageDeletedObject,
error: Option<&EcstoreError>,
synthetic_version_id: bool,
) -> Result<&'a StorageDeletedObject, DeleteBatchError> {
) -> Result<&'a StorageDeletedObject, s3s::dto::Error> {
match delete_objects_reported_storage_error(object, error) {
None => Ok(deleted),
Some(error) => {
let api_error = ApiError::from(error.clone());
Err(DeleteBatchError {
Err(s3s::dto::Error {
code: Some(api_error.code.as_str().to_string()),
key: Some(object.object_name.clone()),
message: Some(api_error.message),
@@ -585,54 +582,50 @@ fn delete_objects_storage_error_counts(
}
impl DefaultObjectUsecase {
#[instrument(level = "debug", skip(self, envelope, input))]
pub(crate) async fn execute_delete_objects(
&self,
envelope: RequestEnvelope,
input: DeleteBatchInput,
) -> S3Result<DeleteBatchOutput> {
#[instrument(level = "debug", skip(self, req))]
pub async fn execute_delete_objects(&self, req: S3Request<DeleteObjectsInput>) -> S3Result<S3Response<DeleteObjectsOutput>> {
let owner = self.clone();
run_owned_mutation(async move { Box::pin(owner.execute_delete_objects_owned(envelope, input)).await }).await
run_owned_mutation(async move { Box::pin(owner.execute_delete_objects_owned(req)).await }).await
}
async fn execute_delete_objects_owned(
&self,
mut envelope: RequestEnvelope,
input: DeleteBatchInput,
) -> S3Result<DeleteBatchOutput> {
mut req: S3Request<DeleteObjectsInput>,
) -> S3Result<S3Response<DeleteObjectsOutput>> {
if let Some(context) = &self.context {
let _ = context.object_store();
}
let view = envelope.audit_view();
let helper =
OperationHelper::from_view(&view, EventName::ObjectRemovedDelete, S3Operation::DeleteObjects).suppress_event();
let request_context = helper.request_context_or_from_view(&view);
// Both edges carry LastModifiedTime and Size. The batch engine only evaluates ETag.
let DeleteBatchInput { bucket, objects, quiet } = input;
let helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObjects).suppress_event();
let request_context = helper.request_context_or_from_request(&req);
let (bucket, delete) = {
let bucket = req.input.bucket.clone();
let delete = req.input.delete.clone();
(bucket, delete)
};
if objects.is_empty() || objects.len() > 1000 {
return Err(rustfs_s3_types::s3_error!(
InvalidArgument,
"No objects to delete or too many objects to delete"
if delete.objects.is_empty() || delete.objects.len() > 1000 {
return Err(S3Error::with_message(
S3ErrorCode::InvalidArgument,
"No objects to delete or too many objects to delete".to_string(),
));
}
let Some(store) = self.object_store() else {
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// Capture the bucket generation before per-object authorization, but
// do not expose a bucket-state error unless at least one object is authorized.
let bucket_generation = load_bucket_generation_from_store(store.as_ref(), &envelope, &bucket).await;
let bucket_generation = load_bucket_generation_from_store(store.as_ref(), &req, &bucket).await;
#[cfg(test)]
wait_for_delete_objects_generation_test_hook(&bucket).await;
let bypass_governance = has_bypass_governance_header(envelope.headers());
let bypass_governance = has_bypass_governance_header(&req.headers);
#[derive(Default, Clone)]
struct DeleteResult {
delete_object: Option<StorageDeletedObject>,
error: Option<DeleteBatchError>,
error: Option<s3s::dto::Error>,
emit_side_effects: bool,
synthetic_version_id: bool,
version_suspended: bool,
@@ -643,40 +636,29 @@ impl DefaultObjectUsecase {
requested_version_id: Option<Uuid>,
}
let mut delete_results = vec![DeleteResult::default(); objects.len()];
let mut delete_results = vec![DeleteResult::default(); delete.objects.len()];
struct AuthorizedDelete {
idx: usize,
object: ObjectToDelete,
}
let mut authorized_deletes = Vec::with_capacity(objects.len());
let mut authorized_deletes = Vec::with_capacity(delete.objects.len());
// Issue #5740: keep the first per-key denial of this bulk request at
// warn and demote the rest to debug, so a denied 1000-key DeleteObjects
// cannot flood the log.
let mut bulk_denial_logged = false;
// Per-key denials are audited once per denied action, in aggregate, after the loop.
let mut bulk_denials = crate::storage::access::BulkDenialAudit::start_owned(&mut envelope)?;
for (idx, obj_id) in objects.iter().enumerate() {
let mut bulk_denials = crate::storage::access::BulkDenialAudit::start(&mut req)?;
for (idx, obj_id) in delete.objects.iter().enumerate() {
// Authorization and every storage side effect must address the same
// normalized object; retain the request key only for response entries.
let key = normalize_delete_object_key(&obj_id.key);
// A rejected wire key has no authorization resource. Its normalized text cannot become a target.
let key_error = if obj_id.key_too_long {
Some((
S3ErrorCode::InvalidArgument,
ApiError::error_code_to_message(&S3ErrorCode::InvalidArgument),
))
} else {
validate_object_key(&key, "DeleteObjects")
.err()
.map(|error| (error.code().clone(), error.to_string()))
};
if let Some((code, message)) = key_error {
delete_results[idx].error = Some(DeleteBatchError {
code: Some(code.as_str().to_string()),
if let Err(err) = validate_object_key(&key, "DeleteObjects") {
delete_results[idx].error = Some(s3s::dto::Error {
code: Some(err.code().as_str().to_string()),
key: Some(obj_id.key.clone()),
message: Some(message),
message: Some(err.to_string()),
version_id: obj_id.version_id.clone(),
});
continue;
@@ -692,7 +674,7 @@ impl DefaultObjectUsecase {
let (version_id, version_uuid) = match normalize_delete_objects_version_id(raw_version_id.clone()) {
Ok(parsed) => parsed,
Err(err) => {
delete_results[idx].error = Some(DeleteBatchError {
delete_results[idx].error = Some(s3s::dto::Error {
code: Some("NoSuchVersion".to_string()),
key: Some(obj_id.key.clone()),
message: Some(err),
@@ -703,29 +685,29 @@ impl DefaultObjectUsecase {
};
{
let req_info = req_info_mut(&mut envelope)?;
let req_info = req_info_mut(&mut req)?;
req_info.bucket = Some(bucket.clone());
req_info.object = Some(key.to_string());
req_info.version_id = version_id.clone();
}
let mut denied_action = delete_object_authorize_action(version_id.as_deref());
let mut auth_res = authorize_request(&mut envelope, denied_action).await;
let mut auth_res = authorize_request(&mut req, denied_action).await;
if auth_res.is_ok()
&& delete_condition
.as_ref()
.is_some_and(DeleteObjectCondition::requires_read_permission)
{
denied_action = Action::S3Action(S3Action::GetObjectAction);
auth_res = authorize_request(&mut envelope, denied_action).await;
auth_res = authorize_request(&mut req, denied_action).await;
}
if auth_res.is_err() {
bulk_denials.denied(denied_action);
if !bulk_denial_logged {
bulk_denial_logged = true;
req_info_mut(&mut envelope)?.suppress_denial_log = true;
req_info_mut(&mut req)?.suppress_denial_log = true;
}
delete_results[idx].error = Some(DeleteBatchError {
delete_results[idx].error = Some(s3s::dto::Error {
code: Some("AccessDenied".to_string()),
key: Some(obj_id.key.clone()),
message: Some("Access Denied".to_string()),
@@ -735,15 +717,14 @@ impl DefaultObjectUsecase {
}
if bypass_governance {
let auth_res =
authorize_request(&mut envelope, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await;
let auth_res = authorize_request(&mut req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await;
if auth_res.is_err() {
bulk_denials.denied(Action::S3Action(S3Action::BypassGovernanceRetentionAction));
if !bulk_denial_logged {
bulk_denial_logged = true;
req_info_mut(&mut envelope)?.suppress_denial_log = true;
req_info_mut(&mut req)?.suppress_denial_log = true;
}
delete_results[idx].error = Some(DeleteBatchError {
delete_results[idx].error = Some(s3s::dto::Error {
code: Some("AccessDenied".to_string()),
key: Some(obj_id.key.clone()),
message: Some("Access Denied".to_string()),
@@ -754,7 +735,7 @@ impl DefaultObjectUsecase {
}
if let Err(err) = validate_table_catalog_object_mutation(&bucket, &key).await {
delete_results[idx].error = Some(DeleteBatchError {
delete_results[idx].error = Some(s3s::dto::Error {
code: Some("InvalidRequest".to_string()),
key: Some(obj_id.key.clone()),
message: Some(err.to_string()),
@@ -777,18 +758,19 @@ impl DefaultObjectUsecase {
authorized_deletes.push(AuthorizedDelete { idx, object });
}
bulk_denials.finish_owned(&mut envelope, RequestEnvelope::audit_view)?;
bulk_denials.finish(&mut req)?;
if authorized_deletes.is_empty() {
let output = DeleteBatchOutput {
deleted: (!quiet.unwrap_or_default()).then(Vec::new),
let output = DeleteObjectsOutput {
deleted: (!delete.quiet.unwrap_or_default()).then(Vec::new),
errors: Some(delete_results.into_iter().filter_map(|result| result.error).collect()),
..Default::default()
};
let result = Ok(output);
let _ = helper.complete_owned(&result, StatusCode::OK, &HeaderMap::new());
let result = Ok(S3Response::new(output));
let _ = helper.complete(&result);
return result;
}
envelope.install_bucket_generation(bucket_generation?);
req.extensions.insert(bucket_generation?);
let bucket_lock_enabled = object_lock_checks_required(&bucket).await;
let delete_config_snapshot = Arc::new(
@@ -813,7 +795,7 @@ impl DefaultObjectUsecase {
// DeleteObjects names each key explicitly. The force-delete header is a
// DeleteObject/DeleteBucket extension and must not change this batch,
// including by rejecting a non-boolean value.
let mut batch_headers = envelope.headers().clone();
let mut batch_headers = req.headers.clone();
batch_headers.remove("x-rustfs-force-delete");
batch_headers.remove("x-minio-force-delete");
let mut prepared_deletes: Vec<PreparedDelete> = Vec::with_capacity(authorized_deletes.len());
@@ -931,7 +913,7 @@ impl DefaultObjectUsecase {
object_lock_delete: Some(StorageObjectLockDeleteOptions { bypass_governance }),
..Default::default()
};
apply_bucket_generation_guard(&envelope, &bucket, &mut storage_delete_opts)?;
apply_bucket_generation_guard(&req, &bucket, &mut storage_delete_opts)?;
let storage_started = std::time::Instant::now();
let (dobjs, errs, accounting) = store
.delete_objects_with_tier_delete_journal_and_accounting(&bucket, object_to_delete.clone(), storage_delete_opts)
@@ -970,8 +952,8 @@ impl DefaultObjectUsecase {
super::super::storage_api::object_usecase::helper::pause_committed_mutation_for_test(&bucket).await;
let _manager = get_concurrency_manager();
if !errs.is_empty() && errs.iter().all(|err| err.as_ref().is_some_and(is_err_bucket_not_found)) {
let result = Err(rustfs_s3_types::s3_error!(NoSuchBucket, "Bucket not found"));
let _ = helper.complete_owned(&result, StatusCode::OK, &HeaderMap::new());
let result = Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "Bucket not found".to_string()));
let _ = helper.complete(&result);
return result;
}
@@ -1019,19 +1001,19 @@ impl DefaultObjectUsecase {
}
}
let deleted = (!quiet.unwrap_or_default()).then(|| {
let deleted = (!delete.quiet.unwrap_or_default()).then(|| {
delete_results
.iter()
.enumerate()
.filter_map(|(idx, result)| result.delete_object.as_ref().map(|object| (idx, result, object)))
.map(|(idx, result, object)| DeleteBatchSuccess {
.map(|(idx, result, object)| DeletedObject {
delete_marker: { if object.delete_marker { Some(true) } else { None } },
delete_marker_version_id: delete_marker_response_version_id(
object.delete_marker_version_id,
object.delete_marker && result.version_suspended && result.requested_version_id.is_none(),
result.synthetic_version_id,
),
key: Some(objects[idx].key.clone()),
key: Some(delete.objects[idx].key.clone()),
version_id: delete_entry_response_version_id(
object.version_id,
object.delete_marker,
@@ -1052,17 +1034,18 @@ impl DefaultObjectUsecase {
.enumerate()
.filter_map(|(idx, result)| {
let mut error = result.error.clone()?;
error.key = Some(objects[idx].key.clone());
error.key = Some(delete.objects[idx].key.clone());
Some(error)
})
.collect::<Vec<_>>();
let output = DeleteBatchOutput {
.collect::<Vec<s3s::dto::Error>>();
let output = DeleteObjectsOutput {
deleted,
errors: Some(errors),
..Default::default()
};
let helper = if helper.wants_audit_object_info() {
let audit_objects =
successful_delete_audit_objects(&objects, delete_results.iter().map(|result| result.delete_object.is_some()));
successful_delete_audit_objects(&delete, delete_results.iter().map(|result| result.delete_object.is_some()));
helper.audit_objects(audit_objects)
} else {
helper
@@ -1091,10 +1074,11 @@ impl DefaultObjectUsecase {
let _ = replication_task.await;
}
let req_headers = envelope.headers().clone();
let req_headers = req.headers.clone();
let notify = current_notify_interface_for_context(self.context.as_deref());
let req_params = rustfs_targets::extract_params_header(&req_headers);
let resp_elements = build_owned_event_resp_elements(&HeaderMap::new(), &request_context.request_id);
let resp_elements =
build_event_resp_elements(&S3Response::new(DeleteObjectsOutput::default()), &request_context.request_id);
let deleted_any = delete_results.iter().any(|result| result.delete_object.is_some());
let delete_producers = delete_results
.iter()
@@ -1159,8 +1143,8 @@ impl DefaultObjectUsecase {
}
});
let result = Ok(output);
let _ = helper.complete_owned(&result, StatusCode::OK, &HeaderMap::new());
let result = Ok(S3Response::new(output));
let _ = helper.complete(&result);
if deleted_any {
rustfs_scanner::record_dirty_usage_bucket_from_producers(&bucket, delete_producers);
}
@@ -1616,20 +1600,12 @@ impl DefaultObjectUsecase {
#[cfg(test)]
impl DefaultObjectUsecase {
pub(crate) async fn execute_delete_objects_from_s3s(
&self,
req: S3Request<DeleteObjectsInput>,
) -> crate::app::storage_api::s3::S3Result<S3Response<DeleteObjectsOutput>> {
let envelope = RequestEnvelope::from_s3s(&req);
Ok(S3Response::new(self.execute_delete_objects(envelope, req.input.into()).await?.into()))
}
/// DeleteObject for a request built as the legacy stack receives it, converted the way its
/// edge converts it.
pub(crate) async fn execute_delete_object_from_s3s(
&self,
req: S3Request<crate::app::storage_api::s3::dto::DeleteObjectInput>,
) -> crate::app::storage_api::s3::S3Result<gateway_dto::DeleteObjectOutput> {
) -> S3Result<gateway_dto::DeleteObjectOutput> {
let envelope = RequestEnvelope::from_s3s(&req);
let input = req.input;
Ok(Box::pin(self.execute_delete_object_fields(envelope, input.bucket, input.key, input.version_id)).await?)
@@ -1867,7 +1843,7 @@ mod tests {
is_owner: true,
..Default::default()
});
let output = Box::pin(usecase.execute_delete_objects_from_s3s(req))
let output = Box::pin(usecase.execute_delete_objects(req))
.await
.expect("batch delete")
.output;
@@ -2395,7 +2371,7 @@ mod tests {
is_owner: true,
..Default::default()
});
let response = Box::pin(usecase.execute_delete_objects_from_s3s(req))
let response = Box::pin(usecase.execute_delete_objects(req))
.await
.expect("batch marker delete must reach the authoritative mutation");
assert!(response.output.errors.as_ref().is_none_or(Vec::is_empty), "{:?}", response.output.errors);
@@ -2576,7 +2552,7 @@ mod tests {
is_owner: true,
..Default::default()
});
let response = Box::pin(usecase.execute_delete_objects_from_s3s(req))
let response = Box::pin(usecase.execute_delete_objects(req))
.await
.expect("a pre-stat failure must not abort independent batch entries");
let deleted = response.output.deleted.expect("per-key successes");
@@ -2610,7 +2586,7 @@ mod tests {
is_owner: true,
..Default::default()
});
let response = Box::pin(usecase.execute_delete_objects_from_s3s(req))
let response = Box::pin(usecase.execute_delete_objects(req))
.await
.expect("all pre-stat failures still produce per-key results");
assert!(response.output.deleted.as_ref().is_none_or(Vec::is_empty));
@@ -2678,7 +2654,7 @@ mod tests {
ApiError::from(StorageError::BucketNotFound("missing".to_string())),
)
.expect_err("shared bucket loss remains a request failure");
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::NoSuchBucket);
assert_eq!(error.code(), &S3ErrorCode::NoSuchBucket);
let version = Uuid::new_v4();
for (version_id, synthetic, expected) in [
@@ -2842,7 +2818,7 @@ mod tests {
let req = build_request(input, Method::POST);
let usecase = DefaultObjectUsecase::without_context();
let err = usecase.execute_delete_objects_from_s3s(req).await.unwrap_err();
let err = usecase.execute_delete_objects(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
}
@@ -2864,7 +2840,7 @@ mod tests {
let req = build_request(input, Method::POST);
let usecase = DefaultObjectUsecase::without_context();
let err = usecase.execute_delete_objects_from_s3s(req).await.unwrap_err();
let err = usecase.execute_delete_objects(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
}
@@ -2886,7 +2862,7 @@ mod tests {
let req = build_request(input, Method::POST);
let usecase = DefaultObjectUsecase::without_context();
let err = usecase.execute_delete_objects_from_s3s(req).await.unwrap_err();
let err = usecase.execute_delete_objects(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(err.message(), Some("Not init"));
}
@@ -2944,7 +2920,7 @@ mod tests {
install_delete_objects_generation_test_hook(bucket.clone(), Arc::clone(&loaded), Arc::clone(&resume));
let usecase = DefaultObjectUsecase::with_context(Some(context));
let delete = tokio::spawn(async move { usecase.execute_delete_objects_from_s3s(req).await });
let delete = tokio::spawn(async move { usecase.execute_delete_objects(req).await });
loaded.wait().await;
store
@@ -3070,8 +3046,13 @@ mod tests {
},
];
let requested: Vec<_> = requested.into_iter().map(Into::into).collect();
let objects = successful_delete_audit_objects(&requested, [true, false, true]);
let objects = successful_delete_audit_objects(
&Delete {
objects: requested,
quiet: Some(true),
},
[true, false, true],
);
assert_eq!(
objects,
@@ -3090,8 +3071,16 @@ mod tests {
..Default::default()
}];
let requested: Vec<_> = requested.into_iter().map(Into::into).collect();
assert!(successful_delete_audit_objects(&requested, [false]).is_empty());
assert!(
successful_delete_audit_objects(
&Delete {
objects: requested,
quiet: None,
},
[false]
)
.is_empty()
);
}
#[test]
@@ -3114,31 +3103,22 @@ mod tests {
key: "key".to_string(),
..Default::default()
};
assert_eq!(
parse_delete_objects_condition(&object.clone().into()).expect("missing ETag is unconditional"),
None
);
assert_eq!(parse_delete_objects_condition(&object).expect("missing ETag is unconditional"), None);
for etag in ["", "\"\"", " \t "] {
let error = parse_delete_objects_condition(
&ObjectIdentifier {
e_tag: Some(ETag::Strong(etag.to_string())),
..object.clone()
}
.into(),
)
let error = parse_delete_objects_condition(&ObjectIdentifier {
e_tag: Some(ETag::Strong(etag.to_string())),
..object.clone()
})
.expect_err("an explicit empty ETag must never become an unconditional delete");
assert_eq!(error.code.as_deref(), Some("InvalidArgument"));
assert_eq!(error.key.as_deref(), Some("key"));
}
for version in ["null", "", "bad-version"] {
let error = parse_delete_objects_condition(
&ObjectIdentifier {
e_tag: Some(ETag::Strong("*".to_string())),
version_id: Some(version.to_string()),
..object.clone()
}
.into(),
)
let error = parse_delete_objects_condition(&ObjectIdentifier {
e_tag: Some(ETag::Strong("*".to_string())),
version_id: Some(version.to_string()),
..object.clone()
})
.expect_err("any explicit VersionId is unsupported with a condition");
assert_eq!(error.code.as_deref(), Some("InvalidRequest"));
assert_eq!(error.version_id.as_deref(), Some(version));
@@ -3179,7 +3159,7 @@ mod tests {
e_tag: Some(ETag::Weak(value.to_string())),
..Default::default()
};
let error = parse_delete_objects_condition(&object.into())
let error = parse_delete_objects_condition(&object)
.expect_err("a weak validator must fail instead of falling back to an unconditional delete");
assert_eq!(error.code.as_deref(), Some("InvalidArgument"));
assert_eq!(error.key.as_deref(), Some("key"));
@@ -3190,7 +3170,7 @@ mod tests {
..Default::default()
};
assert_eq!(
parse_delete_objects_condition(&object.into()).expect("strong validator retains its condition"),
parse_delete_objects_condition(&object).expect("strong validator retains its condition"),
Some(DeleteObjectCondition::MatchEtag("current-etag".to_string()))
);
}
@@ -3418,7 +3398,7 @@ mod tests {
req.extensions.insert(crate::storage::access::ReqInfo::default());
let usecase = DefaultObjectUsecase::from_global();
let response = usecase
.execute_delete_objects_from_s3s(req)
.execute_delete_objects(req)
.await
.expect("mixed batch returns per-entry outcomes");
let deleted = response
@@ -3490,10 +3470,7 @@ mod tests {
.expect("build quiet mixed batch");
let mut quiet_req = build_request(quiet_input, Method::POST);
quiet_req.extensions.insert(crate::storage::access::ReqInfo::default());
let quiet_response = usecase
.execute_delete_objects_from_s3s(quiet_req)
.await
.expect("quiet batch succeeds");
let quiet_response = usecase.execute_delete_objects(quiet_req).await.expect("quiet batch succeeds");
assert!(quiet_response.output.deleted.is_none(), "Quiet hides only successful entries");
let quiet_errors = quiet_response.output.errors.expect("Quiet preserves errors");
assert_eq!(quiet_errors.len(), 1);
@@ -3585,7 +3562,7 @@ mod tests {
let mut request = build_request(input, Method::POST);
request.uri = format!("/{request_bucket}").parse().expect("URI");
request.extensions.insert(crate::storage::access::ReqInfo::default());
usecase.execute_delete_objects_from_s3s(request).await.map(|_| ())
usecase.execute_delete_objects(request).await.map(|_| ())
} else {
let input = DeleteObjectInput::builder()
.bucket(request_bucket.clone())
@@ -3965,7 +3942,7 @@ mod tests {
..Default::default()
});
DefaultObjectUsecase::from_global()
.execute_delete_objects_from_s3s(batch_req)
.execute_delete_objects(batch_req)
.await
.expect("batch compressed delete should succeed");
assert_eq!(
@@ -4345,10 +4322,10 @@ mod bulk_denial_audit_wiring_tests {
.next()
.unwrap();
let started = source
.find("BulkDenialAudit::start_owned(&mut envelope)")
.find("BulkDenialAudit::start(&mut req)")
.expect("per-key denial audit is not suppressed");
let finished = source
.find("bulk_denials.finish_owned(&mut envelope, RequestEnvelope::audit_view)")
.find("bulk_denials.finish(&mut req)")
.expect("the aggregate entry is not emitted");
assert!(started < finished, "the aggregate must be emitted after the per-key checks");
assert_eq!(
@@ -1,63 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Owned input and result fields for the batch deletion engine.
//!
//! Keys and versions stay raw so legacy refusals can echo entries that did not pass validation.
#[derive(Debug, Default)]
pub(crate) struct DeleteBatchInput {
pub(crate) bucket: String,
pub(crate) objects: Vec<DeleteBatchEntry>,
pub(crate) quiet: Option<bool>,
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub(crate) struct DeleteBatchEntry {
pub(crate) key: String,
pub(crate) key_too_long: bool,
pub(crate) version_id: Option<String>,
pub(crate) e_tag: Option<DeleteBatchETag>,
/// Seconds since the Unix epoch and nanoseconds within that second.
pub(crate) _last_modified_time: Option<(i64, u32)>,
pub(crate) _size: Option<i64>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct DeleteBatchETag {
pub(crate) value: String,
pub(crate) weak: bool,
}
#[derive(Debug, Default)]
pub(crate) struct DeleteBatchOutput {
pub(crate) deleted: Option<Vec<DeleteBatchSuccess>>,
pub(crate) errors: Option<Vec<DeleteBatchError>>,
}
#[derive(Debug, Default)]
pub(crate) struct DeleteBatchSuccess {
pub(crate) delete_marker: Option<bool>,
pub(crate) delete_marker_version_id: Option<String>,
pub(crate) key: Option<String>,
pub(crate) version_id: Option<String>,
}
#[derive(Debug, Default, Clone)]
pub(crate) struct DeleteBatchError {
pub(crate) code: Option<String>,
pub(crate) key: Option<String>,
pub(crate) message: Option<String>,
pub(crate) version_id: Option<String>,
}
+2 -3
View File
@@ -86,7 +86,8 @@ use super::storage_api::object_usecase::error::{
};
use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
use super::storage_api::object_usecase::helper::{
OperationHelper, build_owned_event_resp_elements, run_owned_mutation, spawn_background_with_context,
OperationHelper, build_event_resp_elements, build_owned_event_resp_elements, run_owned_mutation,
spawn_background_with_context,
};
use super::storage_api::object_usecase::io::{
DynReader, EtagAlgorithm, HashReader, WritePlan, compression_metadata_value, wrap_reader,
@@ -219,8 +220,6 @@ pub(crate) use copy_input::{CopyObjectWriteInput, CopyObjectWriteOutput, CopySou
#[cfg(all(any(feature = "cuobject", feature = "nixl"), target_os = "linux"))]
pub(crate) mod cuobject;
mod delete;
mod delete_batch_types;
pub(crate) use delete_batch_types::*;
mod extract;
mod get;
mod get_response;
+2 -1
View File
@@ -1638,7 +1638,8 @@ pub(crate) mod helper {
MutationCompletionProbe, capture_mutation_event_for_test, pause_committed_mutation_for_test,
};
pub(crate) use crate::storage::storage_api::helper_consumer::{
OperationHelper, build_owned_event_resp_elements, run_owned_mutation, spawn_background_with_context,
OperationHelper, build_event_resp_elements, build_owned_event_resp_elements, run_owned_mutation,
spawn_background_with_context,
};
}
+111 -275
View File
@@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod owned;
use crate::runtime_sources::current_action_credentials;
#[cfg(feature = "webdav")]
use crate::runtime_sources::current_object_store_handle;
@@ -496,13 +494,38 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
"Protocol storage client request"
);
self.get_object_owned(
bucket,
key,
credentials,
start_pos.map(|first| rustfs_gateway_types::ByteRange::From { first }),
)
.await
let mut builder = GetObjectInput::builder().bucket(bucket.to_string()).key(key.to_string());
// Add range header if start_pos is specified
if let Some(start) = start_pos {
let range = s3s::dto::Range::Int {
first: start,
last: None, // Read to end of file
};
builder = builder.range(Some(range));
}
let input = builder.build().map_err(|e| {
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build GetObjectInput: {}", e))
})?;
let uri = build_object_uri(bucket, key, &[])?;
let req = Self::create_request(
input,
Method::GET,
uri,
RequestParams {
bucket: Some(bucket.to_string()),
object: Some(key.to_string()),
credentials,
},
)?;
match self.fs.get_object(req).await {
Ok(response) => gateway_s3s_compat::generated::ops::get_object::output_from_s3s(response.output)
.map_err(|e| conversion_error("GetObject output", e)),
Err(e) => Err(e.into()),
}
}
async fn put_object(
@@ -565,20 +588,31 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
) -> Result<gateway_dto::DeleteObjectOutput, Self::Error> {
trace_protocol_request("delete_object", Some(bucket), Some(key));
let input = DeleteObjectInput::builder()
.bucket(bucket.to_string())
.key(key.to_string())
.build()
.map_err(|e| {
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build DeleteObjectInput: {}", e))
})?;
let uri = build_object_uri(bucket, key, &[])?;
let envelope = Self::create_envelope(
let req = Self::create_request(
input,
Method::DELETE,
uri,
RequestParams {
bucket: Some(bucket.to_owned()),
object: Some(key.to_owned()),
bucket: Some(bucket.to_string()),
object: Some(key.to_string()),
credentials,
},
);
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::DeleteObject);
self.owned_usecase()
.execute_delete_object_fields(envelope, bucket.to_owned(), key.to_owned(), None)
.await
)?;
match self.fs.delete_object(req).await {
Ok(response) => gateway_s3s_compat::generated::ops::delete_object::output_from_s3s(response.output)
.map_err(|e| conversion_error("DeleteObject output", e)),
Err(e) => Err(e.into()),
}
}
async fn head_object(
@@ -589,32 +623,31 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
) -> Result<gateway_dto::HeadObjectOutput, Self::Error> {
trace_protocol_request("head_object", Some(bucket), Some(key));
let input = HeadObjectInput::builder()
.bucket(bucket.to_string())
.key(key.to_string())
.build()
.map_err(|e| {
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build HeadObjectInput: {}", e))
})?;
let uri = build_object_uri(bucket, key, &[])?;
let envelope = Self::create_envelope(
let req = Self::create_request(
input,
Method::HEAD,
uri,
RequestParams {
bucket: Some(bucket.to_owned()),
object: Some(key.to_owned()),
bucket: Some(bucket.to_string()),
object: Some(key.to_string()),
credentials,
},
);
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::HeadObject);
self.owned_usecase()
.execute_head_object_fields(
envelope,
crate::app::object_usecase::HeadObjectFields {
bucket: bucket.to_owned(),
key: key.to_owned(),
version_id: None,
part_number: None,
range: None,
sse_customer_key: None,
sse_customer_key_md5: None,
},
)
.await
.and_then(|answered| owned::head_output(answered.output))
)?;
match self.fs.head_object(req).await {
Ok(response) => gateway_s3s_compat::generated::ops::head_object::output_from_s3s(response.output)
.map_err(|e| conversion_error("HeadObject output", e)),
Err(e) => Err(e.into()),
}
}
async fn head_bucket(
@@ -858,16 +891,37 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
"Protocol storage client request"
);
self.get_object_owned(
bucket,
key,
credentials,
Some(rustfs_gateway_types::ByteRange::FromTo {
first: start_pos,
last: start_pos + length - 1,
}),
)
.await
let range = s3s::dto::Range::Int {
first: start_pos,
last: Some(start_pos + length - 1),
};
let input = GetObjectInput::builder()
.bucket(bucket.to_string())
.key(key.to_string())
.range(Some(range))
.build()
.map_err(|e| {
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build GetObjectInput: {}", e))
})?;
let uri = build_object_uri(bucket, key, &[])?;
let req = Self::create_request(
input,
Method::GET,
uri,
RequestParams {
bucket: Some(bucket.to_string()),
object: Some(key.to_string()),
credentials,
},
)?;
match self.fs.get_object(req).await {
Ok(response) => gateway_s3s_compat::generated::ops::get_object::output_from_s3s(response.output)
.map_err(|e| conversion_error("GetObject output", e)),
Err(e) => Err(e.into()),
}
}
async fn copy_object(
@@ -1349,30 +1403,16 @@ mod tests {
},
)
.expect("request should build");
let envelope = ProtocolStorageClient::create_envelope(
Method::GET,
http::Uri::from_static("/bucket?list-type=2"),
RequestParams {
bucket: Some("bucket".to_owned()),
object: None,
credentials: &credentials,
},
);
assert_eq!(envelope.uri(), &request.uri);
assert_eq!(envelope.headers(), &request.headers);
for request_info in [
request.extensions.get::<ReqInfo>().expect("request info should be present"),
envelope.req_info().expect("owned request info"),
] {
let copied = request_info.cred.as_ref().expect("credentials should be present");
assert_eq!(copied.access_key, credentials.access_key);
assert_eq!(copied.secret_key, credentials.secret_key);
assert_eq!(copied.session_token, credentials.session_token);
assert_eq!(copied.parent_user, credentials.parent_user);
assert_eq!(copied.groups, credentials.groups);
assert_eq!(copied.claims, Some(claims.clone()));
assert!(copied.is_service_account());
}
let request_info = request.extensions.get::<ReqInfo>().expect("request info should be present");
let copied = request_info.cred.as_ref().expect("credentials should be present");
assert_eq!(copied.access_key, credentials.access_key);
assert_eq!(copied.secret_key, credentials.secret_key);
assert_eq!(copied.session_token, credentials.session_token);
assert_eq!(copied.parent_user, credentials.parent_user);
assert_eq!(copied.groups, credentials.groups);
assert_eq!(copied.claims, Some(claims));
assert!(copied.is_service_account());
}
#[cfg(any(feature = "webdav", feature = "sftp"))]
@@ -1483,208 +1523,4 @@ mod tests {
assert_eq!(uri.to_string(), "/bucket?list-type=2");
}
#[test]
#[serial_test::serial]
fn protocol_mainflow_markers_versions_and_errors_match_legacy() {
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-oracle", || async {
use crate::runtime_sources::ServerContextSlot;
use crate::storage_api::protocols::client::{
BucketOperations, MakeBucketOptions, ObjectIO, StorageObjectOptions, StoragePutObjReader,
};
use http_body_util::BodyExt;
use rustfs_protocols::common::client::s3::StorageBackend;
let root = rustfs_credentials::Credentials {
access_key: "protocol-oracle-root".into(),
secret_key: "protocol-oracle-secret".into(),
status: "on".into(),
..Default::default()
};
let (_temp, context, _iam) = crate::protocols::policy_test::protocol_test_context(root.clone()).await;
let store = context.object_store();
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let fs = FS::with_server_ctx(slot);
let backend = ProtocolStorageClient::new(fs.clone());
for (bucket, key, data, versioned) in [
("protocol-version-oracle", "versioned-key", b"versioned data".as_slice(), true),
("protocol-marker-oracle", "folder/", b"".as_slice(), false),
] {
store
.make_bucket(
bucket,
&MakeBucketOptions {
versioning_enabled: versioned,
..Default::default()
},
)
.await
.expect("create bucket");
let mut reader = StoragePutObjReader::from_vec(data.to_vec());
store
.put_object(
bucket,
key,
&mut reader,
&StorageObjectOptions {
versioned,
mod_time: Some(
time::OffsetDateTime::from_unix_timestamp(1_700_000_000)
.expect("valid test time")
.replace_nanosecond(123_456_789)
.expect("valid nanoseconds"),
),
..Default::default()
},
)
.await
.expect("seed object");
let params = || RequestParams {
bucket: Some(bucket.to_owned()),
object: Some(key.to_owned()),
credentials: &root,
};
let uri = build_object_uri(bucket, key, &[]).expect("object URI");
let old = fs
.get_object(
ProtocolStorageClient::create_request(
GetObjectInput::builder()
.bucket(bucket.to_owned())
.key(key.to_owned())
.build()
.expect("old GET input"),
Method::GET,
uri.clone(),
params(),
)
.expect("old GET request"),
)
.await
.expect("old GET");
let old = gateway_s3s_compat::generated::ops::get_object::output_from_s3s(old.output).expect("old GET output");
if versioned {
assert_eq!(old.last_modified.expect("old LastModified").subsec_nanos(), 123_000_000);
}
let current = backend.get_object(bucket, key, &root, None).await.expect("protocol GET");
assert_eq!(current.last_modified, old.last_modified);
assert_eq!(current.version_id, old.version_id);
assert_eq!(current.version_id.is_some(), versioned);
assert_eq!(current.e_tag, old.e_tag);
assert_eq!(current.content_length, old.content_length);
let collect = |output: gateway_dto::GetObjectOutput| async move {
output
.body
.expect("GET body")
.into_body()
.collect()
.await
.expect("read body")
.to_bytes()
};
assert_eq!(collect(current).await, collect(old).await);
let old = fs
.head_object(
ProtocolStorageClient::create_request(
HeadObjectInput::builder()
.bucket(bucket.to_owned())
.key(key.to_owned())
.build()
.expect("old HEAD input"),
Method::HEAD,
uri,
params(),
)
.expect("old HEAD request"),
)
.await
.expect("old HEAD");
let old = gateway_s3s_compat::generated::ops::head_object::output_from_s3s(old.output).expect("old HEAD output");
if versioned {
assert_eq!(old.last_modified.expect("old LastModified").subsec_nanos(), 123_000_000);
}
let current = backend.head_object(bucket, key, &root).await.expect("protocol HEAD");
assert_eq!(current.last_modified, old.last_modified);
assert_eq!(current.version_id, old.version_id);
assert_eq!(current.e_tag, old.e_tag);
assert_eq!(current.content_length, old.content_length);
let deleted = backend.delete_object(bucket, key, &root).await.expect("protocol DELETE");
if versioned {
assert_eq!(deleted.delete_marker, Some(true));
}
assert_eq!(deleted.version_id.is_some(), versioned);
let old_error: rustfs_s3_types::S3Error = fs
.get_object(
ProtocolStorageClient::create_request(
GetObjectInput::builder()
.bucket(bucket.to_owned())
.key(key.to_owned())
.build()
.expect("old GET input"),
Method::GET,
build_object_uri(bucket, key, &[]).expect("object URI"),
params(),
)
.expect("old GET request"),
)
.await
.expect_err("old GET deleted object")
.into();
let current = backend
.get_object(bucket, key, &root, None)
.await
.expect_err("protocol GET deleted object");
assert_eq!(current.code(), old_error.code());
assert_eq!(current.status_code(), old_error.status_code());
assert_eq!(current.headers(), old_error.headers());
if versioned {
let headers = current.headers().expect("delete marker error headers");
assert_eq!(
headers
.get("x-amz-version-id")
.expect("marker version")
.to_str()
.expect("version text"),
deleted.version_id.as_deref().expect("delete version")
);
}
}
});
}
#[test]
fn protocol_mainflow_retention_timestamp_matches_legacy_precision() {
use crate::app::object_usecase::{GetObjectOutputFacts, HeadObjectOutputFacts};
let instant = rustfs_gateway_types::Timestamp::from_secs_nanos(1_700_000_000, 123_456_789).expect("test timestamp");
let get = || GetObjectOutputFacts {
dto: gateway_dto::GetObjectOutput {
object_lock_retain_until_date: Some(instant),
..Default::default()
},
..Default::default()
};
let head = || HeadObjectOutputFacts {
dto: gateway_dto::HeadObjectOutput {
object_lock_retain_until_date: Some(instant),
..Default::default()
},
etag: None,
};
let old_get =
gateway_s3s_compat::generated::ops::get_object::output_from_s3s(get().into_s3s().expect("old GET projection"))
.expect("old GET output");
let old_head =
gateway_s3s_compat::generated::ops::head_object::output_from_s3s(head().into_s3s().expect("old HEAD projection"))
.expect("old HEAD output");
let old = (old_get.object_lock_retain_until_date, old_head.object_lock_retain_until_date);
let expected =
Some(rustfs_gateway_types::Timestamp::from_secs_nanos(1_700_000_000, 123_000_000).expect("millisecond timestamp"));
assert_eq!(old, (expected, expected));
let current = (
owned::get_output(get())
.expect("protocol GET output")
.object_lock_retain_until_date,
owned::head_output(head())
.expect("protocol HEAD output")
.object_lock_retain_until_date,
);
assert_eq!(current, old);
}
}
-130
View File
@@ -1,130 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Request and response projections for protocol reads and single-object deletion.
//! Authorization stays on the protocol session edge; storage runs in the client's server context.
use super::{ProtocolStorageClient, RequestParams};
use crate::app::object_usecase::{DefaultObjectUsecase, GetObjectBody, GetObjectOutputFacts, HeadObjectOutputFacts};
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
use crate::runtime_sources::current_action_credentials;
use crate::storage_api::protocols::client::{ReqInfo, RequestContext};
use http::{HeaderMap, Method, Uri};
use rustfs_gateway_types::{ETag, Timestamp, dto};
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
impl ProtocolStorageClient {
pub(super) async fn get_object_owned(
&self,
bucket: &str,
key: &str,
credentials: &rustfs_credentials::Credentials,
range: Option<rustfs_gateway_types::ByteRange>,
) -> S3Result<dto::GetObjectOutput> {
let uri = super::build_object_uri(bucket, key, &[])?;
let envelope = Self::create_envelope(
Method::GET,
uri,
RequestParams {
bucket: Some(bucket.to_owned()),
object: Some(key.to_owned()),
credentials,
},
);
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
self.owned_usecase()
.execute_get_object_fields(
envelope,
crate::app::object_usecase::GetObjectFields {
bucket: bucket.to_owned(),
key: key.to_owned(),
version_id: None,
part_number: None,
range,
sse_customer_key: None,
sse_customer_key_md5: None,
},
)
.await
.and_then(|answered| get_output(answered.output))
}
pub(super) fn owned_usecase(&self) -> DefaultObjectUsecase {
DefaultObjectUsecase::with_context(self.fs.server_ctx().app_context())
}
pub(super) fn create_envelope(method: Method, uri: Uri, params: RequestParams<'_>) -> RequestEnvelope {
let is_owner = current_action_credentials().is_some_and(|root| params.credentials.access_key == root.access_key);
let credentials =
RequestCredentials::new(params.credentials.access_key.clone(), params.credentials.secret_key.clone(), None)
.unwrap_or_else(|_| RequestCredentials::without_secret_key(params.credentials.access_key.clone(), None));
let mut extensions = http::Extensions::new();
extensions.insert(ReqInfo {
cred: Some(params.credentials.clone()),
is_owner,
bucket: params.bucket,
object: params.object,
version_id: None,
replication_request_authorized: false,
region: None,
request_context: Some(RequestContext::fallback()),
suppress_denial_log: false,
suppress_denial_audit: false,
});
RequestEnvelope::from_parts(
RequestHead {
method,
uri,
headers: HeaderMap::new(),
credentials: Some(credentials),
region: None,
service: None,
},
extensions,
)
}
}
fn etag(opaque: String, weak: bool) -> S3Result<ETag> {
if weak { ETag::new_weak(opaque) } else { ETag::new(opaque) }
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Object response contains an invalid ETag"))
}
// Protocol clients received millisecond timestamps through the legacy response codec.
fn millisecond_timestamp(value: Timestamp) -> Timestamp {
Timestamp::from_secs_nanos(value.secs(), value.subsec_millis() * 1_000_000)
.expect("a millisecond remainder is below one second") // subsec_millis is at most 999.
}
pub(super) fn get_output(mut output: GetObjectOutputFacts) -> S3Result<dto::GetObjectOutput> {
output.dto.last_modified = output.dto.last_modified.map(millisecond_timestamp);
output.dto.object_lock_retain_until_date = output.dto.object_lock_retain_until_date.map(millisecond_timestamp);
output.dto.e_tag = output.etag.map(|tag| etag(tag.opaque, tag.weak)).transpose()?;
output.dto.body = output
.body
.map(|body| match body {
GetObjectBody::Bytes(bytes) => Ok(rustfs_gateway_stream::ByteStream::from_bytes(bytes)),
GetObjectBody::Stream(stream) => rustfs_gateway_stream::ByteStream::new(stream)
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Invalid GetObject body capabilities")),
})
.transpose()?;
Ok(output.dto)
}
pub(super) fn head_output(mut output: HeadObjectOutputFacts) -> S3Result<dto::HeadObjectOutput> {
output.dto.last_modified = output.dto.last_modified.map(millisecond_timestamp);
output.dto.object_lock_retain_until_date = output.dto.object_lock_retain_until_date.map(millisecond_timestamp);
output.dto.e_tag = output.etag.map(|tag| etag(tag.opaque, tag.weak)).transpose()?;
Ok(output.dto)
}
-198
View File
@@ -1,198 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Protocol mainflow behavior through the installed server's object use case.
//! These probes exercise the protocol client, not the HTTP transport or authorization hook.
use super::ProtocolStorageClient;
use super::policy_test::protocol_test_context;
use crate::runtime_sources::ServerContextSlot;
use crate::storage_api::protocols::client::{
BucketOperations, MakeBucketOptions, ObjectIO, StorageObjectOptions, StoragePutObjReader,
};
use http_body_util::BodyExt;
use rustfs_credentials::Credentials;
use rustfs_protocols::common::client::s3::StorageBackend;
use rustfs_s3_types::S3ErrorCode;
const KEY: &str = "directory/a ?#+%.txt";
async fn object_client(bucket: &str) -> (tempfile::TempDir, Credentials, ProtocolStorageClient, String) {
let root = Credentials {
access_key: "protocol-mainflow-root".into(),
secret_key: "protocol-mainflow-secret".into(),
status: "on".into(),
..Default::default()
};
let (temp, context, _iam) = protocol_test_context(root.clone()).await;
let store = context.object_store();
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let mut reader = StoragePutObjReader::from_vec(b"protocol-data".to_vec());
let stored = store
.put_object(bucket, KEY, &mut reader, &StorageObjectOptions::default())
.await
.expect("seed object");
let slot = ServerContextSlot::new();
assert!(slot.install(context));
(
temp,
root,
crate::init::protocol_storage_client_for_server(slot),
stored.etag.expect("stored ETag"),
)
}
#[test]
#[serial_test::serial]
fn protocol_mainflow_reads_ranges_and_data() {
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-read", || async {
let bucket = "protocol-mainflow-read";
let (_temp, root, backend, stored_etag) = object_client(bucket).await;
for (start, expected) in [(None, b"protocol-data".as_slice()), (Some(9), b"data".as_slice())] {
let output = backend.get_object(bucket, KEY, &root, start).await.expect("protocol GET");
assert_eq!(output.content_length, Some(expected.len() as i64));
assert_eq!(output.e_tag.as_ref().expect("GET ETag").opaque_tag(), stored_etag);
let body = output
.body
.expect("GET body")
.into_body()
.collect()
.await
.expect("consume GET")
.to_bytes();
assert_eq!(body.as_ref(), expected);
assert_eq!(output.version_id, None);
}
let ranged = backend
.get_object_range(bucket, KEY, &root, 2, 4)
.await
.expect("bounded range");
assert_eq!(ranged.content_range.as_deref(), Some("bytes 2-5/13"));
assert_eq!(
ranged
.body
.expect("range body")
.into_body()
.collect()
.await
.expect("consume range")
.to_bytes()
.as_ref(),
b"otoc"
);
});
}
#[test]
#[serial_test::serial]
fn protocol_mainflow_heads_object_metadata() {
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-head", || async {
let bucket = "protocol-mainflow-head";
let (_temp, root, backend, stored_etag) = object_client(bucket).await;
let head = backend.head_object(bucket, KEY, &root).await.expect("protocol HEAD");
assert_eq!(head.content_length, Some(13));
assert_eq!(head.e_tag.as_ref().expect("HEAD ETag").opaque_tag(), stored_etag);
assert_eq!(head.version_id, None);
});
}
#[test]
#[serial_test::serial]
fn protocol_mainflow_deletes_object() {
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-delete", || async {
let bucket = "protocol-mainflow-delete";
let (_temp, root, backend, _stored_etag) = object_client(bucket).await;
let deleted = backend.delete_object(bucket, KEY, &root).await.expect("protocol DELETE");
assert_eq!(deleted.version_id, None);
assert_eq!(deleted.delete_marker, Some(false));
let missing = backend
.get_object(bucket, KEY, &root, None)
.await
.expect_err("deleted object is absent");
assert_eq!(missing.code(), &S3ErrorCode::NoSuchKey);
});
}
#[test]
#[serial_test::serial]
fn protocol_mainflow_refuses_missing_objects_buckets_and_unsatisfied_ranges() {
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-errors", || async {
let bucket = "protocol-mainflow-errors";
let (_temp, root, backend, _stored_etag) = object_client(bucket).await;
for target in [bucket, "protocol-absent-bucket"] {
let expected = if target == bucket {
S3ErrorCode::NoSuchKey
} else {
S3ErrorCode::NoSuchBucket
};
assert_eq!(
backend
.get_object(target, "absent", &root, None)
.await
.expect_err("missing GET")
.code(),
&expected
);
assert_eq!(
backend
.get_object(target, "absent", &root, Some(1))
.await
.expect_err("missing open-range GET")
.code(),
&expected
);
assert_eq!(
backend
.get_object_range(target, "absent", &root, 1, 1)
.await
.expect_err("missing bounded-range GET")
.code(),
&expected
);
assert_eq!(
backend
.head_object(target, "absent", &root)
.await
.expect_err("missing HEAD")
.code(),
&expected
);
}
for start in [13, 14, 1024] {
for error in [
backend
.get_object(bucket, KEY, &root, Some(start))
.await
.expect_err("open range past end"),
backend
.get_object_range(bucket, KEY, &root, start, 1)
.await
.expect_err("bounded range past end"),
] {
assert_eq!(error.code(), &S3ErrorCode::InvalidRange);
}
}
assert_eq!(
backend
.delete_object("protocol-absent-bucket", KEY, &root)
.await
.expect_err("missing DELETE bucket")
.code(),
&S3ErrorCode::NoSuchBucket
);
});
}
-3
View File
@@ -18,6 +18,3 @@ pub use client::ProtocolStorageClient;
#[cfg(test)]
mod policy_test;
#[cfg(test)]
mod mainflow_test;
+1 -1
View File
@@ -54,7 +54,7 @@ fn decode_policy(value: serde_json::Value) -> Result<rustfs_policy::policy::Poli
serde_json::from_str(&value.to_string())
}
pub(super) async fn protocol_test_context(root: Credentials) -> (tempfile::TempDir, Arc<AppContext>, Arc<IamSys<ObjectStore>>) {
async fn protocol_test_context(root: Credentials) -> (tempfile::TempDir, Arc<AppContext>, Arc<IamSys<ObjectStore>>) {
let (temp, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
ObjectStore::new(store.clone())
.save_iam_config(json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
@@ -13,17 +13,17 @@
// limitations under the License.
//! The gateway operations of migration batch E2 (rustfs/backlog#2765): DeleteObject, DeleteObjects, RestoreObject and SelectObjectContent.
//! DeleteObject, RestoreObject and SelectObjectContent are served here.
use super::{OpRow, Served, resp, row};
use crate::app::RequestEnvelope;
use crate::app::dto_values::stored_object_key;
use crate::app::object_usecase::{DefaultObjectUsecase, DeleteBatchETag, DeleteBatchEntry, DeleteBatchInput, DeleteBatchOutput};
use crate::app::object_usecase::DefaultObjectUsecase;
use crate::app::storage_api::object_usecase::bucket::{
RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging,
};
use crate::runtime_sources::AppContext;
use crate::storage_api::server::http::access::access_owned::object_misc::{
authorize_delete_object, authorize_delete_objects, authorize_restore_object, authorize_select_object_content,
authorize_delete_object, authorize_restore_object, authorize_select_object_content,
};
use http::StatusCode;
use rustfs_gateway::{Req, Resp, dto};
@@ -33,7 +33,6 @@ use std::sync::Arc;
pub(super) const ROWS: &[OpRow] = &[
row::<dto::DeleteObject>(),
row::<dto::DeleteObjects>(),
row::<dto::RestoreObject>(),
row::<dto::SelectObjectContent>(),
];
@@ -143,197 +142,3 @@ impl Served for dto::SelectObjectContent {
Ok(resp::<Self>(answered))
}
}
impl Served for dto::DeleteObjects {
const METRIC: S3Operation = S3Operation::DeleteObjects;
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
Ok(authorize_delete_objects(envelope, request.input().bucket.as_str())?)
}
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
let input = delete_batch_input(&request)?;
let output = DefaultObjectUsecase::with_context(context)
.execute_delete_objects(envelope, input)
.await?;
Ok(Resp::new(output.try_into()?))
}
}
fn delete_batch_input(request: &Req<dto::DeleteObjects>) -> S3Result<DeleteBatchInput> {
Ok(DeleteBatchInput {
bucket: request.input().bucket.as_str().to_owned(),
quiet: request.input().delete.quiet,
objects: request
.resources()
.resolve_objects(request.read_proof())
.ok_or_else(|| s3_error!(InternalError, "DeleteObjects resources could not be resolved"))?
.map(Into::into)
.collect(),
})
}
impl From<&dto::ObjectIdentifier> for DeleteBatchEntry {
fn from(object: &dto::ObjectIdentifier) -> Self {
Self {
key: object.key.as_str().to_owned(),
key_too_long: object.key.rejection() == Some(dto::DeleteObjectKeyRejection::TooLong),
version_id: object.version_id.clone(),
e_tag: object.e_tag.as_ref().map(|condition| match condition.etag() {
Some(tag) => DeleteBatchETag {
value: tag.opaque_tag().to_owned(),
weak: tag.is_weak(),
},
None => DeleteBatchETag {
value: String::new(),
weak: false,
},
}),
_last_modified_time: object
.last_modified_time
.as_ref()
.map(|value| (value.secs(), value.subsec_nanos())),
_size: object.size,
}
}
}
impl TryFrom<DeleteBatchOutput> for dto::DeleteObjectsOutput {
type Error = rustfs_s3_types::S3Error;
fn try_from(output: DeleteBatchOutput) -> S3Result<Self> {
// Keep the original decoded key on both successes and per-key errors.
Ok(Self {
deleted: output
.deleted
.unwrap_or_default()
.into_iter()
.map(|entry| {
Ok(dto::DeletedObject {
delete_marker: entry.delete_marker,
delete_marker_version_id: entry.delete_marker_version_id,
key: entry.key.as_deref().map(stored_object_key).transpose()?,
version_id: entry.version_id,
})
})
.collect::<S3Result<_>>()?,
errors: output
.errors
.unwrap_or_default()
.into_iter()
.map(|entry| {
Ok(dto::Error {
code: entry.code,
key: entry
.key
.map(dto::DeleteKeyEcho::new)
.transpose()
.map_err(|error| s3_error!(InternalError, "DeleteObjects error key cannot be encoded: {error}"))?,
message: entry.message,
version_id: entry.version_id,
})
})
.collect::<S3Result<_>>()?,
request_charged: None,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::app::RequestHead;
use crate::server::gateway_handlers::test_support::{Signed, signed_request};
use crate::storage_api::server::http::access::test_support::ReqInfo;
use bytes::Bytes;
use http::{Extensions, HeaderMap, Method, StatusCode};
use http_body_util::{BodyExt as _, Full};
use rustfs_gateway::{
Credentials, Handler, HandlerResult, RegionSet, ServiceBuilder, SigV4Authenticator, StaticCredentials, allow_when,
};
struct BatchProbe;
impl Handler<dto::DeleteObjects> for BatchProbe {
async fn call(&self, request: Req<dto::DeleteObjects>) -> HandlerResult<dto::DeleteObjects> {
let mut envelope = RequestEnvelope::from_parts(
RequestHead {
method: Method::POST,
uri: "/fixture-bucket?delete".parse().unwrap(),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
},
Extensions::new(),
);
envelope.install_req_info(ReqInfo {
bucket: Some("previous-bucket".to_owned()),
object: Some("previous-key".to_owned()),
version_id: Some("previous-version".to_owned()),
..ReqInfo::default()
});
<dto::DeleteObjects as Served>::authorize(&mut envelope, &request)
.await
.unwrap();
let info = envelope.req_info().unwrap();
assert_eq!(info.bucket.as_deref(), Some("fixture-bucket"));
assert_eq!((info.object.as_deref(), info.version_id.as_deref()), (None, None));
let input = delete_batch_input(&request).unwrap();
assert_eq!(input.bucket, "fixture-bucket");
assert_eq!(input.quiet, Some(true));
let first = DeleteBatchEntry {
key: "first".to_owned(),
key_too_long: false,
version_id: Some("null".to_owned()),
e_tag: Some(DeleteBatchETag {
value: "weak-tag".to_owned(),
weak: true,
}),
_last_modified_time: Some((946_684_800, 0)),
_size: Some(-19),
};
let second = DeleteBatchEntry {
key: "second".to_owned(),
e_tag: Some(DeleteBatchETag {
value: "*".to_owned(),
weak: false,
}),
_size: Some(0),
..Default::default()
};
assert_eq!(input.objects, [first.clone(), second, first]);
Ok(Resp::new(dto::DeleteObjectsOutput::default()))
}
}
#[tokio::test]
async fn delete_objects_sets_the_batch_target_and_carries_frozen_entries() {
let access = "batch-fixture";
let secret = "fixture-delete-batch-secret";
let credentials = StaticCredentials::new().with(Credentials::new(access, secret.as_bytes()).unwrap());
let regions = RegionSet::new(["us-east-1"]).unwrap();
let service = ServiceBuilder::new()
.rustfs_profile()
.authenticator(SigV4Authenticator::new(Arc::new(credentials), regions).rustfs_profile())
.authorizer(allow_when(|_| true))
.register::<dto::DeleteObjects, _>(Arc::new(BatchProbe))
.build()
.unwrap();
let first = concat!(
"<Object><Key>first</Key><VersionId>null</VersionId><ETag>W/&quot;weak-tag&quot;</ETag>",
"<LastModifiedTime>Sat, 01 Jan 2000 00:00:00 GMT</LastModifiedTime><Size>-19</Size></Object>"
);
let second = "<Object><Key>second</Key><ETag>*</ETag><Size>0</Size></Object>";
let body = format!("<Delete>{first}{second}{first}<Quiet>true</Quiet></Delete>");
let request = signed_request(Signed::Header, Method::POST, "/fixture-bucket", "delete", access, secret)
.map(|_| Full::new(Bytes::from(body)));
let response = service.call(request).await;
let status = response.status();
let body = response.into_body().collect().await.unwrap().to_bytes();
assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
}
}
+2 -360
View File
@@ -1538,7 +1538,7 @@ type S3EdgeService = S3Service;
/// The S3 service mounted under the middleware: the legacy stack or the gateway stack, selected at
/// startup (`server/stack.rs`).
#[cfg(feature = "gateway-stack")]
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService, NativeConsoleEdgeService>;
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService>;
/// The gateway stack where the legacy service is mounted: the same response and error types, so
/// the layers around it do not change with the stack.
@@ -1566,44 +1566,6 @@ where
}
}
#[cfg(feature = "gateway-stack")]
#[derive(Clone)]
struct NativeConsoleEdgeService(axum::Router);
#[cfg(feature = "gateway-stack")]
impl NativeConsoleEdgeService {
fn for_config(config: &config::Config) -> Option<Self> {
(matches!(config.s3_stack, config::S3Stack::Gateway) && config.console_enable)
.then(|| Self(admin::console::make_console_server()))
}
}
#[cfg(feature = "gateway-stack")]
impl<B> Service<HttpRequest<B>> for NativeConsoleEdgeService
where
B: http_body::Body<Data = Bytes> + Send + 'static,
B::Error: std::error::Error + Send + Sync + 'static,
{
type Response = Response<s3s::Body>;
type Error = s3s::HttpError;
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, request: HttpRequest<B>) -> Self::Future {
let answer = self.0.call(request.map(axum::body::Body::new));
Box::pin(async move {
let response = match answer.await {
Ok(response) => response,
Err(never) => match never {},
};
Ok(response.map(s3s::Body::http_body_unsync))
})
}
}
/// Selects the S3 stack of this server, once. A gateway stack that cannot be assembled fails
/// startup; the legacy stack is never started in its place.
#[cfg(feature = "gateway-stack")]
@@ -1615,8 +1577,7 @@ fn select_s3_stack(
let service = select_stack(config.s3_stack, legacy, || -> Result<GatewayEdgeService> {
let stack = GatewayStack::assemble(assembly()).map_err(Error::other)?;
Ok(GatewayEdgeService(stack))
})?
.with_native_console(NativeConsoleEdgeService::for_config(config));
})?;
info!(
event = EVENT_S3_STACK_ASSEMBLED,
component = LOG_COMPONENT_SERVER,
@@ -4421,325 +4382,6 @@ mod tests {
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[cfg(feature = "gateway-stack")]
mod native_console_tests {
use super::*;
use crate::config::S3Stack;
use std::sync::atomic::{AtomicUsize, Ordering};
use tower::ServiceExt;
async fn assert_console_response_equivalent(
expected: Response<s3s::Body>,
actual: Response<s3s::Body>,
method: &Method,
target: &str,
) {
use http_body_util::BodyExt;
assert_eq!(actual.status(), expected.status(), "{method} {target}");
for header in [http::header::CONTENT_TYPE, http::header::CONTENT_LENGTH, http::header::ALLOW] {
assert_eq!(
actual.headers().get(&header),
expected.headers().get(&header),
"{method} {target}: {header}"
);
}
assert_eq!(
actual.into_body().collect().await.unwrap().to_bytes(),
expected.into_body().collect().await.unwrap().to_bytes(),
"{method} {target}"
);
}
#[tokio::test]
async fn native_console_route_contract() {
use http_body_util::BodyExt;
legacy_console_characterizes_public_static_and_malformed_authorization().await;
let target = format!("{}/route-fixture.txt", crate::server::console_prefix());
let (service, calls) = edge(S3Stack::Gateway, true);
let response = service.oneshot(console_request(Method::GET, &target, false)).await.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response.headers().get(http::header::CONTENT_TYPE).unwrap(), "text/plain");
assert_eq!(response.headers().get(http::header::CONTENT_LENGTH).unwrap(), "28");
assert_eq!(
response.into_body().collect().await.unwrap().to_bytes().as_ref(),
b"console-route-static-content"
);
assert_eq!(counts(&calls), [0, 0, 1]);
public_static_uses_native_console_without_legacy_or_gateway_s3_calls().await;
neighboring_prefixes_admin_sts_s3_and_console_extension_queries_are_excluded().await;
disabled_console_and_legacy_mode_keep_the_existing_entrypoint().await;
}
struct ConsoleTestS3;
#[async_trait::async_trait]
impl s3s::S3 for ConsoleTestS3 {}
fn legacy_console(enabled: bool) -> s3s::service::S3Service {
let mut builder = s3s::service::S3ServiceBuilder::new(ConsoleTestS3);
builder.set_auth(crate::auth::IAMAuth::new("console-test-access", "console-test-secret"));
builder.set_route(crate::admin::router::S3Router::<crate::admin::router::AdminOperation>::new(enabled));
builder.build()
}
fn console_request(method: Method, target: &str, malformed_auth: bool) -> HttpRequest<Empty<Bytes>> {
let mut builder = HttpRequest::builder().method(method).uri(target).header("host", "localhost");
if malformed_auth {
builder = builder.header(http::header::AUTHORIZATION, "invalid");
}
builder.body(Empty::new()).expect("static request")
}
fn static_console_cases() -> [(Method, String); 5] {
let prefix = crate::server::console_prefix();
[
(Method::GET, format!("{prefix}/index.html")),
(Method::HEAD, format!("{prefix}/index.html")),
(Method::OPTIONS, format!("{prefix}/index.html")),
(Method::GET, "/favicon.ico".to_owned()),
(Method::GET, format!("{prefix}/missing-static-file.js")),
]
}
async fn static_response_body(response: Response<s3s::Body>, method: &Method, target: &str) -> Bytes {
use http_body_util::BodyExt;
let (fixture, content_type): (&[u8], &str) = if target == "/favicon.ico" {
(include_bytes!("../../tests/fixtures/native-console/favicon.ico"), "image/x-icon")
} else {
(include_bytes!("../../tests/fixtures/native-console/index.html"), "text/html")
};
assert_eq!(response.status(), StatusCode::OK, "{method} {target}");
if method == Method::OPTIONS {
assert_eq!(
response.headers().get("access-control-allow-methods").unwrap(),
"GET,POST,PUT,DELETE,OPTIONS"
);
assert_eq!(response.headers().get("access-control-allow-headers").unwrap(), "*");
assert!(response.headers().get(http::header::CONTENT_TYPE).is_none());
assert!(response.headers().get(http::header::CONTENT_LENGTH).is_none());
} else {
assert_eq!(response.headers().get(http::header::CONTENT_TYPE).unwrap(), content_type);
assert_eq!(
response
.headers()
.get(http::header::CONTENT_LENGTH)
.unwrap()
.to_str()
.unwrap(),
fixture.len().to_string()
);
}
let body = response.into_body().collect().await.unwrap().to_bytes();
let expected = if method == Method::HEAD || method == Method::OPTIONS {
b"".as_slice()
} else {
fixture
};
assert_eq!(body.as_ref(), expected, "{method} {target}");
body
}
async fn legacy_console_characterizes_public_static_and_malformed_authorization() {
use http_body_util::BodyExt;
for (method, target) in static_console_cases() {
let legacy = legacy_console(true)
.oneshot(console_request(method.clone(), &target, false))
.await
.unwrap();
let native = crate::admin::console::make_console_server()
.oneshot(console_request(method.clone(), &target, false))
.await
.unwrap()
.map(s3s::Body::http_body_unsync);
let malformed_native = crate::admin::console::make_console_server()
.oneshot(console_request(method.clone(), &target, true))
.await
.unwrap()
.map(s3s::Body::http_body_unsync);
assert_eq!(legacy.status(), native.status(), "{method} {target}");
assert_eq!(malformed_native.status(), native.status(), "native {method} {target}");
for header in [http::header::CONTENT_TYPE, http::header::CONTENT_LENGTH, http::header::ALLOW] {
assert_eq!(
legacy.headers().get(&header),
native.headers().get(&header),
"{method} {target}: {header}"
);
assert_eq!(
malformed_native.headers().get(&header),
native.headers().get(&header),
"native malformed auth {method} {target}: {header}"
);
}
let legacy_body = static_response_body(legacy, &method, &target).await;
let native_body = static_response_body(native, &method, &target).await;
let malformed_native_body = static_response_body(malformed_native, &method, &target).await;
assert_eq!(legacy_body, native_body, "{method} {target}");
assert_eq!(malformed_native_body, native_body, "native malformed auth {method} {target}");
let malformed = legacy_console(true)
.oneshot(console_request(method.clone(), &target, true))
.await
.unwrap();
assert_eq!(malformed.status(), StatusCode::BAD_REQUEST, "legacy {method} {target}");
let malformed_body = malformed.into_body().collect().await.unwrap().to_bytes();
if method == Method::HEAD {
assert!(malformed_body.is_empty());
} else {
assert!(String::from_utf8_lossy(&malformed_body).contains("<Code>InvalidRequest</Code>"));
}
}
}
type Probe = tower::util::BoxCloneService<HttpRequest<Empty<Bytes>>, Response<s3s::Body>, s3s::HttpError>;
fn counted<S>(inner: S, calls: Arc<AtomicUsize>) -> Probe
where
S: Service<HttpRequest<Empty<Bytes>>, Response = Response<s3s::Body>, Error = s3s::HttpError>
+ Clone
+ Send
+ 'static,
S::Future: Send + 'static,
{
Probe::new(tower::service_fn(move |request| {
calls.fetch_add(1, Ordering::SeqCst);
inner.clone().oneshot(request)
}))
}
fn edge(stack: S3Stack, enabled: bool) -> (S3StackService<Probe, Probe>, [Arc<AtomicUsize>; 3]) {
let calls = std::array::from_fn(|_| Arc::new(AtomicUsize::new(0)));
let mut config = config::Config::new("127.0.0.1:9001", Vec::new());
config.s3_stack = stack;
config.console_enable = enabled;
let legacy = counted(legacy_console(enabled), calls[0].clone());
let gateway = counted(
tower::service_fn(|_| async {
Ok::<_, s3s::HttpError>(
Response::builder()
.status(StatusCode::IM_A_TEAPOT)
.body(s3s::Body::empty())
.unwrap(),
)
}),
calls[1].clone(),
);
let console = NativeConsoleEdgeService::for_config(&config).map(|service| counted(service, calls[2].clone()));
let edge = select_stack(stack, legacy, || Ok::<_, s3s::HttpError>(gateway))
.unwrap()
.with_native_console(console);
(edge, calls)
}
fn counts(calls: &[Arc<AtomicUsize>; 3]) -> [usize; 3] {
std::array::from_fn(|i| calls[i].load(Ordering::SeqCst))
}
async fn public_static_uses_native_console_without_legacy_or_gateway_s3_calls() {
let (service, calls) = edge(S3Stack::Gateway, true);
let prefix = crate::server::console_prefix();
let cases: Vec<_> = static_console_cases()
.into_iter()
.chain([
(Method::GET, "/apple-touch-icon.png".to_owned()),
(Method::GET, format!("{prefix}/index.html?events")),
(Method::HEAD, format!("{prefix}/index.html?lambdaArn=bad")),
(Method::GET, "/favicon.ico?lambdaArn=bad".to_owned()),
])
.collect();
for (method, target) in &cases {
let actual = service
.clone()
.oneshot(console_request(method.clone(), target, false))
.await
.unwrap();
let expected = legacy_console(true)
.oneshot(console_request(method.clone(), target, false))
.await
.unwrap();
assert_console_response_equivalent(expected, actual, method, target).await;
}
assert_eq!(counts(&calls), [0, 0, cases.len()]);
}
async fn neighboring_prefixes_admin_sts_s3_and_console_extension_queries_are_excluded() {
let (service, calls) = edge(S3Stack::Gateway, true);
let prefix = crate::server::console_prefix();
let cases = [
(Method::GET, format!("{prefix}x/index.html"), false, false),
(Method::GET, "/favicon.icox".to_owned(), false, false),
(Method::GET, "/apple-touch-icon.pngx".to_owned(), false, false),
(Method::GET, "/rustfs/administrator".to_owned(), false, false),
(Method::GET, "/bucket/key".to_owned(), false, false),
(Method::PUT, "/bucket/key".to_owned(), false, false),
(Method::POST, "/".to_owned(), false, false),
(Method::GET, "/rustfs/admin/v3/info".to_owned(), true, false),
(Method::GET, "/minio/admin/v3/info".to_owned(), true, false),
(Method::GET, "/iceberg/v1/config".to_owned(), true, false),
(Method::POST, "/".to_owned(), true, true),
(Method::GET, "/favicon.ico?events".to_owned(), true, false),
(Method::GET, "/favicon.ico?replication-metrics".to_owned(), true, false),
(Method::GET, format!("{prefix}/index.html?lambdaArn="), true, false),
(Method::GET, format!("{prefix}/index.html?lambdaArn=bad"), true, false),
(Method::GET, format!("{prefix}/index.html?lambdaArn=bad&lambdaArn=bad"), true, false),
];
for (method, target, legacy, sts) in &cases {
for malformed_auth in [false, true] {
let make_request = || {
let mut request = console_request(method.clone(), target, malformed_auth);
if *sts {
request.headers_mut().insert(
http::header::CONTENT_TYPE,
http::HeaderValue::from_static("application/x-www-form-urlencoded"),
);
}
request
};
let actual = service.clone().oneshot(make_request()).await.unwrap();
if *legacy {
let expected = legacy_console(true).oneshot(make_request()).await.unwrap();
if target == "/rustfs/admin/v3/info" || target == "/minio/admin/v3/info" {
assert_eq!(
actual.status(),
if malformed_auth {
StatusCode::BAD_REQUEST
} else {
StatusCode::FORBIDDEN
},
"admin on console listener"
);
}
assert_console_response_equivalent(expected, actual, method, target).await;
} else {
assert_eq!(actual.status(), StatusCode::IM_A_TEAPOT, "{method} {target}");
}
}
}
let legacy_cases = cases.iter().filter(|case| case.2).count();
assert_eq!(counts(&calls), [legacy_cases * 2, (cases.len() - legacy_cases) * 2, 0]);
}
async fn disabled_console_and_legacy_mode_keep_the_existing_entrypoint() {
for (stack, enabled) in [(S3Stack::Gateway, false), (S3Stack::Legacy, true)] {
let (service, calls) = edge(stack, enabled);
let cases = static_console_cases();
for (method, target) in &cases {
for malformed_auth in [false, true] {
let actual = service
.clone()
.oneshot(console_request(method.clone(), target, malformed_auth))
.await
.unwrap();
let expected = legacy_console(enabled)
.oneshot(console_request(method.clone(), target, malformed_auth))
.await
.unwrap();
assert_console_response_equivalent(expected, actual, method, target).await;
}
}
assert_eq!(counts(&calls), [cases.len() * 2, 0, 0]);
}
}
}
type RateLimitTestCore = tower::util::BoxCloneService<
HttpRequest<s3s::Body>,
Response<crate::server::hybrid::HybridBody<s3s::Body, Empty<Bytes>>>,
+13 -38
View File
@@ -18,16 +18,16 @@
//! be, and [`select_stack`], the only place a stack is chosen. The whole stack is one or the
//! other: no request is routed between stacks by its operation, and a request the gateway stack
//! refuses is answered by that refusal and never retried on the legacy stack. With the gateway
//! selected, console static requests use the native console service; the legacy service receives
//! the remaining claimed extension and control-plane routes, never S3 operations.
//! selected, the legacy service receives only what the legacy router claims (the RustFS extension
//! routes and the control plane), so no S3 request reaches the legacy S3 implementation.
//! NOT responsible for: assembling either stack (`server/http.rs` builds the legacy service,
//! `server/gateway_stack.rs` the gateway one), or the middleware around them.
//! Upstream: `server/http.rs`. Downstream: the native console, legacy S3 service and gateway stack.
//! Upstream: `server/http.rs`. Downstream: the legacy S3 service and the gateway stack.
//!
//! This file names no stack's types: it is generic over the services with one response
//! This file names neither stack's types: it is generic over two services with one response
//! type, so `server/http.rs` decides what they are and how their bodies meet.
use crate::admin::router::{claims_native_console_route, claims_route};
use crate::admin::router::claims_route;
use crate::config::S3Stack;
use http::Request;
use std::future::Future;
@@ -38,22 +38,12 @@ use tower::Service;
/// The S3 service of one server: the legacy stack, or the gateway stack with the legacy router
/// kept for the routes that are not S3 operations.
#[derive(Clone)]
pub(crate) struct S3StackService<L, G, C = G> {
pub(crate) struct S3StackService<L, G> {
legacy: L,
gateway: Option<G>,
native_console: Option<C>,
}
impl<L, G, C> S3StackService<L, G, C> {
/// Attaches the console service used only by the gateway selection.
pub(crate) fn with_native_console<N>(self, native_console: Option<N>) -> S3StackService<L, G, N> {
S3StackService {
legacy: self.legacy,
gateway: self.gateway,
native_console,
}
}
impl<L, G> S3StackService<L, G> {
/// The stack this service was built with, as the startup line names it: read from the service
/// that answers, never from the configuration that asked for it.
pub(crate) fn assembled(&self) -> &'static str {
@@ -81,21 +71,15 @@ pub(crate) fn select_stack<L, G, E>(
S3Stack::Legacy => None,
S3Stack::Gateway => Some(assemble_gateway()?),
};
Ok(S3StackService {
legacy,
gateway,
native_console: None,
})
Ok(S3StackService { legacy, gateway })
}
impl<L, G, C, B, R, E> Service<Request<B>> for S3StackService<L, G, C>
impl<L, G, B, R, E> Service<Request<B>> for S3StackService<L, G>
where
L: Service<Request<B>, Response = http::Response<R>, Error = E>,
G: Service<Request<B>, Response = http::Response<R>, Error = E>,
C: Service<Request<B>, Response = http::Response<R>, Error = E>,
L::Future: Send + 'static,
G::Future: Send + 'static,
C::Future: Send + 'static,
{
type Response = http::Response<R>;
type Error = E;
@@ -104,25 +88,16 @@ where
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
if let Some(gateway) = &mut self.gateway {
ready!(gateway.poll_ready(cx))?;
if let Some(console) = &mut self.native_console {
ready!(console.poll_ready(cx))?;
}
}
self.legacy.poll_ready(cx)
}
fn call(&mut self, request: Request<B>) -> Self::Future {
if let Some(gateway) = &mut self.gateway {
if !claims_route(request.method(), request.uri(), request.headers()) {
return Box::pin(gateway.call(request));
}
if let Some(console) = &mut self.native_console
&& claims_native_console_route(request.method(), request.uri())
{
return Box::pin(console.call(request));
}
match &mut self.gateway {
// REMOVED BY T2.11: the gateway serves these routes itself once they are migrated.
Some(gateway) if !claims_route(request.method(), request.uri(), request.headers()) => Box::pin(gateway.call(request)),
_ => Box::pin(self.legacy.call(request)),
}
Box::pin(self.legacy.call(request))
}
}
+23 -36
View File
@@ -15,7 +15,7 @@
use self::access_owned::listing_multipart::{
authorize_bucket_listing, authorize_list_multipart_uploads, authorize_list_objects_v2, authorize_list_parts,
};
use self::access_owned::object_misc::{authorize_delete_object, authorize_delete_objects};
use self::access_owned::object_misc::authorize_delete_object;
use super::ObjectOptions;
use super::ecfs::FS;
use super::{ECStore, PolicySys, StorageError, get_lock_acquire_timeout, is_err_bucket_not_found};
@@ -33,7 +33,7 @@ use crate::error::ApiError;
use crate::license::license_check;
use crate::shared_types::RemoteAddr;
use crate::storage::denial_audit::DenialAuditor;
use crate::storage::helper::{AuditRequestView, DenialScope};
use crate::storage::helper::DenialScope;
use crate::storage::request_context::RequestContext;
use crate::storage::storage_api::contract::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, BucketOperations, BucketOptions};
use crate::storage::storage_api::contract::namespace::NamespaceLocking as _;
@@ -2172,15 +2172,15 @@ fn audit_request_denial<R: AuthorizationRequest + ?Sized>(
/// Audits the refused keys of a bulk request (DeleteObjects) in aggregate instead of once per key.
///
/// `start_owned` marks the request so per-key denials are logged but not audited; `finish_owned` clears the
/// `start` marks the request so per-key denials are logged but not audited; `finish` clears the
/// mark and emits one entry per denied action with the number of keys refused for it.
pub(crate) struct BulkDenialAudit {
denied: Vec<(Action, u64)>,
}
impl BulkDenialAudit {
pub(crate) fn start_owned<R: AuthorizationRequest + ?Sized>(request: &mut R) -> S3Result<Self> {
req_info_mut(request)?.suppress_denial_audit = true;
pub(crate) fn start<T>(req: &mut S3Request<T>) -> S3Result<Self> {
ext_req_info_mut(&mut req.extensions)?.suppress_denial_audit = true;
Ok(Self { denied: Vec::new() })
}
@@ -2192,24 +2192,14 @@ impl BulkDenialAudit {
}
}
pub(crate) fn finish_owned<R: AuthorizationRequest + ?Sized>(
self,
request: &mut R,
audit_view: impl FnOnce(&R) -> AuditRequestView<'_>,
) -> S3Result<()> {
self.finish_owned_with(request, audit_view, DenialAuditor::global())
pub(crate) fn finish<T>(self, req: &mut S3Request<T>) -> S3Result<()> {
self.finish_with(req, DenialAuditor::global())
}
fn finish_owned_with<R: AuthorizationRequest + ?Sized>(
self,
request: &mut R,
audit_view: impl FnOnce(&R) -> AuditRequestView<'_>,
auditor: &DenialAuditor,
) -> S3Result<()> {
req_info_mut(request)?.suppress_denial_audit = false;
let view = audit_view(request);
fn finish_with<T>(self, req: &mut S3Request<T>, auditor: &DenialAuditor) -> S3Result<()> {
ext_req_info_mut(&mut req.extensions)?.suppress_denial_audit = false;
for (action, keys) in self.denied {
auditor.audit_owned(&view, &action, "bulk_request_keys_denied", DenialScope::Keys(keys));
auditor.audit(req, &action, "bulk_request_keys_denied", DenialScope::Keys(keys));
}
Ok(())
}
@@ -3583,8 +3573,12 @@ impl S3Access for FS {
///
/// This method returns `Ok(())` by default.
async fn delete_objects(&self, req: &mut S3Request<DeleteObjectsInput>) -> S3Result<()> {
let bucket = req.input.bucket.clone();
authorize_delete_objects(req, &bucket)
let req_info = ext_req_info_mut(&mut req.extensions)?;
req_info.bucket = Some(req.input.bucket.clone());
req_info.object = None;
req_info.version_id = None;
Ok(())
}
/// Checks whether the DeletePublicAccessBlock request has accesses to the resources.
@@ -6890,25 +6884,21 @@ mod tests {
#[test]
fn bulk_denial_is_one_entry_per_denied_action_with_the_key_count() {
let (auditor, captured) = test_auditor(true, 10, 10);
let req = denial_request("denial-bulk", true);
let mut envelope = crate::app::RequestEnvelope::from_s3s(&req);
let mut req = denial_request("denial-bulk", true);
let nothing_denied = BulkDenialAudit::start_owned(&mut envelope).unwrap();
assert!(envelope.req_info().unwrap().suppress_denial_audit);
nothing_denied
.finish_owned_with(&mut envelope, crate::app::RequestEnvelope::audit_view, &auditor)
.unwrap();
assert!(!envelope.req_info().unwrap().suppress_denial_audit);
let nothing_denied = BulkDenialAudit::start(&mut req).unwrap();
assert!(req.extensions.get::<ReqInfo>().unwrap().suppress_denial_audit);
nothing_denied.finish_with(&mut req, &auditor).unwrap();
assert!(!req.extensions.get::<ReqInfo>().unwrap().suppress_denial_audit);
assert!(captured.lock().unwrap().is_empty());
let mut bulk = BulkDenialAudit::start_owned(&mut envelope).unwrap();
let mut bulk = BulkDenialAudit::start(&mut req).unwrap();
let delete = Action::S3Action(S3Action::DeleteObjectAction);
let bypass = Action::S3Action(S3Action::BypassGovernanceRetentionAction);
for action in [delete, delete, bypass, delete] {
bulk.denied(action);
}
bulk.finish_owned_with(&mut envelope, crate::app::RequestEnvelope::audit_view, &auditor)
.unwrap();
bulk.finish_with(&mut req, &auditor).unwrap();
let entries = captured.lock().unwrap();
assert_eq!(entries.len(), 2);
@@ -8606,9 +8596,6 @@ mod tests {
let mut req = build_request(input, Method::POST);
req.extensions.insert(ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
bucket: Some("previous-bucket".to_string()),
object: Some("previous-key".to_string()),
version_id: Some("previous-version".to_string()),
..ReqInfo::default()
});
@@ -38,15 +38,6 @@ use crate::storage::options::destination_version_id;
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, force_delete_header};
/// The bulk engine authorizes each entry after this request-level target setup.
pub(crate) fn authorize_delete_objects<R: AuthorizationRequest + ?Sized>(req: &mut R, bucket: &str) -> S3Result<()> {
let info = req_info_mut(req)?;
info.bucket = Some(bucket.to_owned());
info.object = None;
info.version_id = None;
Ok(())
}
/// Both RestoreObject edges retain the bucket generation read before policy evaluation.
pub(crate) async fn authorize_restore_object<R: AuthorizationRequest + ?Sized>(
server_ctx: &ServerContextSlot,
+17 -13
View File
@@ -27,7 +27,6 @@ use rustfs_audit::{
global::AuditLogger,
};
use rustfs_iam::sys::{OIDC_ISSUER_CLAIM, is_rustfs_oidc_claims};
#[cfg(test)]
use rustfs_io_metrics::record_s3_op;
use rustfs_notify::EventArgsBuilder;
use rustfs_policy::policy::action::Action;
@@ -174,6 +173,10 @@ where
}
/// Builds S3-compatible notification response elements with the canonical request ID.
pub(crate) fn build_event_resp_elements<T>(response: &S3Response<T>, request_id: &str) -> HashMap<String, String> {
build_owned_event_resp_elements(&response.headers, request_id)
}
pub(crate) fn build_owned_event_resp_elements(response_headers: &HeaderMap, request_id: &str) -> HashMap<String, String> {
let mut resp_elements = extract_params_header(response_headers);
resp_elements.insert(AMZ_REQUEST_ID.to_string(), request_id.to_string());
@@ -367,15 +370,15 @@ pub struct EnabledOperationHelper {
}
impl OperationHelper {
/// Builds the legacy request helper for audit and notification fixtures.
#[cfg(test)]
/// Create a new OperationHelper for S3 requests.
pub fn new(req: &S3Request<impl Send + Sync>, event: EventName, op: S3Operation) -> Self {
record_s3_op(op);
Self::from_view(&AuditRequestView::legacy(req), event, op)
}
/// Builds the helper from the audit facts decoded by the operation edge.
/// The edge counts the operation before calling the use case.
/// [`Self::new`] for a request whose audit facts an owned edge decoded
/// (`RequestEnvelope::audit_view`). It does not count the operation: the owned edge already
/// did, once, before calling the use case.
pub(crate) fn from_view(req: &AuditRequestView<'_>, event: EventName, op: S3Operation) -> Self {
Self::build(req, event, op, false)
}
@@ -384,7 +387,8 @@ impl OperationHelper {
///
/// For operations that have no notification event of their own (`UploadPart`, the
/// bucket configuration writes) or whose event is internal (`AbortMultipartUpload`).
/// The handler records the per-operation metric.
/// Unlike `new` it does not call `record_s3_op`, so per-operation metrics stay as the
/// handlers already report them.
pub fn audit_only(req: &S3Request<impl Send + Sync>, op: S3Operation) -> Self {
Self::audit_only_from_view(&AuditRequestView::legacy(req), op)
}
@@ -687,6 +691,11 @@ impl OperationHelper {
/// Returns the operation's canonical context, reading the ingress extension
/// when the disabled fast path holds no request state.
pub(crate) fn request_context_or_from_request<T>(&self, req: &S3Request<T>) -> RequestContext {
self.request_context_or_from_view(&AuditRequestView::legacy(req))
}
/// [`Self::request_context_or_from_request`] for a request whose audit facts an owned edge decoded.
pub(crate) fn request_context_or_from_view(&self, req: &AuditRequestView<'_>) -> RequestContext {
match self {
Self::Enabled(state) => state.request_context.clone(),
@@ -1836,7 +1845,7 @@ mod tests {
});
let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
let request_context = helper.request_context_or_from_view(&super::AuditRequestView::legacy(&req));
let request_context = helper.request_context_or_from_request(&req);
let request_id = request_context.request_id;
assert!(uuid::Uuid::parse_str(&request_id).is_ok());
let result = Ok(S3Response::new(DeleteObjectTaggingOutput::default()));
@@ -1884,12 +1893,7 @@ mod tests {
let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
assert!(matches!(&helper, OperationHelper::Disabled));
assert_eq!(
helper
.request_context_or_from_view(&super::AuditRequestView::legacy(&req))
.request_id,
"server-request-id"
);
assert_eq!(helper.request_context_or_from_request(&req).request_id, "server-request-id");
},
);
}
+3 -67
View File
@@ -26,10 +26,8 @@ use crate::storage::access::legacy_copy_source;
use crate::storage::ecfs::*;
use crate::storage::ecfs_extend::validate_new_object_key;
use crate::storage::helper::audit_only_operation;
use crate::storage::s3_api::{
self, CopyObjectWriteInput, CopyObjectWriteOutput, CopySourceCondition, DefaultObjectUsecase, DeleteBatchETag,
DeleteBatchEntry, DeleteBatchInput, DeleteBatchOutput,
};
use crate::storage::s3_api;
use crate::storage::s3_api::{CopyObjectWriteInput, CopyObjectWriteOutput, CopySourceCondition, DefaultObjectUsecase};
use crate::storage::storage_api::to_s3s_xml_etag;
use http::{HeaderMap, HeaderValue};
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
@@ -194,11 +192,8 @@ impl FS {
&self,
req: S3Request<DeleteObjectsInput>,
) -> S3Result<S3Response<DeleteObjectsOutput>> {
record_s3_op(S3Operation::DeleteObjects);
let usecase = s3_api::object_usecase_for(self);
let envelope = RequestEnvelope::from_s3s(&req);
let output = usecase.execute_delete_objects(envelope, req.input.into()).await?;
Ok(S3Response::new(output.into()))
usecase.execute_delete_objects(req).await
}
pub(in crate::storage::ecfs) async fn restore_object_edge(
@@ -349,62 +344,3 @@ fn copy_output_to_legacy(output: CopyObjectWriteOutput) -> S3Response<CopyObject
..Default::default()
})
}
impl From<DeleteObjectsInput> for DeleteBatchInput {
fn from(input: DeleteObjectsInput) -> Self {
Self {
bucket: input.bucket,
objects: input.delete.objects.into_iter().map(Into::into).collect(),
quiet: input.delete.quiet,
}
}
}
impl From<ObjectIdentifier> for DeleteBatchEntry {
fn from(object: ObjectIdentifier) -> Self {
Self {
key: object.key,
key_too_long: false,
version_id: object.version_id,
e_tag: object.e_tag.map(|tag| match tag {
ETag::Strong(value) => DeleteBatchETag { value, weak: false },
ETag::Weak(value) => DeleteBatchETag { value, weak: true },
}),
_last_modified_time: object.last_modified_time.map(|value| {
let instant = time::OffsetDateTime::from(value);
(instant.unix_timestamp(), instant.nanosecond())
}),
_size: object.size,
}
}
}
impl From<DeleteBatchOutput> for DeleteObjectsOutput {
fn from(output: DeleteBatchOutput) -> Self {
Self {
deleted: output.deleted.map(|entries| {
entries
.into_iter()
.map(|entry| DeletedObject {
delete_marker: entry.delete_marker,
delete_marker_version_id: entry.delete_marker_version_id,
key: entry.key,
version_id: entry.version_id,
})
.collect()
}),
errors: output.errors.map(|entries| {
entries
.into_iter()
.map(|entry| Error {
code: entry.code,
key: entry.key,
message: entry.message,
version_id: entry.version_id,
})
.collect()
}),
request_charged: None,
}
}
}
+2 -3
View File
@@ -31,9 +31,8 @@ pub(crate) use crate::app::object::{
};
pub(crate) use crate::app::object_usecase::DefaultObjectUsecase;
pub(crate) use crate::app::object_usecase::{
DeleteBatchETag, DeleteBatchEntry, DeleteBatchInput, DeleteBatchOutput, GetObjectAttributesFields,
GetObjectAttributesOutputFacts, GetObjectBody, GetObjectFields, GetObjectOutputFacts, HeadObjectFields,
HeadObjectOutputFacts,
GetObjectAttributesFields, GetObjectAttributesOutputFacts, GetObjectBody, GetObjectFields, GetObjectOutputFacts,
HeadObjectFields, HeadObjectOutputFacts,
};
pub(crate) mod bucket;
+2 -1
View File
@@ -257,7 +257,8 @@ pub(crate) mod helper_consumer {
MutationCompletionProbe, capture_mutation_event_for_test, pause_committed_mutation_for_test,
};
pub(crate) use super::super::helper::{
OperationHelper, build_owned_event_resp_elements, run_owned_mutation, spawn_background_with_context,
OperationHelper, build_event_resp_elements, build_owned_event_resp_elements, run_owned_mutation,
spawn_background_with_context,
};
pub(crate) type StorageObjectInfo = super::StorageObjectInfo;
+1 -28
View File
@@ -14440,16 +14440,7 @@ fn assert_object_mutation_reserved_prefix_sources(delete: &str, legacy_delete: &
"fn execute_delete_object_fields(",
"owner.execute_delete_object_owned(envelope, bucket, key, version_id)",
),
(
legacy_delete,
"fn delete_objects_edge(",
"usecase.execute_delete_objects(envelope, req.input.into())",
),
(
delete,
"fn execute_delete_objects(",
"owner.execute_delete_objects_owned(envelope, input)",
),
(delete, "fn execute_delete_objects(", "owner.execute_delete_objects_owned(req)"),
(
legacy_put,
"fn execute_put_object_extract(",
@@ -14560,18 +14551,6 @@ fn object_mutation_reserved_prefix_guard_rejects_broken_delete_chains() {
"owner.execute_delete_object_owned(envelope, bucket, key, version_id)",
"owner.execute_delete_object_unchecked(envelope, bucket, key, version_id)",
),
(
"legacy batch uses a different use case",
true,
"usecase.execute_delete_objects(envelope, req.input.into())",
"usecase.execute_other_batch(envelope, req.input.into())",
),
(
"batch delegate uses different input",
false,
"owner.execute_delete_objects_owned(envelope, input)",
"owner.execute_delete_objects_owned(envelope, changed_input)",
),
("typed method missing", false, "fn execute_delete_object(", "fn missing_delete_object("),
("legacy method missing", true, "fn delete_object_edge(", "fn missing_delete_object_edge("),
(
@@ -14618,12 +14597,6 @@ fn object_mutation_reserved_prefix_guard_rejects_broken_delete_chains() {
"owner.execute_delete_object_owned(envelope, bucket, key, version_id)" => {
"fn execute_delete_object_fields( must call its guarded implementation"
}
"usecase.execute_delete_objects(envelope, req.input.into())" => {
"fn delete_objects_edge( must call its guarded implementation"
}
"owner.execute_delete_objects_owned(envelope, input)" => {
"fn execute_delete_objects( must call its guarded implementation"
}
_ => from,
};
(
@@ -405,22 +405,12 @@ fn single_rule(method: &str, origin: &str) -> CorsConfiguration {
.expect("valid configuration")
}
/// Records the status and request ID headers of the response.
/// Records the status of the one response a request receives.
#[derive(Clone, Debug, Default)]
pub(super) struct StatusCapture(Arc<Mutex<Option<CapturedResponse>>>);
#[derive(Debug)]
pub(super) struct CapturedResponse {
pub(super) status: u16,
pub(super) request_ids: [Option<String>; 2],
}
pub(super) struct StatusCapture(Arc<Mutex<Option<u16>>>);
impl StatusCapture {
pub(super) fn take(&self) -> Option<u16> {
self.take_response().map(|response| response.status)
}
pub(super) fn take_response(&self) -> Option<CapturedResponse> {
self.0.lock().ok().and_then(|mut slot| slot.take())
}
}
@@ -436,15 +426,11 @@ impl Intercept for StatusCapture {
_runtime_components: &RuntimeComponents,
_cfg: &mut ConfigBag,
) -> Result<(), BoxError> {
let response = context.response();
let header = |name| response.headers().get(name).map(str::to_owned);
let status = context.response().status().as_u16();
*self
.0
.lock()
.map_err(|_| std::io::Error::other("status capture mutex was poisoned"))? = Some(CapturedResponse {
status: response.status().as_u16(),
request_ids: [header("x-amz-request-id"), header("x-request-id")],
});
.map_err(|_| std::io::Error::other("status capture mutex was poisoned"))? = Some(status);
Ok(())
}
}
@@ -437,8 +437,7 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
Err(refusal) => refusal,
};
let context = format!("{} as the {caller} caller: {refusal:?}", row.operation);
let per_key = row.metric == S3Operation::DeleteObjects && !by_authentication;
assert_eq!(refusal.status, Some(if per_key { 200 } else { 403 }), "{context}");
assert_eq!(refusal.status, Some(403), "{context}");
if row.has_error_body {
assert_eq!(refusal.code.as_deref(), Some(*code), "{context}");
}
@@ -449,17 +448,7 @@ async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_ad
);
let counted = counted_since(&before);
assert_eq!(counted.values().sum::<u64>(), 1, "counted once: {context} {counted:?}");
let expected_outcome = if per_key { "2xx" } else { "4xx" };
assert!(
counted.keys().all(|(_, _, outcome)| *outcome == expected_outcome),
"{context} {counted:?}"
);
if per_key {
assert!(
counted.keys().all(|(_, label, _)| *label == row.metric.as_str()),
"under the batch operation: {context} {counted:?}"
);
}
assert!(counted.keys().all(|(_, _, outcome)| *outcome == "4xx"), "{context} {counted:?}");
if *by_authentication {
assert!(
counted.keys().all(|(_, label, _)| *label == "unknown"),
@@ -23,8 +23,8 @@ use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata as _;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CsvInput, CsvOutput, Delete, ExpressionType, FileHeaderInfo, InputSerialization, ObjectIdentifier,
OutputSerialization, RestoreRequest, VersioningConfiguration,
BucketVersioningStatus, CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, OutputSerialization,
RestoreRequest, VersioningConfiguration,
};
use bytes::Bytes;
use futures::future::BoxFuture;
@@ -46,12 +46,6 @@ pub(crate) const MATRIX: &[MatrixRow] = &[
has_error_body: true,
call: delete_object,
},
MatrixRow {
operation: "DeleteObjects",
metric: S3Operation::DeleteObjects,
has_error_body: true,
call: delete_objects,
},
MatrixRow {
operation: "RestoreObject",
metric: S3Operation::RestoreObject,
@@ -294,46 +288,6 @@ fn the_legacy_stack_admits_and_renews_restore_object() {
with_stack(LEGACY, restore_admits_once_and_renews_in_place);
}
fn batch(keys: &[&str]) -> aws_sdk_s3::types::builders::DeleteBuilder {
Delete::builder().set_objects(Some(
keys.iter()
.map(|key| ObjectIdentifier::builder().key(*key).build().expect("object key"))
.collect(),
))
}
fn delete_objects<'a>(client: &'a Client, _bucket: &'a str) -> BoxFuture<'a, Outcome> {
Box::pin(async move {
let status = StatusCapture::default();
let result = client
.delete_objects()
.bucket(MATRIX_BUCKET)
.delete(batch(&["matrix-absent"]).build().unwrap())
.customize()
.interceptor(status.clone())
.send()
.await;
let Ok(output) = result else {
return outcome(result);
};
let response = status.take_response().expect("receive DeleteObjects response");
assert_eq!(response.status, 200);
if let [error] = output.errors() {
assert_eq!(error.key(), Some("matrix-absent"));
assert!(output.deleted().is_empty());
return Err(super::Refusal {
code: error.code().map(str::to_owned),
status: Some(response.status),
request_ids: response.request_ids,
});
}
assert!(output.errors().is_empty(), "unexpected response: {output:?}");
assert_eq!(output.deleted().len(), 1);
assert_eq!(output.deleted()[0].key(), Some("matrix-absent"));
Ok(())
})
}
const DELETE_BUCKET: &str = "e2-delete-object";
/// A bucket name RustFS creates and the AWS grammar refuses (`sthree-` is reserved).
const RESERVED_NAME_BUCKET: &str = "sthree-e2-delete";
@@ -738,132 +692,3 @@ fn gateway_select_preserves_the_event_sequence_and_denies_unauthorized_reads() {
fn legacy_select_preserves_the_event_sequence_and_denies_unauthorized_reads() {
with_stack(LEGACY, select_parity);
}
async fn delete_objects_checks_each_key_and_preserves_quiet_errors(server: RustFSServer) {
const BUCKET: &str = "e2-delete-batch-scoped";
const USER: &str = "e2deletebatchscoped";
create_bucket_in_the_store(BUCKET).await;
create_user_without_a_policy(USER).await;
let policy = rustfs_policy::policy::Policy::parse_config(format!(
r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::{BUCKET}/allowed*"}}]}}"#
).as_bytes()).expect("parse batch policy");
let iam = get_global_app_context().expect("embedded server context").iam().handle();
iam.set_policy(USER, policy).await.expect("store batch policy");
iam.policy_db_set(USER, rustfs_iam::store::UserType::Reg, false, USER)
.await
.expect("attach batch policy");
let client = s3_client(&server.endpoint(), USER, USER_SECRET_KEY);
let root = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
for quiet in [false, true] {
put_in_the_store(BUCKET, "allowed").await;
put_in_the_store(BUCKET, "allowed-etag").await;
put_in_the_store(BUCKET, "denied").await;
let head = root
.head_object()
.bucket(BUCKET)
.key("allowed-etag")
.send()
.await
.expect("read fixture ETag");
let objects = [
ObjectIdentifier::builder().key("denied").build().unwrap(),
ObjectIdentifier::builder().key("allowed").e_tag("*").build().unwrap(),
ObjectIdentifier::builder().key("allowed-absent").build().unwrap(),
ObjectIdentifier::builder().key("allowed-absent").build().unwrap(),
ObjectIdentifier::builder()
.key("allowed-etag")
.e_tag(head.e_tag().expect("stored ETag"))
.build()
.unwrap(),
];
let output = client
.delete_objects()
.bucket(BUCKET)
.delete(
Delete::builder()
.set_objects(Some(objects.into()))
.quiet(quiet)
.build()
.unwrap(),
)
.send()
.await
.expect("mixed DeleteObjects returns per-key outcomes");
assert!(is_in_the_store(BUCKET, "denied").await, "key-level denial must preserve the object");
assert!(
is_in_the_store(BUCKET, "allowed-etag").await,
"concrete ETag without GetObject must preserve the object"
);
assert!(
!is_in_the_store(BUCKET, "allowed").await,
"the wildcard needs DeleteObject but not GetObject"
);
let deleted: Vec<_> = output.deleted().iter().map(|object| object.key().unwrap()).collect();
assert_eq!(
deleted,
if quiet {
vec![]
} else {
vec!["allowed", "allowed-absent", "allowed-absent"]
}
);
let errors: Vec<_> = output
.errors()
.iter()
.map(|error| (error.key().unwrap(), error.code().unwrap()))
.collect();
assert_eq!(errors, [("denied", "AccessDenied"), ("allowed-etag", "AccessDenied")]);
}
}
#[test]
fn the_gateway_stack_deletes_only_authorized_batch_entries() {
with_stack(GATEWAY, delete_objects_checks_each_key_and_preserves_quiet_errors);
}
#[test]
fn the_legacy_stack_deletes_only_authorized_batch_entries() {
with_stack(LEGACY, delete_objects_checks_each_key_and_preserves_quiet_errors);
}
#[test]
fn the_gateway_stack_echoes_rejected_entries_without_executing_them() {
with_stack(GATEWAY, |server| async move {
const BUCKET: &str = "e2-delete-batch-refusals";
create_bucket_in_the_store(BUCKET).await;
for key in ["kept", "conditional", "deleted"] {
put_in_the_store(BUCKET, key).await;
}
let oversized_key = format!("{}kept", "/".repeat(1025));
let objects = [
ObjectIdentifier::builder().key("").build().unwrap(),
ObjectIdentifier::builder().key(&oversized_key).build().unwrap(),
ObjectIdentifier::builder().key("conditional").e_tag("\"\"").build().unwrap(),
ObjectIdentifier::builder().key("deleted").build().unwrap(),
];
let root = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
let output = root
.delete_objects()
.bucket(BUCKET)
.delete(Delete::builder().set_objects(Some(objects.into())).build().unwrap())
.send()
.await
.expect("the independent entry still succeeds");
assert_eq!(output.deleted().len(), 1);
assert_eq!(output.deleted()[0].key(), Some("deleted"));
let errors: Vec<_> = output.errors().iter().map(|error| (error.key(), error.code())).collect();
assert_eq!(
errors,
[
(Some(""), Some("InvalidArgument")),
(Some(oversized_key.as_str()), Some("InvalidArgument")),
(Some("conditional"), Some("InvalidArgument")),
],
);
assert!(is_in_the_store(BUCKET, "kept").await);
assert!(is_in_the_store(BUCKET, "conditional").await);
assert!(!is_in_the_store(BUCKET, "deleted").await);
});
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 70 B

-1
View File
@@ -1 +0,0 @@
<!doctype html><html lang="en"><title>Console fixture</title><body>Console static fixture</body></html>
@@ -1 +0,0 @@
console-route-static-content
-18
View File
@@ -22,8 +22,6 @@ classify_source_layer() {
printf 'composition'
elif [[ "$file" == rustfs/src/app/* ]]; then
printf 'app'
elif [[ "$file" == rustfs/src/protocols/client.rs ]] || [[ "$file" == rustfs/src/protocols/client/* ]]; then
printf 'interface'
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] ||
[[ "$file" == rustfs/src/storage/legacy_edge/* ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
[[ "$file" == rustfs/src/storage/s3_compat/* ]]; then
@@ -49,12 +47,6 @@ classify_target_layer() {
# their exported HTTP contracts belong to the interface boundary.
printf 'interface'
;;
protocols)
case "${path#protocols::}" in
client | client::* | ProtocolStorageClient | ProtocolStorageClient::*) printf 'interface' ;;
*) printf 'infra' ;;
esac
;;
app)
printf 'app'
;;
@@ -155,16 +147,6 @@ run_layer_model_self_tests() {
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/tests/object_meta.rs
assert_source_layer 'infra' rustfs/src/storage/legacy_edge.rs
assert_source_layer 'infra' rustfs/src/storage/legacy_edge_helper.rs
assert_source_layer 'interface' rustfs/src/protocols/client.rs
assert_source_layer 'interface' rustfs/src/protocols/client/owned.rs
assert_source_layer 'infra' rustfs/src/protocols/client_helper.rs
assert_target_layer 'interface' protocols::client
assert_target_layer 'interface' protocols::client::ProtocolStorageClient
assert_target_layer 'interface' protocols::ProtocolStorageClient
assert_target_layer 'infra' protocols::client_helper
assert_dependency_direction 'allowed' "$(classify_source_layer rustfs/src/protocols/client/owned.rs)" "$app_target"
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer protocols::client::ProtocolStorageClient)"
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer protocols::ProtocolStorageClient)"
assert_source_layer 'app' rustfs/src/app/bucket_usecase/cors.rs
assert_source_layer 'app' rustfs/src/app/multipart_usecase/complete.rs
assert_target_layer 'interface' storage::legacy_edge
+8 -1
View File
@@ -13,6 +13,7 @@ import re
import subprocess
import sys
import tempfile
import time
import tomllib
import urllib.request
from urllib.parse import urlsplit
@@ -329,7 +330,13 @@ def artifact_list(run_id):
def download_artifact(run, name, destination):
matches = [item for item in artifact_list(run["id"]) if item.get("name") == name]
# A finalized upload can take time to appear in the REST listing. Retry only
# absence; ambiguous identities and invalid metadata must fail immediately.
for attempt in range(6):
matches = [item for item in artifact_list(run["id"]) if item.get("name") == name]
if matches or attempt == 5:
break
time.sleep(2 ** attempt)
require(len(matches) == 1, "candidate artifact is absent or ambiguous")
artifact = validate_artifact(matches[0], run, name)
destination = Path(destination)
+40
View File
@@ -53,6 +53,46 @@ class TransportTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "unsafe"):
candidate.download_artifact(run, "candidate", temp)
def test_download_waits_for_artifact_metadata_visibility(self):
import hashlib
run = {"id": 70, "head_sha": "a" * 40}
stream = io.BytesIO()
with zipfile.ZipFile(stream, "w") as package:
package.writestr("record.json", b"accepted")
content = stream.getvalue()
artifact = {"id": 90, "name": "candidate", "expired": False, "workflow_run": run,
"size_in_bytes": len(content), "digest": "sha256:" + hashlib.sha256(content).hexdigest()}
with tempfile.TemporaryDirectory() as temp, \
patch.object(candidate, "artifact_list", side_effect=[[], [artifact]]) as listing, \
patch("time.sleep") as sleep, \
patch.object(candidate, "github_stream", return_value=io.BytesIO(content)):
self.assertEqual(candidate.download_artifact(run, "candidate", temp), artifact)
self.assertEqual((Path(temp) / "record.json").read_bytes(), b"accepted")
self.assertEqual(listing.call_count, 2)
sleep.assert_called_once_with(1)
def test_missing_artifact_remains_a_bounded_failure(self):
run = {"id": 70, "head_sha": "a" * 40}
with tempfile.TemporaryDirectory() as temp, \
patch.object(candidate, "artifact_list", return_value=[]) as listing, \
patch("time.sleep") as sleep, patch.object(candidate, "github_stream") as download:
with self.assertRaisesRegex(ValueError, "absent"):
candidate.download_artifact(run, "candidate", temp)
self.assertEqual(listing.call_count, 6)
self.assertEqual([call.args[0] for call in sleep.call_args_list], [1, 2, 4, 8, 16])
download.assert_not_called()
def test_ambiguous_artifact_is_rejected_without_retry(self):
run = {"id": 70, "head_sha": "a" * 40}
with tempfile.TemporaryDirectory() as temp, \
patch.object(candidate, "artifact_list", return_value=[{"name": "candidate"}] * 2) as listing, \
patch("time.sleep") as sleep, patch.object(candidate, "github_stream") as download:
with self.assertRaisesRegex(ValueError, "ambiguous"):
candidate.download_artifact(run, "candidate", temp)
listing.assert_called_once_with(70)
sleep.assert_not_called()
download.assert_not_called()
def test_duplicate_json_fields_are_not_silently_accepted(self):
with tempfile.TemporaryDirectory() as temp:
path = Path(temp) / "manifest.json"
-16
View File
@@ -286,18 +286,6 @@ for silent in ${REAL_SPLIT_FILES[@]+"${REAL_SPLIT_FILES[@]}"}; do
done
# --- layer dependencies ------------------------------------------------------
# Protocol edges may call the app; app code cannot call either public client path.
mkdir -p "$FX/rustfs/src/protocols/client"
cat >"$FX/rustfs/src/protocols/client/layer_probe.rs" <<'PROTOCOL_EDGE'
use crate::app::RequestEnvelope;
PROTOCOL_EDGE
cat >"$FX/rustfs/src/protocols/client_helper.rs" <<'PROTOCOL_NEIGHBOR'
use crate::app::RequestEnvelope;
PROTOCOL_NEIGHBOR
cat >"$FX/rustfs/src/app/protocol_client_layer_probe.rs" <<'PROTOCOL_REVERSE'
use crate::protocols::client::ProtocolStorageClient;
use crate::protocols::ProtocolStorageClient;
PROTOCOL_REVERSE
LAYER_OUT="$FX/.layer.out"
if (cd "$FX" && ./scripts/check_layer_dependencies.sh) >"$LAYER_OUT" 2>&1; then
fail "layer guard passed over a planted reverse dependency"
@@ -306,10 +294,6 @@ expect_reported "$LAYER_OUT" "dep|$E|interface->composition|" "crate::startup_se
expect_silent "$LAYER_OUT" "dep|$EDGE_IMPORTS|" "moved ecfs imports stay legal"
expect_silent "$LAYER_OUT" "dep|rustfs/src/app/lifecycle_transition_api_test/" "moved lifecycle transition tests stay excluded"
expect_silent "$LAYER_OUT" "dep|$LISTING|" "moved bucket usecase code keeps the app layer"
expect_silent "$LAYER_OUT" "dep|rustfs/src/protocols/client/layer_probe.rs|" "protocol client edge may use app contracts"
expect_reported "$LAYER_OUT" "dep|rustfs/src/protocols/client_helper.rs|infra->app|" "crate::app::RequestEnvelope" "neighboring protocol helper remains infrastructure"
expect_reported "$LAYER_OUT" "dep|rustfs/src/app/protocol_client_layer_probe.rs|app->interface|" "crate::protocols::client::ProtocolStorageClient" "app cannot import the protocol client module"
expect_reported "$LAYER_OUT" "dep|rustfs/src/app/protocol_client_layer_probe.rs|app->interface|" "crate::protocols::ProtocolStorageClient" "app cannot import the protocol client re-export"
# --- logging hot span --------------------------------------------------------
LOG_OUT="$FX/.logging.out"