mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
feat(gateway): serve GetObjectAttributes through the owned path
Share the read use case and both IAM decisions across the legacy and
gateway handlers. Keep attribute selection, stored metadata and
pagination in the use case, and encode the result at each edge.
The 6f93ac5f control matched 55 legacy wire cases and the restored
sources passed 139 scoped tests. Both stacks exercised 59 signed
requests, including 31 legacy refusals and 32 gateway refusals.
Seventeen injected faults each turned a targeted assertion red. They
cover the four IAM actions, resource and version binding, empty-version
selection, handler authorization, SSE-C, selection, both checksum paths,
paging, ETag, Last-Modified, counting and notification suppression.
This commit is contained in:
@@ -0,0 +1,299 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Read the stored object attributes requested by either S3 edge.
|
||||
|
||||
use super::head::ResponseEtag;
|
||||
use super::*;
|
||||
use crate::app::{Answered, RequestEnvelope};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
|
||||
const ETAG: &str = "ETag";
|
||||
const CHECKSUM: &str = "Checksum";
|
||||
const OBJECT_PARTS: &str = "ObjectParts";
|
||||
const STORAGE_CLASS: &str = "StorageClass";
|
||||
const OBJECT_SIZE: &str = "ObjectSize";
|
||||
|
||||
/// Decoded inputs preserve the version spelling and the raw attribute choices until use.
|
||||
/// Customer key material must never be rendered as Debug output.
|
||||
pub(crate) struct GetObjectAttributesFields {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) key: String,
|
||||
pub(crate) version_id: Option<String>,
|
||||
pub(crate) object_attributes: Vec<String>,
|
||||
pub(crate) max_parts: Option<i32>,
|
||||
pub(crate) part_number_marker: Option<String>,
|
||||
pub(crate) sse_customer_key: Option<String>,
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
}
|
||||
|
||||
/// Stored values are encoded by the response edge after the operation completes.
|
||||
pub(crate) struct GetObjectAttributesOutputFacts {
|
||||
pub(crate) e_tag: Option<ResponseEtag>,
|
||||
pub(crate) last_modified: Option<OffsetDateTime>,
|
||||
pub(crate) version_id: Option<String>,
|
||||
pub(crate) storage_class: Option<String>,
|
||||
pub(crate) object_size: Option<i64>,
|
||||
pub(crate) checksum: Option<ResponseChecksums>,
|
||||
pub(crate) object_parts: Option<AttributesPartsFacts>,
|
||||
}
|
||||
|
||||
pub(crate) struct AttributesPartsFacts {
|
||||
pub(crate) is_truncated: bool,
|
||||
pub(crate) max_parts: i32,
|
||||
pub(crate) next_part_number_marker: Option<i32>,
|
||||
pub(crate) part_number_marker: Option<i32>,
|
||||
pub(crate) parts: Vec<AttributesPartFacts>,
|
||||
pub(crate) total_parts_count: i32,
|
||||
}
|
||||
|
||||
pub(crate) struct AttributesPartFacts {
|
||||
pub(crate) part_number: Option<i32>,
|
||||
pub(crate) size: i64,
|
||||
pub(crate) checksums: ResponseChecksums,
|
||||
}
|
||||
|
||||
impl DefaultObjectUsecase {
|
||||
pub(crate) async fn execute_get_object_attributes(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
input: GetObjectAttributesFields,
|
||||
) -> S3Result<Answered<GetObjectAttributesOutputFacts>> {
|
||||
if let Some(context) = &self.context {
|
||||
let _ = context.object_store();
|
||||
}
|
||||
|
||||
let mut helper = OperationHelper::from_view(
|
||||
&envelope.audit_view(),
|
||||
EventName::ObjectAccessedAttributes,
|
||||
S3Operation::GetObjectAttributes,
|
||||
)
|
||||
.suppress_event();
|
||||
let GetObjectAttributesFields {
|
||||
bucket,
|
||||
key,
|
||||
version_id,
|
||||
object_attributes,
|
||||
max_parts,
|
||||
part_number_marker,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
} = input;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
};
|
||||
|
||||
let headers = envelope.headers();
|
||||
let mut opts = get_opts(&bucket, &key, version_id, None, headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let requested = |name| object_attributes_requested(&object_attributes, name);
|
||||
opts.include_part_checksums = requested(OBJECT_PARTS);
|
||||
|
||||
let info = match store.get_object_info(&bucket, &key, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(err) => {
|
||||
if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
|
||||
if is_dir_object(&key) {
|
||||
let has_children = match probe_prefix_has_children(store, &bucket, &key, false).await {
|
||||
Ok(has_children) => has_children,
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Failed to probe children for object attributes (bucket: {}, key: {}): {}",
|
||||
bucket, key, e
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
let msg = head_prefix_not_found_message(&bucket, &key, has_children);
|
||||
return Err(S3Error::with_message(S3ErrorCode::NoSuchKey, msg));
|
||||
}
|
||||
return Err(S3Error::new(S3ErrorCode::NoSuchKey));
|
||||
}
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
};
|
||||
|
||||
if info.delete_marker {
|
||||
if opts.version_id.is_none() {
|
||||
return Err(S3Error::new(S3ErrorCode::NoSuchKey));
|
||||
}
|
||||
return Err(S3Error::new(S3ErrorCode::MethodNotAllowed));
|
||||
}
|
||||
|
||||
validate_ssec_for_read(&info.user_defined, sse_customer_key.as_ref(), sse_customer_key_md5.as_ref())?;
|
||||
|
||||
let storage_class = response_storage_class_for_object_attributes(&info, &info.user_defined, requested(STORAGE_CLASS));
|
||||
let e_tag = if requested(ETAG) {
|
||||
info.etag.as_deref().map(ResponseEtag::from_stored)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let object_size = if requested(OBJECT_SIZE) {
|
||||
Some(info.get_actual_size().map_err(ApiError::from)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let checksum = if requested(CHECKSUM) {
|
||||
let (checksums, is_multipart) = info.decrypt_checksums(0, headers).map_err(ApiError::from)?;
|
||||
Some(classify_response_checksums(checksums, is_multipart))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let object_parts = if requested(OBJECT_PARTS) && info.is_multipart() {
|
||||
Some(read_parts(&info, headers, part_number_marker.as_deref(), max_parts)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let version_id = s3_response_version_id(info.version_id);
|
||||
let output = GetObjectAttributesOutputFacts {
|
||||
e_tag,
|
||||
last_modified: info.mod_time,
|
||||
version_id: version_id.clone(),
|
||||
storage_class,
|
||||
object_size,
|
||||
checksum,
|
||||
object_parts,
|
||||
};
|
||||
|
||||
helper = helper.object(info).version_id(version_id.unwrap_or_default());
|
||||
let result = Ok(Answered::new(output));
|
||||
let _ = helper.complete_owned(&result, StatusCode::OK, &HeaderMap::new());
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
fn read_parts(
|
||||
info: &ObjectInfo,
|
||||
headers: &HeaderMap,
|
||||
part_number_marker: Option<&str>,
|
||||
max_parts: Option<i32>,
|
||||
) -> S3Result<AttributesPartsFacts> {
|
||||
// Legacy headers have already decoded this as an i32. Native marker text is
|
||||
// only parsed when a stored multipart object actually needs a page.
|
||||
let part_number_marker = part_number_marker
|
||||
.map(str::parse::<i32>)
|
||||
.transpose()
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid part-number-marker"))?;
|
||||
let params = parse_list_parts_params(part_number_marker, max_parts)?;
|
||||
let mut parts = Vec::new();
|
||||
let mut marker = params.part_number_marker;
|
||||
let mut start_at = 0;
|
||||
|
||||
if let Some(value) = marker {
|
||||
if let Some(index) = info.parts.iter().position(|part| part.number == value) {
|
||||
start_at = index + 1;
|
||||
} else {
|
||||
marker = None;
|
||||
}
|
||||
}
|
||||
|
||||
let max_parts = i32::try_from(params.max_parts)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "max-parts value is out of range"))?;
|
||||
let end = (start_at + params.max_parts).min(info.parts.len());
|
||||
|
||||
for part in &info.parts[start_at..end] {
|
||||
let (checksums, is_multipart) = info.decrypt_checksums(part.number, headers).map_err(ApiError::from)?;
|
||||
let checksums = classify_response_checksums(checksums, is_multipart);
|
||||
let size = if part.actual_size > 0 {
|
||||
part.actual_size
|
||||
} else {
|
||||
part.size
|
||||
.try_into()
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Part size value is out of range"))?
|
||||
};
|
||||
parts.push(AttributesPartFacts {
|
||||
part_number: i32::try_from(part.number).ok(),
|
||||
size,
|
||||
checksums,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(AttributesPartsFacts {
|
||||
is_truncated: end < info.parts.len(),
|
||||
max_parts,
|
||||
next_part_number_marker: parts.last().and_then(|part| part.part_number),
|
||||
part_number_marker: marker.and_then(|value| i32::try_from(value).ok()),
|
||||
parts,
|
||||
total_parts_count: i32::try_from(info.parts.len())
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Part count is out of range"))?,
|
||||
})
|
||||
}
|
||||
|
||||
fn object_attributes_requested(object_attributes: &[String], name: &str) -> bool {
|
||||
object_attributes.iter().any(|value| {
|
||||
value.split(',').any(|part| {
|
||||
part.trim_matches(|c: char| c.is_whitespace() || c == '"' || c == '\'')
|
||||
.eq_ignore_ascii_case(name)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_object_attributes_returns_internal_error_when_store_uninitialized() {
|
||||
let fields = GetObjectAttributesFields {
|
||||
bucket: "test-bucket".to_owned(),
|
||||
key: "test-key".to_owned(),
|
||||
version_id: None,
|
||||
object_attributes: vec![],
|
||||
max_parts: None,
|
||||
part_number_marker: None,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
};
|
||||
let request = build_request((), http::Method::GET);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let error = DefaultObjectUsecase::without_context()
|
||||
.execute_get_object_attributes(envelope, fields)
|
||||
.await
|
||||
.err()
|
||||
.expect("uninitialized store");
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_attributes_requested_with_single_value() {
|
||||
let selection = vec![ETAG.to_owned()];
|
||||
assert!(object_attributes_requested(&selection, ETAG));
|
||||
assert!(!object_attributes_requested(&selection, OBJECT_SIZE));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_attributes_requested_with_comma_separated_values() {
|
||||
let selection = vec!["ObjectParts,etag".to_owned(), "StorageClass".to_owned()];
|
||||
assert!(object_attributes_requested(&selection, OBJECT_PARTS));
|
||||
assert!(object_attributes_requested(&selection, ETAG));
|
||||
assert!(!object_attributes_requested(&selection, OBJECT_SIZE));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_attributes_requested_with_quotes_and_spaces() {
|
||||
let selection = vec!["'ObjectSize', \"Checksum\" , \"Etag\"".to_owned()];
|
||||
assert!(object_attributes_requested(&selection, OBJECT_SIZE));
|
||||
assert!(object_attributes_requested(&selection, CHECKSUM));
|
||||
assert!(object_attributes_requested(&selection, ETAG));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_attributes_requested_returns_false_for_missing_name() {
|
||||
let selection = vec!["Checksum".to_owned()];
|
||||
assert!(!object_attributes_requested(&selection, OBJECT_SIZE));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Observe GetObjectAttributes completion at the real server's audit and event boundary.
|
||||
|
||||
use bytes::Bytes;
|
||||
use http_body_util::{BodyExt as _, Empty};
|
||||
use hyper_util::rt::TokioIo;
|
||||
|
||||
fn get_object_attributes_completion(stack: Option<&'static str>) {
|
||||
temp_env::with_vars([("RUSTFS_S3_STACK", stack), ("RUSTFS_AUDIT_ENABLE", Some("true"))], || {
|
||||
crate::app::gating_test_env::run_large_stack_test("attributes-completion", || async {
|
||||
use crate::app::storage_api::object_usecase::helper::MutationCompletionProbe;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::app::storage_api::test::contract::object::ObjectIO as _;
|
||||
use crate::embedded::{RustFSServerBuilder, find_available_port};
|
||||
|
||||
let server = RustFSServerBuilder::new()
|
||||
.address(format!("127.0.0.1:{}", find_available_port().unwrap()))
|
||||
.access_key("attributesprobe")
|
||||
.secret_key("attributes-fixture-test-secret")
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
crate::server::refresh_audit_module_enabled();
|
||||
let context = crate::app::runtime_sources::current_app_context().unwrap();
|
||||
let store = context.object_store();
|
||||
let bucket = "attributes-completion";
|
||||
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": {"AWS": "*"},
|
||||
"Action": ["s3:GetObject", "s3:GetObjectAttributes"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/object"), format!("arn:aws:s3:::{bucket}/missing")],
|
||||
}],
|
||||
});
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
bucket,
|
||||
crate::app::storage_api::object_usecase::bucket::metadata::BUCKET_POLICY_CONFIG,
|
||||
serde_json::to_vec(&policy).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
"object",
|
||||
&mut crate::app::storage_api::object_usecase::StoragePutObjReader::from_vec(b"payload".to_vec()),
|
||||
&Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let probe = MutationCompletionProbe::install(bucket);
|
||||
for (key, expected) in [("object", 200), ("missing", 404), ("other", 403)] {
|
||||
let socket = tokio::net::TcpStream::connect(server.address()).await.unwrap();
|
||||
let (mut sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(socket)).await.unwrap();
|
||||
let transport = tokio::spawn(connection);
|
||||
let request = http::Request::get(format!("/{bucket}/{key}?attributes"))
|
||||
.header(http::header::HOST, "localhost")
|
||||
.header("x-amz-object-attributes", "ObjectSize")
|
||||
.body(Empty::<Bytes>::new())
|
||||
.unwrap();
|
||||
let response = sender.send_request(request).await.unwrap();
|
||||
assert_eq!(response.status().as_u16(), expected);
|
||||
response.into_body().collect().await.unwrap();
|
||||
drop(sender);
|
||||
transport.abort();
|
||||
let _ = transport.await;
|
||||
}
|
||||
probe.wait_for(2, 0).await;
|
||||
let audits = probe
|
||||
.audits
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|entry| entry.api.name.as_deref() == Some(rustfs_s3_ops::S3Operation::GetObjectAttributes.as_str()))
|
||||
.map(|entry| (entry.api.object.clone(), entry.api.status.clone(), entry.api.status_code))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
audits,
|
||||
[
|
||||
(Some("object".to_owned()), Some("success".to_owned()), Some(200)),
|
||||
(Some("missing".to_owned()), Some("failure".to_owned()), Some(500)),
|
||||
],
|
||||
);
|
||||
assert!(probe.events.lock().unwrap().is_empty());
|
||||
server.shutdown().await;
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn legacy_get_object_attributes_completion() {
|
||||
get_object_attributes_completion(None);
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn gateway_get_object_attributes_completion() {
|
||||
get_object_attributes_completion(Some("gateway"));
|
||||
}
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! GetObject / GetObjectAttributes read path: cold fill, resume, stream tuning.
|
||||
//! GetObject read path: cold fill, resume, stream tuning.
|
||||
|
||||
use super::get_response::{GetObjectBody, GetObjectFields, GetObjectOutputFacts};
|
||||
use super::head::ResponseEtag;
|
||||
@@ -29,16 +29,12 @@ use crate::app::storage_api::object_usecase::object_cache::{
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::Body;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
Checksum, ChecksumType, GetObjectAttributesInput, GetObjectAttributesOutput, GetObjectAttributesParts, ObjectAttributes,
|
||||
ObjectPart, Timestamp,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::dto::{GetObjectInput, GetObjectOutput, StreamingBlob};
|
||||
use crate::app::storage_api::s3::header::X_AMZ_RESTORE;
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::stream::ByteStream;
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
use crate::on_demand_migration::WriteBackBody;
|
||||
|
||||
struct ConditionalRead {
|
||||
@@ -4950,221 +4946,6 @@ impl DefaultObjectUsecase {
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_FINALIZE, response_finalize_start);
|
||||
response
|
||||
}
|
||||
|
||||
pub async fn execute_get_object_attributes(
|
||||
&self,
|
||||
req: S3Request<GetObjectAttributesInput>,
|
||||
) -> S3Result<S3Response<GetObjectAttributesOutput>> {
|
||||
if let Some(context) = &self.context {
|
||||
let _ = context.object_store();
|
||||
}
|
||||
|
||||
let mut helper =
|
||||
OperationHelper::new(&req, EventName::ObjectAccessedAttributes, S3Operation::GetObjectAttributes).suppress_event();
|
||||
let GetObjectAttributesInput {
|
||||
bucket,
|
||||
key,
|
||||
max_parts,
|
||||
object_attributes,
|
||||
part_number_marker,
|
||||
version_id,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
opts.include_part_checksums = object_attributes_requested(&object_attributes, ObjectAttributes::OBJECT_PARTS);
|
||||
|
||||
let info = match store.get_object_info(&bucket, &key, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(err) => {
|
||||
if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
|
||||
if is_dir_object(&key) {
|
||||
let has_children = match probe_prefix_has_children(store, &bucket, &key, false).await {
|
||||
Ok(has_children) => has_children,
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Failed to probe children for object attributes (bucket: {}, key: {}): {}",
|
||||
bucket, key, e
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
let msg = head_prefix_not_found_message(&bucket, &key, has_children);
|
||||
return Err(S3Error::with_message(S3ErrorCode::NoSuchKey, msg));
|
||||
}
|
||||
return Err(S3Error::new(S3ErrorCode::NoSuchKey));
|
||||
}
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
};
|
||||
|
||||
if info.delete_marker {
|
||||
if opts.version_id.is_none() {
|
||||
return Err(S3Error::new(S3ErrorCode::NoSuchKey));
|
||||
}
|
||||
return Err(S3Error::new(S3ErrorCode::MethodNotAllowed));
|
||||
}
|
||||
|
||||
validate_ssec_for_read(&info.user_defined, sse_customer_key.as_ref(), sse_customer_key_md5.as_ref())?;
|
||||
|
||||
let metadata_map = info.user_defined.clone();
|
||||
debug!(
|
||||
"GetObjectAttributes raw object_attributes={:?}",
|
||||
object_attributes.iter().map(|value| value.as_str()).collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
let requested = |name: &'static str| -> bool { object_attributes_requested(&object_attributes, name) };
|
||||
let storage_class =
|
||||
response_storage_class_for_object_attributes(&info, &metadata_map, requested(ObjectAttributes::STORAGE_CLASS));
|
||||
|
||||
let e_tag = if requested(ObjectAttributes::ETAG) {
|
||||
info.etag.as_ref().map(|etag| to_s3s_etag(etag))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let object_size = if requested(ObjectAttributes::OBJECT_SIZE) {
|
||||
Some(info.get_actual_size().map_err(ApiError::from)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let checksum = if requested(ObjectAttributes::CHECKSUM) {
|
||||
let (checksums, is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?;
|
||||
// GetObjectAttributes returns checksums in the XML body, and s3s's Checksum
|
||||
// type has no field for the additional algorithms, so `extra` cannot be
|
||||
// surfaced here (unlike the header-based GET/HEAD paths) — an s3s limitation
|
||||
// tracked for when it gains typed fields.
|
||||
let ResponseChecksums {
|
||||
crc32: checksum_crc32,
|
||||
crc32c: checksum_crc32c,
|
||||
sha1: checksum_sha1,
|
||||
sha256: checksum_sha256,
|
||||
crc64nvme: checksum_crc64nvme,
|
||||
checksum_type,
|
||||
..
|
||||
} = classify_response_checksums(checksums, is_multipart);
|
||||
|
||||
Some(Checksum {
|
||||
checksum_crc32,
|
||||
checksum_crc32c,
|
||||
checksum_sha1,
|
||||
checksum_sha256,
|
||||
checksum_crc64nvme,
|
||||
checksum_type: checksum_type.map(ChecksumType::from),
|
||||
..Default::default()
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let object_parts = if requested(ObjectAttributes::OBJECT_PARTS) && info.is_multipart() {
|
||||
let params = parse_list_parts_params(part_number_marker, max_parts)?;
|
||||
let mut parts = Vec::new();
|
||||
let mut marker = params.part_number_marker;
|
||||
let max_parts = params.max_parts;
|
||||
let mut start_at = 0usize;
|
||||
|
||||
if let Some(marker_value) = marker {
|
||||
if let Some(index) = info.parts.iter().position(|part| part.number == marker_value) {
|
||||
start_at = index + 1;
|
||||
} else {
|
||||
marker = None;
|
||||
}
|
||||
}
|
||||
|
||||
let max_parts: i32 = max_parts.try_into().map_err(|_| {
|
||||
S3Error::with_message(S3ErrorCode::InvalidArgument, "max-parts value is out of range".to_string())
|
||||
})?;
|
||||
let end = (start_at + params.max_parts).min(info.parts.len());
|
||||
let is_truncated = end < info.parts.len();
|
||||
|
||||
for part in &info.parts[start_at..end] {
|
||||
let (checksums, is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?;
|
||||
// Additional algorithms cannot be surfaced in the ObjectPart XML body
|
||||
// (s3s has no field); same limitation as the object-level attributes above.
|
||||
let ResponseChecksums {
|
||||
crc32: checksum_crc32,
|
||||
crc32c: checksum_crc32c,
|
||||
sha1: checksum_sha1,
|
||||
sha256: checksum_sha256,
|
||||
crc64nvme: checksum_crc64nvme,
|
||||
..
|
||||
} = classify_response_checksums(checksums, is_multipart);
|
||||
|
||||
let part_size = if part.actual_size > 0 {
|
||||
part.actual_size
|
||||
} else {
|
||||
part.size.try_into().map_err(|_| {
|
||||
S3Error::with_message(S3ErrorCode::InvalidArgument, "Part size value is out of range".to_string())
|
||||
})?
|
||||
};
|
||||
|
||||
parts.push(ObjectPart {
|
||||
checksum_crc32,
|
||||
checksum_crc32c,
|
||||
checksum_sha1,
|
||||
checksum_sha256,
|
||||
checksum_crc64nvme,
|
||||
part_number: i32::try_from(part.number).ok(),
|
||||
size: Some(part_size),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
let part_number_marker = marker.and_then(|v| i32::try_from(v).ok());
|
||||
let next_part_number_marker = parts.last().and_then(|part| part.part_number);
|
||||
|
||||
Some(GetObjectAttributesParts {
|
||||
is_truncated: Some(is_truncated),
|
||||
max_parts: Some(max_parts),
|
||||
next_part_number_marker,
|
||||
part_number_marker,
|
||||
parts: Some(parts),
|
||||
total_parts_count: Some(i32::try_from(info.parts.len()).map_err(|_| {
|
||||
S3Error::with_message(S3ErrorCode::InvalidArgument, "Part count is out of range".to_string())
|
||||
})?),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let version_id = s3_response_version_id(info.version_id);
|
||||
|
||||
let output = GetObjectAttributesOutput {
|
||||
checksum,
|
||||
delete_marker: if info.delete_marker { Some(true) } else { None },
|
||||
e_tag,
|
||||
last_modified: info.mod_time.map(Timestamp::from),
|
||||
object_parts,
|
||||
object_size,
|
||||
storage_class,
|
||||
version_id: version_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
helper = helper.object(info).version_id(version_id.unwrap_or_default());
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'static str) -> bool {
|
||||
object_attributes.iter().any(|value| {
|
||||
value.as_str().split(',').any(|part| {
|
||||
part.trim_matches(|c: char| c.is_whitespace() || c == '"' || c == '\'')
|
||||
.eq_ignore_ascii_case(name)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Bytes the inline tee may queue for the write-back ahead of the client.
|
||||
@@ -12598,57 +12379,6 @@ mod tests {
|
||||
let err = Box::pin(usecase.execute_get_object_from_s3s(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_object_attributes_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetObjectAttributesInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_get_object_attributes(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_attributes_requested_with_single_value() {
|
||||
let object_attributes = vec![ObjectAttributes::from_static(ObjectAttributes::ETAG)];
|
||||
|
||||
assert!(object_attributes_requested(&object_attributes, ObjectAttributes::ETAG));
|
||||
assert!(!object_attributes_requested(&object_attributes, ObjectAttributes::OBJECT_SIZE));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_attributes_requested_with_comma_separated_values() {
|
||||
let object_attributes = vec![
|
||||
ObjectAttributes::from_static("ObjectParts,etag"),
|
||||
ObjectAttributes::from_static("StorageClass"),
|
||||
];
|
||||
|
||||
assert!(object_attributes_requested(&object_attributes, ObjectAttributes::OBJECT_PARTS));
|
||||
assert!(object_attributes_requested(&object_attributes, ObjectAttributes::ETAG));
|
||||
assert!(!object_attributes_requested(&object_attributes, ObjectAttributes::OBJECT_SIZE));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_attributes_requested_with_quotes_and_spaces() {
|
||||
let object_attributes = vec![ObjectAttributes::from_static("'ObjectSize', \"Checksum\" , \"Etag\"")];
|
||||
|
||||
assert!(object_attributes_requested(&object_attributes, ObjectAttributes::OBJECT_SIZE));
|
||||
assert!(object_attributes_requested(&object_attributes, ObjectAttributes::CHECKSUM));
|
||||
assert!(object_attributes_requested(&object_attributes, ObjectAttributes::ETAG));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_attributes_requested_returns_false_for_missing_name() {
|
||||
let object_attributes = vec![ObjectAttributes::from_static("Checksum")];
|
||||
|
||||
assert!(!object_attributes_requested(&object_attributes, ObjectAttributes::OBJECT_SIZE));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -97,6 +97,7 @@ use super::storage_api::object_usecase::object_cache::GetObjectBodySource;
|
||||
#[cfg(test)]
|
||||
use super::storage_api::object_usecase::object_cache::lookup_get_object_body_cache_hook;
|
||||
use super::storage_api::object_usecase::object_cache::{GetObjectBodyCacheHookLookup, get_object_body_cache_plaintext_len};
|
||||
#[cfg(test)]
|
||||
use super::storage_api::object_usecase::object_utils::to_s3s_etag;
|
||||
use super::storage_api::object_usecase::options::{
|
||||
copy_dst_opts_with_replication_authorization, copy_src_opts, del_opts_with_versioning, extract_metadata,
|
||||
@@ -204,8 +205,12 @@ fn remove_source_replication_bookkeeping(user_defined: &mut HashMap<String, Stri
|
||||
}
|
||||
}
|
||||
|
||||
mod attributes;
|
||||
#[cfg(test)]
|
||||
mod attributes_completion_test;
|
||||
#[cfg(test)]
|
||||
mod conditional_read_bench;
|
||||
pub(crate) use attributes::{GetObjectAttributesFields, GetObjectAttributesOutputFacts};
|
||||
mod copy;
|
||||
mod copy_input;
|
||||
pub(crate) use copy_input::{CopyObjectWriteInput, CopyObjectWriteOutput, CopySourceCondition};
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use super::*;
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::dto::ObjectLockRetentionMode;
|
||||
use crate::app::storage_api::s3::dto::{ChecksumAlgorithm, ServerSideEncryption, StorageClass, Timestamp, TimestampFormat};
|
||||
use crate::app::storage_api::s3::dto::{ChecksumAlgorithm, ServerSideEncryption, Timestamp, TimestampFormat};
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::dto::{ObjectLockLegalHoldStatus, ObjectLockMode};
|
||||
use crate::app::storage_api::s3::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
|
||||
@@ -815,7 +815,7 @@ pub(super) fn response_storage_class_for_object_attributes(
|
||||
info: &ObjectInfo,
|
||||
metadata: &HashMap<String, String>,
|
||||
requested: bool,
|
||||
) -> Option<StorageClass> {
|
||||
) -> Option<String> {
|
||||
if !requested {
|
||||
return None;
|
||||
}
|
||||
@@ -828,9 +828,7 @@ pub(super) fn response_storage_class_for_object_attributes(
|
||||
&& !info.transitioned_object.tier.is_empty())
|
||||
.then_some(info.transitioned_object.tier.as_str());
|
||||
|
||||
Some(StorageClass::from(
|
||||
storageclass::effective_class(stored_class, transitioned_tier).to_string(),
|
||||
))
|
||||
Some(storageclass::effective_class(stored_class, transitioned_tier).to_string())
|
||||
}
|
||||
|
||||
// Shared across Object Lock validation paths to keep the client-facing
|
||||
@@ -1953,9 +1951,7 @@ mod tests {
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
response_storage_class_for_object_attributes(&info, &metadata, true)
|
||||
.as_ref()
|
||||
.map(StorageClass::as_str),
|
||||
response_storage_class_for_object_attributes(&info, &metadata, true).as_deref(),
|
||||
Some(storageclass::STANDARD)
|
||||
);
|
||||
|
||||
@@ -1964,9 +1960,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
response_storage_class_for_object_attributes(&legacy_info, &HashMap::new(), true)
|
||||
.as_ref()
|
||||
.map(StorageClass::as_str),
|
||||
response_storage_class_for_object_attributes(&legacy_info, &HashMap::new(), true).as_deref(),
|
||||
Some(storageclass::STANDARD)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1638,7 +1638,9 @@ pub(crate) mod helper {
|
||||
}
|
||||
|
||||
pub(crate) mod object_utils {
|
||||
pub(crate) use crate::storage::storage_api::{to_s3s_etag, to_s3s_xml_etag};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::to_s3s_etag;
|
||||
pub(crate) use crate::storage::storage_api::to_s3s_xml_etag;
|
||||
}
|
||||
|
||||
pub(crate) mod io {
|
||||
@@ -1815,12 +1817,12 @@ pub(crate) mod object_usecase {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::ecfs;
|
||||
pub(crate) use super::{
|
||||
access, bucket, compression, concurrency, data_usage, deadlock_detector, error, head_prefix, helper, io, object_utils,
|
||||
options, request_context, s3_api, set_disk, sse, storage_class, timeout_wrapper,
|
||||
access, bucket, compression, concurrency, data_usage, deadlock_detector, error, head_prefix, helper, io, options,
|
||||
request_context, s3_api, set_disk, sse, storage_class, timeout_wrapper,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{ecfs, object_utils};
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
ECStore, GetObjectReader, OldCurrentSize, RFC1123, StorageDeletedObject, StorageObjectInfo,
|
||||
StorageObjectLockDeleteOptions, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions,
|
||||
|
||||
@@ -13,21 +13,27 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch C (rustfs/backlog#2765): GetObject, HeadObject and GetObjectAttributes.
|
||||
//! GetObject and HeadObject are served; the batch appends the remaining rows here.
|
||||
|
||||
use super::{OpRow, Served, resp, row};
|
||||
use crate::app::object_usecase::{
|
||||
DefaultObjectUsecase, GetObjectBody, GetObjectOutputFacts, HeadObjectOutputFacts, ResponseEtag,
|
||||
DefaultObjectUsecase, GetObjectAttributesFields, GetObjectAttributesOutputFacts, GetObjectBody, GetObjectOutputFacts,
|
||||
HeadObjectOutputFacts, ResponseEtag,
|
||||
};
|
||||
use crate::app::{Answered, RequestEnvelope};
|
||||
use crate::runtime_sources::AppContext;
|
||||
use crate::storage_api::server::http::access::access_owned::object_read::{authorize_get_object, authorize_head_object};
|
||||
use crate::storage_api::server::http::access::access_owned::object_read::{
|
||||
authorize_get_object, authorize_get_object_attributes, authorize_head_object,
|
||||
};
|
||||
use rustfs_gateway::{Req, Resp, dto};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::{S3ErrorCode, S3Result, s3_error};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) const ROWS: &[OpRow] = &[row::<dto::GetObject>(), row::<dto::HeadObject>()];
|
||||
pub(super) const ROWS: &[OpRow] = &[
|
||||
row::<dto::GetObject>(),
|
||||
row::<dto::HeadObject>(),
|
||||
row::<dto::GetObjectAttributes>(),
|
||||
];
|
||||
|
||||
impl Served for dto::HeadObject {
|
||||
const METRIC: S3Operation = S3Operation::HeadObject;
|
||||
@@ -80,6 +86,106 @@ impl Served for dto::GetObject {
|
||||
}
|
||||
}
|
||||
|
||||
impl Served for dto::GetObjectAttributes {
|
||||
const METRIC: S3Operation = S3Operation::GetObjectAttributes;
|
||||
|
||||
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
|
||||
let input = request.input();
|
||||
Ok(
|
||||
authorize_get_object_attributes(envelope, input.bucket.as_str(), input.key.as_str(), input.version_id.as_deref())
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let input = request.into_input();
|
||||
let fields = GetObjectAttributesFields {
|
||||
bucket: input.bucket.as_str().to_owned(),
|
||||
key: input.key.as_str().to_owned(),
|
||||
version_id: input.version_id,
|
||||
object_attributes: input
|
||||
.object_attributes
|
||||
.into_iter()
|
||||
.map(|name| name.as_str().to_owned())
|
||||
.collect(),
|
||||
max_parts: input.max_parts,
|
||||
part_number_marker: input.part_number_marker,
|
||||
sse_customer_key: input.sse_customer_key.map(|key| key.expose_secret().to_owned()),
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
};
|
||||
let Answered { output, status, headers } = DefaultObjectUsecase::with_context(context)
|
||||
.execute_get_object_attributes(envelope, fields)
|
||||
.await?;
|
||||
Ok(resp::<Self>(Answered {
|
||||
output: attributes_output(output)?,
|
||||
status,
|
||||
headers,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn attributes_output(output: GetObjectAttributesOutputFacts) -> S3Result<dto::GetObjectAttributesOutput> {
|
||||
let object_parts = output
|
||||
.object_parts
|
||||
.map(|parts| {
|
||||
let parts_output = parts
|
||||
.parts
|
||||
.into_iter()
|
||||
.map(|part| {
|
||||
Ok(dto::ObjectPart {
|
||||
part_number: part
|
||||
.part_number
|
||||
.ok_or_else(|| s3_error!(InternalError, "Stored part number is out of range"))?,
|
||||
size: part.size,
|
||||
checksum_crc32: part.checksums.crc32,
|
||||
checksum_crc32c: part.checksums.crc32c,
|
||||
checksum_crc64nvme: part.checksums.crc64nvme,
|
||||
checksum_sha1: part.checksums.sha1,
|
||||
checksum_sha256: part.checksums.sha256,
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.collect::<S3Result<_>>()?;
|
||||
Ok::<_, rustfs_s3_types::S3Error>(dto::GetObjectAttributesParts {
|
||||
is_truncated: parts.is_truncated,
|
||||
max_parts: parts.max_parts,
|
||||
next_part_number_marker: parts.next_part_number_marker.map(|marker| marker.to_string()),
|
||||
part_number_marker: parts.part_number_marker.map(|marker| marker.to_string()),
|
||||
total_parts_count: parts.total_parts_count,
|
||||
parts: parts_output,
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
Ok(dto::GetObjectAttributesOutput {
|
||||
e_tag: output.e_tag.map(attributes_etag),
|
||||
last_modified: output.last_modified.map(crate::app::dto_values::timestamp),
|
||||
version_id: output.version_id,
|
||||
storage_class: output.storage_class.map(dto::StorageClass::custom),
|
||||
object_size: output.object_size,
|
||||
checksum: output.checksum.map(|checksum| dto::Checksum {
|
||||
checksum_crc32: checksum.crc32,
|
||||
checksum_crc32c: checksum.crc32c,
|
||||
checksum_crc64nvme: checksum.crc64nvme,
|
||||
checksum_sha1: checksum.sha1,
|
||||
checksum_sha256: checksum.sha256,
|
||||
checksum_type: checksum.checksum_type.map(dto::ChecksumType::custom),
|
||||
..Default::default()
|
||||
}),
|
||||
object_parts,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn attributes_etag(tag: ResponseEtag) -> rustfs_gateway_types::ETag {
|
||||
if tag.weak
|
||||
&& let Ok(weak) = rustfs_gateway_types::ETag::new_weak(tag.opaque.as_str().to_owned())
|
||||
&& weak.opaque_tag() == tag.opaque
|
||||
{
|
||||
return weak;
|
||||
}
|
||||
rustfs_gateway_types::ETag::from_stored(tag.opaque)
|
||||
}
|
||||
|
||||
fn get_result(result: S3Result<Answered<GetObjectOutputFacts>>) -> S3Result<Resp<dto::GetObject>> {
|
||||
match result {
|
||||
Ok(answered) => get_response(answered),
|
||||
|
||||
@@ -3798,20 +3798,8 @@ impl S3Access for FS {
|
||||
///
|
||||
/// This method returns `Ok(())` by default.
|
||||
async fn get_object_attributes(&self, req: &mut S3Request<GetObjectAttributesInput>) -> S3Result<()> {
|
||||
let req_info = ext_req_info_mut(&mut req.extensions)?;
|
||||
req_info.bucket = Some(req.input.bucket.clone());
|
||||
req_info.object = Some(req.input.key.clone());
|
||||
req_info.version_id = req.input.version_id.clone();
|
||||
|
||||
if req.input.version_id.is_some() {
|
||||
authorize_request(req, Action::S3Action(S3Action::GetObjectVersionAttributesAction)).await?;
|
||||
authorize_request(req, Action::S3Action(S3Action::GetObjectVersionAction)).await?;
|
||||
} else {
|
||||
authorize_request(req, Action::S3Action(S3Action::GetObjectAttributesAction)).await?;
|
||||
authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
let (bucket, key, version_id) = (req.input.bucket.clone(), req.input.key.clone(), req.input.version_id.clone());
|
||||
access_owned::object_read::authorize_get_object_attributes(req, &bucket, &key, version_id.as_deref()).await
|
||||
}
|
||||
|
||||
/// Checks whether the GetObjectLegalHold request has accesses to the resources.
|
||||
|
||||
@@ -64,6 +64,22 @@ pub(crate) async fn authorize_head_object<R: AuthorizationRequest + ?Sized>(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Attributes require two IAM decisions on the same resource and original version.
|
||||
pub(crate) async fn authorize_get_object_attributes<R: AuthorizationRequest + ?Sized>(
|
||||
req: &mut R,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
) -> S3Result<()> {
|
||||
let attributes = if version_id.is_some() {
|
||||
S3Action::GetObjectVersionAttributesAction
|
||||
} else {
|
||||
S3Action::GetObjectAttributesAction
|
||||
};
|
||||
authorize_object_request(req, bucket, key, version_id, Action::S3Action(attributes)).await?;
|
||||
authorize_request(req, versioned_read_action(version_id)).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "object_read_test.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -166,3 +166,91 @@ fn get_authorization_binds_version_and_explicit_denial() {
|
||||
assert!(failures.is_empty(), "{}", failures.join("\n"));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn attributes_authorization_requires_each_action_on_the_original_version() {
|
||||
use super::authorize_get_object_attributes;
|
||||
|
||||
crate::app::gating_test_env::run_large_stack_test("attributes-authorization", || async {
|
||||
let context = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = context.object_store();
|
||||
let slot = std::sync::Arc::new(ServerContextSlot::new());
|
||||
assert!(slot.install(Arc::clone(&context)));
|
||||
for (version, required) in [
|
||||
(None, ["s3:GetObjectAttributes", "s3:GetObject"]),
|
||||
(Some(""), ["s3:GetObjectVersionAttributes", "s3:GetObjectVersion"]),
|
||||
(Some("null"), ["s3:GetObjectVersionAttributes", "s3:GetObjectVersion"]),
|
||||
(
|
||||
Some(" aedfa2f8-840d-4a1c-b717-04411d82389e "),
|
||||
["s3:GetObjectVersionAttributes", "s3:GetObjectVersion"],
|
||||
),
|
||||
] {
|
||||
for (allowed, denied, key, permitted) in [
|
||||
(&required[..], None, "key", true),
|
||||
(&required[..1], None, "key", false),
|
||||
(&required[1..], None, "key", false),
|
||||
(&required[..], Some(required[0]), "key", false),
|
||||
(&required[..], Some(required[1]), "key", false),
|
||||
(&required[..], None, "unrelated", false),
|
||||
] {
|
||||
let bucket = format!("attributes-auth-{}", uuid::Uuid::new_v4().simple());
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
let resource = format!("arn:aws:s3:::{bucket}/key");
|
||||
let mut statements = vec![serde_json::json!({
|
||||
"Effect": "Allow",
|
||||
"Principal": {"AWS": "*"},
|
||||
"Action": allowed,
|
||||
"Resource": resource,
|
||||
})];
|
||||
if let Some(action) = denied {
|
||||
statements.push(serde_json::json!({
|
||||
"Effect": "Deny",
|
||||
"Principal": {"AWS": "*"},
|
||||
"Action": action,
|
||||
"Resource": resource,
|
||||
}));
|
||||
}
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG,
|
||||
serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut request = S3Request {
|
||||
input: (),
|
||||
method: Method::GET,
|
||||
uri: Uri::from_static("/bucket/key?attributes"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
request.extensions.insert(ReqInfo::default());
|
||||
request.extensions.insert(Arc::clone(&slot));
|
||||
let mut envelope = RequestEnvelope::from_s3s(&request);
|
||||
let legacy = authorize_get_object_attributes(&mut request, &bucket, key, version).await;
|
||||
let native = authorize_get_object_attributes(&mut envelope, &bucket, key, version).await;
|
||||
let expected = (!permitted).then_some(&S3ErrorCode::AccessDenied);
|
||||
for (edge, result, info) in [
|
||||
("legacy", legacy, request.extensions.get::<ReqInfo>().unwrap()),
|
||||
("native", native, envelope.req_info().unwrap()),
|
||||
] {
|
||||
assert_eq!(
|
||||
result.as_ref().err().map(|error| error.code()),
|
||||
expected,
|
||||
"edge={edge}, allow={allowed:?}, deny={denied:?}, key={key}, version={version:?}",
|
||||
);
|
||||
assert_eq!(
|
||||
(info.bucket.as_deref(), info.object.as_deref(), info.version_id.as_deref()),
|
||||
(Some(bucket.as_str()), Some(key), version)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -64,7 +64,14 @@ impl FS {
|
||||
req: S3Request<GetObjectAttributesInput>,
|
||||
) -> S3Result<S3Response<GetObjectAttributesOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_get_object_attributes(req).await
|
||||
record_s3_op(S3Operation::GetObjectAttributes);
|
||||
through_gateway_seam_answered(
|
||||
req,
|
||||
|input| Ok(s3_api::GetObjectAttributesFields::from_s3s(input)),
|
||||
|output| Ok(s3_api::GetObjectAttributesOutputFacts::into_s3s(output)),
|
||||
|envelope, input| usecase.execute_get_object_attributes(envelope, input),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "head_object", level = "debug", skip(self, req))]
|
||||
@@ -85,6 +92,76 @@ impl FS {
|
||||
}
|
||||
}
|
||||
|
||||
impl s3_api::GetObjectAttributesFields {
|
||||
fn from_s3s(input: GetObjectAttributesInput) -> Self {
|
||||
Self {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
version_id: input.version_id,
|
||||
object_attributes: input
|
||||
.object_attributes
|
||||
.into_iter()
|
||||
.map(|name| name.as_str().to_owned())
|
||||
.collect(),
|
||||
max_parts: input.max_parts,
|
||||
part_number_marker: input.part_number_marker.map(|marker| marker.to_string()),
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl s3_api::GetObjectAttributesOutputFacts {
|
||||
fn into_s3s(self) -> GetObjectAttributesOutput {
|
||||
GetObjectAttributesOutput {
|
||||
e_tag: self.e_tag.map(|tag| {
|
||||
if tag.weak {
|
||||
ETag::Weak(tag.opaque)
|
||||
} else {
|
||||
ETag::Strong(tag.opaque)
|
||||
}
|
||||
}),
|
||||
last_modified: self.last_modified.map(Timestamp::from),
|
||||
version_id: self.version_id,
|
||||
storage_class: self.storage_class.map(StorageClass::from),
|
||||
object_size: self.object_size,
|
||||
checksum: self.checksum.map(|checksum| Checksum {
|
||||
checksum_crc32: checksum.crc32,
|
||||
checksum_crc32c: checksum.crc32c,
|
||||
checksum_crc64nvme: checksum.crc64nvme,
|
||||
checksum_sha1: checksum.sha1,
|
||||
checksum_sha256: checksum.sha256,
|
||||
checksum_type: checksum.checksum_type.map(ChecksumType::from),
|
||||
..Default::default()
|
||||
}),
|
||||
object_parts: self.object_parts.map(|parts| GetObjectAttributesParts {
|
||||
is_truncated: Some(parts.is_truncated),
|
||||
max_parts: Some(parts.max_parts),
|
||||
next_part_number_marker: parts.next_part_number_marker,
|
||||
part_number_marker: parts.part_number_marker,
|
||||
total_parts_count: Some(parts.total_parts_count),
|
||||
parts: Some(
|
||||
parts
|
||||
.parts
|
||||
.into_iter()
|
||||
.map(|part| ObjectPart {
|
||||
part_number: part.part_number,
|
||||
size: Some(part.size),
|
||||
checksum_crc32: part.checksums.crc32,
|
||||
checksum_crc32c: part.checksums.crc32c,
|
||||
checksum_crc64nvme: part.checksums.crc64nvme,
|
||||
checksum_sha1: part.checksums.sha1,
|
||||
checksum_sha256: part.checksums.sha256,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl s3_api::HeadObjectFields {
|
||||
pub(crate) fn from_s3s(input: HeadObjectInput) -> Self {
|
||||
use rustfs_gateway_types::ByteRange;
|
||||
|
||||
@@ -32,7 +32,8 @@ pub(crate) use crate::app::object::{
|
||||
};
|
||||
pub(crate) use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
pub(crate) use crate::app::object_usecase::{
|
||||
GetObjectBody, GetObjectFields, GetObjectOutputFacts, HeadObjectFields, HeadObjectOutputFacts,
|
||||
GetObjectAttributesFields, GetObjectAttributesOutputFacts, GetObjectBody, GetObjectFields, GetObjectOutputFacts,
|
||||
HeadObjectFields, HeadObjectOutputFacts,
|
||||
};
|
||||
|
||||
pub(crate) mod bucket;
|
||||
|
||||
@@ -27,6 +27,7 @@ mod bucket_configs;
|
||||
mod bucket_core;
|
||||
mod bucket_replication_lifecycle;
|
||||
mod listing_multipart;
|
||||
mod object_attributes;
|
||||
mod object_commit;
|
||||
mod object_meta;
|
||||
mod object_misc;
|
||||
|
||||
@@ -0,0 +1,640 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Signed GetObjectAttributes observations against the embedded server.
|
||||
|
||||
use super::{
|
||||
GATEWAY, LEGACY, USER_SECRET_KEY, create_bucket_in_the_store, create_user_without_a_policy, s3_client, with_environment,
|
||||
};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{ChecksumAlgorithm, CompletedMultipartUpload, CompletedPart, ServerSideEncryption};
|
||||
use bytes::Bytes;
|
||||
use http::HeaderMap;
|
||||
use http_body_util::{BodyExt as _, Empty};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use rustfs::app::context::get_global_app_context;
|
||||
use rustfs::embedded::RustFSServer;
|
||||
use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_storage_api::{
|
||||
BucketOperations as _, CompletePart, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, ObjectOperations as _,
|
||||
};
|
||||
|
||||
const BUCKET: &str = "object-attributes";
|
||||
const RESERVED: &str = "sthree-object-attributes";
|
||||
const VERSIONED: &str = "versioned-object-attributes";
|
||||
const ATTRIBUTES: &str = "x-amz-object-attributes";
|
||||
const MARKER: &str = "x-amz-part-number-marker";
|
||||
const MAX: &str = "x-amz-max-parts";
|
||||
const SSE_ALGORITHM: &str = "x-amz-server-side-encryption-customer-algorithm";
|
||||
const SSE_KEY: &str = "x-amz-server-side-encryption-customer-key";
|
||||
const SSE_MD5: &str = "x-amz-server-side-encryption-customer-key-md5";
|
||||
|
||||
fn attributes_count() -> u64 {
|
||||
rustfs_io_metrics::s3_op_metrics_snapshot()
|
||||
.into_iter()
|
||||
.find(|entry| entry.op == S3Operation::GetObjectAttributes.as_str())
|
||||
.map_or(0, |entry| entry.total)
|
||||
}
|
||||
|
||||
struct Observation {
|
||||
status: u16,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
count: u64,
|
||||
}
|
||||
|
||||
impl Observation {
|
||||
fn xml(&self) -> &str {
|
||||
std::str::from_utf8(&self.body).expect("XML response bytes")
|
||||
}
|
||||
|
||||
fn status(&self, expected: u16, count: u64, label: &str) {
|
||||
assert_eq!((self.status, self.count), (expected, count), "{label}: {}", self.xml());
|
||||
}
|
||||
|
||||
fn contains(&self, field: &str, value: &str, label: &str) {
|
||||
assert!(self.xml().contains(&format!("<{field}>{value}</{field}>")), "{label}: {}", self.xml());
|
||||
}
|
||||
|
||||
fn omits(&self, field: &str, label: &str) {
|
||||
assert!(!self.xml().contains(&format!("<{field}>")), "{label}: {}", self.xml());
|
||||
}
|
||||
}
|
||||
|
||||
async fn observe(server: &RustFSServer, path: &str, headers: &[(&str, &str)], user: Option<&str>) -> Observation {
|
||||
let endpoint = server.endpoint();
|
||||
let uri: http::Uri = format!("{endpoint}{path}").parse().unwrap();
|
||||
let mut request = http::Request::get(&uri)
|
||||
.header(http::header::HOST, uri.authority().unwrap().as_str())
|
||||
.header("x-amz-content-sha256", "UNSIGNED-PAYLOAD");
|
||||
for (name, value) in headers {
|
||||
request = request.header(*name, *value);
|
||||
}
|
||||
let (key, secret) = user.map_or((server.access_key(), server.secret_key()), |user| (user, USER_SECRET_KEY));
|
||||
let request = request.body(Empty::<Bytes>::new()).unwrap();
|
||||
let mut request = rustfs_signer::try_sign_v4(request, 0, key, secret, "", "us-east-1").unwrap();
|
||||
*request.uri_mut() = path.parse().unwrap();
|
||||
|
||||
let before = attributes_count();
|
||||
let stream = tokio::net::TcpStream::connect(server.address()).await.unwrap();
|
||||
let (mut sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(stream)).await.unwrap();
|
||||
let transport = tokio::spawn(connection);
|
||||
let (parts, body) = sender.send_request(request).await.unwrap().into_parts();
|
||||
let body = body.collect().await.unwrap().to_bytes();
|
||||
drop(sender);
|
||||
transport.abort();
|
||||
let _ = transport.await;
|
||||
Observation {
|
||||
status: parts.status.as_u16(),
|
||||
headers: parts.headers,
|
||||
body,
|
||||
count: attributes_count() - before,
|
||||
}
|
||||
}
|
||||
|
||||
async fn attributes_wire(server: RustFSServer, stack: Option<&str>) {
|
||||
for bucket in [BUCKET, RESERVED] {
|
||||
create_bucket_in_the_store(bucket).await;
|
||||
}
|
||||
let context = get_global_app_context().unwrap();
|
||||
let store = context.object_store();
|
||||
store
|
||||
.make_bucket(
|
||||
VERSIONED,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let sdk = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
|
||||
let tags = ["abc", "\"abc\"", "W/\"abc\"", "", "\"\"", "a&b", "a\"b"];
|
||||
for (index, tag) in tags.into_iter().enumerate() {
|
||||
let bucket = if index == 0 { RESERVED } else { BUCKET };
|
||||
let key = format!("tag-{index}");
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
&key,
|
||||
&mut PutObjReader::from_vec(b"payload".to_vec()),
|
||||
&ObjectOptions {
|
||||
preserve_etag: Some(tag.to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let observed = observe(
|
||||
&server,
|
||||
&format!("/{bucket}/{key}?attributes"),
|
||||
&[(ATTRIBUTES, "ETag,ObjectSize,StorageClass")],
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!((observed.status, observed.count), (200, 1), "{key}");
|
||||
assert!(observed.headers.contains_key(http::header::LAST_MODIFIED), "{key} date");
|
||||
let expected = ["abc", "abc", "abc", "", "", "a&b", "a"b"];
|
||||
if stack == LEGACY {
|
||||
let body = std::str::from_utf8(&observed.body).unwrap();
|
||||
assert_eq!(
|
||||
body,
|
||||
format!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><GetObjectAttributesResponse xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>{}</ETag><ObjectSize>7</ObjectSize><StorageClass>STANDARD</StorageClass></GetObjectAttributesResponse>",
|
||||
expected[index]
|
||||
),
|
||||
"{key} raw XML"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
sdk.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key("prefix/child")
|
||||
.body(ByteStream::from_static(b"payload"))
|
||||
.checksum_algorithm(ChecksumAlgorithm::Sha256)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let version = sdk
|
||||
.put_object()
|
||||
.bucket(VERSIONED)
|
||||
.key("versioned")
|
||||
.body(ByteStream::from_static(b"versioned"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.version_id
|
||||
.expect("stored version");
|
||||
let delete = sdk
|
||||
.delete_object()
|
||||
.bucket(VERSIONED)
|
||||
.key("versioned")
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.version_id
|
||||
.expect("delete marker version");
|
||||
|
||||
for (label, bucket, key, version, headers) in [
|
||||
("absent-selection", BUCKET, "tag-1", None, vec![]),
|
||||
("empty-selection", BUCKET, "tag-1", None, vec![(ATTRIBUTES, "")]),
|
||||
("unknown-selection", BUCKET, "tag-1", None, vec![(ATTRIBUTES, "Unknown")]),
|
||||
(
|
||||
"normalized-selection",
|
||||
BUCKET,
|
||||
"tag-1",
|
||||
None,
|
||||
vec![(ATTRIBUTES, " 'eTAG' , \"oBJECTsIZE\" , Unknown ")],
|
||||
),
|
||||
("etag-only", BUCKET, "tag-1", None, vec![(ATTRIBUTES, "ETag")]),
|
||||
(
|
||||
"size-only",
|
||||
BUCKET,
|
||||
"tag-1",
|
||||
None,
|
||||
vec![(ATTRIBUTES, "ObjectSize"), (MARKER, "-1"), (MAX, "0")],
|
||||
),
|
||||
("class-only", BUCKET, "tag-1", None, vec![(ATTRIBUTES, "StorageClass")]),
|
||||
("checksum-only", BUCKET, "prefix/child", None, vec![(ATTRIBUTES, "Checksum")]),
|
||||
(
|
||||
"ordinary-object-parts",
|
||||
BUCKET,
|
||||
"prefix/child",
|
||||
None,
|
||||
vec![(ATTRIBUTES, "ObjectParts"), (MARKER, "-1"), (MAX, "0")],
|
||||
),
|
||||
(
|
||||
"preserved-short-tag-parts",
|
||||
BUCKET,
|
||||
"tag-1",
|
||||
None,
|
||||
vec![(ATTRIBUTES, "ObjectParts"), (MARKER, "-1"), (MAX, "0")],
|
||||
),
|
||||
("missing-bucket", "missing-attributes-bucket", "tag-1", None, vec![(ATTRIBUTES, "ETag")]),
|
||||
(
|
||||
"missing-object",
|
||||
BUCKET,
|
||||
"missing",
|
||||
None,
|
||||
vec![(ATTRIBUTES, "ETag"), (SSE_KEY, "invalid")],
|
||||
),
|
||||
("prefix-with-child", BUCKET, "prefix/", None, vec![(ATTRIBUTES, "ETag")]),
|
||||
("prefix-without-child", BUCKET, "empty-prefix/", None, vec![(ATTRIBUTES, "ETag")]),
|
||||
(
|
||||
"bad-version-before-lookup",
|
||||
BUCKET,
|
||||
"missing",
|
||||
Some("invalid"),
|
||||
vec![(ATTRIBUTES, "ETag")],
|
||||
),
|
||||
(
|
||||
"absent-real-version",
|
||||
VERSIONED,
|
||||
"versioned",
|
||||
Some("11111111-1111-4111-8111-111111111111"),
|
||||
vec![(ATTRIBUTES, "ETag")],
|
||||
),
|
||||
("null-version", BUCKET, "tag-1", Some("null"), vec![(ATTRIBUTES, "ETag")]),
|
||||
("empty-version", BUCKET, "tag-1", Some(""), vec![(ATTRIBUTES, "ETag")]),
|
||||
(
|
||||
"existing-version",
|
||||
VERSIONED,
|
||||
"versioned",
|
||||
Some(version.as_str()),
|
||||
vec![(ATTRIBUTES, "ObjectSize")],
|
||||
),
|
||||
(
|
||||
"current-marker",
|
||||
VERSIONED,
|
||||
"versioned",
|
||||
None,
|
||||
vec![(ATTRIBUTES, "ETag"), (SSE_KEY, "invalid")],
|
||||
),
|
||||
(
|
||||
"explicit-marker",
|
||||
VERSIONED,
|
||||
"versioned",
|
||||
Some(delete.as_str()),
|
||||
vec![(ATTRIBUTES, "ETag"), (SSE_KEY, "invalid")],
|
||||
),
|
||||
] {
|
||||
let mut path = format!("/{bucket}/{key}?attributes");
|
||||
if let Some(version) = version {
|
||||
path.push_str(&format!("&versionId={version}"));
|
||||
}
|
||||
let observed = observe(&server, &path, &headers, None).await;
|
||||
if stack == LEGACY {
|
||||
let (status, count, code) = match label {
|
||||
"absent-selection" => (400, 0, "InvalidRequest"),
|
||||
"bad-version-before-lookup" | "empty-version" => (400, 1, "InvalidArgument"),
|
||||
"preserved-short-tag-parts" => (400, 1, "InvalidArgument"),
|
||||
"absent-real-version"
|
||||
| "missing-bucket"
|
||||
| "missing-object"
|
||||
| "current-marker"
|
||||
| "prefix-with-child"
|
||||
| "prefix-without-child" => (404, 1, "NoSuchKey"),
|
||||
"explicit-marker" => (405, 1, "MethodNotAllowed"),
|
||||
_ => (200, 1, ""),
|
||||
};
|
||||
observed.status(status, count, label);
|
||||
if !code.is_empty() {
|
||||
observed.contains("Code", code, label);
|
||||
}
|
||||
}
|
||||
match label {
|
||||
"normalized-selection" | "etag-only" if stack == LEGACY => observed.contains("ETag", "abc", label),
|
||||
"size-only" => {
|
||||
observed.contains("ObjectSize", "7", label);
|
||||
observed.omits("ObjectParts", label);
|
||||
}
|
||||
"class-only" => observed.contains("StorageClass", "STANDARD", label),
|
||||
"checksum-only" => observed.contains("ChecksumSHA256", "I59Z7VXnN8dxR89VrQwbAwttfudIp0JpUvm4UtWpNeU=", label),
|
||||
"ordinary-object-parts" => {
|
||||
observed.status(200, 1, label);
|
||||
observed.omits("ObjectParts", label);
|
||||
}
|
||||
"preserved-short-tag-parts" => {
|
||||
observed.status(400, 1, label);
|
||||
observed.contains("Message", "part-number-marker must be non-negative", label);
|
||||
}
|
||||
"existing-version" => {
|
||||
observed.contains("ObjectSize", "9", label);
|
||||
assert_eq!(observed.headers["x-amz-version-id"], version.unwrap());
|
||||
}
|
||||
"current-marker" | "explicit-marker" => {
|
||||
for header in ["x-amz-delete-marker", "x-amz-version-id", "last-modified"] {
|
||||
assert!(!observed.headers.contains_key(header), "{label}: {header}");
|
||||
}
|
||||
}
|
||||
"prefix-with-child" => assert!(observed.xml().contains("prefix has children objects"), "{label}: {}", observed.xml()),
|
||||
"prefix-without-child" => assert!(
|
||||
observed.xml().contains("no objects exist under this prefix"),
|
||||
"{label}: {}",
|
||||
observed.xml()
|
||||
),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let checksum_algorithm = ChecksumAlgorithm::Sha256;
|
||||
let upload = sdk
|
||||
.create_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key("multipart")
|
||||
.checksum_algorithm(checksum_algorithm.clone())
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.upload_id
|
||||
.unwrap();
|
||||
let mut parts = Vec::new();
|
||||
for (number, data) in [(1, vec![1; 5 * 1024 * 1024]), (3, b"last".to_vec())] {
|
||||
let part = sdk
|
||||
.upload_part()
|
||||
.bucket(BUCKET)
|
||||
.key("multipart")
|
||||
.upload_id(&upload)
|
||||
.part_number(number)
|
||||
.checksum_algorithm(checksum_algorithm.clone())
|
||||
.body(ByteStream::from(data))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
parts.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(number)
|
||||
.set_e_tag(part.e_tag)
|
||||
.set_checksum_sha256(part.checksum_sha256)
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
sdk.complete_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key("multipart")
|
||||
.upload_id(upload)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(parts)).build())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for (label, marker, max) in [
|
||||
("parts-default", None, None),
|
||||
("parts-first", None, Some("1")),
|
||||
("parts-terminal", Some("1"), Some("1")),
|
||||
("parts-empty", Some("3"), Some("1")),
|
||||
("parts-missing-marker", Some("2"), Some("1")),
|
||||
("parts-zero-marker", Some("0"), Some("1000")),
|
||||
("parts-bad-marker-first", Some("-1"), Some("0")),
|
||||
("parts-zero-max", None, Some("0")),
|
||||
("parts-negative-max", None, Some("-1")),
|
||||
("parts-over-max", None, Some("1001")),
|
||||
("parts-marker-overflow", Some("2147483648"), Some("1")),
|
||||
("parts-marker-text", Some("bad"), Some("1")),
|
||||
] {
|
||||
let mut headers = vec![(ATTRIBUTES, "ObjectParts,Checksum,ObjectSize")];
|
||||
headers.extend(marker.map(|value| (MARKER, value)));
|
||||
headers.extend(max.map(|value| (MAX, value)));
|
||||
let observed = observe(&server, &format!("/{BUCKET}/multipart?attributes"), &headers, None).await;
|
||||
let (parts, echoed_marker, next, truncated): (&[&str], Option<&str>, Option<&str>, &str) = match label {
|
||||
"parts-default" | "parts-zero-marker" => (&["1", "3"], None, Some("3"), "false"),
|
||||
"parts-first" | "parts-missing-marker" => (&["1"], None, Some("1"), "true"),
|
||||
"parts-terminal" => (&["3"], Some("1"), Some("3"), "false"),
|
||||
"parts-empty" => (&[], Some("3"), None, "false"),
|
||||
_ => {
|
||||
assert_eq!(observed.status, 400, "{label}: {}", observed.xml());
|
||||
observed.contains("Code", "InvalidArgument", label);
|
||||
match label {
|
||||
"parts-bad-marker-first" => observed.contains("Message", "part-number-marker must be non-negative", label),
|
||||
"parts-over-max" | "parts-negative-max" | "parts-zero-max" => {
|
||||
observed.contains("Message", "max-parts must be between 1 and 1000", label);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
observed.status(200, 1, label);
|
||||
observed.contains("MaxParts", max.unwrap_or("1000"), label);
|
||||
observed.contains("PartsCount", "2", label);
|
||||
observed.contains("IsTruncated", truncated, label);
|
||||
observed.contains("ObjectSize", "5242884", label);
|
||||
assert_eq!(observed.xml().matches("<Part>").count(), parts.len(), "{label}: {}", observed.xml());
|
||||
for number in parts {
|
||||
observed.contains("PartNumber", number, label);
|
||||
}
|
||||
for (field, value) in [("PartNumberMarker", echoed_marker), ("NextPartNumberMarker", next)] {
|
||||
match value {
|
||||
Some(value) => observed.contains(field, value, label),
|
||||
None => observed.omits(field, label),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Persist the per-part checksum so the wire response proves the read requested it.
|
||||
let checksum_key = "stored-part-checksum";
|
||||
let upload = store
|
||||
.new_multipart_upload(BUCKET, checksum_key, &Default::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let part = store
|
||||
.put_object_part(
|
||||
BUCKET,
|
||||
checksum_key,
|
||||
&upload.upload_id,
|
||||
1,
|
||||
&mut PutObjReader::from_vec(b"123456789".to_vec()),
|
||||
&Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut user_defined = std::collections::HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
|
||||
r#"[[1,[["CRC32C","4waSgw=="]]]]"#.to_owned(),
|
||||
);
|
||||
let mut options = ObjectOptions {
|
||||
user_defined,
|
||||
..Default::default()
|
||||
};
|
||||
options.mod_time = Some(time::OffsetDateTime::from_unix_timestamp_nanos(1_609_459_200_123_456_789).unwrap());
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
BUCKET,
|
||||
checksum_key,
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&options,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let stored = store
|
||||
.get_object_info(
|
||||
BUCKET,
|
||||
checksum_key,
|
||||
&ObjectOptions {
|
||||
include_part_checksums: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
stored.parts[0]
|
||||
.checksums
|
||||
.as_ref()
|
||||
.and_then(|checksums| checksums.get("CRC32C"))
|
||||
.map(String::as_str),
|
||||
Some("4waSgw==")
|
||||
);
|
||||
let observed = observe(
|
||||
&server,
|
||||
&format!("/{BUCKET}/{checksum_key}?attributes"),
|
||||
&[(ATTRIBUTES, "ObjectParts")],
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert_eq!((observed.status, observed.count), (200, 1), "stored part checksum");
|
||||
observed.contains("ChecksumCRC32C", "4waSgw==", "stored part checksum");
|
||||
assert_eq!(observed.headers[http::header::LAST_MODIFIED], "Fri, 01 Jan 2021 00:00:00 GMT");
|
||||
|
||||
let customer_key = base64_simd::STANDARD.encode_to_string([0_u8; 32]);
|
||||
let wrong_key = base64_simd::STANDARD.encode_to_string([1_u8; 32]);
|
||||
let customer_md5 = "cLyPS3KoaSFGi/joRB3OUQ==";
|
||||
let wrong_md5 = "4Funlf7OsLF0HL+vKU+fkg==";
|
||||
sdk.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key("ssec")
|
||||
.body(ByteStream::from_static(b"encrypted"))
|
||||
.checksum_algorithm(ChecksumAlgorithm::Sha256)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(customer_md5)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
for (label, key, md5) in [
|
||||
("ssec-valid", Some(customer_key.as_str()), Some(customer_md5)),
|
||||
("ssec-missing", None, None),
|
||||
("ssec-wrong", Some(wrong_key.as_str()), Some(wrong_md5)),
|
||||
("ssec-wrong-md5", Some(wrong_key.as_str()), Some("4FtyTg0JFp3p7ItXND3wOg==")),
|
||||
("ssec-malformed-key", Some("bad"), Some(customer_md5)),
|
||||
("ssec-malformed-md5", Some(customer_key.as_str()), Some("bad")),
|
||||
] {
|
||||
let mut headers = vec![(ATTRIBUTES, "Checksum,ObjectParts,ObjectSize"), (MAX, "0")];
|
||||
if let Some(key) = key {
|
||||
headers.extend([(SSE_ALGORITHM, "AES256"), (SSE_KEY, key)]);
|
||||
}
|
||||
headers.extend(md5.map(|md5| (SSE_MD5, md5)));
|
||||
let observed = observe(&server, &format!("/{BUCKET}/ssec?attributes"), &headers, None).await;
|
||||
if label == "ssec-valid" {
|
||||
observed.status(200, 1, label);
|
||||
observed.contains("ChecksumSHA256", "lU0buD2Au29udGso8N4+xMTtmAz+Z+0jqRWc1GT/M5o=", label);
|
||||
} else {
|
||||
assert_eq!(observed.status, 400, "{label}: {}", observed.xml());
|
||||
if stack == LEGACY {
|
||||
observed.contains("Code", "InvalidRequest", label);
|
||||
if label == "ssec-wrong" {
|
||||
observed.contains(
|
||||
"Message",
|
||||
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
|
||||
label,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let kms = context.kms().handle();
|
||||
kms.reconfigure(rustfs_kms::config::KmsConfig::static_kms(
|
||||
"attributes-test-key".to_owned(),
|
||||
base64_simd::STANDARD.encode_to_string([1_u8; 32]),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
sdk.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key("managed")
|
||||
.body(ByteStream::from_static(b"managed encryption"))
|
||||
.checksum_algorithm(ChecksumAlgorithm::Sha256)
|
||||
.server_side_encryption(ServerSideEncryption::AwsKms)
|
||||
.ssekms_key_id("attributes-test-key")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let observed = observe(
|
||||
&server,
|
||||
&format!("/{BUCKET}/managed?attributes"),
|
||||
&[(ATTRIBUTES, "Checksum,ObjectSize")],
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
observed.status(200, 1, "managed-kms");
|
||||
observed.contains("ChecksumSHA256", "UT6FnUrN4M3d+mdIobVx+iO4EA6lvEyWBtoT7rzzVbg=", "managed-kms");
|
||||
observed.contains("ObjectSize", "18", "managed-kms");
|
||||
kms.stop().await.unwrap();
|
||||
|
||||
for (index, denied_action, version, bucket, object) in [
|
||||
(0, None, None, BUCKET, "tag-1"),
|
||||
(1, Some("s3:GetObjectAttributes"), None, BUCKET, "tag-1"),
|
||||
(2, Some("s3:GetObject"), None, BUCKET, "tag-1"),
|
||||
(3, None, Some(version.as_str()), VERSIONED, "versioned"),
|
||||
(4, Some("s3:GetObjectVersionAttributes"), Some(version.as_str()), VERSIONED, "versioned"),
|
||||
(5, Some("s3:GetObjectVersion"), Some(version.as_str()), VERSIONED, "versioned"),
|
||||
(6, Some("s3:GetObjectVersionAttributes"), Some(""), BUCKET, "tag-1"),
|
||||
(7, Some("s3:GetObjectVersion"), Some("null"), BUCKET, "tag-1"),
|
||||
(8, None, None, RESERVED, "tag-0"),
|
||||
(9, None, None, BUCKET, "tag-2"),
|
||||
(10, Some("s3:GetObjectVersion"), Some(version.as_str()), VERSIONED, "versioned"),
|
||||
] {
|
||||
let user = format!("attributes-user-{index}");
|
||||
create_user_without_a_policy(&user).await;
|
||||
let mut statements = vec![serde_json::json!({
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:GetObjectAttributes", "s3:GetObjectVersion", "s3:GetObjectVersionAttributes"],
|
||||
"Resource": [format!("arn:aws:s3:::{BUCKET}/tag-1"), format!("arn:aws:s3:::{VERSIONED}/versioned")],
|
||||
})];
|
||||
if let Some(action) = denied_action {
|
||||
let mut denial = serde_json::json!({
|
||||
"Effect": "Deny",
|
||||
"Action": [action],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/{object}")],
|
||||
});
|
||||
if index == 10 {
|
||||
denial["Condition"] = serde_json::json!({"StringEquals": {"s3:versionid": version}});
|
||||
}
|
||||
statements.push(denial);
|
||||
}
|
||||
let bytes = serde_json::to_vec(&serde_json::json!({"Version": "2012-10-17", "Statement": statements})).unwrap();
|
||||
let iam = context.iam().handle();
|
||||
let policy = rustfs_policy::policy::Policy::parse_config(&bytes).unwrap();
|
||||
iam.set_policy(&user, policy).await.unwrap();
|
||||
iam.policy_db_set(&user, rustfs_iam::store::UserType::Reg, false, &user)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut path = format!("/{bucket}/{object}?attributes");
|
||||
if let Some(version) = version {
|
||||
path.push_str(&format!("&versionId={version}"));
|
||||
}
|
||||
let observed = observe(&server, &path, &[(ATTRIBUTES, "ObjectSize")], Some(&user)).await;
|
||||
let expected = if matches!(index, 0 | 3) { (200, 1) } else { (403, 0) };
|
||||
assert_eq!((observed.status, observed.count), expected, "IAM row {index}");
|
||||
}
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_get_object_attributes_wire() {
|
||||
with_environment(
|
||||
vec![(super::ENV_S3_STACK, LEGACY), ("RUSTFS_SSE_C_REQUIRE_TLS", Some("false"))],
|
||||
|server| attributes_wire(server, LEGACY),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_get_object_attributes_wire() {
|
||||
with_environment(
|
||||
vec![(super::ENV_S3_STACK, GATEWAY), ("RUSTFS_SSE_C_REQUIRE_TLS", Some("false"))],
|
||||
|server| attributes_wire(server, GATEWAY),
|
||||
);
|
||||
}
|
||||
@@ -27,6 +27,12 @@ pub(crate) const MATRIX: &[super::MatrixRow] = &[
|
||||
has_error_body: true,
|
||||
call: get_object,
|
||||
},
|
||||
super::MatrixRow {
|
||||
operation: "GetObjectAttributes",
|
||||
metric: rustfs_s3_ops::S3Operation::GetObjectAttributes,
|
||||
has_error_body: true,
|
||||
call: get_object_attributes,
|
||||
},
|
||||
];
|
||||
|
||||
fn get_object<'a>(client: &'a aws_sdk_s3::Client, _bucket: &'a str) -> futures::future::BoxFuture<'a, super::Outcome> {
|
||||
@@ -36,6 +42,21 @@ fn get_object<'a>(client: &'a aws_sdk_s3::Client, _bucket: &'a str) -> futures::
|
||||
})
|
||||
}
|
||||
|
||||
fn get_object_attributes<'a>(client: &'a aws_sdk_s3::Client, _bucket: &'a str) -> futures::future::BoxFuture<'a, super::Outcome> {
|
||||
Box::pin(async move {
|
||||
super::put_object_in_the_store(MATRIX_BUCKET, "attributes-matrix").await;
|
||||
super::outcome(
|
||||
client
|
||||
.get_object_attributes()
|
||||
.bucket(MATRIX_BUCKET)
|
||||
.key("attributes-matrix")
|
||||
.object_attributes(aws_sdk_s3::types::ObjectAttributes::ObjectSize)
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) const MATRIX_BUCKET: &str = "object-read-matrix";
|
||||
|
||||
fn head_object<'a>(client: &'a aws_sdk_s3::Client, _bucket: &'a str) -> futures::future::BoxFuture<'a, super::Outcome> {
|
||||
|
||||
Reference in New Issue
Block a user