mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 04:52:36 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f4266a4576 |
+275
-111
@@ -12,6 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod owned;
|
||||
|
||||
use crate::runtime_sources::current_action_credentials;
|
||||
#[cfg(feature = "webdav")]
|
||||
use crate::runtime_sources::current_object_store_handle;
|
||||
@@ -494,38 +496,13 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
"Protocol storage client request"
|
||||
);
|
||||
|
||||
let mut builder = GetObjectInput::builder().bucket(bucket.to_string()).key(key.to_string());
|
||||
|
||||
// Add range header if start_pos is specified
|
||||
if let Some(start) = start_pos {
|
||||
let range = s3s::dto::Range::Int {
|
||||
first: start,
|
||||
last: None, // Read to end of file
|
||||
};
|
||||
builder = builder.range(Some(range));
|
||||
}
|
||||
|
||||
let input = builder.build().map_err(|e| {
|
||||
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build GetObjectInput: {}", e))
|
||||
})?;
|
||||
|
||||
let uri = build_object_uri(bucket, key, &[])?;
|
||||
let req = Self::create_request(
|
||||
input,
|
||||
Method::GET,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_string()),
|
||||
object: Some(key.to_string()),
|
||||
credentials,
|
||||
},
|
||||
)?;
|
||||
|
||||
match self.fs.get_object(req).await {
|
||||
Ok(response) => gateway_s3s_compat::generated::ops::get_object::output_from_s3s(response.output)
|
||||
.map_err(|e| conversion_error("GetObject output", e)),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
self.get_object_owned(
|
||||
bucket,
|
||||
key,
|
||||
credentials,
|
||||
start_pos.map(|first| rustfs_gateway_types::ByteRange::From { first }),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
@@ -588,31 +565,20 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
) -> Result<gateway_dto::DeleteObjectOutput, Self::Error> {
|
||||
trace_protocol_request("delete_object", Some(bucket), Some(key));
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.key(key.to_string())
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build DeleteObjectInput: {}", e))
|
||||
})?;
|
||||
|
||||
let uri = build_object_uri(bucket, key, &[])?;
|
||||
let req = Self::create_request(
|
||||
input,
|
||||
let envelope = Self::create_envelope(
|
||||
Method::DELETE,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_string()),
|
||||
object: Some(key.to_string()),
|
||||
bucket: Some(bucket.to_owned()),
|
||||
object: Some(key.to_owned()),
|
||||
credentials,
|
||||
},
|
||||
)?;
|
||||
|
||||
match self.fs.delete_object(req).await {
|
||||
Ok(response) => gateway_s3s_compat::generated::ops::delete_object::output_from_s3s(response.output)
|
||||
.map_err(|e| conversion_error("DeleteObject output", e)),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
);
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::DeleteObject);
|
||||
self.owned_usecase()
|
||||
.execute_delete_object_fields(envelope, bucket.to_owned(), key.to_owned(), None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn head_object(
|
||||
@@ -623,31 +589,32 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
) -> Result<gateway_dto::HeadObjectOutput, Self::Error> {
|
||||
trace_protocol_request("head_object", Some(bucket), Some(key));
|
||||
|
||||
let input = HeadObjectInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.key(key.to_string())
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build HeadObjectInput: {}", e))
|
||||
})?;
|
||||
|
||||
let uri = build_object_uri(bucket, key, &[])?;
|
||||
let req = Self::create_request(
|
||||
input,
|
||||
let envelope = Self::create_envelope(
|
||||
Method::HEAD,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_string()),
|
||||
object: Some(key.to_string()),
|
||||
bucket: Some(bucket.to_owned()),
|
||||
object: Some(key.to_owned()),
|
||||
credentials,
|
||||
},
|
||||
)?;
|
||||
|
||||
match self.fs.head_object(req).await {
|
||||
Ok(response) => gateway_s3s_compat::generated::ops::head_object::output_from_s3s(response.output)
|
||||
.map_err(|e| conversion_error("HeadObject output", e)),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
);
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::HeadObject);
|
||||
self.owned_usecase()
|
||||
.execute_head_object_fields(
|
||||
envelope,
|
||||
crate::app::object_usecase::HeadObjectFields {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
version_id: None,
|
||||
part_number: None,
|
||||
range: None,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.and_then(|answered| owned::head_output(answered.output))
|
||||
}
|
||||
|
||||
async fn head_bucket(
|
||||
@@ -891,37 +858,16 @@ impl rustfs_protocols::common::client::s3::StorageBackend for ProtocolStorageCli
|
||||
"Protocol storage client request"
|
||||
);
|
||||
|
||||
let range = s3s::dto::Range::Int {
|
||||
first: start_pos,
|
||||
last: Some(start_pos + length - 1),
|
||||
};
|
||||
|
||||
let input = GetObjectInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.key(key.to_string())
|
||||
.range(Some(range))
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
s3s::S3Error::with_message(s3s::S3ErrorCode::InvalidRequest, format!("Failed to build GetObjectInput: {}", e))
|
||||
})?;
|
||||
|
||||
let uri = build_object_uri(bucket, key, &[])?;
|
||||
let req = Self::create_request(
|
||||
input,
|
||||
Method::GET,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_string()),
|
||||
object: Some(key.to_string()),
|
||||
credentials,
|
||||
},
|
||||
)?;
|
||||
|
||||
match self.fs.get_object(req).await {
|
||||
Ok(response) => gateway_s3s_compat::generated::ops::get_object::output_from_s3s(response.output)
|
||||
.map_err(|e| conversion_error("GetObject output", e)),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
self.get_object_owned(
|
||||
bucket,
|
||||
key,
|
||||
credentials,
|
||||
Some(rustfs_gateway_types::ByteRange::FromTo {
|
||||
first: start_pos,
|
||||
last: start_pos + length - 1,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn copy_object(
|
||||
@@ -1403,16 +1349,30 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.expect("request should build");
|
||||
let request_info = request.extensions.get::<ReqInfo>().expect("request info should be present");
|
||||
let copied = request_info.cred.as_ref().expect("credentials should be present");
|
||||
|
||||
assert_eq!(copied.access_key, credentials.access_key);
|
||||
assert_eq!(copied.secret_key, credentials.secret_key);
|
||||
assert_eq!(copied.session_token, credentials.session_token);
|
||||
assert_eq!(copied.parent_user, credentials.parent_user);
|
||||
assert_eq!(copied.groups, credentials.groups);
|
||||
assert_eq!(copied.claims, Some(claims));
|
||||
assert!(copied.is_service_account());
|
||||
let envelope = ProtocolStorageClient::create_envelope(
|
||||
Method::GET,
|
||||
http::Uri::from_static("/bucket?list-type=2"),
|
||||
RequestParams {
|
||||
bucket: Some("bucket".to_owned()),
|
||||
object: None,
|
||||
credentials: &credentials,
|
||||
},
|
||||
);
|
||||
assert_eq!(envelope.uri(), &request.uri);
|
||||
assert_eq!(envelope.headers(), &request.headers);
|
||||
for request_info in [
|
||||
request.extensions.get::<ReqInfo>().expect("request info should be present"),
|
||||
envelope.req_info().expect("owned request info"),
|
||||
] {
|
||||
let copied = request_info.cred.as_ref().expect("credentials should be present");
|
||||
assert_eq!(copied.access_key, credentials.access_key);
|
||||
assert_eq!(copied.secret_key, credentials.secret_key);
|
||||
assert_eq!(copied.session_token, credentials.session_token);
|
||||
assert_eq!(copied.parent_user, credentials.parent_user);
|
||||
assert_eq!(copied.groups, credentials.groups);
|
||||
assert_eq!(copied.claims, Some(claims.clone()));
|
||||
assert!(copied.is_service_account());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "webdav", feature = "sftp"))]
|
||||
@@ -1523,4 +1483,208 @@ mod tests {
|
||||
|
||||
assert_eq!(uri.to_string(), "/bucket?list-type=2");
|
||||
}
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_markers_versions_and_errors_match_legacy() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-oracle", || async {
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::protocols::client::{
|
||||
BucketOperations, MakeBucketOptions, ObjectIO, StorageObjectOptions, StoragePutObjReader,
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_protocols::common::client::s3::StorageBackend;
|
||||
let root = rustfs_credentials::Credentials {
|
||||
access_key: "protocol-oracle-root".into(),
|
||||
secret_key: "protocol-oracle-secret".into(),
|
||||
status: "on".into(),
|
||||
..Default::default()
|
||||
};
|
||||
let (_temp, context, _iam) = crate::protocols::policy_test::protocol_test_context(root.clone()).await;
|
||||
let store = context.object_store();
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let fs = FS::with_server_ctx(slot);
|
||||
let backend = ProtocolStorageClient::new(fs.clone());
|
||||
for (bucket, key, data, versioned) in [
|
||||
("protocol-version-oracle", "versioned-key", b"versioned data".as_slice(), true),
|
||||
("protocol-marker-oracle", "folder/", b"".as_slice(), false),
|
||||
] {
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: versioned,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let mut reader = StoragePutObjReader::from_vec(data.to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
key,
|
||||
&mut reader,
|
||||
&StorageObjectOptions {
|
||||
versioned,
|
||||
mod_time: Some(
|
||||
time::OffsetDateTime::from_unix_timestamp(1_700_000_000)
|
||||
.expect("valid test time")
|
||||
.replace_nanosecond(123_456_789)
|
||||
.expect("valid nanoseconds"),
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed object");
|
||||
let params = || RequestParams {
|
||||
bucket: Some(bucket.to_owned()),
|
||||
object: Some(key.to_owned()),
|
||||
credentials: &root,
|
||||
};
|
||||
let uri = build_object_uri(bucket, key, &[]).expect("object URI");
|
||||
let old = fs
|
||||
.get_object(
|
||||
ProtocolStorageClient::create_request(
|
||||
GetObjectInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.key(key.to_owned())
|
||||
.build()
|
||||
.expect("old GET input"),
|
||||
Method::GET,
|
||||
uri.clone(),
|
||||
params(),
|
||||
)
|
||||
.expect("old GET request"),
|
||||
)
|
||||
.await
|
||||
.expect("old GET");
|
||||
let old = gateway_s3s_compat::generated::ops::get_object::output_from_s3s(old.output).expect("old GET output");
|
||||
if versioned {
|
||||
assert_eq!(old.last_modified.expect("old LastModified").subsec_nanos(), 123_000_000);
|
||||
}
|
||||
let current = backend.get_object(bucket, key, &root, None).await.expect("protocol GET");
|
||||
assert_eq!(current.last_modified, old.last_modified);
|
||||
assert_eq!(current.version_id, old.version_id);
|
||||
assert_eq!(current.version_id.is_some(), versioned);
|
||||
assert_eq!(current.e_tag, old.e_tag);
|
||||
assert_eq!(current.content_length, old.content_length);
|
||||
let collect = |output: gateway_dto::GetObjectOutput| async move {
|
||||
output
|
||||
.body
|
||||
.expect("GET body")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("read body")
|
||||
.to_bytes()
|
||||
};
|
||||
assert_eq!(collect(current).await, collect(old).await);
|
||||
let old = fs
|
||||
.head_object(
|
||||
ProtocolStorageClient::create_request(
|
||||
HeadObjectInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.key(key.to_owned())
|
||||
.build()
|
||||
.expect("old HEAD input"),
|
||||
Method::HEAD,
|
||||
uri,
|
||||
params(),
|
||||
)
|
||||
.expect("old HEAD request"),
|
||||
)
|
||||
.await
|
||||
.expect("old HEAD");
|
||||
let old = gateway_s3s_compat::generated::ops::head_object::output_from_s3s(old.output).expect("old HEAD output");
|
||||
if versioned {
|
||||
assert_eq!(old.last_modified.expect("old LastModified").subsec_nanos(), 123_000_000);
|
||||
}
|
||||
let current = backend.head_object(bucket, key, &root).await.expect("protocol HEAD");
|
||||
assert_eq!(current.last_modified, old.last_modified);
|
||||
assert_eq!(current.version_id, old.version_id);
|
||||
assert_eq!(current.e_tag, old.e_tag);
|
||||
assert_eq!(current.content_length, old.content_length);
|
||||
let deleted = backend.delete_object(bucket, key, &root).await.expect("protocol DELETE");
|
||||
if versioned {
|
||||
assert_eq!(deleted.delete_marker, Some(true));
|
||||
}
|
||||
assert_eq!(deleted.version_id.is_some(), versioned);
|
||||
let old_error: rustfs_s3_types::S3Error = fs
|
||||
.get_object(
|
||||
ProtocolStorageClient::create_request(
|
||||
GetObjectInput::builder()
|
||||
.bucket(bucket.to_owned())
|
||||
.key(key.to_owned())
|
||||
.build()
|
||||
.expect("old GET input"),
|
||||
Method::GET,
|
||||
build_object_uri(bucket, key, &[]).expect("object URI"),
|
||||
params(),
|
||||
)
|
||||
.expect("old GET request"),
|
||||
)
|
||||
.await
|
||||
.expect_err("old GET deleted object")
|
||||
.into();
|
||||
let current = backend
|
||||
.get_object(bucket, key, &root, None)
|
||||
.await
|
||||
.expect_err("protocol GET deleted object");
|
||||
assert_eq!(current.code(), old_error.code());
|
||||
assert_eq!(current.status_code(), old_error.status_code());
|
||||
assert_eq!(current.headers(), old_error.headers());
|
||||
if versioned {
|
||||
let headers = current.headers().expect("delete marker error headers");
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("x-amz-version-id")
|
||||
.expect("marker version")
|
||||
.to_str()
|
||||
.expect("version text"),
|
||||
deleted.version_id.as_deref().expect("delete version")
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
#[test]
|
||||
fn protocol_mainflow_retention_timestamp_matches_legacy_precision() {
|
||||
use crate::app::object_usecase::{GetObjectOutputFacts, HeadObjectOutputFacts};
|
||||
let instant = rustfs_gateway_types::Timestamp::from_secs_nanos(1_700_000_000, 123_456_789).expect("test timestamp");
|
||||
let get = || GetObjectOutputFacts {
|
||||
dto: gateway_dto::GetObjectOutput {
|
||||
object_lock_retain_until_date: Some(instant),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let head = || HeadObjectOutputFacts {
|
||||
dto: gateway_dto::HeadObjectOutput {
|
||||
object_lock_retain_until_date: Some(instant),
|
||||
..Default::default()
|
||||
},
|
||||
etag: None,
|
||||
};
|
||||
let old_get =
|
||||
gateway_s3s_compat::generated::ops::get_object::output_from_s3s(get().into_s3s().expect("old GET projection"))
|
||||
.expect("old GET output");
|
||||
let old_head =
|
||||
gateway_s3s_compat::generated::ops::head_object::output_from_s3s(head().into_s3s().expect("old HEAD projection"))
|
||||
.expect("old HEAD output");
|
||||
let old = (old_get.object_lock_retain_until_date, old_head.object_lock_retain_until_date);
|
||||
let expected =
|
||||
Some(rustfs_gateway_types::Timestamp::from_secs_nanos(1_700_000_000, 123_000_000).expect("millisecond timestamp"));
|
||||
assert_eq!(old, (expected, expected));
|
||||
let current = (
|
||||
owned::get_output(get())
|
||||
.expect("protocol GET output")
|
||||
.object_lock_retain_until_date,
|
||||
owned::head_output(head())
|
||||
.expect("protocol HEAD output")
|
||||
.object_lock_retain_until_date,
|
||||
);
|
||||
assert_eq!(current, old);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Request and response projections for protocol reads and single-object deletion.
|
||||
//! Authorization stays on the protocol session edge; storage runs in the client's server context.
|
||||
|
||||
use super::{ProtocolStorageClient, RequestParams};
|
||||
use crate::app::object_usecase::{DefaultObjectUsecase, GetObjectBody, GetObjectOutputFacts, HeadObjectOutputFacts};
|
||||
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
|
||||
use crate::runtime_sources::current_action_credentials;
|
||||
use crate::storage_api::protocols::client::{ReqInfo, RequestContext};
|
||||
use http::{HeaderMap, Method, Uri};
|
||||
use rustfs_gateway_types::{ETag, Timestamp, dto};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
|
||||
impl ProtocolStorageClient {
|
||||
pub(super) async fn get_object_owned(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
credentials: &rustfs_credentials::Credentials,
|
||||
range: Option<rustfs_gateway_types::ByteRange>,
|
||||
) -> S3Result<dto::GetObjectOutput> {
|
||||
let uri = super::build_object_uri(bucket, key, &[])?;
|
||||
let envelope = Self::create_envelope(
|
||||
Method::GET,
|
||||
uri,
|
||||
RequestParams {
|
||||
bucket: Some(bucket.to_owned()),
|
||||
object: Some(key.to_owned()),
|
||||
credentials,
|
||||
},
|
||||
);
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
|
||||
self.owned_usecase()
|
||||
.execute_get_object_fields(
|
||||
envelope,
|
||||
crate::app::object_usecase::GetObjectFields {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
version_id: None,
|
||||
part_number: None,
|
||||
range,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.and_then(|answered| get_output(answered.output))
|
||||
}
|
||||
|
||||
pub(super) fn owned_usecase(&self) -> DefaultObjectUsecase {
|
||||
DefaultObjectUsecase::with_context(self.fs.server_ctx().app_context())
|
||||
}
|
||||
|
||||
pub(super) fn create_envelope(method: Method, uri: Uri, params: RequestParams<'_>) -> RequestEnvelope {
|
||||
let is_owner = current_action_credentials().is_some_and(|root| params.credentials.access_key == root.access_key);
|
||||
let credentials =
|
||||
RequestCredentials::new(params.credentials.access_key.clone(), params.credentials.secret_key.clone(), None)
|
||||
.unwrap_or_else(|_| RequestCredentials::without_secret_key(params.credentials.access_key.clone(), None));
|
||||
let mut extensions = http::Extensions::new();
|
||||
extensions.insert(ReqInfo {
|
||||
cred: Some(params.credentials.clone()),
|
||||
is_owner,
|
||||
bucket: params.bucket,
|
||||
object: params.object,
|
||||
version_id: None,
|
||||
replication_request_authorized: false,
|
||||
region: None,
|
||||
request_context: Some(RequestContext::fallback()),
|
||||
suppress_denial_log: false,
|
||||
suppress_denial_audit: false,
|
||||
});
|
||||
RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method,
|
||||
uri,
|
||||
headers: HeaderMap::new(),
|
||||
credentials: Some(credentials),
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
extensions,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn etag(opaque: String, weak: bool) -> S3Result<ETag> {
|
||||
if weak { ETag::new_weak(opaque) } else { ETag::new(opaque) }
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Object response contains an invalid ETag"))
|
||||
}
|
||||
|
||||
// Protocol clients received millisecond timestamps through the legacy response codec.
|
||||
fn millisecond_timestamp(value: Timestamp) -> Timestamp {
|
||||
Timestamp::from_secs_nanos(value.secs(), value.subsec_millis() * 1_000_000)
|
||||
.expect("a millisecond remainder is below one second") // subsec_millis is at most 999.
|
||||
}
|
||||
|
||||
pub(super) fn get_output(mut output: GetObjectOutputFacts) -> S3Result<dto::GetObjectOutput> {
|
||||
output.dto.last_modified = output.dto.last_modified.map(millisecond_timestamp);
|
||||
output.dto.object_lock_retain_until_date = output.dto.object_lock_retain_until_date.map(millisecond_timestamp);
|
||||
output.dto.e_tag = output.etag.map(|tag| etag(tag.opaque, tag.weak)).transpose()?;
|
||||
output.dto.body = output
|
||||
.body
|
||||
.map(|body| match body {
|
||||
GetObjectBody::Bytes(bytes) => Ok(rustfs_gateway_stream::ByteStream::from_bytes(bytes)),
|
||||
GetObjectBody::Stream(stream) => rustfs_gateway_stream::ByteStream::new(stream)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Invalid GetObject body capabilities")),
|
||||
})
|
||||
.transpose()?;
|
||||
Ok(output.dto)
|
||||
}
|
||||
|
||||
pub(super) fn head_output(mut output: HeadObjectOutputFacts) -> S3Result<dto::HeadObjectOutput> {
|
||||
output.dto.last_modified = output.dto.last_modified.map(millisecond_timestamp);
|
||||
output.dto.object_lock_retain_until_date = output.dto.object_lock_retain_until_date.map(millisecond_timestamp);
|
||||
output.dto.e_tag = output.etag.map(|tag| etag(tag.opaque, tag.weak)).transpose()?;
|
||||
Ok(output.dto)
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Protocol mainflow behavior through the installed server's object use case.
|
||||
//! These probes exercise the protocol client, not the HTTP transport or authorization hook.
|
||||
|
||||
use super::ProtocolStorageClient;
|
||||
use super::policy_test::protocol_test_context;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::protocols::client::{
|
||||
BucketOperations, MakeBucketOptions, ObjectIO, StorageObjectOptions, StoragePutObjReader,
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_protocols::common::client::s3::StorageBackend;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
const KEY: &str = "directory/a ?#+%.txt";
|
||||
|
||||
async fn object_client(bucket: &str) -> (tempfile::TempDir, Credentials, ProtocolStorageClient, String) {
|
||||
let root = Credentials {
|
||||
access_key: "protocol-mainflow-root".into(),
|
||||
secret_key: "protocol-mainflow-secret".into(),
|
||||
status: "on".into(),
|
||||
..Default::default()
|
||||
};
|
||||
let (temp, context, _iam) = protocol_test_context(root.clone()).await;
|
||||
let store = context.object_store();
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let mut reader = StoragePutObjReader::from_vec(b"protocol-data".to_vec());
|
||||
let stored = store
|
||||
.put_object(bucket, KEY, &mut reader, &StorageObjectOptions::default())
|
||||
.await
|
||||
.expect("seed object");
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
(
|
||||
temp,
|
||||
root,
|
||||
crate::init::protocol_storage_client_for_server(slot),
|
||||
stored.etag.expect("stored ETag"),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_reads_ranges_and_data() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-read", || async {
|
||||
let bucket = "protocol-mainflow-read";
|
||||
let (_temp, root, backend, stored_etag) = object_client(bucket).await;
|
||||
for (start, expected) in [(None, b"protocol-data".as_slice()), (Some(9), b"data".as_slice())] {
|
||||
let output = backend.get_object(bucket, KEY, &root, start).await.expect("protocol GET");
|
||||
assert_eq!(output.content_length, Some(expected.len() as i64));
|
||||
assert_eq!(output.e_tag.as_ref().expect("GET ETag").opaque_tag(), stored_etag);
|
||||
let body = output
|
||||
.body
|
||||
.expect("GET body")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("consume GET")
|
||||
.to_bytes();
|
||||
assert_eq!(body.as_ref(), expected);
|
||||
assert_eq!(output.version_id, None);
|
||||
}
|
||||
let ranged = backend
|
||||
.get_object_range(bucket, KEY, &root, 2, 4)
|
||||
.await
|
||||
.expect("bounded range");
|
||||
assert_eq!(ranged.content_range.as_deref(), Some("bytes 2-5/13"));
|
||||
assert_eq!(
|
||||
ranged
|
||||
.body
|
||||
.expect("range body")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("consume range")
|
||||
.to_bytes()
|
||||
.as_ref(),
|
||||
b"otoc"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_heads_object_metadata() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-head", || async {
|
||||
let bucket = "protocol-mainflow-head";
|
||||
let (_temp, root, backend, stored_etag) = object_client(bucket).await;
|
||||
let head = backend.head_object(bucket, KEY, &root).await.expect("protocol HEAD");
|
||||
assert_eq!(head.content_length, Some(13));
|
||||
assert_eq!(head.e_tag.as_ref().expect("HEAD ETag").opaque_tag(), stored_etag);
|
||||
assert_eq!(head.version_id, None);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_deletes_object() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-delete", || async {
|
||||
let bucket = "protocol-mainflow-delete";
|
||||
let (_temp, root, backend, _stored_etag) = object_client(bucket).await;
|
||||
let deleted = backend.delete_object(bucket, KEY, &root).await.expect("protocol DELETE");
|
||||
assert_eq!(deleted.version_id, None);
|
||||
assert_eq!(deleted.delete_marker, Some(false));
|
||||
let missing = backend
|
||||
.get_object(bucket, KEY, &root, None)
|
||||
.await
|
||||
.expect_err("deleted object is absent");
|
||||
assert_eq!(missing.code(), &S3ErrorCode::NoSuchKey);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn protocol_mainflow_refuses_missing_objects_buckets_and_unsatisfied_ranges() {
|
||||
crate::app::gating_test_env::run_large_stack_test("protocol-mainflow-errors", || async {
|
||||
let bucket = "protocol-mainflow-errors";
|
||||
let (_temp, root, backend, _stored_etag) = object_client(bucket).await;
|
||||
for target in [bucket, "protocol-absent-bucket"] {
|
||||
let expected = if target == bucket {
|
||||
S3ErrorCode::NoSuchKey
|
||||
} else {
|
||||
S3ErrorCode::NoSuchBucket
|
||||
};
|
||||
assert_eq!(
|
||||
backend
|
||||
.get_object(target, "absent", &root, None)
|
||||
.await
|
||||
.expect_err("missing GET")
|
||||
.code(),
|
||||
&expected
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.get_object(target, "absent", &root, Some(1))
|
||||
.await
|
||||
.expect_err("missing open-range GET")
|
||||
.code(),
|
||||
&expected
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.get_object_range(target, "absent", &root, 1, 1)
|
||||
.await
|
||||
.expect_err("missing bounded-range GET")
|
||||
.code(),
|
||||
&expected
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.head_object(target, "absent", &root)
|
||||
.await
|
||||
.expect_err("missing HEAD")
|
||||
.code(),
|
||||
&expected
|
||||
);
|
||||
}
|
||||
for start in [13, 14, 1024] {
|
||||
for error in [
|
||||
backend
|
||||
.get_object(bucket, KEY, &root, Some(start))
|
||||
.await
|
||||
.expect_err("open range past end"),
|
||||
backend
|
||||
.get_object_range(bucket, KEY, &root, start, 1)
|
||||
.await
|
||||
.expect_err("bounded range past end"),
|
||||
] {
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidRange);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
backend
|
||||
.delete_object("protocol-absent-bucket", KEY, &root)
|
||||
.await
|
||||
.expect_err("missing DELETE bucket")
|
||||
.code(),
|
||||
&S3ErrorCode::NoSuchBucket
|
||||
);
|
||||
});
|
||||
}
|
||||
@@ -18,3 +18,6 @@ pub use client::ProtocolStorageClient;
|
||||
|
||||
#[cfg(test)]
|
||||
mod policy_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod mainflow_test;
|
||||
|
||||
@@ -54,7 +54,7 @@ fn decode_policy(value: serde_json::Value) -> Result<rustfs_policy::policy::Poli
|
||||
serde_json::from_str(&value.to_string())
|
||||
}
|
||||
|
||||
async fn protocol_test_context(root: Credentials) -> (tempfile::TempDir, Arc<AppContext>, Arc<IamSys<ObjectStore>>) {
|
||||
pub(super) async fn protocol_test_context(root: Credentials) -> (tempfile::TempDir, Arc<AppContext>, Arc<IamSys<ObjectStore>>) {
|
||||
let (temp, _paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
ObjectStore::new(store.clone())
|
||||
.save_iam_config(json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
|
||||
|
||||
@@ -22,6 +22,8 @@ classify_source_layer() {
|
||||
printf 'composition'
|
||||
elif [[ "$file" == rustfs/src/app/* ]]; then
|
||||
printf 'app'
|
||||
elif [[ "$file" == rustfs/src/protocols/client.rs ]] || [[ "$file" == rustfs/src/protocols/client/* ]]; then
|
||||
printf 'interface'
|
||||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] ||
|
||||
[[ "$file" == rustfs/src/storage/legacy_edge/* ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
|
||||
[[ "$file" == rustfs/src/storage/s3_compat/* ]]; then
|
||||
@@ -47,6 +49,12 @@ classify_target_layer() {
|
||||
# their exported HTTP contracts belong to the interface boundary.
|
||||
printf 'interface'
|
||||
;;
|
||||
protocols)
|
||||
case "${path#protocols::}" in
|
||||
client | client::* | ProtocolStorageClient | ProtocolStorageClient::*) printf 'interface' ;;
|
||||
*) printf 'infra' ;;
|
||||
esac
|
||||
;;
|
||||
app)
|
||||
printf 'app'
|
||||
;;
|
||||
@@ -147,6 +155,16 @@ run_layer_model_self_tests() {
|
||||
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/tests/object_meta.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge_helper.rs
|
||||
assert_source_layer 'interface' rustfs/src/protocols/client.rs
|
||||
assert_source_layer 'interface' rustfs/src/protocols/client/owned.rs
|
||||
assert_source_layer 'infra' rustfs/src/protocols/client_helper.rs
|
||||
assert_target_layer 'interface' protocols::client
|
||||
assert_target_layer 'interface' protocols::client::ProtocolStorageClient
|
||||
assert_target_layer 'interface' protocols::ProtocolStorageClient
|
||||
assert_target_layer 'infra' protocols::client_helper
|
||||
assert_dependency_direction 'allowed' "$(classify_source_layer rustfs/src/protocols/client/owned.rs)" "$app_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer protocols::client::ProtocolStorageClient)"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer protocols::ProtocolStorageClient)"
|
||||
assert_source_layer 'app' rustfs/src/app/bucket_usecase/cors.rs
|
||||
assert_source_layer 'app' rustfs/src/app/multipart_usecase/complete.rs
|
||||
assert_target_layer 'interface' storage::legacy_edge
|
||||
|
||||
@@ -286,6 +286,18 @@ for silent in ${REAL_SPLIT_FILES[@]+"${REAL_SPLIT_FILES[@]}"}; do
|
||||
done
|
||||
|
||||
# --- layer dependencies ------------------------------------------------------
|
||||
# Protocol edges may call the app; app code cannot call either public client path.
|
||||
mkdir -p "$FX/rustfs/src/protocols/client"
|
||||
cat >"$FX/rustfs/src/protocols/client/layer_probe.rs" <<'PROTOCOL_EDGE'
|
||||
use crate::app::RequestEnvelope;
|
||||
PROTOCOL_EDGE
|
||||
cat >"$FX/rustfs/src/protocols/client_helper.rs" <<'PROTOCOL_NEIGHBOR'
|
||||
use crate::app::RequestEnvelope;
|
||||
PROTOCOL_NEIGHBOR
|
||||
cat >"$FX/rustfs/src/app/protocol_client_layer_probe.rs" <<'PROTOCOL_REVERSE'
|
||||
use crate::protocols::client::ProtocolStorageClient;
|
||||
use crate::protocols::ProtocolStorageClient;
|
||||
PROTOCOL_REVERSE
|
||||
LAYER_OUT="$FX/.layer.out"
|
||||
if (cd "$FX" && ./scripts/check_layer_dependencies.sh) >"$LAYER_OUT" 2>&1; then
|
||||
fail "layer guard passed over a planted reverse dependency"
|
||||
@@ -294,6 +306,10 @@ expect_reported "$LAYER_OUT" "dep|$E|interface->composition|" "crate::startup_se
|
||||
expect_silent "$LAYER_OUT" "dep|$EDGE_IMPORTS|" "moved ecfs imports stay legal"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/app/lifecycle_transition_api_test/" "moved lifecycle transition tests stay excluded"
|
||||
expect_silent "$LAYER_OUT" "dep|$LISTING|" "moved bucket usecase code keeps the app layer"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/protocols/client/layer_probe.rs|" "protocol client edge may use app contracts"
|
||||
expect_reported "$LAYER_OUT" "dep|rustfs/src/protocols/client_helper.rs|infra->app|" "crate::app::RequestEnvelope" "neighboring protocol helper remains infrastructure"
|
||||
expect_reported "$LAYER_OUT" "dep|rustfs/src/app/protocol_client_layer_probe.rs|app->interface|" "crate::protocols::client::ProtocolStorageClient" "app cannot import the protocol client module"
|
||||
expect_reported "$LAYER_OUT" "dep|rustfs/src/app/protocol_client_layer_probe.rs|app->interface|" "crate::protocols::ProtocolStorageClient" "app cannot import the protocol client re-export"
|
||||
|
||||
# --- logging hot span --------------------------------------------------------
|
||||
LOG_OUT="$FX/.logging.out"
|
||||
|
||||
Reference in New Issue
Block a user