mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 00:26:53 +00:00
refactor: Restructure project layout and clean up dependencies (#30)
This commit introduces a significant reorganization of the project structure to improve maintainability and clarity. Key changes include: - Adjusted the directory layout for a more logical module organization. - Removed unused crate dependencies, reducing the overall project size and potentially speeding up build times. - Updated import paths and configuration files to reflect the structural changes.
This commit is contained in:
houseme
@@ -0,0 +1,47 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use http::status::StatusCode;
|
||||
use std::fmt::{self, Display, Formatter};
|
||||
|
||||
#[derive(Default, thiserror::Error, Debug, Clone, PartialEq)]
|
||||
pub struct AdminError {
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
pub status_code: StatusCode,
|
||||
}
|
||||
|
||||
impl Display for AdminError {
|
||||
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
|
||||
write!(f, "{}", self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminError {
|
||||
pub fn new(code: &str, message: &str, status_code: StatusCode) -> Self {
|
||||
Self {
|
||||
code: code.to_string(),
|
||||
message: message.to_string(),
|
||||
status_code,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn msg(message: &str) -> Self {
|
||||
Self {
|
||||
code: "InternalError".to_string(),
|
||||
message: message.to_string(),
|
||||
status_code: StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, StatusCode};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
|
||||
if policy == "" {
|
||||
return self.remove_bucket_policy(bucket_name).await;
|
||||
}
|
||||
|
||||
self.put_bucket_policy(bucket_name, policy).await
|
||||
}
|
||||
|
||||
pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let mut req_metadata = RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
|
||||
content_length: policy.len() as i64,
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_md5_base64: "".to_string(),
|
||||
content_sha256_hex: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
};
|
||||
|
||||
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
|
||||
//defer closeResponse(resp)
|
||||
//if resp != nil {
|
||||
if resp.status() != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
|
||||
}
|
||||
//}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::DELETE,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
//defer closeResponse(resp)
|
||||
|
||||
if resp.status() != StatusCode::NO_CONTENT {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?;
|
||||
Ok(bucket_policy)
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let policy = String::from_utf8_lossy(&resp.body().bytes().expect("err").to_vec()).to_string();
|
||||
Ok(policy)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{de::Deserializer, ser::Serializer};
|
||||
use std::fmt::Display;
|
||||
|
||||
use s3s::Body;
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
const _REPORT_ISSUE: &str = "Please report this issue at https://github.com/rustfs/rustfs/issues.";
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, thiserror::Error, PartialEq, Eq)]
|
||||
#[serde(default, rename_all = "PascalCase")]
|
||||
pub struct ErrorResponse {
|
||||
#[serde(serialize_with = "serialize_code", deserialize_with = "deserialize_code")]
|
||||
pub code: S3ErrorCode,
|
||||
pub message: String,
|
||||
pub bucket_name: String,
|
||||
pub key: String,
|
||||
pub resource: String,
|
||||
pub request_id: String,
|
||||
pub host_id: String,
|
||||
pub region: String,
|
||||
pub server: String,
|
||||
#[serde(skip)]
|
||||
pub status_code: StatusCode,
|
||||
}
|
||||
|
||||
fn serialize_code<S>(_data: &S3ErrorCode, s: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
s.serialize_str("")
|
||||
}
|
||||
|
||||
fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
|
||||
}
|
||||
|
||||
impl Default for ErrorResponse {
|
||||
fn default() -> Self {
|
||||
ErrorResponse {
|
||||
code: S3ErrorCode::Custom("".into()),
|
||||
message: Default::default(),
|
||||
bucket_name: Default::default(),
|
||||
key: Default::default(),
|
||||
resource: Default::default(),
|
||||
request_id: Default::default(),
|
||||
host_id: Default::default(),
|
||||
region: Default::default(),
|
||||
server: Default::default(),
|
||||
status_code: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for ErrorResponse {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.message)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_error_response(err: &std::io::Error) -> ErrorResponse {
|
||||
if let Some(err) = err.get_ref() {
|
||||
if err.is::<ErrorResponse>() {
|
||||
err.downcast_ref::<ErrorResponse>().expect("err!").clone()
|
||||
} else {
|
||||
ErrorResponse::default()
|
||||
}
|
||||
} else {
|
||||
ErrorResponse::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn http_resp_to_error_response(
|
||||
resp: http::Response<Body>,
|
||||
b: Vec<u8>,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
) -> ErrorResponse {
|
||||
let err_body = String::from_utf8(b).unwrap();
|
||||
let err_resp_ = serde_xml_rs::from_str::<ErrorResponse>(&err_body);
|
||||
let mut err_resp = ErrorResponse::default();
|
||||
if err_resp_.is_err() {
|
||||
match resp.status() {
|
||||
StatusCode::NOT_FOUND => {
|
||||
if object_name == "" {
|
||||
err_resp = ErrorResponse {
|
||||
status_code: resp.status(),
|
||||
code: S3ErrorCode::NoSuchBucket,
|
||||
message: "The specified bucket does not exist.".to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
} else {
|
||||
err_resp = ErrorResponse {
|
||||
status_code: resp.status(),
|
||||
code: S3ErrorCode::NoSuchKey,
|
||||
message: "The specified key does not exist.".to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
key: object_name.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
}
|
||||
StatusCode::FORBIDDEN => {
|
||||
err_resp = ErrorResponse {
|
||||
status_code: resp.status(),
|
||||
code: S3ErrorCode::AccessDenied,
|
||||
message: "Access Denied.".to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
key: object_name.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
StatusCode::CONFLICT => {
|
||||
err_resp = ErrorResponse {
|
||||
status_code: resp.status(),
|
||||
code: S3ErrorCode::BucketNotEmpty,
|
||||
message: "Bucket not empty.".to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
StatusCode::PRECONDITION_FAILED => {
|
||||
err_resp = ErrorResponse {
|
||||
status_code: resp.status(),
|
||||
code: S3ErrorCode::PreconditionFailed,
|
||||
message: "Pre condition failed.".to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
key: object_name.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
_ => {
|
||||
let mut msg = resp.status().to_string();
|
||||
if err_body.len() > 0 {
|
||||
msg = err_body;
|
||||
}
|
||||
err_resp = ErrorResponse {
|
||||
status_code: resp.status(),
|
||||
code: S3ErrorCode::Custom(resp.status().to_string().into()),
|
||||
message: msg,
|
||||
bucket_name: bucket_name.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err_resp = err_resp_.unwrap();
|
||||
}
|
||||
err_resp.status_code = resp.status();
|
||||
if let Some(server_name) = resp.headers().get("Server") {
|
||||
err_resp.server = server_name.to_str().expect("err").to_string();
|
||||
}
|
||||
|
||||
let code = resp.headers().get("x-minio-error-code");
|
||||
if code.is_some() {
|
||||
err_resp.code = S3ErrorCode::Custom(code.expect("err").to_str().expect("err").into());
|
||||
}
|
||||
let desc = resp.headers().get("x-minio-error-desc");
|
||||
if desc.is_some() {
|
||||
err_resp.message = desc.expect("err").to_str().expect("err").trim_matches('"').to_string();
|
||||
}
|
||||
|
||||
if err_resp.request_id == "" {
|
||||
if let Some(x_amz_request_id) = resp.headers().get("x-amz-request-id") {
|
||||
err_resp.request_id = x_amz_request_id.to_str().expect("err").to_string();
|
||||
}
|
||||
}
|
||||
if err_resp.host_id == "" {
|
||||
if let Some(x_amz_id_2) = resp.headers().get("x-amz-id-2") {
|
||||
err_resp.host_id = x_amz_id_2.to_str().expect("err").to_string();
|
||||
}
|
||||
}
|
||||
if err_resp.region == "" {
|
||||
if let Some(x_amz_bucket_region) = resp.headers().get("x-amz-bucket-region") {
|
||||
err_resp.region = x_amz_bucket_region.to_str().expect("err").to_string();
|
||||
}
|
||||
}
|
||||
if err_resp.code == S3ErrorCode::InvalidLocationConstraint/*InvalidRegion*/ && err_resp.region != "" {
|
||||
err_resp.message = format!("Region does not match, expecting region ‘{}’.", err_resp.region);
|
||||
}
|
||||
|
||||
err_resp
|
||||
}
|
||||
|
||||
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
code: S3ErrorCode::InvalidArgument,
|
||||
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’."
|
||||
.to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
||||
let msg = format!(
|
||||
"Your proposed upload size ‘{}’ exceeds the maximum allowed object size ‘{}’ for single PUT operation.",
|
||||
total_size, max_object_size
|
||||
);
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
code: S3ErrorCode::EntityTooLarge,
|
||||
message: msg,
|
||||
bucket_name: bucket_name.to_string(),
|
||||
key: object_name.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_entity_too_small(total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
||||
let msg = format!(
|
||||
"Your proposed upload size ‘{}’ is below the minimum allowed object size ‘0B’ for single PUT operation.",
|
||||
total_size
|
||||
);
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
code: S3ErrorCode::EntityTooSmall,
|
||||
message: msg,
|
||||
bucket_name: bucket_name.to_string(),
|
||||
key: object_name.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_unexpected_eof(total_read: i64, total_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
|
||||
let msg = format!(
|
||||
"Data read ‘{}’ is not equal to the size ‘{}’ of the input Reader.",
|
||||
total_read, total_size
|
||||
);
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
code: S3ErrorCode::Custom("UnexpectedEOF".into()),
|
||||
message: msg,
|
||||
bucket_name: bucket_name.to_string(),
|
||||
key: object_name.to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_invalid_argument(message: &str) -> ErrorResponse {
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::BAD_REQUEST,
|
||||
code: S3ErrorCode::InvalidArgument,
|
||||
message: message.to_string(),
|
||||
request_id: "rustfs".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn err_api_not_supported(message: &str) -> ErrorResponse {
|
||||
ErrorResponse {
|
||||
status_code: StatusCode::NOT_IMPLEMENTED,
|
||||
code: S3ErrorCode::Custom("APINotSupported".into()),
|
||||
message: message.to_string(),
|
||||
request_id: "rustfs".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use bytes::Bytes;
|
||||
use http::HeaderMap;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::BufReader;
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::err_invalid_argument,
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
|
||||
};
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
|
||||
impl TransitionClient {
|
||||
pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub async fn get_object_inner(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
opts: &GetObjectOptions,
|
||||
) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: opts.to_query_values(),
|
||||
custom_header: opts.header(),
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp = &resp;
|
||||
let object_stat = to_object_info(bucket_name, object_name, resp.headers())?;
|
||||
|
||||
let b = resp.body().bytes().expect("err").to_vec();
|
||||
Ok((object_stat, resp.headers().clone(), BufReader::new(Cursor::new(b))))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct GetRequest {
|
||||
pub buffer: Vec<u8>,
|
||||
pub offset: i64,
|
||||
pub did_offset_change: bool,
|
||||
pub been_read: bool,
|
||||
pub is_read_at: bool,
|
||||
pub is_read_op: bool,
|
||||
pub is_first_req: bool,
|
||||
pub setting_object_info: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct GetResponse {
|
||||
pub size: i64,
|
||||
//pub error: error,
|
||||
pub did_read: bool,
|
||||
pub object_info: ObjectInfo,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Object {
|
||||
//pub reqch: chan<- getRequest,
|
||||
//pub resch: <-chan getResponse,
|
||||
//pub cancel: context.CancelFunc,
|
||||
pub curr_offset: i64,
|
||||
pub object_info: ObjectInfo,
|
||||
pub seek_data: bool,
|
||||
pub is_closed: bool,
|
||||
pub is_started: bool,
|
||||
//pub prev_err: error,
|
||||
pub been_read: bool,
|
||||
pub object_info_set: bool,
|
||||
}
|
||||
|
||||
impl Object {
|
||||
pub fn new() -> Object {
|
||||
Self { ..Default::default() }
|
||||
}
|
||||
|
||||
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
||||
self.curr_offset += bytes_read;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
|
||||
let mut read_req = GetRequest {
|
||||
is_read_op: true,
|
||||
been_read: self.been_read,
|
||||
buffer: b.to_vec(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if !self.is_started {
|
||||
read_req.is_first_req = true;
|
||||
}
|
||||
|
||||
read_req.did_offset_change = self.seek_data;
|
||||
read_req.offset = self.curr_offset;
|
||||
|
||||
let response = self.do_get_request(&read_req)?;
|
||||
|
||||
let bytes_read = response.size;
|
||||
|
||||
let oerr = self.set_offset(bytes_read);
|
||||
|
||||
Ok(response.size)
|
||||
}
|
||||
|
||||
fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
|
||||
if !self.is_started || !self.object_info_set {
|
||||
let _ = self.do_get_request(&GetRequest {
|
||||
is_first_req: !self.is_started,
|
||||
setting_object_info: !self.object_info_set,
|
||||
..Default::default()
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(self.object_info.clone())
|
||||
}
|
||||
|
||||
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
|
||||
self.curr_offset = offset;
|
||||
|
||||
let mut read_at_req = GetRequest {
|
||||
is_read_op: true,
|
||||
is_read_at: true,
|
||||
did_offset_change: true,
|
||||
been_read: self.been_read,
|
||||
offset,
|
||||
buffer: b.to_vec(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if !self.is_started {
|
||||
read_at_req.is_first_req = true;
|
||||
}
|
||||
|
||||
let response = self.do_get_request(&read_at_req)?;
|
||||
let bytes_read = response.size;
|
||||
if !self.object_info_set {
|
||||
self.curr_offset += bytes_read;
|
||||
} else {
|
||||
let oerr = self.set_offset(bytes_read);
|
||||
}
|
||||
Ok(response.size)
|
||||
}
|
||||
|
||||
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
|
||||
if !self.is_started || !self.object_info_set {
|
||||
let seek_req = GetRequest {
|
||||
is_read_op: false,
|
||||
offset,
|
||||
is_first_req: true,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = self.do_get_request(&seek_req);
|
||||
}
|
||||
|
||||
let mut new_offset = self.curr_offset;
|
||||
|
||||
match whence {
|
||||
0 => {
|
||||
new_offset = offset;
|
||||
}
|
||||
1 => {
|
||||
new_offset += offset;
|
||||
}
|
||||
2 => {
|
||||
new_offset = self.object_info.size as i64 + offset as i64;
|
||||
}
|
||||
_ => {
|
||||
return Err(std::io::Error::other(err_invalid_argument(&format!("Invalid whence {}", whence))));
|
||||
}
|
||||
}
|
||||
|
||||
self.seek_data = (new_offset != self.curr_offset) || self.seek_data;
|
||||
self.curr_offset = new_offset;
|
||||
|
||||
Ok(self.curr_offset)
|
||||
}
|
||||
|
||||
fn close(&mut self) -> Result<(), std::io::Error> {
|
||||
self.is_closed = true;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::client::api_error_response::err_invalid_argument;
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedGetOptions {
|
||||
replication_deletemarker: bool,
|
||||
is_replication_ready_for_deletemarker: bool,
|
||||
replication_proxy_request: String,
|
||||
}
|
||||
|
||||
pub struct GetObjectOptions {
|
||||
pub headers: HashMap<String, String>,
|
||||
pub req_params: HashMap<String, String>,
|
||||
//pub server_side_encryption: encrypt.ServerSide,
|
||||
pub version_id: String,
|
||||
pub part_number: i64,
|
||||
pub checksum: bool,
|
||||
pub internal: AdvancedGetOptions,
|
||||
}
|
||||
|
||||
impl Default for GetObjectOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
headers: HashMap::new(),
|
||||
req_params: HashMap::new(),
|
||||
//server_side_encryption: encrypt.ServerSide::default(),
|
||||
version_id: "".to_string(),
|
||||
part_number: 0,
|
||||
checksum: false,
|
||||
internal: AdvancedGetOptions::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GetObjectOptions {
|
||||
pub fn header(&self) -> HeaderMap {
|
||||
let mut headers: HeaderMap = HeaderMap::with_capacity(self.headers.len());
|
||||
for (k, v) in &self.headers {
|
||||
if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
|
||||
headers.insert(header_name, v.parse().expect("err"));
|
||||
} else {
|
||||
warn!("Invalid header name: {}", k);
|
||||
}
|
||||
}
|
||||
if self.checksum {
|
||||
headers.insert("x-amz-checksum-mode", "ENABLED".parse().expect("err"));
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
pub fn set(&self, key: &str, value: &str) {
|
||||
//self.headers[http.CanonicalHeaderKey(key)] = value;
|
||||
}
|
||||
|
||||
pub fn set_req_param(&mut self, key: &str, value: &str) {
|
||||
self.req_params.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
|
||||
pub fn add_req_param(&mut self, key: &str, value: &str) {
|
||||
self.req_params.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
|
||||
pub fn set_match_etag(&mut self, etag: &str) -> Result<(), std::io::Error> {
|
||||
self.set("If-Match", &format!("\"{etag}\""));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_match_etag_except(&mut self, etag: &str) -> Result<(), std::io::Error> {
|
||||
self.set("If-None-Match", &format!("\"{etag}\""));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_unmodified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
|
||||
if mod_time.unix_timestamp() == 0 {
|
||||
return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
|
||||
}
|
||||
self.set("If-Unmodified-Since", &mod_time.to_string());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_modified(&mut self, mod_time: OffsetDateTime) -> Result<(), std::io::Error> {
|
||||
if mod_time.unix_timestamp() == 0 {
|
||||
return Err(std::io::Error::other(err_invalid_argument("Modified since cannot be empty.")));
|
||||
}
|
||||
self.set("If-Modified-Since", &mod_time.to_string());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_range(&mut self, start: i64, end: i64) -> Result<(), std::io::Error> {
|
||||
if start == 0 && end < 0 {
|
||||
self.set("Range", &format!("bytes={}", end));
|
||||
} else if 0 < start && end == 0 {
|
||||
self.set("Range", &format!("bytes={}-", start));
|
||||
} else if 0 <= start && start <= end {
|
||||
self.set("Range", &format!("bytes={}-{}", start, end));
|
||||
} else {
|
||||
return Err(std::io::Error::other(err_invalid_argument(&format!(
|
||||
"Invalid range specified: start={} end={}",
|
||||
start, end
|
||||
))));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn to_query_values(&self) -> HashMap<String, String> {
|
||||
let mut url_values = HashMap::new();
|
||||
if self.version_id != "" {
|
||||
url_values.insert("versionId".to_string(), self.version_id.clone());
|
||||
}
|
||||
if self.part_number > 0 {
|
||||
url_values.insert("partNumber".to_string(), self.part_number.to_string());
|
||||
}
|
||||
|
||||
for (key, value) in self.req_params.iter() {
|
||||
url_values.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
|
||||
url_values
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, StatusCode};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
api_s3_datatypes::{
|
||||
ListBucketResult, ListBucketV2Result, ListMultipartUploadsResult, ListObjectPartsResult, ListVersionsResult, ObjectPart,
|
||||
},
|
||||
credentials,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use crate::store_api::BucketInfo;
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
|
||||
impl TransitionClient {
|
||||
pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub async fn list_objects_v2_query(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_prefix: &str,
|
||||
continuation_token: &str,
|
||||
fetch_owner: bool,
|
||||
metadata: bool,
|
||||
delimiter: &str,
|
||||
start_after: &str,
|
||||
max_keys: i64,
|
||||
headers: HeaderMap,
|
||||
) -> Result<ListBucketV2Result, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
|
||||
url_values.insert("list-type".to_string(), "2".to_string());
|
||||
if metadata {
|
||||
url_values.insert("metadata".to_string(), "true".to_string());
|
||||
}
|
||||
if start_after != "" {
|
||||
url_values.insert("start-after".to_string(), start_after.to_string());
|
||||
}
|
||||
url_values.insert("encoding-type".to_string(), "url".to_string());
|
||||
url_values.insert("prefix".to_string(), object_prefix.to_string());
|
||||
url_values.insert("delimiter".to_string(), delimiter.to_string());
|
||||
|
||||
if continuation_token != "" {
|
||||
url_values.insert("continuation-token".to_string(), continuation_token.to_string());
|
||||
}
|
||||
|
||||
if fetch_owner {
|
||||
url_values.insert("fetch-owner".to_string(), "true".to_string());
|
||||
}
|
||||
|
||||
if max_keys > 0 {
|
||||
url_values.insert("max-keys".to_string(), max_keys.to_string());
|
||||
}
|
||||
|
||||
let mut resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: "".to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
custom_header: headers,
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if resp.status() != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, "")));
|
||||
}
|
||||
|
||||
//let mut list_bucket_result = ListBucketV2Result::default();
|
||||
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
|
||||
let mut list_bucket_result = match serde_xml_rs::from_str::<ListBucketV2Result>(&String::from_utf8(b).unwrap()) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
//println!("list_bucket_result: {:?}", list_bucket_result);
|
||||
|
||||
if list_bucket_result.is_truncated && list_bucket_result.next_continuation_token == "" {
|
||||
return Err(std::io::Error::other(credentials::ErrorResponse {
|
||||
sts_error: credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: "Truncated response should have continuation token set".to_string(),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
}));
|
||||
}
|
||||
|
||||
for (i, obj) in list_bucket_result.contents.iter_mut().enumerate() {
|
||||
obj.name = decode_s3_name(&obj.name, &list_bucket_result.encoding_type)?;
|
||||
//list_bucket_result.contents[i].mod_time = list_bucket_result.contents[i].mod_time.Truncate(time.Millisecond);
|
||||
}
|
||||
|
||||
for (i, obj) in list_bucket_result.common_prefixes.iter_mut().enumerate() {
|
||||
obj.prefix = decode_s3_name(&obj.prefix, &list_bucket_result.encoding_type)?;
|
||||
}
|
||||
|
||||
Ok(list_bucket_result)
|
||||
}
|
||||
|
||||
pub fn list_object_versions_query(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
opts: &ListObjectsOptions,
|
||||
key_marker: &str,
|
||||
version_id_marker: &str,
|
||||
delimiter: &str,
|
||||
) -> Result<ListVersionsResult, std::io::Error> {
|
||||
/*if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
||||
return ListVersionsResult{}, err
|
||||
}
|
||||
if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
|
||||
return ListVersionsResult{}, err
|
||||
}
|
||||
urlValues := make(url.Values)
|
||||
|
||||
urlValues.Set("versions", "")
|
||||
|
||||
urlValues.Set("prefix", opts.Prefix)
|
||||
|
||||
urlValues.Set("delimiter", delimiter)
|
||||
|
||||
if keyMarker != "" {
|
||||
urlValues.Set("key-marker", keyMarker)
|
||||
}
|
||||
|
||||
if opts.max_keys > 0 {
|
||||
urlValues.Set("max-keys", fmt.Sprintf("%d", opts.max_keys))
|
||||
}
|
||||
|
||||
if versionIDMarker != "" {
|
||||
urlValues.Set("version-id-marker", versionIDMarker)
|
||||
}
|
||||
|
||||
if opts.WithMetadata {
|
||||
urlValues.Set("metadata", "true")
|
||||
}
|
||||
|
||||
urlValues.Set("encoding-type", "url")
|
||||
|
||||
let resp = self.executeMethod(http::Method::GET, &mut RequestMetadata{
|
||||
bucketName: bucketName,
|
||||
queryValues: urlValues,
|
||||
contentSHA256Hex: emptySHA256Hex,
|
||||
customHeader: opts.headers,
|
||||
}).await?;
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return ListVersionsResult{}, err
|
||||
}
|
||||
if resp != nil {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "")
|
||||
}
|
||||
}
|
||||
|
||||
listObjectVersionsOutput := ListVersionsResult{}
|
||||
err = xml_decoder(resp.Body, &listObjectVersionsOutput)
|
||||
if err != nil {
|
||||
return ListVersionsResult{}, err
|
||||
}
|
||||
|
||||
for i, obj := range listObjectVersionsOutput.Versions {
|
||||
listObjectVersionsOutput.Versions[i].Key, err = decode_s3_name(obj.Key, listObjectVersionsOutput.EncodingType)
|
||||
if err != nil {
|
||||
return listObjectVersionsOutput, err
|
||||
}
|
||||
}
|
||||
|
||||
for i, obj := range listObjectVersionsOutput.CommonPrefixes {
|
||||
listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decode_s3_name(obj.Prefix, listObjectVersionsOutput.EncodingType)
|
||||
if err != nil {
|
||||
return listObjectVersionsOutput, err
|
||||
}
|
||||
}
|
||||
|
||||
if listObjectVersionsOutput.NextKeyMarker != "" {
|
||||
listObjectVersionsOutput.NextKeyMarker, err = decode_s3_name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType)
|
||||
if err != nil {
|
||||
return listObjectVersionsOutput, err
|
||||
}
|
||||
}
|
||||
|
||||
Ok(listObjectVersionsOutput)*/
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub fn list_objects_query(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_prefix: &str,
|
||||
object_marker: &str,
|
||||
delimiter: &str,
|
||||
max_keys: i64,
|
||||
headers: HeaderMap,
|
||||
) -> Result<ListBucketResult, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub fn list_multipart_uploads_query(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
key_marker: &str,
|
||||
upload_id_marker: &str,
|
||||
prefix: &str,
|
||||
delimiter: &str,
|
||||
max_uploads: i64,
|
||||
) -> Result<ListMultipartUploadsResult, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub fn list_object_parts(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
upload_id: &str,
|
||||
) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result<Vec<String>, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub async fn list_object_parts_query(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
upload_id: &str,
|
||||
part_number_marker: i64,
|
||||
max_parts: i64,
|
||||
) -> Result<ListObjectPartsResult, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct ListObjectsOptions {
|
||||
reverse_versions: bool,
|
||||
with_versions: bool,
|
||||
with_metadata: bool,
|
||||
prefix: String,
|
||||
recursive: bool,
|
||||
max_keys: i64,
|
||||
start_after: String,
|
||||
use_v1: bool,
|
||||
headers: HeaderMap,
|
||||
}
|
||||
|
||||
impl ListObjectsOptions {
|
||||
pub fn set(&mut self, key: &str, value: &str) {
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Error> {
|
||||
match encoding_type {
|
||||
"url" => {
|
||||
//return url::QueryUnescape(name);
|
||||
return Ok(name.to_string());
|
||||
}
|
||||
_ => {
|
||||
return Ok(name.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::{Duration, OffsetDateTime, macros::format_description};
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use rustfs_utils::hasher::Hasher;
|
||||
use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
|
||||
use s3s::header::{
|
||||
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
|
||||
X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
|
||||
};
|
||||
//use crate::disk::{BufferReader, Reader};
|
||||
use crate::checksum::ChecksumMode;
|
||||
use crate::client::{
|
||||
api_error_response::{err_entity_too_large, err_invalid_argument},
|
||||
api_put_object_common::optimal_part_info,
|
||||
api_put_object_multipart::UploadPartParams,
|
||||
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
||||
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
|
||||
credentials::SignatureType,
|
||||
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
||||
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
||||
};
|
||||
use rustfs_utils::crypto::base64_encode;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AdvancedPutOptions {
|
||||
pub source_version_id: String,
|
||||
pub source_etag: String,
|
||||
pub replication_status: ReplicationStatus,
|
||||
pub source_mtime: OffsetDateTime,
|
||||
pub replication_request: bool,
|
||||
pub retention_timestamp: OffsetDateTime,
|
||||
pub tagging_timestamp: OffsetDateTime,
|
||||
pub legalhold_timestamp: OffsetDateTime,
|
||||
pub replication_validity_check: bool,
|
||||
}
|
||||
|
||||
impl Default for AdvancedPutOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
source_version_id: "".to_string(),
|
||||
source_etag: "".to_string(),
|
||||
replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
|
||||
source_mtime: OffsetDateTime::now_utc(),
|
||||
replication_request: false,
|
||||
retention_timestamp: OffsetDateTime::now_utc(),
|
||||
tagging_timestamp: OffsetDateTime::now_utc(),
|
||||
legalhold_timestamp: OffsetDateTime::now_utc(),
|
||||
replication_validity_check: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PutObjectOptions {
|
||||
pub user_metadata: HashMap<String, String>,
|
||||
pub user_tags: HashMap<String, String>,
|
||||
//pub progress: ReaderImpl,
|
||||
pub content_type: String,
|
||||
pub content_encoding: String,
|
||||
pub content_disposition: String,
|
||||
pub content_language: String,
|
||||
pub cache_control: String,
|
||||
pub expires: OffsetDateTime,
|
||||
pub mode: ObjectLockRetentionMode,
|
||||
pub retain_until_date: OffsetDateTime,
|
||||
//pub server_side_encryption: encrypt.ServerSide,
|
||||
pub num_threads: u64,
|
||||
pub storage_class: String,
|
||||
pub website_redirect_location: String,
|
||||
pub part_size: u64,
|
||||
pub legalhold: ObjectLockLegalHoldStatus,
|
||||
pub send_content_md5: bool,
|
||||
pub disable_content_sha256: bool,
|
||||
pub disable_multipart: bool,
|
||||
pub auto_checksum: ChecksumMode,
|
||||
pub checksum: ChecksumMode,
|
||||
pub concurrent_stream_parts: bool,
|
||||
pub internal: AdvancedPutOptions,
|
||||
pub custom_header: HeaderMap,
|
||||
}
|
||||
|
||||
impl Default for PutObjectOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
user_metadata: HashMap::new(),
|
||||
user_tags: HashMap::new(),
|
||||
//progress: ReaderImpl::Body(Bytes::new()),
|
||||
content_type: "".to_string(),
|
||||
content_encoding: "".to_string(),
|
||||
content_disposition: "".to_string(),
|
||||
content_language: "".to_string(),
|
||||
cache_control: "".to_string(),
|
||||
expires: OffsetDateTime::UNIX_EPOCH,
|
||||
mode: ObjectLockRetentionMode::from_static(""),
|
||||
retain_until_date: OffsetDateTime::UNIX_EPOCH,
|
||||
//server_side_encryption: encrypt.ServerSide::default(),
|
||||
num_threads: 0,
|
||||
storage_class: "".to_string(),
|
||||
website_redirect_location: "".to_string(),
|
||||
part_size: 0,
|
||||
legalhold: ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF),
|
||||
send_content_md5: false,
|
||||
disable_content_sha256: false,
|
||||
disable_multipart: false,
|
||||
auto_checksum: ChecksumMode::ChecksumNone,
|
||||
checksum: ChecksumMode::ChecksumNone,
|
||||
concurrent_stream_parts: false,
|
||||
internal: AdvancedPutOptions::default(),
|
||||
custom_header: HeaderMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjectOptions {
|
||||
fn set_matche_tag(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
.insert("If-Match", HeaderValue::from_str("*").expect("err"));
|
||||
} else {
|
||||
self.custom_header
|
||||
.insert("If-Match", HeaderValue::from_str(&format!("\"{}\"", etag)).expect("err"));
|
||||
}
|
||||
}
|
||||
|
||||
fn set_matche_tag_except(&mut self, etag: &str) {
|
||||
if etag == "*" {
|
||||
self.custom_header
|
||||
.insert("If-None-Match", HeaderValue::from_str("*").expect("err"));
|
||||
} else {
|
||||
self.custom_header
|
||||
.insert("If-None-Match", HeaderValue::from_str(&format!("\"{etag}\"")).expect("err"));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn header(&self) -> HeaderMap {
|
||||
let mut header = HeaderMap::new();
|
||||
|
||||
let mut content_type = self.content_type.clone();
|
||||
if content_type == "" {
|
||||
content_type = "application/octet-stream".to_string();
|
||||
}
|
||||
header.insert("Content-Type", HeaderValue::from_str(&content_type).expect("err"));
|
||||
|
||||
if self.content_encoding != "" {
|
||||
header.insert("Content-Encoding", HeaderValue::from_str(&self.content_encoding).expect("err"));
|
||||
}
|
||||
if self.content_disposition != "" {
|
||||
header.insert("Content-Disposition", HeaderValue::from_str(&self.content_disposition).expect("err"));
|
||||
}
|
||||
if self.content_language != "" {
|
||||
header.insert("Content-Language", HeaderValue::from_str(&self.content_language).expect("err"));
|
||||
}
|
||||
if self.cache_control != "" {
|
||||
header.insert("Cache-Control", HeaderValue::from_str(&self.cache_control).expect("err"));
|
||||
}
|
||||
|
||||
if self.expires.unix_timestamp() != 0 {
|
||||
header.insert(
|
||||
"Expires",
|
||||
HeaderValue::from_str(&self.expires.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
|
||||
); //rustfs invalid heade
|
||||
}
|
||||
|
||||
if self.mode.as_str() != "" {
|
||||
header.insert(X_AMZ_OBJECT_LOCK_MODE, HeaderValue::from_str(self.mode.as_str()).expect("err"));
|
||||
}
|
||||
|
||||
if self.retain_until_date.unix_timestamp() != 0 {
|
||||
header.insert(
|
||||
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
|
||||
HeaderValue::from_str(&self.retain_until_date.format(ISO8601_DATEFORMAT).unwrap()).expect("err"),
|
||||
);
|
||||
}
|
||||
|
||||
if self.legalhold.as_str() != "" {
|
||||
header.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD, HeaderValue::from_str(self.legalhold.as_str()).expect("err"));
|
||||
}
|
||||
|
||||
if self.storage_class != "" {
|
||||
header.insert(X_AMZ_STORAGE_CLASS, HeaderValue::from_str(&self.storage_class).expect("err"));
|
||||
}
|
||||
|
||||
if self.website_redirect_location != "" {
|
||||
header.insert(
|
||||
X_AMZ_WEBSITE_REDIRECT_LOCATION,
|
||||
HeaderValue::from_str(&self.website_redirect_location).expect("err"),
|
||||
);
|
||||
}
|
||||
|
||||
if !self.internal.replication_status.as_str().is_empty() {
|
||||
header.insert(
|
||||
X_AMZ_REPLICATION_STATUS,
|
||||
HeaderValue::from_str(self.internal.replication_status.as_str()).expect("err"),
|
||||
);
|
||||
}
|
||||
|
||||
for (k, v) in &self.user_metadata {
|
||||
if is_amz_header(k) || is_standard_header(k) || is_storageclass_header(k) || is_rustfs_header(k) || is_minio_header(k)
|
||||
{
|
||||
if let Ok(header_name) = HeaderName::from_bytes(k.as_bytes()) {
|
||||
header.insert(header_name, HeaderValue::from_str(&v).unwrap());
|
||||
}
|
||||
} else if let Ok(header_name) = HeaderName::from_bytes(format!("x-amz-meta-{}", k).as_bytes()) {
|
||||
header.insert(header_name, HeaderValue::from_str(&v).unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
for (k, v) in self.custom_header.iter() {
|
||||
header.insert(k.clone(), v.clone());
|
||||
}
|
||||
|
||||
header
|
||||
}
|
||||
|
||||
fn validate(&self, c: TransitionClient) -> Result<(), std::io::Error> {
|
||||
//if self.checksum.is_set() {
|
||||
/*if !self.trailing_header_support {
|
||||
return Err(Error::from(err_invalid_argument("Checksum requires Client with TrailingHeaders enabled")));
|
||||
}*/
|
||||
/*else if self.override_signer_type == SignatureType::SignatureV2 {
|
||||
return Err(Error::from(err_invalid_argument("Checksum cannot be used with v2 signatures")));
|
||||
}*/
|
||||
//}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn put_object(
|
||||
self: Arc<Self>,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
reader: ReaderImpl,
|
||||
object_size: i64,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
if object_size < 0 && opts.disable_multipart {
|
||||
return Err(std::io::Error::other("object size must be provided with disable multipart upload"));
|
||||
}
|
||||
|
||||
self.put_object_common(bucket_name, object_name, reader, object_size, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn put_object_common(
|
||||
self: Arc<Self>,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
reader: ReaderImpl,
|
||||
size: i64,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
if size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other(err_entity_too_large(
|
||||
size,
|
||||
MAX_MULTIPART_PUT_OBJECT_SIZE,
|
||||
bucket_name,
|
||||
object_name,
|
||||
)));
|
||||
}
|
||||
let mut opts = opts.clone();
|
||||
opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
|
||||
|
||||
let mut part_size = opts.part_size as i64;
|
||||
if opts.part_size == 0 {
|
||||
part_size = MIN_PART_SIZE;
|
||||
}
|
||||
|
||||
if SignatureType::SignatureV2 == self.override_signer_type {
|
||||
if size >= 0 && size < part_size || opts.disable_multipart {
|
||||
return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
|
||||
}
|
||||
return self.put_object_multipart(bucket_name, object_name, reader, size, &opts).await;
|
||||
}
|
||||
|
||||
if size < 0 {
|
||||
if opts.disable_multipart {
|
||||
return Err(std::io::Error::other("no length provided and multipart disabled"));
|
||||
}
|
||||
if opts.concurrent_stream_parts && opts.num_threads > 1 {
|
||||
return self
|
||||
.put_object_multipart_stream_parallel(bucket_name, object_name, reader, &opts)
|
||||
.await;
|
||||
}
|
||||
return self
|
||||
.put_object_multipart_stream_no_length(bucket_name, object_name, reader, &opts)
|
||||
.await;
|
||||
}
|
||||
|
||||
if size <= part_size || opts.disable_multipart {
|
||||
return self.put_object_gcs(bucket_name, object_name, reader, size, &opts).await;
|
||||
}
|
||||
|
||||
self.put_object_multipart_stream(bucket_name, object_name, reader, size, &opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn put_object_multipart_stream_no_length(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
mut reader: ReaderImpl,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let mut total_uploaded_size: i64 = 0;
|
||||
|
||||
let mut compl_multipart_upload = CompleteMultipartUpload::default();
|
||||
|
||||
let (total_parts_count, part_size, _) = optimal_part_info(-1, opts.part_size)?;
|
||||
|
||||
let mut opts = opts.clone();
|
||||
|
||||
if opts.checksum.is_set() {
|
||||
opts.send_content_md5 = false;
|
||||
opts.auto_checksum = opts.checksum.clone();
|
||||
}
|
||||
if !opts.send_content_md5 {
|
||||
//add_auto_checksum_headers(&mut opts);
|
||||
}
|
||||
|
||||
let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
|
||||
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
|
||||
|
||||
let mut part_number = 1;
|
||||
let mut parts_info = HashMap::<i64, ObjectPart>::new();
|
||||
let mut buf = Vec::<u8>::with_capacity(part_size as usize);
|
||||
|
||||
let mut custom_header = HeaderMap::new();
|
||||
|
||||
while part_number <= total_parts_count {
|
||||
buf = match &mut reader {
|
||||
ReaderImpl::Body(content_body) => content_body.to_vec(),
|
||||
ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
|
||||
};
|
||||
let length = buf.len();
|
||||
|
||||
let mut md5_base64: String = "".to_string();
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = self.md5_hasher.lock().unwrap();
|
||||
let hash = md5_hasher.as_mut().expect("err");
|
||||
hash.write(&buf[..length]);
|
||||
md5_base64 = base64_encode(hash.sum().as_bytes());
|
||||
} else {
|
||||
let csum;
|
||||
{
|
||||
let mut crc = opts.auto_checksum.hasher()?;
|
||||
crc.reset();
|
||||
crc.write(&buf[..length]);
|
||||
csum = crc.sum();
|
||||
}
|
||||
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
|
||||
custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().unwrap());
|
||||
} else {
|
||||
warn!("Invalid header name: {}", opts.auto_checksum.key());
|
||||
}
|
||||
}
|
||||
|
||||
//let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
|
||||
let rd = ReaderImpl::Body(Bytes::from(buf));
|
||||
|
||||
let mut p = UploadPartParams {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
upload_id: upload_id.clone(),
|
||||
reader: rd,
|
||||
part_number,
|
||||
md5_base64,
|
||||
size: length as i64,
|
||||
//sse: opts.server_side_encryption,
|
||||
stream_sha256: !opts.disable_content_sha256,
|
||||
custom_header: custom_header.clone(),
|
||||
sha256_hex: Default::default(),
|
||||
trailer: Default::default(),
|
||||
};
|
||||
let obj_part = self.upload_part(&mut p).await?;
|
||||
|
||||
parts_info.entry(part_number).or_insert(obj_part);
|
||||
total_uploaded_size += length as i64;
|
||||
part_number += 1;
|
||||
}
|
||||
|
||||
let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
|
||||
for i in 1..part_number {
|
||||
let part = parts_info[&i].clone();
|
||||
all_parts.push(part.clone());
|
||||
compl_multipart_upload.parts.push(CompletePart {
|
||||
etag: part.etag,
|
||||
part_num: part.part_num,
|
||||
checksum_crc32: part.checksum_crc32,
|
||||
checksum_crc32c: part.checksum_crc32c,
|
||||
checksum_sha1: part.checksum_sha1,
|
||||
checksum_sha256: part.checksum_sha256,
|
||||
checksum_crc64nvme: part.checksum_crc64nvme,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
compl_multipart_upload.parts.sort();
|
||||
|
||||
let opts = PutObjectOptions {
|
||||
//server_side_encryption: opts.server_side_encryption,
|
||||
auto_checksum: opts.auto_checksum,
|
||||
..Default::default()
|
||||
};
|
||||
//apply_auto_checksum(&mut opts, all_parts);
|
||||
|
||||
let mut upload_info = self
|
||||
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
|
||||
.await?;
|
||||
|
||||
upload_info.size = total_uploaded_size;
|
||||
Ok(upload_info)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::{err_entity_too_large, err_invalid_argument},
|
||||
api_put_object::PutObjectOptions,
|
||||
constants::{ABS_MIN_PART_SIZE, MAX_MULTIPART_PUT_OBJECT_SIZE, MAX_PART_SIZE, MAX_PARTS_COUNT, MIN_PART_SIZE},
|
||||
transition_api::ReaderImpl,
|
||||
transition_api::TransitionClient,
|
||||
};
|
||||
|
||||
pub fn is_object(reader: &ReaderImpl) -> bool {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub fn is_read_at(reader: ReaderImpl) -> bool {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
||||
let unknown_size;
|
||||
let mut object_size = object_size;
|
||||
if object_size == -1 {
|
||||
unknown_size = true;
|
||||
object_size = MAX_MULTIPART_PUT_OBJECT_SIZE;
|
||||
} else {
|
||||
unknown_size = false;
|
||||
}
|
||||
|
||||
if object_size > MAX_MULTIPART_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other(err_entity_too_large(
|
||||
object_size,
|
||||
MAX_MULTIPART_PUT_OBJECT_SIZE,
|
||||
"",
|
||||
"",
|
||||
)));
|
||||
}
|
||||
|
||||
let mut part_size_flt: f64;
|
||||
if configured_part_size > 0 {
|
||||
if configured_part_size as i64 > object_size {
|
||||
return Err(std::io::Error::other(err_entity_too_large(
|
||||
configured_part_size as i64,
|
||||
object_size,
|
||||
"",
|
||||
"",
|
||||
)));
|
||||
}
|
||||
|
||||
if !unknown_size && object_size > (configured_part_size as i64 * MAX_PARTS_COUNT) {
|
||||
return Err(std::io::Error::other(err_invalid_argument(
|
||||
"Part size * max_parts(10000) is lesser than input objectSize.",
|
||||
)));
|
||||
}
|
||||
|
||||
if (configured_part_size as i64) < ABS_MIN_PART_SIZE {
|
||||
return Err(std::io::Error::other(err_invalid_argument(
|
||||
"Input part size is smaller than allowed minimum of 5MiB.",
|
||||
)));
|
||||
}
|
||||
|
||||
if configured_part_size as i64 > MAX_PART_SIZE {
|
||||
return Err(std::io::Error::other(err_invalid_argument(
|
||||
"Input part size is bigger than allowed maximum of 5GiB.",
|
||||
)));
|
||||
}
|
||||
|
||||
part_size_flt = configured_part_size as f64;
|
||||
if unknown_size {
|
||||
object_size = configured_part_size as i64 * MAX_PARTS_COUNT;
|
||||
}
|
||||
} else {
|
||||
let mut configured_part_size = configured_part_size;
|
||||
configured_part_size = MIN_PART_SIZE as u64;
|
||||
part_size_flt = (object_size / MAX_PARTS_COUNT) as f64;
|
||||
part_size_flt = (part_size_flt / configured_part_size as f64) * configured_part_size as f64;
|
||||
}
|
||||
|
||||
let total_parts_count = (object_size as f64 / part_size_flt).ceil() as i64;
|
||||
let part_size = part_size_flt.ceil() as i64;
|
||||
let last_part_size = object_size - (total_parts_count - 1) * part_size;
|
||||
Ok((total_parts_count, part_size, last_part_size))
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn new_upload_id(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<String, std::io::Error> {
|
||||
let init_multipart_upload_result = self.initiate_multipart_upload(bucket_name, object_name, opts).await?;
|
||||
Ok(init_multipart_upload_result.upload_id)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
||||
use s3s::S3ErrorCode;
|
||||
use std::io::Read;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::{OffsetDateTime, format_description};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
use tracing::{error, info};
|
||||
use url::form_urlencoded::Serializer;
|
||||
use uuid::Uuid;
|
||||
|
||||
use rustfs_utils::hasher::Hasher;
|
||||
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
|
||||
use s3s::{Body, dto::StreamingBlob};
|
||||
//use crate::disk::{Reader, BufferReader};
|
||||
use crate::client::{
|
||||
api_error_response::{
|
||||
err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
|
||||
},
|
||||
api_put_object::PutObjectOptions,
|
||||
api_put_object_common::optimal_part_info,
|
||||
api_s3_datatypes::{
|
||||
CompleteMultipartUpload, CompleteMultipartUploadResult, CompletePart, InitiateMultipartUploadResult, ObjectPart,
|
||||
},
|
||||
constants::{ABS_MIN_PART_SIZE, ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
|
||||
};
|
||||
use crate::{
|
||||
checksum::ChecksumMode,
|
||||
disk::DiskAPI,
|
||||
store_api::{GetObjectReader, StorageAPI},
|
||||
};
|
||||
use rustfs_utils::{crypto::base64_encode, path::trim_etag};
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn put_object_multipart(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
mut reader: ReaderImpl,
|
||||
size: i64,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let info = self
|
||||
.put_object_multipart_no_stream(bucket_name, object_name, &mut reader, opts)
|
||||
.await;
|
||||
if let Err(err) = &info {
|
||||
let err_resp = to_error_response(err);
|
||||
if err_resp.code == S3ErrorCode::AccessDenied && err_resp.message.contains("Access Denied") {
|
||||
if size > MAX_SINGLE_PUT_OBJECT_SIZE {
|
||||
return Err(std::io::Error::other(err_entity_too_large(
|
||||
size,
|
||||
MAX_SINGLE_PUT_OBJECT_SIZE,
|
||||
bucket_name,
|
||||
object_name,
|
||||
)));
|
||||
}
|
||||
return self.put_object_gcs(bucket_name, object_name, reader, size, opts).await;
|
||||
}
|
||||
}
|
||||
Ok(info?)
|
||||
}
|
||||
|
||||
pub async fn put_object_multipart_no_stream(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
reader: &mut ReaderImpl,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let mut total_uploaded_size: i64 = 0;
|
||||
let mut compl_multipart_upload = CompleteMultipartUpload::default();
|
||||
|
||||
let ret = optimal_part_info(-1, opts.part_size)?;
|
||||
let (total_parts_count, part_size, _) = ret;
|
||||
|
||||
let (mut hash_algos, mut hash_sums) = self.hash_materials(opts.send_content_md5, !opts.disable_content_sha256);
|
||||
let upload_id = self.new_upload_id(bucket_name, object_name, opts).await?;
|
||||
let mut opts = opts.clone();
|
||||
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
|
||||
|
||||
let mut part_number = 1;
|
||||
let mut parts_info = HashMap::<i64, ObjectPart>::new();
|
||||
let mut buf = Vec::<u8>::with_capacity(part_size as usize);
|
||||
let mut custom_header = HeaderMap::new();
|
||||
while part_number <= total_parts_count {
|
||||
match reader {
|
||||
ReaderImpl::Body(content_body) => {
|
||||
buf = content_body.to_vec();
|
||||
}
|
||||
ReaderImpl::ObjectBody(content_body) => {
|
||||
buf = content_body.read_all().await?;
|
||||
}
|
||||
}
|
||||
let length = buf.len();
|
||||
|
||||
for (k, v) in hash_algos.iter_mut() {
|
||||
v.write(&buf[..length]);
|
||||
hash_sums.insert(k.to_string(), Vec::try_from(v.sum().as_bytes()).unwrap());
|
||||
}
|
||||
|
||||
//let rd = newHook(bytes.NewReader(buf[..length]), opts.progress);
|
||||
let rd = Bytes::from(buf.clone());
|
||||
|
||||
let md5_base64: String;
|
||||
let sha256_hex: String;
|
||||
|
||||
//if hash_sums["md5"] != nil {
|
||||
md5_base64 = base64_encode(&hash_sums["md5"]);
|
||||
//}
|
||||
//if hash_sums["sha256"] != nil {
|
||||
sha256_hex = hex_simd::encode_to_string(hash_sums["sha256"].clone(), hex_simd::AsciiCase::Lower);
|
||||
//}
|
||||
if hash_sums.len() == 0 {
|
||||
let csum;
|
||||
{
|
||||
let mut crc = opts.auto_checksum.hasher()?;
|
||||
crc.reset();
|
||||
crc.write(&buf[..length]);
|
||||
csum = crc.sum();
|
||||
}
|
||||
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
|
||||
custom_header.insert(header_name, base64_encode(csum.as_bytes()).parse().expect("err"));
|
||||
} else {
|
||||
warn!("Invalid header name: {}", opts.auto_checksum.key());
|
||||
}
|
||||
}
|
||||
|
||||
let mut p = UploadPartParams {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
upload_id: upload_id.clone(),
|
||||
reader: ReaderImpl::Body(rd),
|
||||
part_number,
|
||||
md5_base64,
|
||||
sha256_hex,
|
||||
size: length as i64,
|
||||
//sse: opts.server_side_encryption,
|
||||
stream_sha256: !opts.disable_content_sha256,
|
||||
custom_header: custom_header.clone(),
|
||||
trailer: HeaderMap::new(),
|
||||
};
|
||||
let obj_part = self.upload_part(&mut p).await?;
|
||||
|
||||
parts_info.insert(part_number, obj_part);
|
||||
total_uploaded_size += length as i64;
|
||||
part_number += 1;
|
||||
}
|
||||
|
||||
let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
|
||||
for i in 1..part_number {
|
||||
let part = parts_info[&i].clone();
|
||||
all_parts.push(part.clone());
|
||||
compl_multipart_upload.parts.push(CompletePart {
|
||||
etag: part.etag,
|
||||
part_num: part.part_num,
|
||||
checksum_crc32: part.checksum_crc32,
|
||||
checksum_crc32c: part.checksum_crc32c,
|
||||
checksum_sha1: part.checksum_sha1,
|
||||
checksum_sha256: part.checksum_sha256,
|
||||
checksum_crc64nvme: part.checksum_crc64nvme,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
compl_multipart_upload.parts.sort();
|
||||
let opts = PutObjectOptions {
|
||||
//server_side_encryption: opts.server_side_encryption,
|
||||
auto_checksum: opts.auto_checksum,
|
||||
..Default::default()
|
||||
};
|
||||
//apply_auto_checksum(&mut opts, all_parts);
|
||||
|
||||
let mut upload_info = self
|
||||
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
|
||||
.await?;
|
||||
|
||||
upload_info.size = total_uploaded_size;
|
||||
Ok(upload_info)
|
||||
}
|
||||
|
||||
pub async fn initiate_multipart_upload(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<InitiateMultipartUploadResult, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("uploads".to_string(), "".to_string());
|
||||
|
||||
if opts.internal.source_version_id != "" {
|
||||
if !opts.internal.source_version_id.is_empty() {
|
||||
if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
|
||||
return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
|
||||
}
|
||||
}
|
||||
url_values.insert("versionId".to_string(), opts.internal.source_version_id.clone());
|
||||
}
|
||||
|
||||
let custom_header = opts.header();
|
||||
|
||||
let mut req_metadata = RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header,
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
content_sha256_hex: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
};
|
||||
|
||||
let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
|
||||
//if resp.is_none() {
|
||||
if resp.status() != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
|
||||
}
|
||||
//}
|
||||
let initiate_multipart_upload_result = InitiateMultipartUploadResult::default();
|
||||
Ok(initiate_multipart_upload_result)
|
||||
}
|
||||
|
||||
pub async fn upload_part(&self, p: &mut UploadPartParams) -> Result<ObjectPart, std::io::Error> {
|
||||
if p.size > MAX_PART_SIZE {
|
||||
return Err(std::io::Error::other(err_entity_too_large(
|
||||
p.size,
|
||||
MAX_PART_SIZE,
|
||||
&p.bucket_name,
|
||||
&p.object_name,
|
||||
)));
|
||||
}
|
||||
if p.size <= -1 {
|
||||
return Err(std::io::Error::other(err_entity_too_small(p.size, &p.bucket_name, &p.object_name)));
|
||||
}
|
||||
if p.part_number <= 0 {
|
||||
return Err(std::io::Error::other(err_invalid_argument(
|
||||
"Part number cannot be negative or equal to zero.",
|
||||
)));
|
||||
}
|
||||
if p.upload_id == "" {
|
||||
return Err(std::io::Error::other(err_invalid_argument("UploadID cannot be empty.")));
|
||||
}
|
||||
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("partNumber".to_string(), p.part_number.to_string());
|
||||
url_values.insert("uploadId".to_string(), p.upload_id.clone());
|
||||
|
||||
let buf = match &mut p.reader {
|
||||
ReaderImpl::Body(content_body) => content_body.to_vec(),
|
||||
ReaderImpl::ObjectBody(content_body) => content_body.read_all().await?,
|
||||
};
|
||||
let mut req_metadata = RequestMetadata {
|
||||
bucket_name: p.bucket_name.clone(),
|
||||
object_name: p.object_name.clone(),
|
||||
query_values: url_values,
|
||||
custom_header: p.custom_header.clone(),
|
||||
content_body: ReaderImpl::Body(Bytes::from(buf)),
|
||||
content_length: p.size,
|
||||
content_md5_base64: p.md5_base64.clone(),
|
||||
content_sha256_hex: p.sha256_hex.clone(),
|
||||
stream_sha256: p.stream_sha256,
|
||||
trailer: p.trailer.clone(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
};
|
||||
|
||||
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
|
||||
//defer closeResponse(resp)
|
||||
//if resp.is_none() {
|
||||
if resp.status() != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp,
|
||||
vec![],
|
||||
&p.bucket_name.clone(),
|
||||
&p.object_name,
|
||||
)));
|
||||
}
|
||||
//}
|
||||
let h = resp.headers();
|
||||
let mut obj_part = ObjectPart {
|
||||
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
|
||||
h_checksum_crc32.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
|
||||
h_checksum_crc32c.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
|
||||
h_checksum_sha1.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
|
||||
h_checksum_sha256.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
|
||||
h_checksum_crc64nvme.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
obj_part.size = p.size;
|
||||
obj_part.part_num = p.part_number;
|
||||
obj_part.etag = if let Some(h_etag) = h.get("ETag") {
|
||||
h_etag.to_str().expect("err").trim_matches('"').to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
};
|
||||
Ok(obj_part)
|
||||
}
|
||||
|
||||
pub async fn complete_multipart_upload(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
upload_id: &str,
|
||||
complete: CompleteMultipartUpload,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("uploadId".to_string(), upload_id.to_string());
|
||||
let complete_multipart_upload_bytes = complete.marshal_msg()?.as_bytes().to_vec();
|
||||
|
||||
let headers = opts.header();
|
||||
|
||||
let complete_multipart_upload_buffer = Bytes::from(complete_multipart_upload_bytes);
|
||||
let mut req_metadata = RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_body: ReaderImpl::Body(complete_multipart_upload_buffer),
|
||||
content_length: 100, //complete_multipart_upload_bytes.len(),
|
||||
content_sha256_hex: "".to_string(), //hex_simd::encode_to_string(complete_multipart_upload_bytes, hex_simd::AsciiCase::Lower),
|
||||
custom_header: headers,
|
||||
stream_sha256: Default::default(),
|
||||
trailer: Default::default(),
|
||||
content_md5_base64: "".to_string(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
};
|
||||
|
||||
let resp = self.execute_method(http::Method::POST, &mut req_metadata).await?;
|
||||
|
||||
let b = resp.body().bytes().expect("err").to_vec();
|
||||
let complete_multipart_upload_result: CompleteMultipartUploadResult = CompleteMultipartUploadResult::default();
|
||||
|
||||
let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
|
||||
(
|
||||
OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
|
||||
"".to_string(),
|
||||
)
|
||||
} else {
|
||||
(OffsetDateTime::now_utc(), "".to_string())
|
||||
};
|
||||
|
||||
let h = resp.headers();
|
||||
Ok(UploadInfo {
|
||||
bucket: complete_multipart_upload_result.bucket,
|
||||
key: complete_multipart_upload_result.key,
|
||||
etag: trim_etag(&complete_multipart_upload_result.etag),
|
||||
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
|
||||
h_x_amz_version_id.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
location: complete_multipart_upload_result.location,
|
||||
expiration: exp_time,
|
||||
expiration_rule_id: rule_id,
|
||||
checksum_sha256: complete_multipart_upload_result.checksum_sha256,
|
||||
checksum_sha1: complete_multipart_upload_result.checksum_sha1,
|
||||
checksum_crc32: complete_multipart_upload_result.checksum_crc32,
|
||||
checksum_crc32c: complete_multipart_upload_result.checksum_crc32c,
|
||||
checksum_crc64nvme: complete_multipart_upload_result.checksum_crc64nvme,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UploadPartParams {
|
||||
pub bucket_name: String,
|
||||
pub object_name: String,
|
||||
pub upload_id: String,
|
||||
pub reader: ReaderImpl,
|
||||
pub part_number: i64,
|
||||
pub md5_base64: String,
|
||||
pub sha256_hex: String,
|
||||
pub size: i64,
|
||||
//pub sse: encrypt.ServerSide,
|
||||
pub stream_sha256: bool,
|
||||
pub custom_header: HeaderMap,
|
||||
pub trailer: HeaderMap,
|
||||
}
|
||||
@@ -0,0 +1,544 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::future::join_all;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
||||
use std::sync::RwLock;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::{OffsetDateTime, format_description};
|
||||
use tokio::{select, sync::mpsc};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
|
||||
use crate::client::{
|
||||
api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
|
||||
api_put_object::PutObjectOptions,
|
||||
api_put_object_common::{is_object, optimal_part_info},
|
||||
api_put_object_multipart::UploadPartParams,
|
||||
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
|
||||
constants::ISO8601_DATEFORMAT,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
|
||||
};
|
||||
use rustfs_utils::hasher::Hasher;
|
||||
use rustfs_utils::{crypto::base64_encode, path::trim_etag};
|
||||
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
|
||||
|
||||
pub struct UploadedPartRes {
|
||||
pub error: std::io::Error,
|
||||
pub part_num: i64,
|
||||
pub size: i64,
|
||||
pub part: ObjectPart,
|
||||
}
|
||||
|
||||
pub struct UploadPartReq {
|
||||
pub part_num: i64,
|
||||
pub part: ObjectPart,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn put_object_multipart_stream(
|
||||
self: Arc<Self>,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
reader: ReaderImpl,
|
||||
size: i64,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let info: UploadInfo;
|
||||
if opts.concurrent_stream_parts && opts.num_threads > 1 {
|
||||
info = self
|
||||
.put_object_multipart_stream_parallel(bucket_name, object_name, reader, opts)
|
||||
.await?;
|
||||
} else if !is_object(&reader) && !opts.send_content_md5 {
|
||||
info = self
|
||||
.put_object_multipart_stream_from_readat(bucket_name, object_name, reader, size, opts)
|
||||
.await?;
|
||||
} else {
|
||||
info = self
|
||||
.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, opts)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
pub async fn put_object_multipart_stream_from_readat(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
reader: ReaderImpl,
|
||||
size: i64,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let ret = optimal_part_info(size, opts.part_size)?;
|
||||
let (total_parts_count, part_size, lastpart_size) = ret;
|
||||
let mut opts = opts.clone();
|
||||
if opts.checksum.is_set() {
|
||||
opts.auto_checksum = opts.checksum.clone();
|
||||
}
|
||||
let with_checksum = self.trailing_header_support;
|
||||
let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
|
||||
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
|
||||
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub async fn put_object_multipart_stream_optional_checksum(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
mut reader: ReaderImpl,
|
||||
size: i64,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let mut opts = opts.clone();
|
||||
if opts.checksum.is_set() {
|
||||
opts.auto_checksum = opts.checksum.clone();
|
||||
opts.send_content_md5 = false;
|
||||
}
|
||||
|
||||
if !opts.send_content_md5 {
|
||||
add_auto_checksum_headers(&mut opts);
|
||||
}
|
||||
|
||||
let ret = optimal_part_info(size, opts.part_size)?;
|
||||
let (total_parts_count, mut part_size, lastpart_size) = ret;
|
||||
let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
|
||||
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
|
||||
|
||||
let mut custom_header = opts.header().clone();
|
||||
|
||||
let mut total_uploaded_size: i64 = 0;
|
||||
|
||||
let mut parts_info = HashMap::<i64, ObjectPart>::new();
|
||||
let mut buf = Vec::<u8>::with_capacity(part_size as usize);
|
||||
|
||||
let mut md5_base64: String = "".to_string();
|
||||
for part_number in 1..=total_parts_count {
|
||||
if part_number == total_parts_count {
|
||||
part_size = lastpart_size;
|
||||
}
|
||||
|
||||
match &mut reader {
|
||||
ReaderImpl::Body(content_body) => {
|
||||
buf = content_body.to_vec();
|
||||
}
|
||||
ReaderImpl::ObjectBody(content_body) => {
|
||||
buf = content_body.read_all().await?;
|
||||
}
|
||||
}
|
||||
let length = buf.len();
|
||||
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = self.md5_hasher.lock().unwrap();
|
||||
let md5_hash = md5_hasher.as_mut().expect("err");
|
||||
md5_hash.reset();
|
||||
md5_hash.write(&buf[..length]);
|
||||
md5_base64 = base64_encode(md5_hash.sum().as_bytes());
|
||||
} else {
|
||||
let csum;
|
||||
{
|
||||
let mut crc = opts.auto_checksum.hasher()?;
|
||||
crc.reset();
|
||||
crc.write(&buf[..length]);
|
||||
csum = crc.sum();
|
||||
}
|
||||
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key_capitalized().as_bytes()) {
|
||||
custom_header.insert(header_name, HeaderValue::from_str(&base64_encode(csum.as_bytes())).expect("err"));
|
||||
} else {
|
||||
warn!("Invalid header name: {}", opts.auto_checksum.key_capitalized());
|
||||
}
|
||||
}
|
||||
|
||||
let hooked = ReaderImpl::Body(Bytes::from(buf)); //newHook(BufferReader::new(buf), opts.progress);
|
||||
let mut p = UploadPartParams {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
upload_id: upload_id.clone(),
|
||||
reader: hooked,
|
||||
part_number,
|
||||
md5_base64: md5_base64.clone(),
|
||||
size: part_size,
|
||||
//sse: opts.server_side_encryption,
|
||||
stream_sha256: !opts.disable_content_sha256,
|
||||
custom_header: custom_header.clone(),
|
||||
sha256_hex: "".to_string(),
|
||||
trailer: HeaderMap::new(),
|
||||
};
|
||||
let obj_part = self.upload_part(&mut p).await?;
|
||||
|
||||
parts_info.entry(part_number).or_insert(obj_part);
|
||||
|
||||
total_uploaded_size += part_size as i64;
|
||||
}
|
||||
|
||||
if size > 0 && total_uploaded_size != size {
|
||||
return Err(std::io::Error::other(err_unexpected_eof(
|
||||
total_uploaded_size,
|
||||
size,
|
||||
bucket_name,
|
||||
object_name,
|
||||
)));
|
||||
}
|
||||
|
||||
let mut compl_multipart_upload = CompleteMultipartUpload::default();
|
||||
|
||||
let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.len());
|
||||
let part_number = total_parts_count;
|
||||
for i in 1..part_number {
|
||||
let part = parts_info[&i].clone();
|
||||
|
||||
all_parts.push(part.clone());
|
||||
compl_multipart_upload.parts.push(CompletePart {
|
||||
etag: part.etag,
|
||||
part_num: part.part_num,
|
||||
checksum_crc32: part.checksum_crc32,
|
||||
checksum_crc32c: part.checksum_crc32c,
|
||||
checksum_sha1: part.checksum_sha1,
|
||||
checksum_sha256: part.checksum_sha256,
|
||||
checksum_crc64nvme: part.checksum_crc64nvme,
|
||||
});
|
||||
}
|
||||
|
||||
compl_multipart_upload.parts.sort();
|
||||
|
||||
let mut opts = PutObjectOptions {
|
||||
//server_side_encryption: opts.server_side_encryption,
|
||||
auto_checksum: opts.auto_checksum,
|
||||
..Default::default()
|
||||
};
|
||||
apply_auto_checksum(&mut opts, &mut all_parts);
|
||||
let mut upload_info = self
|
||||
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
|
||||
.await?;
|
||||
|
||||
upload_info.size = total_uploaded_size;
|
||||
Ok(upload_info)
|
||||
}
|
||||
|
||||
pub async fn put_object_multipart_stream_parallel(
|
||||
self: Arc<Self>,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
mut reader: ReaderImpl, /*GetObjectReader*/
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let mut opts = opts.clone();
|
||||
if opts.checksum.is_set() {
|
||||
opts.send_content_md5 = false;
|
||||
opts.auto_checksum = opts.checksum.clone();
|
||||
}
|
||||
if !opts.send_content_md5 {
|
||||
add_auto_checksum_headers(&mut opts);
|
||||
}
|
||||
|
||||
let ret = optimal_part_info(-1, opts.part_size)?;
|
||||
let (total_parts_count, part_size, _) = ret;
|
||||
|
||||
let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
|
||||
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
|
||||
|
||||
let mut total_uploaded_size: i64 = 0;
|
||||
let parts_info = Arc::new(RwLock::new(HashMap::<i64, ObjectPart>::new()));
|
||||
|
||||
let n_buffers = opts.num_threads;
|
||||
let (bufs_tx, mut bufs_rx) = mpsc::channel(n_buffers as usize);
|
||||
//let all = Vec::<u8>::with_capacity(n_buffers as usize * part_size as usize);
|
||||
for i in 0..n_buffers {
|
||||
//bufs_tx.send(&all[i * part_size..i * part_size + part_size]);
|
||||
bufs_tx.send(Vec::<u8>::with_capacity(part_size as usize));
|
||||
}
|
||||
|
||||
let mut futures = Vec::with_capacity(total_parts_count as usize);
|
||||
let (err_tx, mut err_rx) = mpsc::channel(opts.num_threads as usize);
|
||||
let cancel_token = CancellationToken::new();
|
||||
|
||||
//reader = newHook(reader, opts.progress);
|
||||
|
||||
for part_number in 1..=total_parts_count {
|
||||
let mut buf = Vec::<u8>::new();
|
||||
select! {
|
||||
buf = bufs_rx.recv() => {}
|
||||
err = err_rx.recv() => {
|
||||
//cancel_token.cancel();
|
||||
//wg.Wait()
|
||||
return Err(err.expect("err"));
|
||||
}
|
||||
else => (),
|
||||
}
|
||||
|
||||
if buf.len() != part_size as usize {
|
||||
return Err(std::io::Error::other(format!(
|
||||
"read buffer < {} than expected partSize: {}",
|
||||
buf.len(),
|
||||
part_size
|
||||
)));
|
||||
}
|
||||
|
||||
match &mut reader {
|
||||
ReaderImpl::Body(content_body) => {
|
||||
buf = content_body.to_vec();
|
||||
}
|
||||
ReaderImpl::ObjectBody(content_body) => {
|
||||
buf = content_body.read_all().await?;
|
||||
}
|
||||
}
|
||||
let length = buf.len();
|
||||
|
||||
let mut custom_header = HeaderMap::new();
|
||||
if !opts.send_content_md5 {
|
||||
let csum;
|
||||
{
|
||||
let mut crc = opts.auto_checksum.hasher()?;
|
||||
crc.reset();
|
||||
crc.write(&buf[..length]);
|
||||
csum = crc.sum();
|
||||
}
|
||||
if let Ok(header_name) = HeaderName::from_bytes(opts.auto_checksum.key().as_bytes()) {
|
||||
if let Ok(header_value) = HeaderValue::from_str(&base64_encode(csum.as_bytes())) {
|
||||
custom_header.insert(header_name, header_value);
|
||||
}
|
||||
} else {
|
||||
warn!("Invalid header name: {}", opts.auto_checksum.key());
|
||||
}
|
||||
}
|
||||
|
||||
let clone_bufs_tx = bufs_tx.clone();
|
||||
let clone_parts_info = parts_info.clone();
|
||||
let clone_upload_id = upload_id.clone();
|
||||
let clone_self = self.clone();
|
||||
futures.push(async move {
|
||||
let mut md5_base64: String = "".to_string();
|
||||
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
|
||||
let md5_hash = md5_hasher.as_mut().expect("err");
|
||||
md5_hash.write(&buf[..length]);
|
||||
md5_base64 = base64_encode(md5_hash.sum().as_bytes());
|
||||
}
|
||||
|
||||
//defer wg.Done()
|
||||
let mut p = UploadPartParams {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
upload_id: clone_upload_id,
|
||||
reader: ReaderImpl::Body(Bytes::from(buf.clone())),
|
||||
part_number,
|
||||
md5_base64,
|
||||
size: length as i64,
|
||||
//sse: opts.server_side_encryption,
|
||||
stream_sha256: !opts.disable_content_sha256,
|
||||
custom_header,
|
||||
sha256_hex: "".to_string(),
|
||||
trailer: HeaderMap::new(),
|
||||
};
|
||||
let obj_part = clone_self.upload_part(&mut p).await.expect("err");
|
||||
|
||||
let mut clone_parts_info = clone_parts_info.write().unwrap();
|
||||
clone_parts_info.entry(part_number).or_insert(obj_part);
|
||||
|
||||
clone_bufs_tx.send(buf);
|
||||
});
|
||||
|
||||
total_uploaded_size += length as i64;
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
|
||||
select! {
|
||||
err = err_rx.recv() => {
|
||||
return Err(err.expect("err"));
|
||||
}
|
||||
else => (),
|
||||
}
|
||||
|
||||
let mut compl_multipart_upload = CompleteMultipartUpload::default();
|
||||
|
||||
let part_number: i64 = total_parts_count;
|
||||
let mut all_parts = Vec::<ObjectPart>::with_capacity(parts_info.read().unwrap().len());
|
||||
for i in 1..part_number {
|
||||
let part = parts_info.read().unwrap()[&i].clone();
|
||||
|
||||
all_parts.push(part.clone());
|
||||
compl_multipart_upload.parts.push(CompletePart {
|
||||
etag: part.etag,
|
||||
part_num: part.part_num,
|
||||
checksum_crc32: part.checksum_crc32,
|
||||
checksum_crc32c: part.checksum_crc32c,
|
||||
checksum_sha1: part.checksum_sha1,
|
||||
checksum_sha256: part.checksum_sha256,
|
||||
checksum_crc64nvme: part.checksum_crc64nvme,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
compl_multipart_upload.parts.sort();
|
||||
|
||||
let mut opts = PutObjectOptions {
|
||||
//server_side_encryption: opts.server_side_encryption,
|
||||
auto_checksum: opts.auto_checksum,
|
||||
..Default::default()
|
||||
};
|
||||
apply_auto_checksum(&mut opts, &mut all_parts);
|
||||
|
||||
let mut upload_info = self
|
||||
.complete_multipart_upload(bucket_name, object_name, &upload_id, compl_multipart_upload, &opts)
|
||||
.await?;
|
||||
|
||||
upload_info.size = total_uploaded_size;
|
||||
Ok(upload_info)
|
||||
}
|
||||
|
||||
pub async fn put_object_gcs(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
reader: ReaderImpl,
|
||||
size: i64,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let mut opts = opts.clone();
|
||||
if opts.checksum.is_set() {
|
||||
opts.send_content_md5 = false;
|
||||
}
|
||||
|
||||
let md5_base64: String = "".to_string();
|
||||
let progress_reader = reader; //newHook(reader, opts.progress);
|
||||
|
||||
self.put_object_do(bucket_name, object_name, progress_reader, &md5_base64, "", size, &opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn put_object_do(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
reader: ReaderImpl,
|
||||
md5_base64: &str,
|
||||
sha256_hex: &str,
|
||||
size: i64,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<UploadInfo, std::io::Error> {
|
||||
let custom_header = opts.header();
|
||||
|
||||
let mut req_metadata = RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
custom_header,
|
||||
content_body: reader,
|
||||
content_length: size,
|
||||
content_md5_base64: md5_base64.to_string(),
|
||||
content_sha256_hex: sha256_hex.to_string(),
|
||||
stream_sha256: !opts.disable_content_sha256,
|
||||
add_crc: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
pre_sign_url: Default::default(),
|
||||
query_values: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
expires: Default::default(),
|
||||
trailer: Default::default(),
|
||||
};
|
||||
let mut add_crc = false; //self.trailing_header_support && md5_base64 == "" && !s3utils.IsGoogleEndpoint(self.endpoint_url) && (opts.disable_content_sha256 || self.secure);
|
||||
let mut opts = opts.clone();
|
||||
if opts.checksum.is_set() {
|
||||
req_metadata.add_crc = opts.checksum;
|
||||
} else if add_crc {
|
||||
for (k, _) in opts.user_metadata {
|
||||
if k.to_lowercase().starts_with("x-amz-checksum-") {
|
||||
add_crc = false;
|
||||
}
|
||||
}
|
||||
if add_crc {
|
||||
opts.auto_checksum.set_default(ChecksumMode::ChecksumCRC32C);
|
||||
req_metadata.add_crc = opts.auto_checksum;
|
||||
}
|
||||
}
|
||||
|
||||
if opts.internal.source_version_id != "" {
|
||||
if !opts.internal.source_version_id.is_empty() {
|
||||
if let Err(err) = Uuid::parse_str(&opts.internal.source_version_id) {
|
||||
return Err(std::io::Error::other(err_invalid_argument(&err.to_string())));
|
||||
}
|
||||
}
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("versionId".to_string(), opts.internal.source_version_id);
|
||||
req_metadata.query_values = url_values;
|
||||
}
|
||||
|
||||
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
|
||||
|
||||
if resp.status() != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
|
||||
}
|
||||
|
||||
let (exp_time, rule_id) = if let Some(h_x_amz_expiration) = resp.headers().get(X_AMZ_EXPIRATION) {
|
||||
(
|
||||
OffsetDateTime::parse(h_x_amz_expiration.to_str().unwrap(), ISO8601_DATEFORMAT).unwrap(),
|
||||
"".to_string(),
|
||||
)
|
||||
} else {
|
||||
(OffsetDateTime::now_utc(), "".to_string())
|
||||
};
|
||||
let h = resp.headers();
|
||||
Ok(UploadInfo {
|
||||
bucket: bucket_name.to_string(),
|
||||
key: object_name.to_string(),
|
||||
etag: trim_etag(h.get("ETag").expect("err").to_str().expect("err")),
|
||||
version_id: if let Some(h_x_amz_version_id) = h.get(X_AMZ_VERSION_ID) {
|
||||
h_x_amz_version_id.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
size,
|
||||
expiration: exp_time,
|
||||
expiration_rule_id: rule_id,
|
||||
checksum_crc32: if let Some(h_checksum_crc32) = h.get(ChecksumMode::ChecksumCRC32.key()) {
|
||||
h_checksum_crc32.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_crc32c: if let Some(h_checksum_crc32c) = h.get(ChecksumMode::ChecksumCRC32C.key()) {
|
||||
h_checksum_crc32c.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_sha1: if let Some(h_checksum_sha1) = h.get(ChecksumMode::ChecksumSHA1.key()) {
|
||||
h_checksum_sha1.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_sha256: if let Some(h_checksum_sha256) = h.get(ChecksumMode::ChecksumSHA256.key()) {
|
||||
h_checksum_sha256.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
checksum_crc64nvme: if let Some(h_checksum_crc64nvme) = h.get(ChecksumMode::ChecksumCRC64NVME.key()) {
|
||||
h_checksum_crc64nvme.to_str().expect("err").to_string()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,492 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue, Method, StatusCode};
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::ReplicationStatus;
|
||||
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
|
||||
use std::fmt::Display;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use crate::{
|
||||
disk::DiskAPI,
|
||||
store_api::{GetObjectReader, ObjectInfo, StorageAPI},
|
||||
};
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
use rustfs_utils::hasher::{sum_md5_base64, sum_sha256_hex};
|
||||
|
||||
pub struct RemoveBucketOptions {
|
||||
_forced_elete: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedRemoveOptions {
|
||||
replication_delete_marker: bool,
|
||||
replication_status: ReplicationStatus,
|
||||
replication_mtime: OffsetDateTime,
|
||||
replication_request: bool,
|
||||
replication_validity_check: bool,
|
||||
}
|
||||
|
||||
impl Default for AdvancedRemoveOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
replication_delete_marker: false,
|
||||
replication_status: ReplicationStatus::from_static(ReplicationStatus::PENDING),
|
||||
replication_mtime: OffsetDateTime::now_utc(),
|
||||
replication_request: false,
|
||||
replication_validity_check: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct RemoveObjectOptions {
|
||||
pub force_delete: bool,
|
||||
pub governance_bypass: bool,
|
||||
pub version_id: String,
|
||||
pub internal: AdvancedRemoveOptions,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn remove_bucket_with_options(&self, bucket_name: &str, opts: &RemoveBucketOptions) -> Result<(), std::io::Error> {
|
||||
let headers = HeaderMap::new();
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
Method::DELETE,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
custom_header: headers,
|
||||
object_name: "".to_string(),
|
||||
query_values: Default::default(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
{
|
||||
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
|
||||
bucket_loc_cache.delete(bucket_name);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn remove_bucket(&self, bucket_name: &str) -> Result<(), std::io::Error> {
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::DELETE,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
custom_header: Default::default(),
|
||||
object_name: "".to_string(),
|
||||
query_values: Default::default(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
{
|
||||
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
|
||||
bucket_loc_cache.delete(bucket_name);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn remove_object(&self, bucket_name: &str, object_name: &str, opts: RemoveObjectOptions) -> Option<std::io::Error> {
|
||||
let res = self.remove_object_inner(bucket_name, object_name, opts).await.expect("err");
|
||||
res.err
|
||||
}
|
||||
|
||||
pub async fn remove_object_inner(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
opts: RemoveObjectOptions,
|
||||
) -> Result<RemoveObjectResult, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
|
||||
if opts.version_id != "" {
|
||||
url_values.insert("versionId".to_string(), opts.version_id.clone());
|
||||
}
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
if opts.governance_bypass {
|
||||
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err")); //amzBypassGovernance
|
||||
}
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::DELETE,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: headers,
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(RemoveObjectResult {
|
||||
object_name: object_name.to_string(),
|
||||
object_version_id: opts.version_id,
|
||||
delete_marker: resp.headers().get("x-amz-delete-marker").expect("err") == "true",
|
||||
delete_marker_version_id: resp
|
||||
.headers()
|
||||
.get("x-amz-version-id")
|
||||
.expect("err")
|
||||
.to_str()
|
||||
.expect("err")
|
||||
.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn remove_objects_with_result(
|
||||
self: Arc<Self>,
|
||||
bucket_name: &str,
|
||||
objects_rx: Receiver<ObjectInfo>,
|
||||
opts: RemoveObjectsOptions,
|
||||
) -> Receiver<RemoveObjectResult> {
|
||||
let (result_tx, result_rx) = mpsc::channel(1);
|
||||
|
||||
let self_clone = Arc::clone(&self);
|
||||
let bucket_name_owned = bucket_name.to_string();
|
||||
|
||||
tokio::spawn(async move {
|
||||
self_clone
|
||||
.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
|
||||
.await;
|
||||
});
|
||||
result_rx
|
||||
}
|
||||
|
||||
pub async fn remove_objects(
|
||||
self: Arc<Self>,
|
||||
bucket_name: &str,
|
||||
objects_rx: Receiver<ObjectInfo>,
|
||||
opts: RemoveObjectsOptions,
|
||||
) -> Receiver<RemoveObjectError> {
|
||||
let (error_tx, error_rx) = mpsc::channel(1);
|
||||
|
||||
let self_clone = Arc::clone(&self);
|
||||
let bucket_name_owned = bucket_name.to_string();
|
||||
|
||||
let (result_tx, mut result_rx) = mpsc::channel(1);
|
||||
tokio::spawn(async move {
|
||||
self_clone
|
||||
.remove_objects_inner(&bucket_name_owned, objects_rx, &result_tx, opts)
|
||||
.await;
|
||||
});
|
||||
tokio::spawn(async move {
|
||||
while let Some(res) = result_rx.recv().await {
|
||||
if res.err.is_none() {
|
||||
continue;
|
||||
}
|
||||
error_tx
|
||||
.send(RemoveObjectError {
|
||||
object_name: res.object_name,
|
||||
version_id: res.object_version_id,
|
||||
err: res.err,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
});
|
||||
|
||||
error_rx
|
||||
}
|
||||
|
||||
pub async fn remove_objects_inner(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
mut objects_rx: Receiver<ObjectInfo>,
|
||||
result_tx: &Sender<RemoveObjectResult>,
|
||||
opts: RemoveObjectsOptions,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let max_entries = 1000;
|
||||
let mut finish = false;
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("delete".to_string(), "".to_string());
|
||||
|
||||
loop {
|
||||
if finish {
|
||||
break;
|
||||
}
|
||||
let mut count = 0;
|
||||
let mut batch = Vec::<ObjectInfo>::new();
|
||||
|
||||
while let Some(object) = objects_rx.recv().await {
|
||||
if has_invalid_xml_char(&object.name) {
|
||||
let remove_result = self
|
||||
.remove_object_inner(
|
||||
bucket_name,
|
||||
&object.name,
|
||||
RemoveObjectOptions {
|
||||
version_id: object.version_id.expect("err").to_string(),
|
||||
governance_bypass: opts.governance_bypass,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let remove_result_clone = remove_result.clone();
|
||||
if !remove_result.err.is_none() {
|
||||
match to_error_response(&remove_result.err.expect("err")).code {
|
||||
S3ErrorCode::InvalidArgument | S3ErrorCode::NoSuchVersion => {
|
||||
continue;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
result_tx.send(remove_result_clone.clone()).await;
|
||||
}
|
||||
|
||||
result_tx.send(remove_result_clone).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
batch.push(object);
|
||||
count += 1;
|
||||
if count >= max_entries {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
if count < max_entries {
|
||||
finish = true;
|
||||
}
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
if opts.governance_bypass {
|
||||
headers.insert(X_AMZ_BYPASS_GOVERNANCE_RETENTION, "true".parse().expect("err"));
|
||||
}
|
||||
|
||||
let remove_bytes = generate_remove_multi_objects_request(&batch);
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::POST,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values.clone(),
|
||||
content_body: ReaderImpl::Body(Bytes::from(remove_bytes.clone())),
|
||||
content_length: remove_bytes.len() as i64,
|
||||
content_md5_base64: sum_md5_base64(&remove_bytes),
|
||||
content_sha256_hex: sum_sha256_hex(&remove_bytes),
|
||||
custom_header: headers,
|
||||
object_name: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let body_bytes: Vec<u8> = resp.body().bytes().expect("err").to_vec();
|
||||
process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_bytes)), result_tx.clone());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn remove_incomplete_upload(&self, bucket_name: &str, object_name: &str) -> Result<(), std::io::Error> {
|
||||
let upload_ids = self.find_upload_ids(bucket_name, object_name)?;
|
||||
for upload_id in upload_ids {
|
||||
self.abort_multipart_upload(bucket_name, object_name, &upload_id).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn abort_multipart_upload(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
upload_id: &str,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("uploadId".to_string(), upload_id.to_string());
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::DELETE,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
//if resp.is_some() {
|
||||
if resp.status() != StatusCode::NO_CONTENT {
|
||||
let error_response: ErrorResponse;
|
||||
match resp.status() {
|
||||
StatusCode::NOT_FOUND => {
|
||||
error_response = ErrorResponse {
|
||||
code: S3ErrorCode::NoSuchUpload,
|
||||
message: "The specified multipart upload does not exist.".to_string(),
|
||||
bucket_name: bucket_name.to_string(),
|
||||
key: object_name.to_string(),
|
||||
request_id: resp
|
||||
.headers()
|
||||
.get("x-amz-request-id")
|
||||
.expect("err")
|
||||
.to_str()
|
||||
.expect("err")
|
||||
.to_string(),
|
||||
host_id: resp
|
||||
.headers()
|
||||
.get("x-amz-id-2")
|
||||
.expect("err")
|
||||
.to_str()
|
||||
.expect("err")
|
||||
.to_string(),
|
||||
region: resp
|
||||
.headers()
|
||||
.get("x-amz-bucket-region")
|
||||
.expect("err")
|
||||
.to_str()
|
||||
.expect("err")
|
||||
.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
_ => {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(resp, vec![], bucket_name, object_name)));
|
||||
}
|
||||
}
|
||||
return Err(std::io::Error::other(error_response));
|
||||
}
|
||||
//}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct RemoveObjectError {
|
||||
object_name: String,
|
||||
#[allow(dead_code)]
|
||||
version_id: String,
|
||||
err: Option<std::io::Error>,
|
||||
}
|
||||
|
||||
impl Display for RemoveObjectError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
if self.err.is_none() {
|
||||
return write!(f, "unexpected remove object error result");
|
||||
}
|
||||
write!(f, "{}", self.err.as_ref().expect("err").to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct RemoveObjectResult {
|
||||
pub object_name: String,
|
||||
pub object_version_id: String,
|
||||
pub delete_marker: bool,
|
||||
pub delete_marker_version_id: String,
|
||||
pub err: Option<std::io::Error>,
|
||||
}
|
||||
|
||||
impl Clone for RemoveObjectResult {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
object_name: self.object_name.clone(),
|
||||
object_version_id: self.object_version_id.clone(),
|
||||
delete_marker: self.delete_marker,
|
||||
delete_marker_version_id: self.delete_marker_version_id.clone(),
|
||||
err: None, //err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RemoveObjectsOptions {
|
||||
pub governance_bypass: bool,
|
||||
}
|
||||
|
||||
pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub fn process_remove_multi_objects_response(body: ReaderImpl, result_tx: Sender<RemoveObjectResult>) {
|
||||
todo!();
|
||||
}
|
||||
|
||||
fn has_invalid_xml_char(str: &str) -> bool {
|
||||
false
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use s3s::dto::Owner;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::checksum::ChecksumMode;
|
||||
use crate::client::transition_api::ObjectMultipartInfo;
|
||||
use rustfs_utils::crypto::base64_decode;
|
||||
|
||||
use super::transition_api;
|
||||
|
||||
pub struct ListAllMyBucketsResult {
|
||||
pub owner: Owner,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct CommonPrefix {
|
||||
pub prefix: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "PascalCase")]
|
||||
pub struct ListBucketV2Result {
|
||||
pub common_prefixes: Vec<CommonPrefix>,
|
||||
pub contents: Vec<transition_api::ObjectInfo>,
|
||||
pub delimiter: String,
|
||||
pub encoding_type: String,
|
||||
pub is_truncated: bool,
|
||||
pub max_keys: i64,
|
||||
pub name: String,
|
||||
pub next_continuation_token: String,
|
||||
pub continuation_token: String,
|
||||
pub prefix: String,
|
||||
pub fetch_owner: String,
|
||||
pub start_after: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub struct Version {
|
||||
etag: String,
|
||||
is_latest: bool,
|
||||
key: String,
|
||||
last_modified: OffsetDateTime,
|
||||
owner: Owner,
|
||||
size: i64,
|
||||
storage_class: String,
|
||||
version_id: String,
|
||||
user_metadata: HashMap<String, String>,
|
||||
user_tags: HashMap<String, String>,
|
||||
is_delete_marker: bool,
|
||||
}
|
||||
|
||||
pub struct ListVersionsResult {
|
||||
versions: Vec<Version>,
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
name: String,
|
||||
prefix: String,
|
||||
delimiter: String,
|
||||
max_keys: i64,
|
||||
encoding_type: String,
|
||||
is_truncated: bool,
|
||||
key_marker: String,
|
||||
version_id_marker: String,
|
||||
next_key_marker: String,
|
||||
next_version_id_marker: String,
|
||||
}
|
||||
|
||||
pub struct ListBucketResult {
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
contents: Vec<transition_api::ObjectInfo>,
|
||||
delimiter: String,
|
||||
encoding_type: String,
|
||||
is_truncated: bool,
|
||||
marker: String,
|
||||
max_keys: i64,
|
||||
name: String,
|
||||
next_marker: String,
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
pub struct ListMultipartUploadsResult {
|
||||
bucket: String,
|
||||
key_marker: String,
|
||||
upload_id_marker: String,
|
||||
next_key_marker: String,
|
||||
next_upload_id_marker: String,
|
||||
encoding_type: String,
|
||||
max_uploads: i64,
|
||||
is_truncated: bool,
|
||||
uploads: Vec<ObjectMultipartInfo>,
|
||||
prefix: String,
|
||||
delimiter: String,
|
||||
common_prefixes: Vec<CommonPrefix>,
|
||||
}
|
||||
|
||||
pub struct Initiator {
|
||||
id: String,
|
||||
display_name: String,
|
||||
}
|
||||
|
||||
pub struct CopyObjectResult {
|
||||
pub etag: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ObjectPart {
|
||||
pub etag: String,
|
||||
pub part_num: i64,
|
||||
pub last_modified: OffsetDateTime,
|
||||
pub size: i64,
|
||||
pub checksum_crc32: String,
|
||||
pub checksum_crc32c: String,
|
||||
pub checksum_sha1: String,
|
||||
pub checksum_sha256: String,
|
||||
pub checksum_crc64nvme: String,
|
||||
}
|
||||
|
||||
impl Default for ObjectPart {
|
||||
fn default() -> Self {
|
||||
ObjectPart {
|
||||
etag: Default::default(),
|
||||
part_num: 0,
|
||||
last_modified: OffsetDateTime::now_utc(),
|
||||
size: 0,
|
||||
checksum_crc32: Default::default(),
|
||||
checksum_crc32c: Default::default(),
|
||||
checksum_sha1: Default::default(),
|
||||
checksum_sha256: Default::default(),
|
||||
checksum_crc64nvme: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ObjectPart {
|
||||
fn checksum(&self, t: &ChecksumMode) -> String {
|
||||
match t {
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
return self.checksum_crc32c.clone();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC32 => {
|
||||
return self.checksum_crc32.clone();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA1 => {
|
||||
return self.checksum_sha1.clone();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA256 => {
|
||||
return self.checksum_sha256.clone();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC64NVME => {
|
||||
return self.checksum_crc64nvme.clone();
|
||||
}
|
||||
_ => {
|
||||
return "".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn checksum_raw(&self, t: &ChecksumMode) -> Result<Vec<u8>, std::io::Error> {
|
||||
let b = self.checksum(t);
|
||||
if b == "" {
|
||||
return Err(std::io::Error::other("no checksum set"));
|
||||
}
|
||||
let decoded = match base64_decode(b.as_bytes()) {
|
||||
Ok(b) => b,
|
||||
Err(e) => return Err(std::io::Error::other(e)),
|
||||
};
|
||||
if decoded.len() != t.raw_byte_len() as usize {
|
||||
return Err(std::io::Error::other("checksum length mismatch"));
|
||||
}
|
||||
Ok(decoded)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ListObjectPartsResult {
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub upload_id: String,
|
||||
pub initiator: Initiator,
|
||||
pub owner: Owner,
|
||||
pub storage_class: String,
|
||||
pub part_number_marker: i32,
|
||||
pub next_part_number_marker: i32,
|
||||
pub max_parts: i32,
|
||||
pub checksum_algorithm: String,
|
||||
pub checksum_type: String,
|
||||
pub is_truncated: bool,
|
||||
pub object_parts: Vec<ObjectPart>,
|
||||
pub encoding_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct InitiateMultipartUploadResult {
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub upload_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct CompleteMultipartUploadResult {
|
||||
pub location: String,
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub etag: String,
|
||||
pub checksum_crc32: String,
|
||||
pub checksum_crc32c: String,
|
||||
pub checksum_sha1: String,
|
||||
pub checksum_sha256: String,
|
||||
pub checksum_crc64nvme: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
|
||||
pub struct CompletePart {
|
||||
//api has
|
||||
pub etag: String,
|
||||
pub part_num: i64,
|
||||
pub checksum_crc32: String,
|
||||
pub checksum_crc32c: String,
|
||||
pub checksum_sha1: String,
|
||||
pub checksum_sha256: String,
|
||||
pub checksum_crc64nvme: String,
|
||||
}
|
||||
|
||||
impl CompletePart {
|
||||
fn checksum(&self, t: &ChecksumMode) -> String {
|
||||
match t {
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
return self.checksum_crc32c.clone();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC32 => {
|
||||
return self.checksum_crc32.clone();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA1 => {
|
||||
return self.checksum_sha1.clone();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA256 => {
|
||||
return self.checksum_sha256.clone();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC64NVME => {
|
||||
return self.checksum_crc64nvme.clone();
|
||||
}
|
||||
_ => {
|
||||
return "".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CopyObjectPartResult {
|
||||
pub etag: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize)]
|
||||
pub struct CompleteMultipartUpload {
|
||||
pub parts: Vec<CompletePart>,
|
||||
}
|
||||
|
||||
impl CompleteMultipartUpload {
|
||||
pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
|
||||
//let buf = serde_json::to_string(self)?;
|
||||
let buf = match serde_xml_rs::to_string(self) {
|
||||
Ok(buf) => buf,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e));
|
||||
}
|
||||
};
|
||||
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CreateBucketConfiguration {
|
||||
pub location: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct DeleteObject {
|
||||
//api has
|
||||
pub key: String,
|
||||
pub version_id: String,
|
||||
}
|
||||
|
||||
pub struct DeletedObject {
|
||||
//s3s has
|
||||
pub key: String,
|
||||
pub version_id: String,
|
||||
pub deletemarker: bool,
|
||||
pub deletemarker_version_id: String,
|
||||
}
|
||||
|
||||
pub struct NonDeletedObject {
|
||||
pub key: String,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
pub version_id: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct DeleteMultiObjects {
|
||||
pub quiet: bool,
|
||||
pub objects: Vec<DeleteObject>,
|
||||
}
|
||||
|
||||
impl DeleteMultiObjects {
|
||||
pub fn marshal_msg(&self) -> Result<String, std::io::Error> {
|
||||
//let buf = serde_json::to_string(self)?;
|
||||
let buf = match serde_xml_rs::to_string(self) {
|
||||
Ok(buf) => buf,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e));
|
||||
}
|
||||
};
|
||||
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DeleteMultiObjectsResult {
|
||||
pub deleted_objects: Vec<DeletedObject>,
|
||||
pub undeleted_objects: Vec<NonDeletedObject>,
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::Request;
|
||||
use hyper::StatusCode;
|
||||
use hyper::body::Incoming;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use tracing::warn;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::{http_resp_to_error_response, to_error_response},
|
||||
transition_api::{Document, TransitionClient},
|
||||
};
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
use rustfs_utils::hasher::{Hasher, Sha256};
|
||||
use s3s::Body;
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
use super::constants::UNSIGNED_PAYLOAD;
|
||||
use super::credentials::SignatureType;
|
||||
|
||||
pub struct BucketLocationCache {
|
||||
items: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl BucketLocationCache {
|
||||
pub fn new() -> BucketLocationCache {
|
||||
BucketLocationCache { items: HashMap::new() }
|
||||
}
|
||||
|
||||
pub fn get(&self, bucket_name: &str) -> Option<String> {
|
||||
self.items.get(bucket_name).map(|s| s.clone())
|
||||
}
|
||||
|
||||
pub fn set(&mut self, bucket_name: &str, location: &str) {
|
||||
self.items.insert(bucket_name.to_string(), location.to_string());
|
||||
}
|
||||
|
||||
pub fn delete(&mut self, bucket_name: &str) {
|
||||
self.items.remove(bucket_name);
|
||||
}
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn get_bucket_location(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
Ok(self.get_bucket_location_inner(bucket_name).await?)
|
||||
}
|
||||
|
||||
async fn get_bucket_location_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
if self.region != "" {
|
||||
return Ok(self.region.clone());
|
||||
}
|
||||
|
||||
let mut location;
|
||||
{
|
||||
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
|
||||
let ret = bucket_loc_cache.get(bucket_name);
|
||||
if let Some(location) = ret {
|
||||
return Ok(location);
|
||||
}
|
||||
//location = ret?;
|
||||
}
|
||||
|
||||
let req = self.get_bucket_location_request(bucket_name)?;
|
||||
|
||||
let mut resp = self.doit(req).await?;
|
||||
location = process_bucket_location_response(resp, bucket_name).await?;
|
||||
{
|
||||
let mut bucket_loc_cache = self.bucket_loc_cache.lock().unwrap();
|
||||
bucket_loc_cache.set(bucket_name, &location);
|
||||
}
|
||||
Ok(location)
|
||||
}
|
||||
|
||||
fn get_bucket_location_request(&self, bucket_name: &str) -> Result<http::Request<Body>, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("location".to_string(), "".to_string());
|
||||
|
||||
let mut target_url = self.endpoint_url.clone();
|
||||
let scheme = self.endpoint_url.scheme();
|
||||
let h = target_url.host().expect("host is none.");
|
||||
let default_port = if scheme == "https" { 443 } else { 80 };
|
||||
let p = target_url.port().unwrap_or(default_port);
|
||||
|
||||
let is_virtual_style = self.is_virtual_host_style_request(&target_url, bucket_name);
|
||||
|
||||
let mut url_str: String = "".to_string();
|
||||
|
||||
if is_virtual_style {
|
||||
url_str = scheme.to_string();
|
||||
url_str.push_str("://");
|
||||
url_str.push_str(bucket_name);
|
||||
url_str.push_str(".");
|
||||
url_str.push_str(target_url.host_str().expect("err"));
|
||||
url_str.push_str("/?location");
|
||||
} else {
|
||||
let mut path = bucket_name.to_string();
|
||||
path.push_str("/");
|
||||
target_url.set_path(&path);
|
||||
{
|
||||
let mut q = target_url.query_pairs_mut();
|
||||
for (k, v) in url_values {
|
||||
q.append_pair(&k, &urlencoding::encode(&v));
|
||||
}
|
||||
}
|
||||
url_str = target_url.to_string();
|
||||
}
|
||||
|
||||
let mut req_builder = Request::builder().method(http::Method::GET).uri(url_str);
|
||||
|
||||
self.set_user_agent(&mut req_builder);
|
||||
|
||||
let value;
|
||||
{
|
||||
let mut creds_provider = self.creds_provider.lock().unwrap();
|
||||
value = match creds_provider.get_with_context(Some(self.cred_context())) {
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let mut signer_type = value.signer_type.clone();
|
||||
let mut access_key_id = value.access_key_id;
|
||||
let mut secret_access_key = value.secret_access_key;
|
||||
let mut session_token = value.session_token;
|
||||
|
||||
if self.override_signer_type != SignatureType::SignatureDefault {
|
||||
signer_type = self.override_signer_type.clone();
|
||||
}
|
||||
|
||||
if value.signer_type == SignatureType::SignatureAnonymous {
|
||||
signer_type = SignatureType::SignatureAnonymous
|
||||
}
|
||||
|
||||
if signer_type == SignatureType::SignatureAnonymous {
|
||||
let req = match req_builder.body(Body::empty()) {
|
||||
Ok(req) => return Ok(req),
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
if signer_type == SignatureType::SignatureV2 {
|
||||
let req_builder = rustfs_signer::sign_v2(req_builder, 0, &access_key_id, &secret_access_key, is_virtual_style);
|
||||
let req = match req_builder.body(Body::empty()) {
|
||||
Ok(req) => return Ok(req),
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let mut content_sha256 = EMPTY_STRING_SHA256_HASH.to_string();
|
||||
if self.secure {
|
||||
content_sha256 = UNSIGNED_PAYLOAD.to_string();
|
||||
}
|
||||
|
||||
req_builder
|
||||
.headers_mut()
|
||||
.expect("err")
|
||||
.insert("X-Amz-Content-Sha256", content_sha256.parse().unwrap());
|
||||
let req_builder = rustfs_signer::sign_v4(req_builder, 0, &access_key_id, &secret_access_key, &session_token, "us-east-1");
|
||||
let req = match req_builder.body(Body::empty()) {
|
||||
Ok(req) => return Ok(req),
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err));
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_bucket_location_response(mut resp: http::Response<Body>, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
//if resp != nil {
|
||||
if resp.status() != StatusCode::OK {
|
||||
let err_resp = http_resp_to_error_response(resp, vec![], bucket_name, "");
|
||||
match err_resp.code {
|
||||
S3ErrorCode::NotImplemented => {
|
||||
match err_resp.server.as_str() {
|
||||
"AmazonSnowball" => {
|
||||
return Ok("snowball".to_string());
|
||||
}
|
||||
"cloudflare" => {
|
||||
return Ok("us-east-1".to_string());
|
||||
}
|
||||
_ => {
|
||||
return Err(std::io::Error::other(err_resp));
|
||||
}
|
||||
}
|
||||
}
|
||||
S3ErrorCode::AuthorizationHeaderMalformed |
|
||||
//S3ErrorCode::InvalidRegion |
|
||||
S3ErrorCode::AccessDenied => {
|
||||
if err_resp.region == "" {
|
||||
return Ok("us-east-1".to_string());
|
||||
}
|
||||
return Ok(err_resp.region);
|
||||
}
|
||||
_ => {
|
||||
return Err(std::io::Error::other(err_resp));
|
||||
}
|
||||
}
|
||||
}
|
||||
//}
|
||||
|
||||
let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
|
||||
let Document(location_constraint) = serde_xml_rs::from_str::<Document>(&String::from_utf8(b).unwrap()).unwrap();
|
||||
|
||||
let mut location = location_constraint;
|
||||
if location == "" {
|
||||
location = "us-east-1".to_string();
|
||||
}
|
||||
|
||||
if location == "EU" {
|
||||
location = "eu-west-1".to_string();
|
||||
}
|
||||
|
||||
Ok(location)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::{format_description::FormatItem, macros::format_description};
|
||||
|
||||
pub const ABS_MIN_PART_SIZE: i64 = 1024 * 1024 * 5;
|
||||
pub const MAX_PARTS_COUNT: i64 = 10000;
|
||||
pub const MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
pub const MIN_PART_SIZE: i64 = 1024 * 1024 * 16;
|
||||
|
||||
pub const MAX_SINGLE_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
|
||||
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
|
||||
|
||||
pub const TOTAL_WORKERS: i64 = 4;
|
||||
|
||||
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
||||
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
||||
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
||||
|
||||
pub const GET_OBJECT_ATTRIBUTES_TAGS: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
|
||||
pub const GET_OBJECT_ATTRIBUTES_MAX_PARTS: i64 = 1000;
|
||||
@@ -0,0 +1,163 @@
|
||||
#![allow(unused_imports)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::fmt::{Display, Formatter};
|
||||
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[derive(Debug, Default, Clone, Eq, PartialEq)]
|
||||
pub enum SignatureType {
|
||||
#[default]
|
||||
SignatureDefault,
|
||||
SignatureV4,
|
||||
SignatureV2,
|
||||
SignatureV4Streaming,
|
||||
SignatureAnonymous,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Credentials<P: Provider + Default> {
|
||||
creds: Value,
|
||||
force_refresh: bool,
|
||||
provider: P,
|
||||
}
|
||||
|
||||
impl<P: Provider + Default> Credentials<P> {
|
||||
pub fn new(provider: P) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
force_refresh: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get(&mut self) -> Result<Value, std::io::Error> {
|
||||
self.get_with_context(None)
|
||||
}
|
||||
|
||||
pub fn get_with_context(&mut self, mut cc: Option<CredContext>) -> Result<Value, std::io::Error> {
|
||||
if self.is_expired() {
|
||||
let creds = self.provider.retrieve_with_cred_context(cc.expect("err"));
|
||||
self.creds = creds;
|
||||
self.force_refresh = false;
|
||||
}
|
||||
|
||||
Ok(self.creds.clone())
|
||||
}
|
||||
|
||||
fn expire(&mut self) {
|
||||
self.force_refresh = true;
|
||||
}
|
||||
|
||||
pub fn is_expired(&self) -> bool {
|
||||
self.force_refresh || self.provider.is_expired()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Value {
|
||||
pub access_key_id: String,
|
||||
pub secret_access_key: String,
|
||||
pub session_token: String,
|
||||
pub expiration: OffsetDateTime,
|
||||
pub signer_type: SignatureType,
|
||||
}
|
||||
|
||||
impl Default for Value {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
access_key_id: "".to_string(),
|
||||
secret_access_key: "".to_string(),
|
||||
session_token: "".to_string(),
|
||||
expiration: OffsetDateTime::now_utc(),
|
||||
signer_type: SignatureType::SignatureDefault,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CredContext {
|
||||
//pub client: SendRequest,
|
||||
pub endpoint: String,
|
||||
}
|
||||
|
||||
pub trait Provider {
|
||||
fn retrieve(&self) -> Value;
|
||||
fn retrieve_with_cred_context(&self, _: CredContext) -> Value;
|
||||
fn is_expired(&self) -> bool;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Static(pub Value);
|
||||
|
||||
impl Provider for Static {
|
||||
fn retrieve(&self) -> Value {
|
||||
if self.0.access_key_id == "" || self.0.secret_access_key == "" {
|
||||
return Value {
|
||||
signer_type: SignatureType::SignatureAnonymous,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
self.0.clone()
|
||||
}
|
||||
|
||||
fn retrieve_with_cred_context(&self, _: CredContext) -> Value {
|
||||
self.retrieve()
|
||||
}
|
||||
|
||||
fn is_expired(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct STSError {
|
||||
pub r#type: String,
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
pub struct ErrorResponse {
|
||||
pub sts_error: STSError,
|
||||
pub request_id: String,
|
||||
}
|
||||
|
||||
impl Display for ErrorResponse {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.error())
|
||||
}
|
||||
}
|
||||
|
||||
impl ErrorResponse {
|
||||
fn error(&self) -> String {
|
||||
if self.sts_error.message == "" {
|
||||
return format!("Error response code {}.", self.sts_error.code);
|
||||
}
|
||||
return self.sts_error.message.clone();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use crate::{
|
||||
disk::{
|
||||
error::{is_unformatted_disk, DiskError},
|
||||
format::{DistributionAlgoVersion, FormatV3},
|
||||
new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore,
|
||||
},
|
||||
store_api::{
|
||||
BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
|
||||
ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult,
|
||||
ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI,
|
||||
},
|
||||
credentials::{Credentials, SignatureType,},
|
||||
api_put_object_multipart::UploadPartParams,
|
||||
};
|
||||
|
||||
use http::HeaderMap;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
use tracing::{error, info};
|
||||
use url::Url;
|
||||
|
||||
struct HookReader {
|
||||
source: GetObjectReader,
|
||||
hook: GetObjectReader,
|
||||
}
|
||||
|
||||
impl HookReader {
|
||||
pub fn new(source: GetObjectReader, hook: GetObjectReader) -> HookReader {
|
||||
HookReader {
|
||||
source,
|
||||
hook,
|
||||
}
|
||||
}
|
||||
|
||||
fn seek(&self, offset: i64, whence: i64) -> Result<i64> {
|
||||
todo!();
|
||||
}
|
||||
|
||||
fn read(&self, b: &[u8]) -> Result<i64> {
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod admin_handler_utils;
|
||||
pub mod api_bucket_policy;
|
||||
pub mod api_error_response;
|
||||
pub mod api_get_object;
|
||||
pub mod api_get_options;
|
||||
pub mod api_list;
|
||||
pub mod api_put_object;
|
||||
pub mod api_put_object_common;
|
||||
pub mod api_put_object_multipart;
|
||||
pub mod api_put_object_streaming;
|
||||
pub mod api_remove;
|
||||
pub mod api_s3_datatypes;
|
||||
pub mod bucket_cache;
|
||||
pub mod constants;
|
||||
pub mod credentials;
|
||||
pub mod object_api_utils;
|
||||
pub mod object_handlers_common;
|
||||
pub mod transition_api;
|
||||
pub mod utils;
|
||||
@@ -0,0 +1,157 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::HeaderMap;
|
||||
use std::io::Cursor;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use tokio::io::BufReader;
|
||||
|
||||
use crate::error::ErrorResponse;
|
||||
use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions};
|
||||
use rustfs_filemeta::fileinfo::ObjectPartInfo;
|
||||
use rustfs_rio::HashReader;
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
//#[derive(Clone)]
|
||||
pub struct PutObjReader {
|
||||
pub reader: HashReader,
|
||||
pub raw_reader: HashReader,
|
||||
//pub sealMD5Fn: SealMD5CurrFn,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjReader {
|
||||
pub fn new(raw_reader: HashReader) -> Self {
|
||||
todo!();
|
||||
}
|
||||
|
||||
fn md5_current_hex_string(&self) -> String {
|
||||
todo!();
|
||||
}
|
||||
|
||||
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
||||
self.reader = enc_reader;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub type ObjReaderFn = Arc<dyn Fn(BufReader<Cursor<Vec<u8>>>, HeaderMap) -> GetObjectReader + 'static>;
|
||||
|
||||
fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRangeSpec> {
|
||||
if oi.size == 0 || oi.parts.len() == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut start: i64 = 0;
|
||||
let mut end: i64 = -1;
|
||||
let mut i = 0;
|
||||
while i < oi.parts.len() && i < part_number {
|
||||
start = end + 1;
|
||||
end = start + oi.parts[i].actual_size as i64 - 1;
|
||||
i += 1;
|
||||
}
|
||||
|
||||
Some(HTTPRangeSpec {
|
||||
start,
|
||||
end,
|
||||
is_suffix_length: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
|
||||
let mut skip_length: i64 = 0;
|
||||
let mut cumulative_actual_size: i64 = 0;
|
||||
let mut first_part_idx: i64 = 0;
|
||||
let mut compressed_offset: i64 = 0;
|
||||
let mut part_skip: i64 = 0;
|
||||
let mut decrypt_skip: i64 = 0;
|
||||
let mut seq_num: u64 = 0;
|
||||
for (i, part) in oi.parts.iter().enumerate() {
|
||||
cumulative_actual_size += part.actual_size as i64;
|
||||
if cumulative_actual_size <= offset {
|
||||
compressed_offset += part.size as i64;
|
||||
} else {
|
||||
first_part_idx = i as i64;
|
||||
skip_length = cumulative_actual_size - part.actual_size as i64;
|
||||
break;
|
||||
}
|
||||
}
|
||||
skip_length = offset - skip_length;
|
||||
|
||||
let parts: &[ObjectPartInfo] = &oi.parts;
|
||||
if skip_length > 0
|
||||
&& parts.len() > first_part_idx as usize
|
||||
&& parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0
|
||||
{
|
||||
todo!();
|
||||
}
|
||||
|
||||
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
|
||||
}
|
||||
|
||||
pub fn new_getobjectreader(
|
||||
rs: HTTPRangeSpec,
|
||||
oi: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
h: &HeaderMap,
|
||||
) -> Result<(ObjReaderFn, i64, i64), ErrorResponse> {
|
||||
//let (_, mut is_encrypted) = crypto.is_encrypted(oi.user_defined)?;
|
||||
let mut is_encrypted = false;
|
||||
let is_compressed = false; //oi.is_compressed_ok();
|
||||
|
||||
let mut get_fn: ObjReaderFn;
|
||||
|
||||
let (off, length) = match rs.get_offset_length(oi.size) {
|
||||
Ok(x) => x,
|
||||
Err(err) => {
|
||||
return Err(ErrorResponse {
|
||||
code: S3ErrorCode::InvalidRange,
|
||||
message: err.to_string(),
|
||||
key: None,
|
||||
bucket_name: None,
|
||||
region: None,
|
||||
request_id: None,
|
||||
host_id: "".to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
get_fn = Arc::new(move |input_reader: BufReader<Cursor<Vec<u8>>>, _: HeaderMap| {
|
||||
//Box::pin({
|
||||
/*let r = GetObjectReader {
|
||||
object_info: oi.clone(),
|
||||
stream: StreamingBlob::new(HashReader::new(input_reader, 10, None, None, 10)),
|
||||
};
|
||||
r*/
|
||||
todo!();
|
||||
//})
|
||||
});
|
||||
|
||||
Ok((get_fn, off as i64, length as i64))
|
||||
}
|
||||
|
||||
pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
|
||||
if let Some(etag) = metadata.get("etag") {
|
||||
etag.clone()
|
||||
} else {
|
||||
metadata["md5Sum"].clone()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::StorageAPI;
|
||||
use crate::bucket::lifecycle::lifecycle;
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{ObjectOptions, ObjectToDelete};
|
||||
use rustfs_lock::local_locker::MAX_DELETE_LIST;
|
||||
|
||||
pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
|
||||
let mut remaining = to_del;
|
||||
loop {
|
||||
let mut to_del = remaining;
|
||||
if to_del.len() > MAX_DELETE_LIST {
|
||||
remaining = &to_del[MAX_DELETE_LIST..];
|
||||
to_del = &to_del[..MAX_DELETE_LIST];
|
||||
} else {
|
||||
remaining = &[];
|
||||
}
|
||||
let vc = BucketVersioningSys::get(bucket).await.expect("err!");
|
||||
let _deleted_objs = api.delete_objects(
|
||||
bucket,
|
||||
to_del.to_vec(),
|
||||
ObjectOptions {
|
||||
//prefix_enabled_fn: vc.prefix_enabled(""),
|
||||
version_suspended: vc.suspended(),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,92 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use s3s::header::X_AMZ_STORAGE_CLASS;
|
||||
|
||||
lazy_static! {
|
||||
static ref SUPPORTED_QUERY_VALUES: HashMap<String, bool> = {
|
||||
let mut m = HashMap::new();
|
||||
m.insert("attributes".to_string(), true);
|
||||
m.insert("partNumber".to_string(), true);
|
||||
m.insert("versionId".to_string(), true);
|
||||
m.insert("response-cache-control".to_string(), true);
|
||||
m.insert("response-content-disposition".to_string(), true);
|
||||
m.insert("response-content-encoding".to_string(), true);
|
||||
m.insert("response-content-language".to_string(), true);
|
||||
m.insert("response-content-type".to_string(), true);
|
||||
m.insert("response-expires".to_string(), true);
|
||||
m
|
||||
};
|
||||
static ref SUPPORTED_HEADERS: HashMap<String, bool> = {
|
||||
let mut m = HashMap::new();
|
||||
m.insert("content-type".to_string(), true);
|
||||
m.insert("cache-control".to_string(), true);
|
||||
m.insert("content-encoding".to_string(), true);
|
||||
m.insert("content-disposition".to_string(), true);
|
||||
m.insert("content-language".to_string(), true);
|
||||
m.insert("x-amz-website-redirect-location".to_string(), true);
|
||||
m.insert("x-amz-object-lock-mode".to_string(), true);
|
||||
m.insert("x-amz-metadata-directive".to_string(), true);
|
||||
m.insert("x-amz-object-lock-retain-until-date".to_string(), true);
|
||||
m.insert("expires".to_string(), true);
|
||||
m.insert("x-amz-replication-status".to_string(), true);
|
||||
m
|
||||
};
|
||||
static ref SSE_HEADERS: HashMap<String, bool> = {
|
||||
let mut m = HashMap::new();
|
||||
m.insert("x-amz-server-side-encryption".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-context".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
|
||||
m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
|
||||
m
|
||||
};
|
||||
}
|
||||
|
||||
pub fn is_standard_query_value(qs_key: &str) -> bool {
|
||||
SUPPORTED_QUERY_VALUES[qs_key]
|
||||
}
|
||||
|
||||
pub fn is_storageclass_header(header_key: &str) -> bool {
|
||||
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
|
||||
}
|
||||
|
||||
pub fn is_standard_header(header_key: &str) -> bool {
|
||||
*SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
|
||||
}
|
||||
|
||||
pub fn is_sse_header(header_key: &str) -> bool {
|
||||
*SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
|
||||
}
|
||||
|
||||
pub fn is_amz_header(header_key: &str) -> bool {
|
||||
let key = header_key.to_lowercase();
|
||||
key.starts_with("x-amz-meta-")
|
||||
|| key.starts_with("x-amz-grant-")
|
||||
|| key == "x-amz-acl"
|
||||
|| is_sse_header(header_key)
|
||||
|| key.starts_with("x-amz-checksum-")
|
||||
}
|
||||
|
||||
pub fn is_rustfs_header(header_key: &str) -> bool {
|
||||
header_key.to_lowercase().starts_with("x-rustfs-")
|
||||
}
|
||||
|
||||
pub fn is_minio_header(header_key: &str) -> bool {
|
||||
header_key.to_lowercase().starts_with("x-minio-")
|
||||
}
|
||||
Reference in New Issue
Block a user