mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-19 19:16:17 +00:00
refactor: stabilize heal format recovery integration tests (#1984)
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
@@ -41,7 +41,11 @@ use tokio_util::io::ReaderStream;
|
||||
|
||||
impl TransitionClient {
|
||||
pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
|
||||
todo!();
|
||||
let _ = opts;
|
||||
Err(std::io::Error::new(
|
||||
IoErrorKind::Unsupported,
|
||||
format!("get_object is not implemented for {bucket_name}/{object_name}"),
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn get_object_inner(
|
||||
@@ -131,7 +135,18 @@ impl Object {
|
||||
}
|
||||
|
||||
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
|
||||
todo!()
|
||||
let _ = request.did_offset_change;
|
||||
let _ = request.offset;
|
||||
let _ = request.is_first_req;
|
||||
let _ = request.is_read_at;
|
||||
let _ = request.setting_object_info;
|
||||
let _ = request.is_read_op;
|
||||
let _ = request.been_read;
|
||||
let _ = request.buffer.len();
|
||||
Err(std::io::Error::new(
|
||||
IoErrorKind::Unsupported,
|
||||
"read-path for Object in api_get_object is not implemented",
|
||||
))
|
||||
}
|
||||
|
||||
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
|
||||
|
||||
@@ -34,10 +34,21 @@ use hyper::body::Bytes;
|
||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
use std::collections::HashMap;
|
||||
use std::io::ErrorKind;
|
||||
|
||||
impl TransitionClient {
|
||||
pub fn list_buckets(&self) -> Result<Vec<BucketInfo>, std::io::Error> {
|
||||
todo!();
|
||||
Err(std::io::Error::new(
|
||||
ErrorKind::Unsupported,
|
||||
credentials::ErrorResponse {
|
||||
sts_error: credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: "The list_buckets API is not implemented in this build.".to_string(),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn list_objects_v2_query(
|
||||
@@ -237,7 +248,17 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
Ok(listObjectVersionsOutput)*/
|
||||
todo!();
|
||||
Err(std::io::Error::new(
|
||||
ErrorKind::Unsupported,
|
||||
credentials::ErrorResponse {
|
||||
sts_error: credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: format!("list_object_versions_query is not implemented for bucket {bucket_name}"),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub fn list_objects_query(
|
||||
@@ -249,7 +270,17 @@ impl TransitionClient {
|
||||
max_keys: i64,
|
||||
headers: HeaderMap,
|
||||
) -> Result<ListBucketResult, std::io::Error> {
|
||||
todo!();
|
||||
Err(std::io::Error::new(
|
||||
ErrorKind::Unsupported,
|
||||
credentials::ErrorResponse {
|
||||
sts_error: credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: format!("list_objects_query is not implemented for bucket {bucket_name}"),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub fn list_multipart_uploads_query(
|
||||
@@ -261,7 +292,17 @@ impl TransitionClient {
|
||||
delimiter: &str,
|
||||
max_uploads: i64,
|
||||
) -> Result<ListMultipartUploadsResult, std::io::Error> {
|
||||
todo!();
|
||||
Err(std::io::Error::new(
|
||||
ErrorKind::Unsupported,
|
||||
credentials::ErrorResponse {
|
||||
sts_error: credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: format!("list_multipart_uploads_query is not implemented for bucket {bucket_name}"),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub fn list_object_parts(
|
||||
@@ -270,11 +311,33 @@ impl TransitionClient {
|
||||
object_name: &str,
|
||||
upload_id: &str,
|
||||
) -> Result<HashMap<i64, ObjectPart>, std::io::Error> {
|
||||
todo!();
|
||||
Err(std::io::Error::new(
|
||||
ErrorKind::Unsupported,
|
||||
credentials::ErrorResponse {
|
||||
sts_error: credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: format!(
|
||||
"list_object_parts is not implemented for bucket {bucket_name}, object {object_name}, upload_id {upload_id}"
|
||||
),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub fn find_upload_ids(&self, bucket_name: &str, object_name: &str) -> Result<Vec<String>, std::io::Error> {
|
||||
todo!();
|
||||
Err(std::io::Error::new(
|
||||
ErrorKind::Unsupported,
|
||||
credentials::ErrorResponse {
|
||||
sts_error: credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: format!("find_upload_ids is not implemented for bucket {bucket_name}, object {object_name}"),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn list_object_parts_query(
|
||||
@@ -285,7 +348,19 @@ impl TransitionClient {
|
||||
part_number_marker: i64,
|
||||
max_parts: i64,
|
||||
) -> Result<ListObjectPartsResult, std::io::Error> {
|
||||
todo!();
|
||||
Err(std::io::Error::new(
|
||||
ErrorKind::Unsupported,
|
||||
credentials::ErrorResponse {
|
||||
sts_error: credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: format!(
|
||||
"list_object_parts_query is not implemented for bucket {bucket_name}, object {object_name}, upload_id {upload_id}"
|
||||
),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
},
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,7 +379,44 @@ pub struct ListObjectsOptions {
|
||||
|
||||
impl ListObjectsOptions {
|
||||
pub fn set(&mut self, key: &str, value: &str) {
|
||||
todo!();
|
||||
match key {
|
||||
"prefix" => {
|
||||
self.prefix = value.to_string();
|
||||
}
|
||||
"start-after" => {
|
||||
self.start_after = value.to_string();
|
||||
}
|
||||
"max-keys" => {
|
||||
if let Ok(v) = value.parse::<i64>() {
|
||||
self.max_keys = v;
|
||||
}
|
||||
}
|
||||
"delimiter" => {
|
||||
// delimiter is currently kept in request only; this option structure does not persist it yet.
|
||||
}
|
||||
"reverse" | "versions" | "metadata" | "recursive" | "use-v1" => {
|
||||
if let Some(v) = value.strip_prefix("v").or_else(|| value.strip_prefix("V")) {
|
||||
let v = v.eq_ignore_ascii_case("true");
|
||||
match key {
|
||||
"reverse" => self.reverse_versions = v,
|
||||
"versions" => self.with_versions = v,
|
||||
"metadata" => self.with_metadata = v,
|
||||
"recursive" => self.recursive = v,
|
||||
_ => self.use_v1 = v,
|
||||
}
|
||||
} else {
|
||||
let v = value.eq_ignore_ascii_case("true");
|
||||
match key {
|
||||
"reverse" => self.reverse_versions = v,
|
||||
"versions" => self.with_versions = v,
|
||||
"metadata" => self.with_metadata = v,
|
||||
"recursive" => self.recursive = v,
|
||||
_ => self.use_v1 = v,
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -27,11 +27,11 @@ use crate::client::{
|
||||
};
|
||||
|
||||
pub fn is_object(reader: &ReaderImpl) -> bool {
|
||||
todo!();
|
||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||
}
|
||||
|
||||
pub fn is_read_at(reader: ReaderImpl) -> bool {
|
||||
todo!();
|
||||
matches!(reader, ReaderImpl::ObjectBody(_))
|
||||
}
|
||||
|
||||
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
|
||||
|
||||
@@ -97,11 +97,11 @@ impl TransitionClient {
|
||||
if opts.checksum.is_set() {
|
||||
opts.auto_checksum = opts.checksum.clone();
|
||||
}
|
||||
let with_checksum = self.trailing_header_support;
|
||||
let upload_id = self.new_upload_id(bucket_name, object_name, &opts).await?;
|
||||
|
||||
opts.user_metadata.remove("X-Amz-Checksum-Algorithm");
|
||||
|
||||
todo!();
|
||||
self.put_object_multipart_stream_optional_checksum(bucket_name, object_name, reader, size, &opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn put_object_multipart_stream_optional_checksum(
|
||||
|
||||
@@ -26,14 +26,19 @@ use rustfs_utils::HashAlgorithm;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::ReplicationStatus;
|
||||
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
|
||||
use serde::Deserialize;
|
||||
use std::fmt::Display;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
sync::Arc,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{
|
||||
api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
|
||||
api_s3_datatypes::{DeleteMultiObjects, DeleteObject},
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use crate::{
|
||||
@@ -46,6 +51,8 @@ pub struct RemoveBucketOptions {
|
||||
_forced_delete: bool,
|
||||
}
|
||||
|
||||
const DELETE_RESPONSE_PREVIEW_LEN: usize = 1024;
|
||||
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)]
|
||||
pub struct AdvancedRemoveOptions {
|
||||
@@ -354,7 +361,7 @@ impl TransitionClient {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_vec)), result_tx.clone());
|
||||
process_remove_multi_objects_response(ReaderImpl::Body(Bytes::from(body_vec)), &batch, result_tx.clone()).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -498,11 +505,239 @@ pub struct RemoveObjectsOptions {
|
||||
}
|
||||
|
||||
pub fn generate_remove_multi_objects_request(objects: &[ObjectInfo]) -> Vec<u8> {
|
||||
todo!();
|
||||
let escape_xml = |value: &str| -> String {
|
||||
value
|
||||
.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
.replace('\"', """)
|
||||
.replace('\'', "'")
|
||||
};
|
||||
|
||||
let request: DeleteMultiObjects = DeleteMultiObjects {
|
||||
quiet: false,
|
||||
objects: objects
|
||||
.iter()
|
||||
.map(|object| DeleteObject {
|
||||
key: object.name.clone(),
|
||||
version_id: object.version_id.map(|v| v.to_string()).unwrap_or_default(),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
|
||||
match request.marshal_msg() {
|
||||
Ok(body) => body.into_bytes(),
|
||||
Err(_) => {
|
||||
let mut body = String::new();
|
||||
body.push_str("<Delete><Quiet>false</Quiet>");
|
||||
for object in objects {
|
||||
body.push_str("<Object>");
|
||||
body.push_str("<Key>");
|
||||
body.push_str(&escape_xml(&object.name));
|
||||
body.push_str("</Key>");
|
||||
if object.version_id.is_some() {
|
||||
body.push_str("<VersionId>");
|
||||
body.push_str(&escape_xml(&object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default()));
|
||||
body.push_str("</VersionId>");
|
||||
}
|
||||
body.push_str("</Object>");
|
||||
}
|
||||
body.push_str("</Delete>");
|
||||
body.into_bytes()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn process_remove_multi_objects_response(body: ReaderImpl, result_tx: Sender<RemoveObjectResult>) {
|
||||
todo!();
|
||||
pub async fn process_remove_multi_objects_response(
|
||||
body: ReaderImpl,
|
||||
objects: &[ObjectInfo],
|
||||
result_tx: Sender<RemoveObjectResult>,
|
||||
) {
|
||||
let mut body_vec = Vec::new();
|
||||
match body {
|
||||
ReaderImpl::Body(content_body) => {
|
||||
body_vec = content_body.to_vec();
|
||||
}
|
||||
ReaderImpl::ObjectBody(mut object_body) => match object_body.read_all().await {
|
||||
Ok(content) => {
|
||||
body_vec = content;
|
||||
}
|
||||
Err(err) => {
|
||||
for object in objects {
|
||||
let version_id = object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default();
|
||||
let _ = result_tx
|
||||
.send(RemoveObjectResult {
|
||||
object_name: object.name.clone(),
|
||||
object_version_id: version_id,
|
||||
err: Some(std::io::Error::other(ErrorResponse {
|
||||
code: S3ErrorCode::Custom("ReadDeleteResponseFailed".into()),
|
||||
message: format!("read multi remove response failed: {err}"),
|
||||
bucket_name: object.bucket.clone(),
|
||||
key: object.name.clone(),
|
||||
resource: "".to_string(),
|
||||
request_id: "".to_string(),
|
||||
host_id: "".to_string(),
|
||||
region: "".to_string(),
|
||||
server: "".to_string(),
|
||||
status_code: StatusCode::OK,
|
||||
})),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
return;
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename = "DeleteResult")]
|
||||
struct Deleted {
|
||||
#[serde(rename = "Deleted", default)]
|
||||
deleted: Vec<DeleteResultDeleted>,
|
||||
#[serde(rename = "Error", default)]
|
||||
error: Vec<DeleteResultError>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DeleteResultDeleted {
|
||||
#[serde(rename = "Key")]
|
||||
key: String,
|
||||
#[serde(rename = "VersionId", default)]
|
||||
version_id: String,
|
||||
#[serde(rename = "DeleteMarker")]
|
||||
deletemarker: bool,
|
||||
#[serde(rename = "DeleteMarkerVersionId", default)]
|
||||
deletemarker_version_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DeleteResultError {
|
||||
#[serde(rename = "Key")]
|
||||
key: String,
|
||||
#[serde(rename = "VersionId", default)]
|
||||
version_id: String,
|
||||
#[serde(rename = "Code")]
|
||||
code: String,
|
||||
#[serde(rename = "Message")]
|
||||
message: String,
|
||||
}
|
||||
|
||||
let mut pending = HashSet::with_capacity(objects.len());
|
||||
for object in objects {
|
||||
pending.insert((object.name.clone(), object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default()));
|
||||
}
|
||||
|
||||
let body = String::from_utf8_lossy(&body_vec).into_owned();
|
||||
let parsed: Deleted = match quick_xml::de::from_str(&body) {
|
||||
Ok(parsed) => parsed,
|
||||
Err(err) => {
|
||||
for object in objects {
|
||||
let version_id = object.version_id.as_ref().map(|v| v.to_string()).unwrap_or_default();
|
||||
let _ = result_tx
|
||||
.send(RemoveObjectResult {
|
||||
object_name: object.name.clone(),
|
||||
object_version_id: version_id,
|
||||
err: Some(std::io::Error::other(ErrorResponse {
|
||||
code: S3ErrorCode::Custom("UnmarshalDeleteResponseFailed".into()),
|
||||
message: format!(
|
||||
"unmarshal multi remove response failed: {err}; response_body={}",
|
||||
body.chars().take(DELETE_RESPONSE_PREVIEW_LEN).collect::<String>()
|
||||
),
|
||||
bucket_name: object.bucket.clone(),
|
||||
key: object.name.clone(),
|
||||
resource: "".to_string(),
|
||||
request_id: "".to_string(),
|
||||
host_id: "".to_string(),
|
||||
region: "".to_string(),
|
||||
server: "".to_string(),
|
||||
status_code: StatusCode::OK,
|
||||
})),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
for deleted in parsed.deleted {
|
||||
if !pending.remove(&(deleted.key.clone(), deleted.version_id.clone())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let _ = result_tx
|
||||
.send(RemoveObjectResult {
|
||||
object_name: deleted.key,
|
||||
object_version_id: deleted.version_id,
|
||||
delete_marker: deleted.deletemarker,
|
||||
delete_marker_version_id: deleted.deletemarker_version_id,
|
||||
err: None,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
for removed in parsed.error {
|
||||
if !pending.remove(&(removed.key.clone(), removed.version_id.clone())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let _ = result_tx
|
||||
.send(RemoveObjectResult {
|
||||
object_name: removed.key.clone(),
|
||||
object_version_id: removed.version_id,
|
||||
err: Some(std::io::Error::other(ErrorResponse {
|
||||
code: S3ErrorCode::Custom(removed.code.into()),
|
||||
message: removed.message,
|
||||
bucket_name: "".to_string(),
|
||||
key: removed.key,
|
||||
resource: "".to_string(),
|
||||
request_id: "".to_string(),
|
||||
host_id: "".to_string(),
|
||||
region: "".to_string(),
|
||||
server: "".to_string(),
|
||||
status_code: StatusCode::OK,
|
||||
})),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
for (object_name, object_version_id) in pending {
|
||||
let bucket_name = objects
|
||||
.iter()
|
||||
.find(|object| {
|
||||
object.name == object_name && object.version_id.as_ref().map(|v| v.to_string()) == Some(object_version_id.clone())
|
||||
})
|
||||
.map(|o| o.bucket.clone())
|
||||
.unwrap_or_default();
|
||||
let object_name = object_name;
|
||||
let object_version_id = object_version_id;
|
||||
let error_message = format!(
|
||||
"remove response did not contain an entry for object {} with version {}",
|
||||
object_name, object_version_id
|
||||
);
|
||||
|
||||
let _ = result_tx
|
||||
.send(RemoveObjectResult {
|
||||
object_name: object_name.clone(),
|
||||
object_version_id: object_version_id.clone(),
|
||||
err: Some(std::io::Error::other(ErrorResponse {
|
||||
code: S3ErrorCode::Custom("UnmatchedDeleteResponseEntry".into()),
|
||||
message: error_message,
|
||||
bucket_name,
|
||||
key: object_name,
|
||||
resource: "".to_string(),
|
||||
request_id: "".to_string(),
|
||||
host_id: "".to_string(),
|
||||
region: "".to_string(),
|
||||
server: "".to_string(),
|
||||
status_code: StatusCode::OK,
|
||||
})),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn has_invalid_xml_char(str: &str) -> bool {
|
||||
|
||||
@@ -290,7 +290,50 @@ impl CompleteMultipartUpload {
|
||||
}
|
||||
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||
todo!();
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct WirePart {
|
||||
#[serde(rename = "ETag")]
|
||||
etag: String,
|
||||
#[serde(rename = "PartNumber")]
|
||||
part_num: i64,
|
||||
#[serde(rename = "ChecksumCRC32")]
|
||||
checksum_crc32: String,
|
||||
#[serde(rename = "ChecksumCRC32C")]
|
||||
checksum_crc32c: String,
|
||||
#[serde(rename = "ChecksumSHA1")]
|
||||
checksum_sha1: String,
|
||||
#[serde(rename = "ChecksumSHA256")]
|
||||
checksum_sha256: String,
|
||||
#[serde(rename = "ChecksumCRC64NVME")]
|
||||
checksum_crc64nvme: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename = "CompleteMultipartUpload")]
|
||||
struct WireCompleteMultipartUpload {
|
||||
#[serde(rename = "Part", default)]
|
||||
parts: Vec<WirePart>,
|
||||
}
|
||||
|
||||
let body = String::from_utf8_lossy(buf);
|
||||
let wire: WireCompleteMultipartUpload = quick_xml::de::from_str(&body)
|
||||
.map_err(|err| std::io::Error::other(format!("failed to parse CompleteMultipartUpload XML: {err}; body: {body}")))?;
|
||||
|
||||
Ok(Self {
|
||||
parts: wire
|
||||
.parts
|
||||
.into_iter()
|
||||
.map(|p| CompletePart {
|
||||
etag: p.etag,
|
||||
part_num: p.part_num,
|
||||
checksum_crc32: p.checksum_crc32,
|
||||
checksum_crc32c: p.checksum_crc32c,
|
||||
checksum_sha1: p.checksum_sha1,
|
||||
checksum_sha256: p.checksum_sha256,
|
||||
checksum_crc64nvme: p.checksum_crc64nvme,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -340,7 +383,37 @@ impl DeleteMultiObjects {
|
||||
}
|
||||
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
|
||||
todo!();
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct WireDeleteObject {
|
||||
#[serde(rename = "Key")]
|
||||
key: String,
|
||||
#[serde(rename = "VersionId")]
|
||||
version_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename = "Delete")]
|
||||
struct WireDeleteMultiObjects {
|
||||
#[serde(rename = "Quiet", default)]
|
||||
quiet: bool,
|
||||
#[serde(rename = "Object", default)]
|
||||
objects: Vec<WireDeleteObject>,
|
||||
}
|
||||
|
||||
let body = String::from_utf8_lossy(buf);
|
||||
let wire: WireDeleteMultiObjects = quick_xml::de::from_str(&body).map_err(|err| std::io::Error::other(err))?;
|
||||
|
||||
Ok(Self {
|
||||
quiet: wire.quiet,
|
||||
objects: wire
|
||||
.objects
|
||||
.into_iter()
|
||||
.map(|o| DeleteObject {
|
||||
key: o.key,
|
||||
version_id: o.version_id,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -248,8 +248,12 @@ impl ChecksumMode {
|
||||
}
|
||||
});
|
||||
let c = self.base();
|
||||
let crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
|
||||
let mut crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
|
||||
let mut h = self.hasher()?;
|
||||
for part in p.iter() {
|
||||
let part_checksum = part.checksum_raw(&c)?;
|
||||
crc_bytes.extend(part_checksum);
|
||||
}
|
||||
h.update(crc_bytes.as_ref());
|
||||
let hash = h.finalize();
|
||||
Ok(Checksum {
|
||||
@@ -260,7 +264,11 @@ impl ChecksumMode {
|
||||
}
|
||||
|
||||
pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
|
||||
todo!();
|
||||
if !self.can_merge_crc() {
|
||||
return Err(std::io::Error::other("cannot do full-object checksum"));
|
||||
}
|
||||
|
||||
self.composite_checksum(p)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,10 @@
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use quick_xml;
|
||||
use serde::de::Deserialize;
|
||||
use std::fmt::{Display, Formatter};
|
||||
use std::io::{Error, ErrorKind};
|
||||
|
||||
use time::OffsetDateTime;
|
||||
|
||||
@@ -154,10 +157,21 @@ impl ErrorResponse {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, std::io::Error> {
|
||||
todo!();
|
||||
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, Error>
|
||||
where
|
||||
for<'de> T: Deserialize<'de>,
|
||||
{
|
||||
match std::str::from_utf8(body) {
|
||||
Ok(xml_body) => quick_xml::de::from_str::<T>(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||
Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error> {
|
||||
todo!();
|
||||
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error>
|
||||
where
|
||||
for<'de> T: Deserialize<'de>,
|
||||
{
|
||||
let body = body_reader.to_vec();
|
||||
let parsed = xml_decoder(&body)?;
|
||||
Ok((body, parsed))
|
||||
}
|
||||
|
||||
@@ -33,18 +33,17 @@ use s3s::S3ErrorCode;
|
||||
//#[derive(Clone)]
|
||||
pub struct PutObjReader {
|
||||
pub reader: HashReader,
|
||||
pub raw_reader: HashReader,
|
||||
//pub sealMD5Fn: SealMD5CurrFn,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl PutObjReader {
|
||||
pub fn new(raw_reader: HashReader) -> Self {
|
||||
todo!();
|
||||
pub fn new(reader: HashReader) -> Self {
|
||||
Self { reader }
|
||||
}
|
||||
|
||||
fn md5_current_hex_string(&self) -> String {
|
||||
todo!();
|
||||
self.reader.checksum().map(|v| v.encoded).unwrap_or_default()
|
||||
}
|
||||
|
||||
fn with_encryption(&mut self, enc_reader: HashReader) -> Result<(), std::io::Error> {
|
||||
@@ -100,9 +99,11 @@ fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u
|
||||
let parts: &[ObjectPartInfo] = &oi.parts;
|
||||
if skip_length > 0
|
||||
&& parts.len() > first_part_idx as usize
|
||||
&& parts[first_part_idx as usize].index.as_ref().expect("err").len() > 0
|
||||
&& parts[first_part_idx as usize].index.as_ref().is_some_and(|idx| idx.len() > 0)
|
||||
{
|
||||
todo!();
|
||||
let _ = part_skip;
|
||||
let _ = decrypt_skip;
|
||||
let _ = seq_num;
|
||||
}
|
||||
|
||||
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
|
||||
|
||||
@@ -243,11 +243,13 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
|
||||
todo!();
|
||||
let mut endpoint = self.s3_accelerate_endpoint.lock().unwrap();
|
||||
*endpoint = accelerate_endpoint.to_string();
|
||||
}
|
||||
|
||||
fn set_s3_enable_dual_stack(&self, enabled: bool) {
|
||||
todo!();
|
||||
let mut dual_stack = self.s3_dual_stack_enabled.lock().unwrap();
|
||||
*dual_stack = enabled;
|
||||
}
|
||||
|
||||
pub fn hash_materials(
|
||||
@@ -255,7 +257,22 @@ impl TransitionClient {
|
||||
is_md5_requested: bool,
|
||||
is_sha256_requested: bool,
|
||||
) -> (HashMap<String, HashAlgorithm>, HashMap<String, Vec<u8>>) {
|
||||
todo!()
|
||||
// `hash_algos` declares which algorithms are active for this multipart upload.
|
||||
// `hash_sums` keeps the current part digest bytes and is refreshed on every loop.
|
||||
let mut hash_algos = HashMap::new();
|
||||
let mut hash_sums = HashMap::new();
|
||||
|
||||
if is_md5_requested {
|
||||
hash_algos.insert("md5".to_string(), HashAlgorithm::Md5);
|
||||
hash_sums.insert("md5".to_string(), vec![]);
|
||||
}
|
||||
|
||||
if is_sha256_requested {
|
||||
hash_algos.insert("sha256".to_string(), HashAlgorithm::SHA256);
|
||||
hash_sums.insert("sha256".to_string(), vec![]);
|
||||
}
|
||||
|
||||
(hash_algos, hash_sums)
|
||||
}
|
||||
|
||||
fn is_online(&self) -> bool {
|
||||
@@ -272,7 +289,7 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
fn health_check(hc_duration: Duration) {
|
||||
todo!();
|
||||
let _ = hc_duration;
|
||||
}
|
||||
|
||||
fn dump_http(&self, req: &http::Request<s3s::Body>, resp: &http::Response<Incoming>) -> Result<(), std::io::Error> {
|
||||
@@ -734,7 +751,19 @@ impl TransitionCore {
|
||||
) -> Result<CompletePart, std::io::Error> {
|
||||
//self.0.copy_object_part_do(src_bucket, src_object, dest_bucket, dest_object, upload_id,
|
||||
// part_id, start_offset, length, metadata)
|
||||
todo!();
|
||||
Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
crate::client::credentials::ErrorResponse {
|
||||
sts_error: crate::client::credentials::STSError {
|
||||
r#type: "".to_string(),
|
||||
code: "NotImplemented".to_string(),
|
||||
message: format!(
|
||||
"copy_object_part is not implemented for {src_bucket}/{src_object} -> {dest_bucket}/{dest_object}"
|
||||
),
|
||||
},
|
||||
request_id: "".to_string(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn put_object(
|
||||
|
||||
Reference in New Issue
Block a user