use crate::heal::heal_ops::HealSequence; use crate::store_utils::clean_metadata; use crate::{ disk::DiskStore, error::{Error, Result}, heal::heal_commands::HealOpts, utils::path::decode_dir_object, xhttp, }; use futures::StreamExt; use http::{HeaderMap, HeaderValue}; use madmin::heal_commands::HealResultItem; use rmp_serde::Serializer; use s3s::{dto::StreamingBlob, Body}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use time::OffsetDateTime; use uuid::Uuid; pub const ERASURE_ALGORITHM: &str = "rs-vandermonde"; pub const BLOCK_SIZE_V2: usize = 1048576; // 1M pub const RESERVED_METADATA_PREFIX: &str = "X-Rustfs-Internal-"; pub const RESERVED_METADATA_PREFIX_LOWER: &str = "X-Rustfs-Internal-"; pub const RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing"; pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov"; // #[derive(Debug, Clone)] #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] pub struct FileInfo { pub volume: String, pub name: String, pub version_id: Option, pub is_latest: bool, pub deleted: bool, // TransitionStatus // TransitionedObjName // TransitionTier // TransitionVersionID // ExpireRestored pub data_dir: Option, pub mod_time: Option, pub size: usize, // Mode pub metadata: Option>, pub parts: Vec, pub erasure: ErasureInfo, // MarkDeleted // ReplicationState pub data: Option>, pub num_versions: usize, pub successor_mod_time: Option, pub fresh: bool, pub idx: usize, // Checksum pub versioned: bool, } impl FileInfo { pub fn new(object: &str, data_blocks: usize, parity_blocks: usize) -> Self { let indexs = { let cardinality = data_blocks + parity_blocks; let mut nums = vec![0; cardinality]; let key_crc = crc32fast::hash(object.as_bytes()); let start = key_crc as usize % cardinality; for i in 1..=cardinality { nums[i - 1] = 1 + ((start + i) % cardinality); } nums }; Self { erasure: ErasureInfo { algorithm: String::from(ERASURE_ALGORITHM), data_blocks, parity_blocks, block_size: BLOCK_SIZE_V2, distribution: indexs, ..Default::default() }, ..Default::default() } } pub fn is_valid(&self) -> bool { if self.deleted { return true; } let data_blocks = self.erasure.data_blocks; let parity_blocks = self.erasure.parity_blocks; (data_blocks >= parity_blocks) && (data_blocks > 0) && (self.erasure.index > 0 && self.erasure.index <= data_blocks + parity_blocks && self.erasure.distribution.len() == (data_blocks + parity_blocks)) } pub fn is_remote(&self) -> bool { // TODO: when lifecycle false } pub fn get_etag(&self) -> Option { if let Some(meta) = &self.metadata { meta.get("etag").cloned() } else { None } } pub fn write_quorum(&self, quorum: usize) -> usize { if self.deleted { return quorum; } if self.erasure.data_blocks == self.erasure.parity_blocks { return self.erasure.data_blocks + 1; } self.erasure.data_blocks } pub fn marshal_msg(&self) -> Result> { let mut buf = Vec::new(); self.serialize(&mut Serializer::new(&mut buf))?; Ok(buf) } pub fn unmarshal(buf: &[u8]) -> Result { let t: FileInfo = rmp_serde::from_slice(buf)?; Ok(t) } pub fn add_object_part( &mut self, num: usize, e_tag: Option, part_size: usize, mod_time: Option, actual_size: usize, ) { let part = ObjectPartInfo { e_tag, number: num, size: part_size, mod_time, actual_size, }; for p in self.parts.iter_mut() { if p.number == num { *p = part; return; } } self.parts.push(part); self.parts.sort_by(|a, b| a.number.cmp(&b.number)); } pub fn to_object_info(&self, bucket: &str, object: &str, versioned: bool) -> ObjectInfo { let name = decode_dir_object(object); let mut version_id = self.version_id; if versioned && version_id.is_none() { version_id = Some(Uuid::nil()) } // etag let (content_type, content_encoding, etag) = { if let Some(ref meta) = self.metadata { let content_type = meta.get("content-type").cloned(); let content_encoding = meta.get("content-encoding").cloned(); let etag = meta.get("etag").cloned(); (content_type, content_encoding, etag) } else { (None, None, None) } }; // tags let user_tags = self .metadata .as_ref() .map(|m| { if let Some(tags) = m.get(xhttp::AMZ_OBJECT_TAGGING) { tags.clone() } else { "".to_string() } }) .unwrap_or_default(); let inlined = self.inline_data(); // TODO:expires // TODO:ReplicationState // TODO:TransitionedObject let metadata = self.metadata.clone().map(|mut v| { clean_metadata(&mut v); v }); ObjectInfo { bucket: bucket.to_string(), name, is_dir: object.starts_with('/'), parity_blocks: self.erasure.parity_blocks, data_blocks: self.erasure.data_blocks, version_id, delete_marker: self.deleted, mod_time: self.mod_time, size: self.size, parts: self.parts.clone(), is_latest: self.is_latest, user_tags, content_type, content_encoding, num_versions: self.num_versions, successor_mod_time: self.successor_mod_time, etag, inlined, user_defined: metadata, ..Default::default() } } // to_part_offset 取offset 所在的part index, 返回part index, offset pub fn to_part_offset(&self, offset: usize) -> Result<(usize, usize)> { if offset == 0 { return Ok((0, 0)); } let mut part_offset = offset; for (i, part) in self.parts.iter().enumerate() { let part_index = i; if part_offset < part.size { return Ok((part_index, part_offset)); } part_offset -= part.size } Err(Error::msg("part not found")) } pub fn set_healing(&mut self) { if self.metadata.is_none() { self.metadata = Some(HashMap::new()); } if let Some(metadata) = self.metadata.as_mut() { metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string()); } } pub fn set_inline_data(&mut self) { if let Some(meta) = self.metadata.as_mut() { meta.insert("x-rustfs-inline-data".to_owned(), "true".to_owned()); } else { let mut meta = HashMap::new(); meta.insert("x-rustfs-inline-data".to_owned(), "true".to_owned()); self.metadata = Some(meta); } } pub fn inline_data(&self) -> bool { if let Some(ref meta) = self.metadata { if let Some(val) = meta.get("x-rustfs-inline-data") { val.as_str() == "true" } else { false } } else { false } } } #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] pub struct ObjectPartInfo { pub e_tag: Option, pub number: usize, pub size: usize, pub actual_size: usize, // 源数据大小 pub mod_time: Option, // pub index: Option>, // pub checksums: Option>, } // impl Default for ObjectPartInfo { // fn default() -> Self { // Self { // number: Default::default(), // size: Default::default(), // mod_time: OffsetDateTime::UNIX_EPOCH, // actual_size: Default::default(), // } // } // } #[derive(Default, Serialize, Deserialize)] pub struct RawFileInfo { pub buf: Vec, } #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] // ErasureInfo holds erasure coding and bitrot related information. pub struct ErasureInfo { // Algorithm is the String representation of erasure-coding-algorithm pub algorithm: String, // DataBlocks is the number of data blocks for erasure-coding pub data_blocks: usize, // ParityBlocks is the number of parity blocks for erasure-coding pub parity_blocks: usize, // BlockSize is the size of one erasure-coded block pub block_size: usize, // Index is the index of the current disk pub index: usize, // Distribution is the distribution of the data and parity blocks pub distribution: Vec, // Checksums holds all bitrot checksums of all erasure encoded blocks pub checksums: Vec, } impl ErasureInfo { pub fn get_checksum_info(&self, part_number: usize) -> ChecksumInfo { for sum in &self.checksums { if sum.part_number == part_number { return sum.clone(); } } ChecksumInfo { algorithm: DEFAULT_BITROT_ALGO, ..Default::default() } } // 算出每个分片大小 pub fn shard_size(&self, data_size: usize) -> usize { data_size.div_ceil(self.data_blocks) } // returns final erasure size from original size. pub fn shard_file_size(&self, total_size: usize) -> usize { if total_size == 0 { return 0; } let num_shards = total_size / self.block_size; let last_block_size = total_size % self.block_size; let last_shard_size = last_block_size.div_ceil(self.data_blocks); num_shards * self.shard_size(self.block_size) + last_shard_size // // 因为写入的时候ec需要补全,所以最后一个长度应该也是一样的 // if last_block_size != 0 { // num_shards += 1 // } // num_shards * self.shard_size(self.block_size) } } #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] // ChecksumInfo - carries checksums of individual scattered parts per disk. pub struct ChecksumInfo { pub part_number: usize, pub algorithm: BitrotAlgorithm, pub hash: Vec, } pub const DEFAULT_BITROT_ALGO: BitrotAlgorithm = BitrotAlgorithm::HighwayHash256S; #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)] // BitrotAlgorithm specifies a algorithm used for bitrot protection. pub enum BitrotAlgorithm { // SHA256 represents the SHA-256 hash function SHA256, // HighwayHash256 represents the HighwayHash-256 hash function HighwayHash256, // HighwayHash256S represents the Streaming HighwayHash-256 hash function #[default] HighwayHash256S, // BLAKE2b512 represents the BLAKE2b-512 hash function BLAKE2b512, } #[derive(Debug, Default, Serialize, Deserialize)] pub struct MakeBucketOptions { pub lock_enabled: bool, pub versioning_enabled: bool, pub force_create: bool, // Create buckets even if they are already created. pub created_at: Option, // only for site replication pub no_lock: bool, } #[derive(Debug, Default, Clone)] pub enum SRBucketDeleteOp { #[default] NoOp, MarkDelete, Purge, } #[derive(Debug, Default, Clone)] pub struct DeleteBucketOptions { pub no_lock: bool, pub no_recreate: bool, pub force: bool, // Force deletion pub srdelete_op: SRBucketDeleteOp, } #[derive(Debug)] pub struct PutObjReader { pub stream: StreamingBlob, pub content_length: usize, } impl PutObjReader { pub fn new(stream: StreamingBlob, content_length: usize) -> Self { PutObjReader { stream, content_length } } pub fn from_vec(data: Vec) -> Self { let content_length = data.len(); PutObjReader { stream: Body::from(data).into(), content_length, } } } pub struct GetObjectReader { pub stream: StreamingBlob, pub object_info: ObjectInfo, } impl GetObjectReader { #[tracing::instrument(level = "debug", skip(reader))] pub fn new( reader: StreamingBlob, rs: Option, oi: &ObjectInfo, opts: &ObjectOptions, _h: &HeaderMap, ) -> Result<(Self, usize, usize)> { let mut rs = rs; if let Some(part_number) = opts.part_number { if rs.is_none() { rs = HTTPRangeSpec::from_object_info(oi, part_number); } } if let Some(rs) = rs { let (off, length) = rs.get_offset_length(oi.size)?; return Ok(( GetObjectReader { stream: reader, object_info: oi.clone(), }, off, length, )); } else { return Ok(( GetObjectReader { stream: reader, object_info: oi.clone(), }, 0, oi.size, )); } } pub async fn read_all(&mut self) -> Result> { let mut data = Vec::new(); while let Some(x) = self.stream.next().await { let buf = match x { Ok(res) => res, Err(e) => return Err(Error::msg(e.to_string())), }; data.extend_from_slice(buf.as_ref()); } Ok(data) } } #[derive(Debug)] pub struct HTTPRangeSpec { pub is_suffix_length: bool, pub start: usize, pub end: Option, } impl HTTPRangeSpec { pub fn from_object_info(oi: &ObjectInfo, part_number: usize) -> Option { if oi.size == 0 || oi.parts.is_empty() { return None; } let mut start = 0; let mut end = -1; for i in 0..oi.parts.len().min(part_number) { start = end + 1; end = start + oi.parts[i].size as i64 - 1 } Some(HTTPRangeSpec { is_suffix_length: false, start: start as usize, end: { if end < 0 { None } else { Some(end as usize) } }, }) } pub fn get_offset_length(&self, res_size: usize) -> Result<(usize, usize)> { let len = self.get_length(res_size)?; let mut start = self.start; if self.is_suffix_length { start = res_size - self.start } Ok((start, len)) } pub fn get_length(&self, res_size: usize) -> Result { if self.is_suffix_length { let specified_len = self.start; // 假设 h.start 是一个 i64 类型 let mut range_length = specified_len; if specified_len > res_size { range_length = res_size; } return Ok(range_length); } if self.start >= res_size { return Err(Error::msg("The requested range is not satisfiable")); } if let Some(end) = self.end { let mut end = end; if res_size <= end { end = res_size - 1; } let range_length = end - self.start + 1; return Ok(range_length); } if self.end.is_none() { let range_length = res_size - self.start; return Ok(range_length); } Err(Error::msg("range value invaild")) } } #[derive(Debug, Default, Clone)] pub struct ObjectOptions { // Use the maximum parity (N/2), used when saving server configuration files pub max_parity: bool, pub mod_time: Option, pub part_number: Option, pub delete_prefix: bool, pub delete_prefix_object: bool, pub version_id: Option, pub no_lock: bool, pub versioned: bool, pub version_suspended: bool, pub skip_decommissioned: bool, pub skip_rebalancing: bool, pub data_movement: bool, pub src_pool_idx: usize, pub user_defined: Option>, pub preserve_etag: Option, pub metadata_chg: bool, pub replication_request: bool, pub delete_marker: bool, } // impl Default for ObjectOptions { // fn default() -> Self { // Self { // max_parity: Default::default(), // mod_time: OffsetDateTime::UNIX_EPOCH, // part_number: Default::default(), // } // } // } #[derive(Debug, Default, Serialize, Deserialize)] pub struct BucketOptions { pub deleted: bool, // true only when site replication is enabled pub cached: bool, // true only when we are requesting a cached response instead of hitting the disk for example ListBuckets() call. pub no_metadata: bool, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct BucketInfo { pub name: String, pub created: Option, pub deleted: Option, pub versionning: bool, pub object_locking: bool, } #[derive(Debug)] pub struct MultipartUploadResult { pub upload_id: String, } #[derive(Debug)] pub struct PartInfo { pub part_num: usize, pub last_mod: Option, pub size: usize, pub etag: Option, } #[derive(Debug, Clone)] pub struct CompletePart { pub part_num: usize, pub e_tag: Option, } impl From for CompletePart { fn from(value: s3s::dto::CompletedPart) -> Self { Self { part_num: value.part_number.unwrap_or_default() as usize, e_tag: value.e_tag, } } } #[derive(Debug, Default)] pub struct ObjectInfo { pub bucket: String, pub name: String, pub mod_time: Option, pub size: usize, // Actual size is the real size of the object uploaded by client. pub actual_size: Option, pub is_dir: bool, pub user_defined: Option>, pub parity_blocks: usize, pub data_blocks: usize, pub version_id: Option, pub delete_marker: bool, pub user_tags: String, pub parts: Vec, pub is_latest: bool, pub content_type: Option, pub content_encoding: Option, pub num_versions: usize, pub successor_mod_time: Option, pub put_object_reader: Option, pub etag: Option, pub inlined: bool, pub metadata_only: bool, pub version_only: bool, } impl Clone for ObjectInfo { fn clone(&self) -> Self { Self { bucket: self.bucket.clone(), name: self.name.clone(), mod_time: self.mod_time, size: self.size, actual_size: self.actual_size, is_dir: self.is_dir, user_defined: self.user_defined.clone(), parity_blocks: self.parity_blocks, data_blocks: self.data_blocks, version_id: self.version_id, delete_marker: self.delete_marker, user_tags: self.user_tags.clone(), parts: self.parts.clone(), is_latest: self.is_latest, content_type: self.content_type.clone(), content_encoding: self.content_encoding.clone(), num_versions: self.num_versions, successor_mod_time: self.successor_mod_time, put_object_reader: None, // reader can not clone etag: self.etag.clone(), inlined: self.inlined, metadata_only: self.metadata_only, version_only: self.version_only, } } } impl ObjectInfo { pub fn is_compressed(&self) -> bool { if let Some(meta) = &self.user_defined { meta.contains_key(&format!("{}compression", RESERVED_METADATA_PREFIX)) } else { false } } pub fn get_actual_size(&self) -> Result { if let Some(actual_size) = self.actual_size { return Ok(actual_size); } if self.is_compressed() { if let Some(meta) = &self.user_defined { if let Some(size_str) = meta.get(&format!("{}actual-size", RESERVED_METADATA_PREFIX)) { if !size_str.is_empty() { // Todo: deal with error let size = size_str.parse::()?; return Ok(size); } } } let mut actual_size = 0; self.parts.iter().for_each(|part| { actual_size += part.actual_size; }); if actual_size == 0 && actual_size != self.size { return Err(Error::from_string("invalid decompressed size")); } return Ok(actual_size); } // TODO: IsEncrypted Ok(self.size) } } #[derive(Debug, Default)] pub struct ListObjectsInfo { // Indicates whether the returned list objects response is truncated. A // value of true indicates that the list was truncated. The list can be truncated // if the number of objects exceeds the limit allowed or specified // by max keys. pub is_truncated: bool, // When response is truncated (the IsTruncated element value in the response // is true), you can use the key name in this field as marker in the subsequent // request to get next set of objects. pub next_marker: Option, // List of objects info for this request. pub objects: Vec, // List of prefixes for this request. pub prefixes: Vec, } #[derive(Debug, Default)] pub struct ListObjectsV2Info { // Indicates whether the returned list objects response is truncated. A // value of true indicates that the list was truncated. The list can be truncated // if the number of objects exceeds the limit allowed or specified // by max keys. pub is_truncated: bool, // When response is truncated (the IsTruncated element value in the response // is true), you can use the key name in this field as marker in the subsequent // request to get next set of objects. // // NOTE: This element is returned only if you have delimiter request parameter // specified. pub continuation_token: Option, pub next_continuation_token: Option, // List of objects info for this request. pub objects: Vec, // List of prefixes for this request. pub prefixes: Vec, } #[derive(Debug, Clone, Default)] pub struct MultipartInfo { // Name of the bucket. pub bucket: String, // Name of the object. pub object: String, // Upload ID identifying the multipart upload whose parts are being listed. pub upload_id: String, // Date and time at which the multipart upload was initiated. pub initiated: Option, // Any metadata set during InitMultipartUpload, including encryption headers. pub user_defined: HashMap, } // ListMultipartsInfo - represents bucket resources for incomplete multipart uploads. #[derive(Debug, Clone, Default)] pub struct ListMultipartsInfo { // Together with upload-id-marker, this parameter specifies the multipart upload // after which listing should begin. pub key_marker: Option, // Together with key-marker, specifies the multipart upload after which listing // should begin. If key-marker is not specified, the upload-id-marker parameter // is ignored. pub upload_id_marker: Option, // When a list is truncated, this element specifies the value that should be // used for the key-marker request parameter in a subsequent request. pub next_key_marker: Option, // When a list is truncated, this element specifies the value that should be // used for the upload-id-marker request parameter in a subsequent request. pub next_upload_id_marker: Option, // Maximum number of multipart uploads that could have been included in the // response. pub max_uploads: usize, // Indicates whether the returned list of multipart uploads is truncated. A // value of true indicates that the list was truncated. The list can be truncated // if the number of multipart uploads exceeds the limit allowed or specified // by max uploads. pub is_truncated: bool, // List of all pending uploads. pub uploads: Vec, // When a prefix is provided in the request, The result contains only keys // starting with the specified prefix. pub prefix: String, // A character used to truncate the object prefixes. // NOTE: only supported delimiter is '/'. pub delimiter: Option, // CommonPrefixes contains all (if there are any) keys between Prefix and the // next occurrence of the string specified by delimiter. pub common_prefixes: Vec, // encoding_type: String, // Not supported yet. } #[derive(Debug, Default, Clone)] pub struct ObjectToDelete { pub object_name: String, pub version_id: Option, } #[derive(Debug, Default, Clone)] pub struct DeletedObject { pub delete_marker: bool, pub delete_marker_version_id: Option, pub object_name: String, pub version_id: Option, // MTime of DeleteMarker on source that needs to be propagated to replica pub delete_marker_mtime: Option, // to support delete marker replication // pub replication_state: ReplicationState, } pub struct ListObjectVersionsInfo { pub is_truncated: bool, pub next_marker: Option, pub next_version_idmarker: Option, pub objects: Vec, pub prefixes: Vec, } #[async_trait::async_trait] pub trait ObjectIO: Send + Sync + 'static { // GetObjectNInfo FIXME: async fn get_object_reader( &self, bucket: &str, object: &str, range: Option, h: HeaderMap, opts: &ObjectOptions, ) -> Result; // PutObject async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result; } #[async_trait::async_trait] #[allow(clippy::too_many_arguments)] pub trait StorageAPI: ObjectIO { // NewNSLock TODO: // Shutdown TODO: // NSScanner TODO: async fn backend_info(&self) -> madmin::BackendInfo; async fn storage_info(&self) -> madmin::StorageInfo; async fn local_storage_info(&self) -> madmin::StorageInfo; async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>; async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result; async fn list_bucket(&self, opts: &BucketOptions) -> Result>; async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>; // ListObjects TODO: FIXME: async fn list_objects_v2( self: Arc, bucket: &str, prefix: &str, continuation_token: Option, delimiter: Option, max_keys: i32, fetch_owner: bool, start_after: Option, ) -> Result; // ListObjectVersions TODO: FIXME: async fn list_object_versions( self: Arc, bucket: &str, prefix: &str, marker: Option, version_marker: Option, delimiter: Option, max_keys: i32, ) -> Result; // Walk TODO: // GetObjectNInfo ObjectIO async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; // PutObject ObjectIO // CopyObject async fn copy_object( &self, src_bucket: &str, src_object: &str, dst_bucket: &str, dst_object: &str, src_info: &mut ObjectInfo, src_opts: &ObjectOptions, dst_opts: &ObjectOptions, ) -> Result; async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result; async fn delete_objects( &self, bucket: &str, objects: Vec, opts: ObjectOptions, ) -> Result<(Vec, Vec>)>; // TransitionObject TODO: // RestoreTransitionedObject TODO: // ListMultipartUploads async fn list_multipart_uploads( &self, bucket: &str, prefix: &str, key_marker: Option, upload_id_marker: Option, delimiter: Option, max_uploads: usize, ) -> Result; async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; // CopyObjectPart async fn copy_object_part( &self, src_bucket: &str, src_object: &str, dst_bucket: &str, dst_object: &str, upload_id: &str, part_id: usize, start_offset: i64, length: i64, src_info: &ObjectInfo, src_opts: &ObjectOptions, dst_opts: &ObjectOptions, ) -> Result<()>; async fn put_object_part( &self, bucket: &str, object: &str, upload_id: &str, part_id: usize, data: &mut PutObjReader, opts: &ObjectOptions, ) -> Result; // GetMultipartInfo async fn get_multipart_info( &self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions, ) -> Result; // ListObjectParts async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()>; async fn complete_multipart_upload( &self, bucket: &str, object: &str, upload_id: &str, uploaded_parts: Vec, opts: &ObjectOptions, ) -> Result; // GetDisks async fn get_disks(&self, pool_idx: usize, set_idx: usize) -> Result>>; // SetDriveCounts fn set_drive_counts(&self) -> Vec; // Health TODO: // PutObjectMetadata async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; // DecomTieredObject async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result; async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)>; async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; async fn heal_object( &self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts, ) -> Result<(HealResultItem, Option)>; async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, hs: Arc, is_meta: bool) -> Result<()>; async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)>; async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()>; }