merge main

This commit is contained in:
houseme
2025-05-28 17:26:31 +08:00
100 changed files with 10881 additions and 2464 deletions
+4 -4
View File
@@ -209,12 +209,12 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
let mut versions = madmin::Versions::default();
let mut delete_markers = madmin::DeleteMarkers::default();
let mut usage = madmin::Usage::default();
let mut mode = madmin::ITEM_INITIALIZING;
let mut mode = ITEM_INITIALIZING;
let mut backend = madmin::ErasureBackend::default();
let mut pools: HashMap<i32, HashMap<i32, madmin::ErasureSetInfo>> = HashMap::new();
let mut pools: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
if let Some(store) = new_object_layer_fn() {
mode = madmin::ITEM_ONLINE;
mode = ITEM_ONLINE;
match load_data_usage_from_backend(store.clone()).await {
Ok(res) => {
buckets.count = res.buckets_count;
@@ -242,7 +242,7 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
warn!("backend_info end {:?}", after4 - after3);
let mut all_disks: Vec<madmin::Disk> = Vec::new();
let mut all_disks: Vec<Disk> = Vec::new();
for server in servers.iter() {
all_disks.extend(server.disks.clone());
}
+2 -3
View File
@@ -546,8 +546,7 @@ impl Writer for BitrotFileWriter {
hasher.update(h_buf);
hasher.finalize()
})
.await
.unwrap();
.await?;
if let Some(f) = self.inner.as_mut() {
f.write_all(&hash_bytes).await?;
@@ -775,7 +774,7 @@ mod test {
if !algo.available() || *algo != BitrotAlgorithm::HighwayHash256 {
continue;
}
let checksum = decode_to_vec(checksums.get(algo).unwrap()).unwrap();
let checksum = decode_to_vec(checksums.get(algo).unwrap())?;
let mut h = algo.new_hasher();
let mut msg = Vec::with_capacity(h.size() * h.block_size());
+1 -6
View File
@@ -288,12 +288,7 @@ impl BucketMetadata {
}
pub fn set_created(&mut self, created: Option<OffsetDateTime>) {
self.created = {
match created {
Some(t) => t,
None => OffsetDateTime::now_utc(),
}
}
self.created = created.unwrap_or_else(OffsetDateTime::now_utc)
}
pub async fn save(&mut self) -> Result<()> {
+39 -39
View File
@@ -359,10 +359,10 @@ impl BucketMetadataSys {
let bm = match load_bucket_metadata(self.api.clone(), bucket).await {
Ok(res) => res,
Err(err) => {
if *self.initialized.read().await {
return Err(Error::msg("errBucketMetadataNotInitialized"));
return if *self.initialized.read().await {
Err(Error::msg("errBucketMetadataNotInitialized"))
} else {
return Err(err);
Err(err)
}
}
};
@@ -381,11 +381,11 @@ impl BucketMetadataSys {
Ok((res, _)) => res,
Err(err) => {
warn!("get_versioning_config err {:?}", &err);
if config::error::is_err_config_not_found(&err) {
return Ok((VersioningConfiguration::default(), OffsetDateTime::UNIX_EPOCH));
return if config::error::is_err_config_not_found(&err) {
Ok((VersioningConfiguration::default(), OffsetDateTime::UNIX_EPOCH))
} else {
return Err(err);
}
Err(err)
};
}
};
@@ -401,11 +401,11 @@ impl BucketMetadataSys {
Ok((res, _)) => res,
Err(err) => {
warn!("get_bucket_policy err {:?}", &err);
if config::error::is_err_config_not_found(&err) {
return Err(Error::new(BucketMetadataError::BucketPolicyNotFound));
return if config::error::is_err_config_not_found(&err) {
Err(Error::new(BucketMetadataError::BucketPolicyNotFound))
} else {
return Err(err);
}
Err(err)
};
}
};
@@ -421,11 +421,11 @@ impl BucketMetadataSys {
Ok((res, _)) => res,
Err(err) => {
warn!("get_tagging_config err {:?}", &err);
if config::error::is_err_config_not_found(&err) {
return Err(Error::new(BucketMetadataError::TaggingNotFound));
return if config::error::is_err_config_not_found(&err) {
Err(Error::new(BucketMetadataError::TaggingNotFound))
} else {
return Err(err);
}
Err(err)
};
}
};
@@ -441,11 +441,11 @@ impl BucketMetadataSys {
Ok((res, _)) => res,
Err(err) => {
warn!("get_object_lock_config err {:?}", &err);
if config::error::is_err_config_not_found(&err) {
return Err(Error::new(BucketMetadataError::BucketObjectLockConfigNotFound));
return if config::error::is_err_config_not_found(&err) {
Err(Error::new(BucketMetadataError::BucketObjectLockConfigNotFound))
} else {
return Err(err);
}
Err(err)
};
}
};
@@ -461,11 +461,11 @@ impl BucketMetadataSys {
Ok((res, _)) => res,
Err(err) => {
warn!("get_lifecycle_config err {:?}", &err);
if config::error::is_err_config_not_found(&err) {
return Err(Error::new(BucketMetadataError::BucketLifecycleNotFound));
return if config::error::is_err_config_not_found(&err) {
Err(Error::new(BucketMetadataError::BucketLifecycleNotFound))
} else {
return Err(err);
}
Err(err)
};
}
};
@@ -501,11 +501,11 @@ impl BucketMetadataSys {
Ok((res, _)) => res,
Err(err) => {
warn!("get_sse_config err {:?}", &err);
if config::error::is_err_config_not_found(&err) {
return Err(Error::new(BucketMetadataError::BucketSSEConfigNotFound));
return if config::error::is_err_config_not_found(&err) {
Err(Error::new(BucketMetadataError::BucketSSEConfigNotFound))
} else {
return Err(err);
}
Err(err)
};
}
};
@@ -532,11 +532,11 @@ impl BucketMetadataSys {
Ok((res, _)) => res,
Err(err) => {
warn!("get_quota_config err {:?}", &err);
if config::error::is_err_config_not_found(&err) {
return Err(Error::new(BucketMetadataError::BucketQuotaConfigNotFound));
return if config::error::is_err_config_not_found(&err) {
Err(Error::new(BucketMetadataError::BucketQuotaConfigNotFound))
} else {
return Err(err);
}
Err(err)
};
}
};
@@ -552,11 +552,11 @@ impl BucketMetadataSys {
Ok(res) => res,
Err(err) => {
warn!("get_replication_config err {:?}", &err);
if config::error::is_err_config_not_found(&err) {
return Err(Error::new(BucketMetadataError::BucketReplicationConfigNotFound));
return if config::error::is_err_config_not_found(&err) {
Err(Error::new(BucketMetadataError::BucketReplicationConfigNotFound))
} else {
return Err(err);
}
Err(err)
};
}
};
@@ -576,11 +576,11 @@ impl BucketMetadataSys {
Ok(res) => res,
Err(err) => {
warn!("get_replication_config err {:?}", &err);
if config::error::is_err_config_not_found(&err) {
return Err(Error::new(BucketMetadataError::BucketRemoteTargetNotFound));
return if config::error::is_err_config_not_found(&err) {
Err(Error::new(BucketMetadataError::BucketRemoteTargetNotFound))
} else {
return Err(err);
}
Err(err)
};
}
};
+3 -3
View File
@@ -2,16 +2,16 @@ use common::error::Result;
use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize};
// 定义QuotaType枚举类型
// 定义 QuotaType 枚举类型
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum QuotaType {
Hard,
}
// 定义BucketQuota结构体
// 定义 BucketQuota 结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct BucketQuota {
quota: Option<u64>, // 使用Option来表示可能不存在的字段
quota: Option<u64>, // 使用 Option 来表示可能不存在的字段
size: u64,
+1 -1
View File
@@ -25,7 +25,7 @@ pub struct LatencyStat {
max: Duration, // 最大延迟
}
// 定义BucketTarget结构体
// 定义 BucketTarget 结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct BucketTarget {
source_bucket: String,
+4 -4
View File
@@ -14,10 +14,6 @@ impl VersioningApi for VersioningConfiguration {
fn enabled(&self) -> bool {
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
}
fn suspended(&self) -> bool {
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
}
fn prefix_enabled(&self, prefix: &str) -> bool {
if self.status != Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
return false;
@@ -46,6 +42,7 @@ impl VersioningApi for VersioningConfiguration {
true
}
fn prefix_suspended(&self, prefix: &str) -> bool {
if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)) {
return true;
@@ -79,4 +76,7 @@ impl VersioningApi for VersioningConfiguration {
fn versioned(&self, prefix: &str) -> bool {
self.prefix_enabled(prefix) || self.prefix_suspended(prefix)
}
fn suspended(&self) -> bool {
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
}
}
+5 -5
View File
@@ -64,7 +64,7 @@
// // content_length,
// // );
// // 填充0
// // 填充 0
// if !need_padding {
// y.yield_ok(prev_bytes).await;
// break;
@@ -129,7 +129,7 @@
// combined.extend_from_slice(&data);
// // debug!(
// // "取到的长度大于所需,取出需要的长度:{},与上一次合并得到:{},bytes剩余:{}",
// // "取到的长度大于所需,取出需要的长度:{},与上一次合并得到:{},bytes 剩余:{}",
// // need_size,
// // combined.len(),
// // bytes.len(),
@@ -142,7 +142,7 @@
// combined.extend_from_slice(&bytes);
// // debug!(
// // "取到的长度小于所需,取出需要的长度:{},与上一次合并得到:{},bytes剩余:{},直接返回",
// // "取到的长度小于所需,取出需要的长度:{},与上一次合并得到:{},bytes 剩余:{},直接返回",
// // need_size,
// // combined.len(),
// // bytes.len(),
@@ -152,14 +152,14 @@
// }
// }
// // 取到的数据比需要的块大,从bytes中截取需要的块大小
// // 取到的数据比需要的块大,从 bytes 中截取需要的块大小
// if data_size <= bytes.len() {
// let n = bytes.len() / data_size;
// for _ in 0..n {
// let data = bytes.split_to(data_size);
// // println!("bytes_buffer.push: {} 剩余:{}", data.len(), bytes.len());
// // println!("bytes_buffer.push: {},剩余:{}", data.len(), bytes.len());
// bytes_buffer.push(data);
// }
+34 -101
View File
@@ -1,5 +1,5 @@
use super::error::{is_err_config_not_found, ConfigError};
use super::{storageclass, Config, GLOBAL_StorageClass, KVS};
use super::{storageclass, Config, GLOBAL_StorageClass};
use crate::disk::RUSTFS_META_BUCKET;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use crate::store_err::is_err_object_not_found;
@@ -12,62 +12,25 @@ use std::io::Cursor;
use std::sync::Arc;
use tracing::{error, warn};
/// * config prefix
pub const CONFIG_PREFIX: &str = "config";
/// * config file
const CONFIG_FILE: &str = "config.json";
/// * config class sub system
pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
/// * config class sub system
pub const DEFAULT_KV_KEY: &str = "_";
lazy_static! {
/// * config bucket
static ref CONFIG_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, CONFIG_PREFIX);
/// * config file
static ref SubSystemsDynamic: HashSet<String> = {
let mut h = HashSet::new();
h.insert(STORAGE_CLASS_SUB_SYS.to_owned());
h
};
}
/// Reads configuration file content from the storage system
///
/// This function reads a specified configuration file through the provided storage API interface
/// and returns its raw byte content. It's a basic configuration reading function that returns
/// only the configuration data without any metadata.
///
/// # Parameters
/// * `api` - An Arc smart pointer containing an implementation of the StorageAPI trait
/// * `file` - The name of the configuration file to read
///
/// # Returns
/// * `Result<Vec<u8>>` - Returns the configuration file contents as bytes on success, or an error on failure
///
/// # Errors
/// May return the following errors:
/// * `ConfigError::NotFound` - When the requested configuration file does not exist
/// * Other storage operation related errors
///
pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
Ok(data)
}
/// read config with metadata with api,file and opts
///
/// # Parameters
/// - `api`: StorageAPI
/// - `file`: file name
/// - `opts`: object options
///
/// # Returns
/// Result<(Vec<u8>, ObjectInfo)>
///
/// # Errors
/// - ConfigError::NotFound
pub async fn read_config_with_metadata<S: StorageAPI>(
api: Arc<S>,
file: &str,
@@ -94,16 +57,6 @@ pub async fn read_config_with_metadata<S: StorageAPI>(
Ok((data, rd.object_info))
}
/// save config with api,file and data
///
/// # Parameters
///
/// - `api`: StorageAPI
/// - `file`: file name
/// - `data`: data to save
///
/// # Returns
/// Result
pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
save_config_with_opts(
api,
@@ -117,15 +70,6 @@ pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>)
.await
}
/// delete config with api and file
///
/// # Parameters
/// - `api`: StorageAPI
/// - `file`: file name
///
/// # Returns
/// Result
///
pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
match api
.delete_object(
@@ -150,15 +94,6 @@ pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()>
}
}
/// save config with opts
///
/// # Parameters
/// - `api`: StorageAPI
/// - `file`: file name
/// - `data`: data to save
///
/// # Returns
/// Result
pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
let size = data.len();
let _ = api
@@ -179,45 +114,46 @@ async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<Config
Ok(cfg)
}
fn get_config_file() -> String {
format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE)
}
/// * read config without migrate
pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<Config> {
let config_file = get_config_file();
let data = match read_config_and_init_if_not_found(api.clone(), config_file.as_str()).await? {
Some(data) => data,
None => return Ok(new_and_save_server_config(api).await?),
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE);
let data = match read_config(api.clone(), config_file.as_str()).await {
Ok(res) => res,
Err(err) => {
return if is_err_config_not_found(&err) {
warn!("config not found, start to init");
let cfg = new_and_save_server_config(api).await?;
warn!("config init done");
Ok(cfg)
} else {
error!("read config err {:?}", &err);
Err(err)
}
}
};
read_server_config(api, data.as_slice()).await
}
/// read config and init if not found
async fn read_config_and_init_if_not_found<S: StorageAPI>(api: Arc<S>, config_file: &str) -> Result<Option<Vec<u8>>> {
match read_config(api.clone(), config_file).await {
Ok(data) => Ok(Some(data)),
Err(err) => {
if is_err_config_not_found(&err) {
warn!("config not found, start to init");
return Ok(None);
}
error!("read config err {:?}", &err);
Err(err)
}
}
}
async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<Config> {
let cfg = {
if data.is_empty() {
let config_file = get_config_file();
let cfg_data = match read_config_and_init_if_not_found(api.clone(), config_file.as_str()).await? {
Some(data) => data,
None => return Ok(new_and_save_server_config(api).await?),
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE);
let cfg_data = match read_config(api.clone(), config_file.as_str()).await {
Ok(res) => res,
Err(err) => {
return if is_err_config_not_found(&err) {
warn!("config not found init start");
let cfg = new_and_save_server_config(api).await?;
warn!("config not found init done");
Ok(cfg)
} else {
error!("read config err {:?}", &err);
Err(err)
}
}
};
// TODO: decrypt
Config::unmarshal(cfg_data.as_slice())?
} else {
Config::unmarshal(data)?
@@ -230,12 +166,11 @@ async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<C
async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &Config) -> Result<()> {
let data = cfg.marshal()?;
let config_file = get_config_file();
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE);
save_config(api, &config_file, data).await
}
/// * lookup_configs
pub async fn lookup_configs<S: StorageAPI>(cfg: &mut Config, api: Arc<S>) {
// TODO: from etcd
if let Err(err) = apply_dynamic_config(cfg, api).await {
@@ -251,12 +186,10 @@ async fn apply_dynamic_config<S: StorageAPI>(cfg: &mut Config, api: Arc<S>) -> R
Ok(())
}
async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api: Arc<S>, sub_sys: &str) -> Result<()> {
async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api: Arc<S>, subsys: &str) -> Result<()> {
let set_drive_counts = api.set_drive_counts();
if sub_sys == STORAGE_CLASS_SUB_SYS {
let kvs = cfg
.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_KV_KEY)
.unwrap_or_else(|| KVS::new());
if subsys == STORAGE_CLASS_SUB_SYS {
let kvs = cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_KV_KEY).unwrap_or_default();
for (i, count) in set_drive_counts.iter().enumerate() {
match storageclass::lookup_config(&kvs, *count) {
+7 -8
View File
@@ -17,7 +17,7 @@ pub enum EndpointType {
/// any type of endpoint.
#[derive(Debug, PartialEq, Eq, Clone, Hash)]
pub struct Endpoint {
pub url: url::Url,
pub url: Url,
pub is_local: bool,
pub pool_idx: i32,
@@ -179,8 +179,8 @@ impl Endpoint {
}
}
/// parse a file path into an URL.
fn url_parse_from_file_path(value: &str) -> Result<url::Url> {
/// parse a file path into a URL.
fn url_parse_from_file_path(value: &str) -> Result<Url> {
// Only check if the arg is an ip address and ask for scheme since its absent.
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
// /mnt/export1. So we go ahead and start the rustfs server in FS modes in these cases.
@@ -202,7 +202,6 @@ fn url_parse_from_file_path(value: &str) -> Result<url::Url> {
#[cfg(test)]
mod test {
use super::*;
#[test]
@@ -215,10 +214,10 @@ mod test {
expected_err: Option<Error>,
}
let u2 = url::Url::parse("https://example.org/path").unwrap();
let u4 = url::Url::parse("http://192.168.253.200/path").unwrap();
let u6 = url::Url::parse("http://server:/path").unwrap();
let root_slash_foo = url::Url::from_file_path("/foo").unwrap();
let u2 = Url::parse("https://example.org/path").unwrap();
let u4 = Url::parse("http://192.168.253.200/path").unwrap();
let u6 = Url::parse("http://server:/path").unwrap();
let root_slash_foo = Url::from_file_path("/foo").unwrap();
let test_cases = [
TestCase {
+3 -3
View File
@@ -301,8 +301,8 @@ pub fn clone_disk_err(e: &DiskError) -> Error {
pub fn os_err_to_file_err(e: io::Error) -> Error {
match e.kind() {
io::ErrorKind::NotFound => Error::new(DiskError::FileNotFound),
io::ErrorKind::PermissionDenied => Error::new(DiskError::FileAccessDenied),
ErrorKind::NotFound => Error::new(DiskError::FileNotFound),
ErrorKind::PermissionDenied => Error::new(DiskError::FileAccessDenied),
// io::ErrorKind::ConnectionRefused => todo!(),
// io::ErrorKind::ConnectionReset => todo!(),
// io::ErrorKind::HostUnreachable => todo!(),
@@ -350,7 +350,7 @@ pub fn os_err_to_file_err(e: io::Error) -> Error {
pub struct FileAccessDeniedWithContext {
pub path: PathBuf,
#[source]
pub source: std::io::Error,
pub source: io::Error,
}
impl std::fmt::Display for FileAccessDeniedWithContext {
+1 -1
View File
@@ -40,7 +40,7 @@ pub struct FormatErasureV3 {
pub this: Uuid,
/// Sets field carries the input disk order generated the first
/// time when fresh disks were supplied, it is a two dimensional
/// time when fresh disks were supplied, it is a two-dimensional
/// array second dimension represents list of disks used per set.
pub sets: Vec<Vec<Uuid>>,
+69 -80
View File
@@ -43,7 +43,7 @@ use crate::utils::fs::{
};
use crate::utils::os::get_info;
use crate::utils::path::{
self, clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, GLOBAL_DIR_SUFFIX,
clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, GLOBAL_DIR_SUFFIX,
GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR,
};
use crate::{
@@ -69,7 +69,7 @@ use tokio::fs::{self, File};
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind};
use tokio::sync::mpsc::Sender;
use tokio::sync::RwLock;
use tracing::{error, info, warn};
use tracing::{debug, error, info, warn};
use uuid::Uuid;
#[derive(Debug)]
@@ -127,7 +127,7 @@ impl LocalDisk {
// TODO: 删除 tmp 数据
}
let format_path = Path::new(super::RUSTFS_META_BUCKET)
let format_path = Path::new(RUSTFS_META_BUCKET)
.join(Path::new(super::FORMAT_CONFIG_FILE))
.absolutize_virtually(&root)?
.into_owned();
@@ -192,7 +192,7 @@ impl LocalDisk {
let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default());
// TODO: DIRECT suport
// TODO: DIRECT support
// TODD: DiskInfo
let mut disk = Self {
root: root.clone(),
@@ -272,10 +272,10 @@ impl LocalDisk {
Ok(md)
}
async fn make_meta_volumes(&self) -> Result<()> {
let buckets = format!("{}/{}", super::RUSTFS_META_BUCKET, super::BUCKET_META_PREFIX);
let multipart = format!("{}/{}", super::RUSTFS_META_BUCKET, "multipart");
let config = format!("{}/{}", super::RUSTFS_META_BUCKET, "config");
let tmp = format!("{}/{}", super::RUSTFS_META_BUCKET, "tmp");
let buckets = format!("{}/{}", RUSTFS_META_BUCKET, BUCKET_META_PREFIX);
let multipart = format!("{}/{}", RUSTFS_META_BUCKET, "multipart");
let config = format!("{}/{}", RUSTFS_META_BUCKET, "config");
let tmp = format!("{}/{}", RUSTFS_META_BUCKET, "tmp");
let defaults = vec![buckets.as_str(), multipart.as_str(), config.as_str(), tmp.as_str()];
self.make_volumes(defaults).await
@@ -341,7 +341,7 @@ impl LocalDisk {
rename(&delete_path, &trash_path).await.map_err(Error::new).err()
};
if immediate_purge || delete_path.to_string_lossy().ends_with(path::SLASH_SEPARATOR) {
if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR) {
warn!("move_to_trash immediate_purge {:?}", &delete_path.to_string_lossy());
let trash_path2 = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
let _ = rename_all(
@@ -443,7 +443,7 @@ impl LocalDisk {
return Err(Error::new(DiskError::FileNotFound));
}
let meta_path = file_path.as_ref().join(Path::new(super::STORAGE_FORMAT_FILE));
let meta_path = file_path.as_ref().join(Path::new(STORAGE_FORMAT_FILE));
let res = {
if read_data {
@@ -530,12 +530,12 @@ impl LocalDisk {
volume_dir: impl AsRef<Path>,
file_path: impl AsRef<Path>,
) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
let mut f = match utils::fs::open_file(file_path.as_ref(), utils::fs::O_RDONLY).await {
let mut f = match utils::fs::open_file(file_path.as_ref(), O_RDONLY).await {
Ok(f) => f,
Err(e) => {
if os_is_not_exist(&e) {
if !skip_access_checks(volume) {
if let Err(er) = utils::fs::access(volume_dir.as_ref()).await {
if let Err(er) = access(volume_dir.as_ref()).await {
if os_is_not_exist(&er) {
warn!("read_all_data_with_dmtime os err {:?}", &er);
return Err(Error::new(DiskError::VolumeNotFound));
@@ -553,7 +553,7 @@ impl LocalDisk {
} else if is_sys_err_too_many_files(&e) {
return Err(Error::new(DiskError::TooManyOpenFiles));
} else if is_sys_err_invalid_arg(&e) {
if let Ok(meta) = utils::fs::lstat(file_path.as_ref()).await {
if let Ok(meta) = lstat(file_path.as_ref()).await {
if meta.is_dir() {
return Err(Error::new(DiskError::FileNotFound));
}
@@ -587,7 +587,7 @@ impl LocalDisk {
async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &Vec<FileInfo>) -> Result<()> {
let volume_dir = self.get_bucket_path(volume)?;
let xlpath = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?;
let xlpath = self.get_object_path(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str())?;
let data = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
Ok((data, _)) => data,
@@ -627,7 +627,7 @@ impl LocalDisk {
};
if let Some(dir) = data_dir {
let vid = fi.version_id.unwrap_or(Uuid::nil());
let vid = fi.version_id.unwrap_or_default();
let _ = fm.data.remove(vec![vid, dir]);
let dir_path = self.get_object_path(volume, format!("{}/{}", path, dir).as_str())?;
@@ -650,14 +650,8 @@ impl LocalDisk {
let volume_dir = self.get_bucket_path(volume)?;
self.write_all_private(
volume,
format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(),
&buf,
true,
volume_dir,
)
.await?;
self.write_all_private(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str(), &buf, true, volume_dir)
.await?;
Ok(())
}
@@ -672,12 +666,12 @@ impl LocalDisk {
self.write_all_internal(&tmp_file_path, buf, sync, tmp_volume_dir).await?;
os::rename_all(tmp_file_path, file_path, volume_dir).await
rename_all(tmp_file_path, file_path, volume_dir).await
}
// write_all_public for trail
async fn write_all_public(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE {
if volume == RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE {
let mut format_info = self.format_info.write().await;
format_info.data.clone_from(&data);
}
@@ -713,7 +707,7 @@ impl LocalDisk {
sync: bool,
skip_parent: impl AsRef<Path>,
) -> Result<()> {
let flags = utils::fs::O_CREATE | utils::fs::O_WRONLY | utils::fs::O_TRUNC;
let flags = O_CREATE | O_WRONLY | utils::fs::O_TRUNC;
let mut f = {
if sync {
@@ -924,7 +918,7 @@ impl LocalDisk {
continue;
}
let name = path::path_join_buf(&[current, entry]);
let name = path_join_buf(&[current, entry]);
if !dir_stack.is_empty() {
if let Some(pop) = dir_stack.pop() {
@@ -1068,7 +1062,7 @@ fn skip_access_checks(p: impl AsRef<str>) -> bool {
super::RUSTFS_META_TMP_DELETED_BUCKET,
super::RUSTFS_META_TMP_BUCKET,
super::RUSTFS_META_MULTIPART_BUCKET,
super::RUSTFS_META_BUCKET,
RUSTFS_META_BUCKET,
];
for v in vols.iter() {
@@ -1203,7 +1197,7 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(skip(self))]
#[must_use]
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE {
if volume == RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE {
let format_info = self.format_info.read().await;
if !format_info.data.is_empty() {
return Ok(format_info.data.clone());
@@ -1225,7 +1219,7 @@ impl DiskAPI for LocalDisk {
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
if let Err(e) = utils::fs::access(&volume_dir).await {
if let Err(e) = access(&volume_dir).await {
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
}
@@ -1243,7 +1237,7 @@ impl DiskAPI for LocalDisk {
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
if let Err(e) = utils::fs::access(&volume_dir).await {
if let Err(e) = access(&volume_dir).await {
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
}
@@ -1259,7 +1253,7 @@ impl DiskAPI for LocalDisk {
.join(path)
.join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string()))
.join(format!("part.{}", part.number));
let err = (self
let err = self
.bitrot_verify(
&part_path,
erasure.shard_file_size(part.size),
@@ -1267,7 +1261,7 @@ impl DiskAPI for LocalDisk {
&checksum_info.hash,
erasure.shard_size(erasure.block_size),
)
.await)
.await
.err();
resp.results[i] = conv_part_err_to_int(&err);
if resp.results[i] == CHECK_PART_UNKNOWN {
@@ -1384,7 +1378,7 @@ impl DiskAPI for LocalDisk {
}
}
if let Err(e) = utils::fs::remove_std(&dst_file_path) {
if let Err(e) = remove_std(&dst_file_path) {
if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) {
warn!("rename_part remove dst failed {:?} err {:?}", &dst_file_path, e);
return Err(Error::new(DiskError::FileAccessDenied));
@@ -1396,7 +1390,7 @@ impl DiskAPI for LocalDisk {
}
}
if let Err(err) = os::rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await {
if let Err(err) = rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await {
if let Some(e) = err.to_io_err() {
if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) {
warn!("rename_part rename all failed {:?} err {:?}", &dst_file_path, e);
@@ -1429,7 +1423,7 @@ impl DiskAPI for LocalDisk {
let src_volume_dir = self.get_bucket_path(src_volume)?;
let dst_volume_dir = self.get_bucket_path(dst_volume)?;
if !skip_access_checks(src_volume) {
if let Err(e) = utils::fs::access(&src_volume_dir).await {
if let Err(e) = access(&src_volume_dir).await {
if os_is_not_exist(&e) {
return Err(Error::from(DiskError::VolumeNotFound));
} else if is_sys_err_io(&e) {
@@ -1440,7 +1434,7 @@ impl DiskAPI for LocalDisk {
}
}
if !skip_access_checks(dst_volume) {
if let Err(e) = utils::fs::access(&dst_volume_dir).await {
if let Err(e) = access(&dst_volume_dir).await {
if os_is_not_exist(&e) {
return Err(Error::from(DiskError::VolumeNotFound));
} else if is_sys_err_io(&e) {
@@ -1484,7 +1478,7 @@ impl DiskAPI for LocalDisk {
}
}
if let Err(e) = utils::fs::remove(&dst_file_path).await {
if let Err(e) = remove(&dst_file_path).await {
if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) {
return Err(Error::new(DiskError::FileAccessDenied));
} else if is_sys_err_io(&e) {
@@ -1495,7 +1489,7 @@ impl DiskAPI for LocalDisk {
}
}
if let Err(err) = os::rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await {
if let Err(err) = rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await {
if let Some(e) = err.to_io_err() {
if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) {
return Err(Error::new(DiskError::FileAccessDenied));
@@ -1521,7 +1515,7 @@ impl DiskAPI for LocalDisk {
if !origvolume.is_empty() {
let origvolume_dir = self.get_bucket_path(origvolume)?;
if !skip_access_checks(origvolume) {
if let Err(e) = utils::fs::access(origvolume_dir).await {
if let Err(e) = access(origvolume_dir).await {
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
}
@@ -1552,7 +1546,7 @@ impl DiskAPI for LocalDisk {
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
if let Err(e) = utils::fs::access(&volume_dir).await {
if let Err(e) = access(&volume_dir).await {
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
}
@@ -1571,7 +1565,7 @@ impl DiskAPI for LocalDisk {
// warn!("disk read_file: volume: {}, path: {}", volume, path);
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
if let Err(e) = utils::fs::access(&volume_dir).await {
if let Err(e) = access(&volume_dir).await {
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
}
@@ -1609,7 +1603,7 @@ impl DiskAPI for LocalDisk {
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
if let Err(e) = utils::fs::access(&volume_dir).await {
if let Err(e) = access(&volume_dir).await {
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
}
@@ -1655,7 +1649,7 @@ impl DiskAPI for LocalDisk {
if !origvolume.is_empty() {
let origvolume_dir = self.get_bucket_path(origvolume)?;
if !skip_access_checks(origvolume) {
if let Err(e) = utils::fs::access(origvolume_dir).await {
if let Err(e) = access(origvolume_dir).await {
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
}
@@ -1668,7 +1662,7 @@ impl DiskAPI for LocalDisk {
Ok(res) => res,
Err(e) => {
if is_err_file_not_found(&e) && !skip_access_checks(volume) {
if let Err(e) = utils::fs::access(&volume_dir).await {
if let Err(e) = access(&volume_dir).await {
return Err(convert_access_error(e, DiskError::VolumeAccessDenied));
}
}
@@ -1750,8 +1744,8 @@ impl DiskAPI for LocalDisk {
}
// xl.meta 路径
let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str()));
let dst_file_path = dst_volume_dir.join(Path::new(format!("{}/{}", &dst_path, super::STORAGE_FORMAT_FILE).as_str()));
let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, STORAGE_FORMAT_FILE).as_str()));
let dst_file_path = dst_volume_dir.join(Path::new(format!("{}/{}", &dst_path, STORAGE_FORMAT_FILE).as_str()));
// data_dir 路径
let has_data_dir_path = {
@@ -1846,7 +1840,7 @@ impl DiskAPI for LocalDisk {
let new_dst_buf = xlmeta.marshal_msg()?;
self.write_all(src_volume, format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str(), new_dst_buf)
self.write_all(src_volume, format!("{}/{}", &src_path, STORAGE_FORMAT_FILE).as_str(), new_dst_buf)
.await
.map_err(|err| {
if let Some(e) = err.to_io_err() {
@@ -1858,7 +1852,7 @@ impl DiskAPI for LocalDisk {
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
let no_inline = fi.data.is_none() && fi.size > 0;
if no_inline {
if let Err(err) = os::rename_all(&src_data_path, &dst_data_path, &skip_parent).await {
if let Err(err) = rename_all(&src_data_path, &dst_data_path, &skip_parent).await {
let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await;
info!(
"rename all failed src_data_path: {:?}, dst_data_path: {:?}, err: {:?}",
@@ -1881,7 +1875,7 @@ impl DiskAPI for LocalDisk {
if let Err(err) = self
.write_all_private(
dst_volume,
format!("{}/{}/{}", &dst_path, &old_data_dir.to_string(), super::STORAGE_FORMAT_FILE).as_str(),
format!("{}/{}/{}", &dst_path, &old_data_dir.to_string(), STORAGE_FORMAT_FILE).as_str(),
&dst_buf,
true,
&skip_parent,
@@ -1900,7 +1894,7 @@ impl DiskAPI for LocalDisk {
}
}
if let Err(err) = os::rename_all(&src_file_path, &dst_file_path, &skip_parent).await {
if let Err(err) = rename_all(&src_file_path, &dst_file_path, &skip_parent).await {
if let Some((_, dst_data_path)) = has_data_dir_path.as_ref() {
let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await;
}
@@ -1916,7 +1910,7 @@ impl DiskAPI for LocalDisk {
if let Some(src_file_path_parent) = src_file_path.parent() {
if src_volume != super::RUSTFS_META_MULTIPART_BUCKET {
let _ = utils::fs::remove_std(src_file_path_parent);
let _ = remove_std(src_file_path_parent);
} else {
let _ = self
.delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false)
@@ -1951,7 +1945,7 @@ impl DiskAPI for LocalDisk {
let volume_dir = self.get_bucket_path(volume)?;
if let Err(e) = utils::fs::access(&volume_dir).await {
if let Err(e) = access(&volume_dir).await {
if os_is_not_exist(&e) {
os::make_dir_all(&volume_dir, self.root.as_path()).await?;
return Ok(());
@@ -1981,7 +1975,7 @@ impl DiskAPI for LocalDisk {
})?;
for entry in entries {
if !utils::path::has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(utils::path::clean(&entry).as_str()) {
if !has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(clean(&entry).as_str()) {
continue;
}
@@ -1997,17 +1991,17 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(skip(self))]
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
let volume_dir = self.get_bucket_path(volume)?;
let meta = match utils::fs::lstat(&volume_dir).await {
let meta = match lstat(&volume_dir).await {
Ok(res) => res,
Err(e) => {
if os_is_not_exist(&e) {
return Err(Error::new(DiskError::VolumeNotFound));
return if os_is_not_exist(&e) {
Err(Error::new(DiskError::VolumeNotFound))
} else if os_is_permission(&e) {
return Err(Error::new(DiskError::DiskAccessDenied));
Err(Error::new(DiskError::DiskAccessDenied))
} else if is_sys_err_io(&e) {
return Err(Error::new(DiskError::FaultyDisk));
Err(Error::new(DiskError::FaultyDisk))
} else {
return Err(Error::new(e));
Err(Error::new(e))
}
}
};
@@ -2027,7 +2021,7 @@ impl DiskAPI for LocalDisk {
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
utils::fs::access(&volume_dir)
access(&volume_dir)
.await
.map_err(|e| convert_access_error(e, DiskError::VolumeAccessDenied))?
}
@@ -2052,7 +2046,7 @@ impl DiskAPI for LocalDisk {
check_path_length(file_path.to_string_lossy().as_ref())?;
let buf = self
.read_all(volume, format!("{}/{}", &path, super::STORAGE_FORMAT_FILE).as_str())
.read_all(volume, format!("{}/{}", &path, STORAGE_FORMAT_FILE).as_str())
.await
.map_err(|e| {
if is_err_file_not_found(&e) && fi.version_id.is_some() {
@@ -2073,12 +2067,7 @@ impl DiskAPI for LocalDisk {
let wbuf = xl_meta.marshal_msg()?;
return self
.write_all_meta(
volume,
format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(),
&wbuf,
!opts.no_persistence,
)
.write_all_meta(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str(), &wbuf, !opts.no_persistence)
.await;
}
@@ -2087,7 +2076,7 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
let p = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?;
let p = self.get_object_path(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str())?;
let mut meta = FileMeta::new();
if !fi.fresh {
@@ -2103,10 +2092,10 @@ impl DiskAPI for LocalDisk {
let fm_data = meta.marshal_msg()?;
self.write_all(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(), fm_data)
self.write_all(volume, format!("{}/{}", path, STORAGE_FORMAT_FILE).as_str(), fm_data)
.await?;
return Ok(());
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
@@ -2182,11 +2171,11 @@ impl DiskAPI for LocalDisk {
return self.write_metadata("", volume, path, fi).await;
}
if fi.version_id.is_some() {
return Err(Error::new(DiskError::FileVersionNotFound));
return if fi.version_id.is_some() {
Err(Error::new(DiskError::FileVersionNotFound))
} else {
return Err(Error::new(DiskError::FileNotFound));
}
Err(Error::new(DiskError::FileNotFound))
};
}
};
@@ -2194,7 +2183,7 @@ impl DiskAPI for LocalDisk {
let old_dir = meta.delete_version(&fi)?;
if let Some(uuid) = old_dir {
let vid = fi.version_id.unwrap_or(Uuid::nil());
let vid = fi.version_id.unwrap_or_default();
let _ = meta.data.remove(vec![vid, uuid])?;
let old_path = file_path.join(Path::new(uuid.to_string().as_str()));
@@ -2357,7 +2346,7 @@ impl DiskAPI for LocalDisk {
self.scanning.fetch_add(1, Ordering::SeqCst);
defer!(|| { self.scanning.fetch_sub(1, Ordering::SeqCst) });
// must befor metadata_sys
// must before metadata_sys
let Some(store) = new_object_layer_fn() else { return Err(Error::msg("errServerNotInitialized")) };
let mut cache = cache.clone();
@@ -2375,7 +2364,7 @@ impl DiskAPI for LocalDisk {
}
}
let vcfg = (BucketVersioningSys::get(&cache.info.name).await).ok();
let vcfg = BucketVersioningSys::get(&cache.info.name).await.ok();
let loc = self.get_disk_location();
let disks = store.get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()).await?;
@@ -2489,7 +2478,7 @@ impl DiskAPI for LocalDisk {
)
.await?;
data_usage_info.info.last_update = Some(SystemTime::now());
info!("ns_scanner completed: {data_usage_info:?}");
debug!("ns_scanner completed: {data_usage_info:?}");
Ok(data_usage_info)
}
@@ -2546,7 +2535,7 @@ mod test {
super::super::RUSTFS_META_TMP_DELETED_BUCKET,
super::super::RUSTFS_META_TMP_BUCKET,
super::super::RUSTFS_META_MULTIPART_BUCKET,
super::super::RUSTFS_META_BUCKET,
RUSTFS_META_BUCKET,
];
let paths: Vec<_> = vols.iter().map(|v| Path::new(v).join("test")).collect();
+191 -191
View File
@@ -57,6 +57,14 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(skip(self))]
async fn is_online(&self) -> bool {
match self {
Disk::Local(local_disk) => local_disk.is_online().await,
Disk::Remote(remote_disk) => remote_disk.is_online().await,
}
}
#[tracing::instrument(skip(self))]
fn is_local(&self) -> bool {
match self {
@@ -73,14 +81,6 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(skip(self))]
async fn is_online(&self) -> bool {
match self {
Disk::Local(local_disk) => local_disk.is_online().await,
Disk::Remote(remote_disk) => remote_disk.is_online().await,
}
}
#[tracing::instrument(skip(self))]
fn endpoint(&self) -> Endpoint {
match self {
@@ -97,22 +97,6 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(skip(self))]
fn path(&self) -> PathBuf {
match self {
Disk::Local(local_disk) => local_disk.path(),
Disk::Remote(remote_disk) => remote_disk.path(),
}
}
#[tracing::instrument(skip(self))]
fn get_disk_location(&self) -> DiskLocation {
match self {
Disk::Local(local_disk) => local_disk.get_disk_location(),
Disk::Remote(remote_disk) => remote_disk.get_disk_location(),
}
}
#[tracing::instrument(skip(self))]
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
match self {
@@ -130,133 +114,18 @@ impl DiskAPI for Disk {
}
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
fn path(&self) -> PathBuf {
match self {
Disk::Local(local_disk) => local_disk.read_all(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.read_all(volume, path).await,
Disk::Local(local_disk) => local_disk.path(),
Disk::Remote(remote_disk) => remote_disk.path(),
}
}
#[tracing::instrument(skip(self))]
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
fn get_disk_location(&self) -> DiskLocation {
match self {
Disk::Local(local_disk) => local_disk.write_all(volume, path, data).await,
Disk::Remote(remote_disk) => remote_disk.write_all(volume, path, data).await,
}
}
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete(volume, path, opt).await,
Disk::Remote(remote_disk) => remote_disk.delete(volume, path, opt).await,
}
}
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
match self {
Disk::Local(local_disk) => local_disk.verify_file(volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.verify_file(volume, path, fi).await,
}
}
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
match self {
Disk::Local(local_disk) => local_disk.check_parts(volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.check_parts(volume, path, fi).await,
}
}
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.rename_part(src_volume, src_path, dst_volume, dst_path, meta).await,
Disk::Remote(remote_disk) => {
remote_disk
.rename_part(src_volume, src_path, dst_volume, dst_path, meta)
.await
}
}
}
#[tracing::instrument(skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await,
Disk::Remote(remote_disk) => remote_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await,
}
}
#[tracing::instrument(skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result<FileWriter> {
match self {
Disk::Local(local_disk) => local_disk.create_file(_origvolume, volume, path, _file_size).await,
Disk::Remote(remote_disk) => remote_disk.create_file(_origvolume, volume, path, _file_size).await,
}
}
#[tracing::instrument(skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
match self {
Disk::Local(local_disk) => local_disk.append_file(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.append_file(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
match self {
Disk::Local(local_disk) => local_disk.read_file(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.read_file(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
match self {
Disk::Local(local_disk) => local_disk.read_file_stream(volume, path, offset, length).await,
Disk::Remote(remote_disk) => remote_disk.read_file_stream(volume, path, offset, length).await,
}
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
match self {
Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
Disk::Remote(remote_disk) => remote_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
}
}
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.walk_dir(opts, wr).await,
Disk::Remote(remote_disk) => remote_disk.walk_dir(opts, wr).await,
}
}
#[tracing::instrument(skip(self))]
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
match self {
Disk::Local(local_disk) => local_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
Disk::Remote(remote_disk) => remote_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
}
}
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.make_volumes(volumes).await,
Disk::Remote(remote_disk) => remote_disk.make_volumes(volumes).await,
Disk::Local(local_disk) => local_disk.get_disk_location(),
Disk::Remote(remote_disk) => remote_disk.get_disk_location(),
}
}
@@ -268,6 +137,14 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.make_volumes(volumes).await,
Disk::Remote(remote_disk) => remote_disk.make_volumes(volumes).await,
}
}
#[tracing::instrument(skip(self))]
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
match self {
@@ -285,49 +162,18 @@ impl DiskAPI for Disk {
}
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
async fn delete_volume(&self, volume: &str) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete_paths(volume, paths).await,
Disk::Remote(remote_disk) => remote_disk.delete_paths(volume, paths).await,
Disk::Local(local_disk) => local_disk.delete_volume(volume).await,
Disk::Remote(remote_disk) => remote_disk.delete_volume(volume).await,
}
}
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
#[tracing::instrument(skip(self, wr))]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.update_metadata(volume, path, fi, opts).await,
Disk::Remote(remote_disk) => remote_disk.update_metadata(volume, path, fi, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.write_metadata(_org_volume, volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.write_metadata(_org_volume, volume, path, fi).await,
}
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_version(
&self,
_org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
match self {
Disk::Local(local_disk) => local_disk.read_version(_org_volume, volume, path, version_id, opts).await,
Disk::Remote(remote_disk) => remote_disk.read_version(_org_volume, volume, path, version_id, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
match self {
Disk::Local(local_disk) => local_disk.read_xl(volume, path, read_data).await,
Disk::Remote(remote_disk) => remote_disk.read_xl(volume, path, read_data).await,
Disk::Local(local_disk) => local_disk.walk_dir(opts, wr).await,
Disk::Remote(remote_disk) => remote_disk.walk_dir(opts, wr).await,
}
}
@@ -359,6 +205,152 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(skip(self))]
async fn delete_paths(&self, volume: &str, paths: &[String]) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete_paths(volume, paths).await,
Disk::Remote(remote_disk) => remote_disk.delete_paths(volume, paths).await,
}
}
#[tracing::instrument(skip(self))]
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.write_metadata(_org_volume, volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.write_metadata(_org_volume, volume, path, fi).await,
}
}
#[tracing::instrument(skip(self))]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.update_metadata(volume, path, fi, opts).await,
Disk::Remote(remote_disk) => remote_disk.update_metadata(volume, path, fi, opts).await,
}
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_version(
&self,
_org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
match self {
Disk::Local(local_disk) => local_disk.read_version(_org_volume, volume, path, version_id, opts).await,
Disk::Remote(remote_disk) => remote_disk.read_version(_org_volume, volume, path, version_id, opts).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
match self {
Disk::Local(local_disk) => local_disk.read_xl(volume, path, read_data).await,
Disk::Remote(remote_disk) => remote_disk.read_xl(volume, path, read_data).await,
}
}
#[tracing::instrument(skip(self, fi))]
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
match self {
Disk::Local(local_disk) => local_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
Disk::Remote(remote_disk) => remote_disk.rename_data(src_volume, src_path, fi, dst_volume, dst_path).await,
}
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
match self {
Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
Disk::Remote(remote_disk) => remote_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
match self {
Disk::Local(local_disk) => local_disk.read_file(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.read_file(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
match self {
Disk::Local(local_disk) => local_disk.read_file_stream(volume, path, offset, length).await,
Disk::Remote(remote_disk) => remote_disk.read_file_stream(volume, path, offset, length).await,
}
}
#[tracing::instrument(skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
match self {
Disk::Local(local_disk) => local_disk.append_file(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.append_file(volume, path).await,
}
}
#[tracing::instrument(skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result<FileWriter> {
match self {
Disk::Local(local_disk) => local_disk.create_file(_origvolume, volume, path, _file_size).await,
Disk::Remote(remote_disk) => remote_disk.create_file(_origvolume, volume, path, _file_size).await,
}
}
#[tracing::instrument(skip(self))]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await,
Disk::Remote(remote_disk) => remote_disk.rename_file(src_volume, src_path, dst_volume, dst_path).await,
}
}
#[tracing::instrument(skip(self))]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec<u8>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.rename_part(src_volume, src_path, dst_volume, dst_path, meta).await,
Disk::Remote(remote_disk) => {
remote_disk
.rename_part(src_volume, src_path, dst_volume, dst_path, meta)
.await
}
}
}
#[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete(volume, path, opt).await,
Disk::Remote(remote_disk) => remote_disk.delete(volume, path, opt).await,
}
}
#[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
match self {
Disk::Local(local_disk) => local_disk.verify_file(volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.verify_file(volume, path, fi).await,
}
}
#[tracing::instrument(skip(self))]
async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
match self {
Disk::Local(local_disk) => local_disk.check_parts(volume, path, fi).await,
Disk::Remote(remote_disk) => remote_disk.check_parts(volume, path, fi).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
match self {
@@ -368,10 +360,18 @@ impl DiskAPI for Disk {
}
#[tracing::instrument(skip(self))]
async fn delete_volume(&self, volume: &str) -> Result<()> {
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.delete_volume(volume).await,
Disk::Remote(remote_disk) => remote_disk.delete_volume(volume).await,
Disk::Local(local_disk) => local_disk.write_all(volume, path, data).await,
Disk::Remote(remote_disk) => remote_disk.write_all(volume, path, data).await,
}
}
#[tracing::instrument(skip(self))]
async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>> {
match self {
Disk::Local(local_disk) => local_disk.read_all(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.read_all(volume, path).await,
}
}
@@ -406,12 +406,12 @@ impl DiskAPI for Disk {
}
}
pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result<DiskStore> {
pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result<DiskStore> {
if ep.is_local {
let s = local::LocalDisk::new(ep, opt.cleanup).await?;
let s = LocalDisk::new(ep, opt.cleanup).await?;
Ok(Arc::new(Disk::Local(Box::new(s))))
} else {
let remote_disk = remote::RemoteDisk::new(ep, opt).await?;
let remote_disk = RemoteDisk::new(ep, opt).await?;
Ok(Arc::new(Disk::Remote(Box::new(remote_disk))))
}
}
@@ -601,7 +601,7 @@ impl FileInfoVersions {
return None;
}
let vid = Uuid::parse_str(v).unwrap_or(Uuid::nil());
let vid = Uuid::parse_str(v).unwrap_or_default();
self.versions.iter().position(|v| v.version_id == Some(vid))
}
@@ -763,7 +763,7 @@ impl MetaCacheEntry {
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
return Ok(fi);
Ok(fi)
}
pub fn file_info_versions(&self, bucket: &str) -> Result<FileInfoVersions> {
+1 -1
View File
@@ -199,7 +199,7 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
}
if let Some(parent) = dir_path.as_ref().parent() {
// 不支持递归,直接create_dir_all了
// 不支持递归,直接 create_dir_all
if let Err(e) = utils::fs::make_dir_all(&parent).await {
if os_is_exist(&e) {
return Ok(());
+580 -580
View File
File diff suppressed because it is too large Load Diff
+6 -10
View File
@@ -94,13 +94,10 @@ impl DisksLayout {
let is_ellipses = args.iter().any(|v| has_ellipses(&[v]));
let set_drive_count_env = match env::var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT) {
Ok(res) => res,
Err(err) => {
debug!("{} not set use default:0, {:?}", ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, err);
"0".to_string()
}
};
let set_drive_count_env = env::var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT).unwrap_or_else(|err| {
debug!("{} not set use default:0, {:?}", ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, err);
"0".to_string()
});
let set_drive_count: usize = set_drive_count_env.parse()?;
// None of the args have ellipses use the old style.
@@ -290,7 +287,7 @@ impl EndpointSet {
}
}
/// returns a greatest common divisor of all the ellipses sizes.
/// returns the greatest common divisor of all the ellipses sizes.
fn get_divisible_size(total_sizes: &[usize]) -> usize {
fn gcd(mut x: usize, mut y: usize) -> usize {
while y != 0 {
@@ -446,7 +443,6 @@ fn get_total_sizes(arg_patterns: &[ArgPattern]) -> Vec<usize> {
#[cfg(test)]
mod test {
use super::*;
impl PartialEq for EndpointSet {
@@ -868,7 +864,7 @@ mod test {
},
success: true,
},
// More than 1 ellipses per argument for standalone setup.
// More than an ellipse per argument for standalone setup.
TestCase {
num: 15,
arg: "/export{1...10}/disk{1...10}",
+4 -4
View File
@@ -113,7 +113,7 @@ impl Erasure {
let blocks_inner = blocks.clone();
async move {
if let Some(w) = w_op {
(w.write(blocks_inner[i_inner].clone()).await).err()
w.write(blocks_inner[i_inner].clone()).await.err()
} else {
Some(Error::new(DiskError::DiskNotFound))
}
@@ -302,7 +302,7 @@ impl Erasure {
// ec encode, 结果会写进 data_buffer
let data_slices: SmallVec<[&mut [u8]; 16]> = data_buffer.chunks_exact_mut(shard_size).collect();
// partiy 数量大于0 才ec
// partiy 数量大于 0 才 ec
if self.parity_shards > 0 {
self.encoder.as_ref().unwrap().encode(data_slices)?;
}
@@ -348,7 +348,7 @@ impl Erasure {
let last_shard_size = last_block_size.div_ceil(self.data_shards);
num_shards * self.shard_size(self.block_size) + last_shard_size
// // 因为写入的时候ec需要补全,所以最后一个长度应该也是一样的
// // 因为写入的时候 ec 需要补全,所以最后一个长度应该也是一样的
// if last_block_size != 0 {
// num_shards += 1
// }
@@ -446,7 +446,7 @@ pub struct ShardReader {
parity_block_count: usize,
shard_size: usize, // 每个分片的块大小 一次读取一块
shard_file_size: usize, // 分片文件总长度
offset: usize, // 在分片中的offset
offset: usize, // 在分片中的 offset
}
impl ShardReader {
+173 -95
View File
@@ -73,7 +73,7 @@ impl FileMeta {
// check_xl2_v1 读 xl 文件头,返回后续内容,版本信息
// checkXL2V1
#[tracing::instrument]
#[tracing::instrument(level = "debug", skip_all)]
pub fn check_xl2_v1(buf: &[u8]) -> Result<(&[u8], u16, u16)> {
if buf.len() < 8 {
return Err(Error::msg("xl file header not exists"));
@@ -94,6 +94,13 @@ impl FileMeta {
// 固定 u32
pub fn read_bytes_header(buf: &[u8]) -> Result<(u32, &[u8])> {
if buf.len() < 5 {
return Err(Error::new(io::Error::new(
io::ErrorKind::UnexpectedEof,
format!("Buffer too small: {} bytes, need at least 5", buf.len()),
)));
}
let (mut size_buf, _) = buf.split_at(5);
// 取 meta 数据,buf = crc + data
@@ -169,7 +176,7 @@ impl FileMeta {
}
// decode_xl_headers 解析 meta 头,返回 (versions 数量,xl_header_version, xl_meta_version, 已读数据长度)
#[tracing::instrument]
#[tracing::instrument(level = "debug", skip_all)]
fn decode_xl_headers(buf: &[u8]) -> Result<(usize, u8, u8, &[u8])> {
let mut cur = Cursor::new(buf);
@@ -349,9 +356,9 @@ impl FileMeta {
self.versions.sort_by(|a, b| {
match (a.header.mod_time, b.header.mod_time) {
(Some(a_time), Some(b_time)) => b_time.cmp(&a_time), // Descending order
(Some(_), None) => std::cmp::Ordering::Less,
(None, Some(_)) => std::cmp::Ordering::Greater,
(None, None) => std::cmp::Ordering::Equal,
(Some(_), None) => Ordering::Less,
(None, Some(_)) => Ordering::Greater,
(None, None) => Ordering::Equal,
}
});
}
@@ -446,7 +453,7 @@ impl FileMeta {
let vid = fi.version_id;
if let Some(ref data) = fi.data {
let key = vid.unwrap_or(Uuid::nil()).to_string();
let key = vid.unwrap_or_default().to_string();
self.data.replace(&key, data.clone())?;
}
@@ -515,18 +522,18 @@ impl FileMeta {
continue;
}
match ver.header.version_type {
VersionType::Invalid => return Err(Error::msg("invalid file meta version")),
VersionType::Delete => return Ok(None),
return match ver.header.version_type {
VersionType::Invalid => Err(Error::msg("invalid file meta version")),
VersionType::Delete => Ok(None),
VersionType::Object => {
let v = self.get_idx(i)?;
self.versions.remove(i);
let a = v.object.map(|v| v.data_dir).unwrap_or_default();
return Ok(a);
Ok(a)
}
}
};
}
Err(Error::new(DiskError::FileVersionNotFound))
@@ -574,7 +581,7 @@ impl FileMeta {
fi.successor_mod_time = succ_mod_time;
}
if read_data {
fi.data = self.data.find(fi.version_id.unwrap_or(Uuid::nil()).to_string().as_str())?;
fi.data = self.data.find(fi.version_id.unwrap_or_default().to_string().as_str())?;
}
fi.num_versions = self.versions.len();
@@ -1004,7 +1011,7 @@ impl FileMetaVersionHeader {
rmp::encode::write_array_len(&mut wr, 7)?;
// version_id
rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or(Uuid::nil()).as_bytes())?;
rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?;
// mod_time
rmp::encode::write_i64(&mut wr, self.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp_nanos() as i64)?;
// signature
@@ -1079,20 +1086,20 @@ impl PartialOrd for FileMetaVersionHeader {
impl Ord for FileMetaVersionHeader {
fn cmp(&self, other: &Self) -> Ordering {
match self.mod_time.cmp(&other.mod_time) {
core::cmp::Ordering::Equal => {}
Ordering::Equal => {}
ord => return ord,
}
match self.version_type.cmp(&other.version_type) {
core::cmp::Ordering::Equal => {}
Ordering::Equal => {}
ord => return ord,
}
match self.signature.cmp(&other.signature) {
core::cmp::Ordering::Equal => {}
Ordering::Equal => {}
ord => return ord,
}
match self.version_id.cmp(&other.version_id) {
core::cmp::Ordering::Equal => {}
Ordering::Equal => {}
ord => return ord,
}
self.flags.cmp(&other.flags)
@@ -1430,11 +1437,11 @@ impl MetaObject {
// string "ID"
rmp::encode::write_str(&mut wr, "ID")?;
rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or(Uuid::nil()).as_bytes())?;
rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?;
// string "DDir"
rmp::encode::write_str(&mut wr, "DDir")?;
rmp::encode::write_bin(&mut wr, self.data_dir.unwrap_or(Uuid::nil()).as_bytes())?;
rmp::encode::write_bin(&mut wr, self.data_dir.unwrap_or_default().as_bytes())?;
// string "EcAlgo"
rmp::encode::write_str(&mut wr, "EcAlgo")?;
@@ -1754,7 +1761,7 @@ impl MetaDeleteMarker {
// string "ID"
rmp::encode::write_str(&mut wr, "ID")?;
rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or(Uuid::nil()).as_bytes())?;
rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?;
// string "MTime"
rmp::encode::write_str(&mut wr, "MTime")?;
@@ -2174,6 +2181,7 @@ pub async fn read_xl_meta_no_data<R: AsyncRead + Unpin>(reader: &mut R, size: us
}
}
#[cfg(test)]
#[allow(clippy::field_reassign_with_default)]
mod test {
use super::*;
@@ -2241,7 +2249,7 @@ mod test {
#[test]
#[tracing::instrument]
fn test_marshal_metaversion() {
let mut fi = FileInfo::new("tset", 3, 2);
let mut fi = FileInfo::new("test", 3, 2);
fi.version_id = Some(Uuid::new_v4());
fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(OffsetDateTime::now_utc().unix_timestamp()).unwrap());
let mut obj = FileMetaVersion::from(fi);
@@ -2392,10 +2400,12 @@ mod test {
#[test]
fn test_file_meta_version_header_methods() {
let mut header = FileMetaVersionHeader::default();
header.ec_n = 4;
header.ec_m = 2;
header.flags = XL_FLAG_FREE_VERSION;
let mut header = FileMetaVersionHeader {
ec_n: 4,
ec_m: 2,
flags: XL_FLAG_FREE_VERSION,
..Default::default()
};
// Test has_ec
assert!(header.has_ec());
@@ -2413,13 +2423,17 @@ mod test {
#[test]
fn test_file_meta_version_header_comparison() {
let mut header1 = FileMetaVersionHeader::default();
header1.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000).unwrap());
header1.version_id = Some(Uuid::new_v4());
let mut header1 = FileMetaVersionHeader {
mod_time: Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()),
version_id: Some(Uuid::new_v4()),
..Default::default()
};
let mut header2 = FileMetaVersionHeader::default();
header2.mod_time = Some(OffsetDateTime::from_unix_timestamp(2000).unwrap());
header2.version_id = Some(Uuid::new_v4());
let mut header2 = FileMetaVersionHeader {
mod_time: Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()),
version_id: Some(Uuid::new_v4()),
..Default::default()
};
// Test sorts_before - header2 should sort before header1 (newer mod_time)
assert!(!header1.sorts_before(&header2));
@@ -2469,9 +2483,11 @@ mod test {
#[test]
fn test_meta_object_methods() {
let mut obj = MetaObject::default();
obj.data_dir = Some(Uuid::new_v4());
obj.size = 1024;
let mut obj = MetaObject {
data_dir: Some(Uuid::new_v4()),
size: 1024,
..Default::default()
};
// Test use_data_dir
assert!(obj.use_data_dir());
@@ -2667,16 +2683,20 @@ mod test {
fn test_decode_data_dir_from_meta() {
// Test with valid metadata containing data_dir
let data_dir = Some(Uuid::new_v4());
let mut obj = MetaObject::default();
obj.data_dir = data_dir;
obj.mod_time = Some(OffsetDateTime::now_utc());
obj.erasure_algorithm = ErasureAlgo::ReedSolomon;
obj.bitrot_checksum_algo = ChecksumAlgo::HighwayHash;
let obj = MetaObject {
data_dir,
mod_time: Some(OffsetDateTime::now_utc()),
erasure_algorithm: ErasureAlgo::ReedSolomon,
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
..Default::default()
};
// Create a valid FileMetaVersion with the object
let mut version = FileMetaVersion::default();
version.version_type = VersionType::Object;
version.object = Some(obj);
let version = FileMetaVersion {
version_type: VersionType::Object,
object: Some(obj),
..Default::default()
};
let encoded = version.marshal_msg().unwrap();
let result = FileMetaVersion::decode_data_dir_from_meta(&encoded);
@@ -2832,18 +2852,32 @@ fn test_file_meta_load_function() {
#[test]
fn test_file_meta_read_bytes_header() {
// Test read_bytes_header function - it expects the first 5 bytes to be msgpack bin length
// Create a buffer with proper msgpack bin format for a 9-byte binary
let mut buf = vec![0xc4, 0x09]; // msgpack bin8 format for 9 bytes
buf.extend_from_slice(b"test data"); // 9 bytes of data
buf.extend_from_slice(b"extra"); // additional data
// Create a real FileMeta and marshal it to get proper format
let mut fm = FileMeta::new();
let mut fi = FileInfo::new("test", 4, 2);
fi.version_id = Some(Uuid::new_v4());
fi.mod_time = Some(OffsetDateTime::now_utc());
fm.add_version(fi).unwrap();
let result = FileMeta::read_bytes_header(&buf);
let marshaled = fm.marshal_msg().unwrap();
// First call check_xl2_v1 to get the buffer after XL header validation
let (after_xl_header, _major, _minor) = FileMeta::check_xl2_v1(&marshaled).unwrap();
// Ensure we have at least 5 bytes for read_bytes_header
if after_xl_header.len() < 5 {
panic!("Buffer too small: {} bytes, need at least 5", after_xl_header.len());
}
// Now call read_bytes_header on the remaining buffer
let result = FileMeta::read_bytes_header(after_xl_header);
assert!(result.is_ok());
let (length, remaining) = result.unwrap();
assert_eq!(length, 9); // "test data" length
// remaining should be everything after the 5-byte header (but we only have 2-byte header)
assert_eq!(remaining.len(), buf.len() - 5);
// The length should be greater than 0 for real data
assert!(length > 0);
// remaining should be everything after the 5-byte header
assert_eq!(remaining.len(), after_xl_header.len() - 5);
// Test with buffer too small
let small_buf = vec![0u8; 2];
@@ -2868,8 +2902,10 @@ fn test_file_meta_get_set_idx() {
assert!(result.is_err());
// Test set_idx
let mut new_version = FileMetaVersion::default();
new_version.version_type = VersionType::Object;
let new_version = FileMetaVersion {
version_type: VersionType::Object,
..Default::default()
};
let result = fm.set_idx(0, new_version);
assert!(result.is_ok());
@@ -2983,10 +3019,12 @@ fn test_file_meta_version_header_from_version() {
#[test]
fn test_meta_object_into_fileinfo() {
let mut obj = MetaObject::default();
obj.version_id = Some(Uuid::new_v4());
obj.size = 1024;
obj.mod_time = Some(OffsetDateTime::now_utc());
let obj = MetaObject {
version_id: Some(Uuid::new_v4()),
size: 1024,
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
let version_id = obj.version_id;
let expected_version_id = version_id;
@@ -3014,9 +3052,11 @@ fn test_meta_object_from_fileinfo() {
#[test]
fn test_meta_delete_marker_into_fileinfo() {
let mut marker = MetaDeleteMarker::default();
marker.version_id = Some(Uuid::new_v4());
marker.mod_time = Some(OffsetDateTime::now_utc());
let marker = MetaDeleteMarker {
version_id: Some(Uuid::new_v4()),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
};
let version_id = marker.version_id;
let expected_version_id = version_id;
@@ -3049,30 +3089,42 @@ fn test_flags_enum() {
#[test]
fn test_file_meta_version_header_user_data_dir() {
let mut header = FileMetaVersionHeader::default();
let header = FileMetaVersionHeader {
flags: 0,
..Default::default()
};
// Test without UsesDataDir flag
header.flags = 0;
assert!(!header.user_data_dir());
// Test with UsesDataDir flag
header.flags = Flags::UsesDataDir as u8;
let header = FileMetaVersionHeader {
flags: Flags::UsesDataDir as u8,
..Default::default()
};
assert!(header.user_data_dir());
// Test with multiple flags including UsesDataDir
header.flags = Flags::UsesDataDir as u8 | Flags::FreeVersion as u8;
let header = FileMetaVersionHeader {
flags: Flags::UsesDataDir as u8 | Flags::FreeVersion as u8,
..Default::default()
};
assert!(header.user_data_dir());
}
#[test]
fn test_file_meta_version_header_ordering() {
let mut header1 = FileMetaVersionHeader::default();
header1.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000).unwrap());
header1.version_id = Some(Uuid::new_v4());
let header1 = FileMetaVersionHeader {
mod_time: Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()),
version_id: Some(Uuid::new_v4()),
..Default::default()
};
let mut header2 = FileMetaVersionHeader::default();
header2.mod_time = Some(OffsetDateTime::from_unix_timestamp(2000).unwrap());
header2.version_id = Some(Uuid::new_v4());
let header2 = FileMetaVersionHeader {
mod_time: Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()),
version_id: Some(Uuid::new_v4()),
..Default::default()
};
// Test partial_cmp
assert!(header1.partial_cmp(&header2).is_some());
@@ -3200,64 +3252,90 @@ async fn test_file_info_from_raw_edge_cases() {
#[test]
fn test_file_meta_version_invalid_cases() {
// Test invalid version
let mut version = FileMetaVersion::default();
version.version_type = VersionType::Invalid;
let version = FileMetaVersion {
version_type: VersionType::Invalid,
..Default::default()
};
assert!(!version.valid());
// Test version with neither object nor delete marker
version.version_type = VersionType::Object;
version.object = None;
version.delete_marker = None;
let version = FileMetaVersion {
version_type: VersionType::Object,
object: None,
delete_marker: None,
..Default::default()
};
assert!(!version.valid());
}
#[test]
fn test_meta_object_edge_cases() {
let mut obj = MetaObject::default();
let obj = MetaObject {
data_dir: None,
..Default::default()
};
// Test use_data_dir with None (use_data_dir always returns true)
obj.data_dir = None;
assert!(obj.use_data_dir());
// Test use_inlinedata (always returns false in current implementation)
obj.size = 128 * 1024; // 128KB threshold
let obj = MetaObject {
size: 128 * 1024, // 128KB threshold
..Default::default()
};
assert!(!obj.use_inlinedata()); // Should be false
obj.size = 128 * 1024 - 1;
let obj = MetaObject {
size: 128 * 1024 - 1,
..Default::default()
};
assert!(!obj.use_inlinedata()); // Should also be false (always false)
}
#[test]
fn test_file_meta_version_header_edge_cases() {
let mut header = FileMetaVersionHeader::default();
let header = FileMetaVersionHeader {
ec_n: 0,
ec_m: 0,
..Default::default()
};
// Test has_ec with zero values
header.ec_n = 0;
header.ec_m = 0;
assert!(!header.has_ec());
// Test matches_not_strict with different signatures but same version_id
let mut other = FileMetaVersionHeader::default();
let version_id = Some(Uuid::new_v4());
header.version_id = version_id;
other.version_id = version_id;
header.version_type = VersionType::Object;
other.version_type = VersionType::Object;
header.signature = [1, 2, 3, 4];
other.signature = [5, 6, 7, 8];
let header = FileMetaVersionHeader {
version_id,
version_type: VersionType::Object,
signature: [1, 2, 3, 4],
..Default::default()
};
let other = FileMetaVersionHeader {
version_id,
version_type: VersionType::Object,
signature: [5, 6, 7, 8],
..Default::default()
};
// Should match because they have same version_id and type
assert!(header.matches_not_strict(&other));
// Test sorts_before with same mod_time but different version_id
let time = OffsetDateTime::from_unix_timestamp(1000).unwrap();
header.mod_time = Some(time);
other.mod_time = Some(time);
header.version_id = Some(Uuid::new_v4());
other.version_id = Some(Uuid::new_v4());
let header_time1 = FileMetaVersionHeader {
mod_time: Some(time),
version_id: Some(Uuid::new_v4()),
..Default::default()
};
let header_time2 = FileMetaVersionHeader {
mod_time: Some(time),
version_id: Some(Uuid::new_v4()),
..Default::default()
};
// Should use version_id for comparison when mod_time is same
let sorts_before = header.sorts_before(&other);
assert!(sorts_before || other.sorts_before(&header)); // One should sort before the other
let sorts_before = header_time1.sorts_before(&header_time2);
assert!(sorts_before || header_time2.sorts_before(&header_time1)); // One should sort before the other
}
#[test]
+9 -9
View File
@@ -151,7 +151,7 @@ pub async fn init_data_scanner() {
let mut r = rand::thread_rng();
r.gen_range(0.0..1.0)
};
let duration = Duration::from_secs_f64(random * (SCANNER_CYCLE.load(std::sync::atomic::Ordering::SeqCst) as f64));
let duration = Duration::from_secs_f64(random * (SCANNER_CYCLE.load(Ordering::SeqCst) as f64));
let sleep_duration = if duration < Duration::new(1, 0) {
Duration::new(1, 0)
} else {
@@ -227,7 +227,7 @@ async fn run_data_scanner() {
}
}
stop_fn(&res).await;
sleep(Duration::from_secs(SCANNER_CYCLE.load(std::sync::atomic::Ordering::SeqCst))).await;
sleep(Duration::from_secs(SCANNER_CYCLE.load(Ordering::SeqCst))).await;
}
}
@@ -383,7 +383,7 @@ impl CurrentScannerCycle {
Deserialize::deserialize(&mut Deserializer::new(&buf[..])).expect("Deserialization failed");
self.cycle_completed = u;
}
name => return Err(Error::msg(format!("not suport field name {}", name))),
name => return Err(Error::msg(format!("not support field name {}", name))),
}
}
@@ -435,8 +435,8 @@ impl ScannerItem {
path_join(&[PathBuf::from(self.prefix.clone()), PathBuf::from(self.object_name.clone())])
}
pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?;
pub async fn apply_versions_actions(&self, fives: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
let obj_infos = self.apply_newer_noncurrent_version_limit(fives).await?;
if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst) as usize {
// todo
}
@@ -453,16 +453,16 @@ impl ScannerItem {
Ok(obj_infos)
}
pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
pub async fn apply_newer_noncurrent_version_limit(&self, fives: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
// let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent);
let versioned = match BucketVersioningSys::get(&self.bucket).await {
Ok(vcfg) => vcfg.versioned(self.object_path().to_str().unwrap_or_default()),
Err(_) => false,
};
let mut object_infos = Vec::with_capacity(fivs.len());
let mut object_infos = Vec::with_capacity(fives.len());
if self.lifecycle.is_none() {
for info in fivs.iter() {
for info in fives.iter() {
object_infos.push(info.to_object_info(&self.bucket, &self.object_path().to_string_lossy(), versioned));
}
return Ok(object_infos);
@@ -1000,7 +1000,7 @@ impl FolderScanner {
if !into.compacted {
self.new_cache.reduce_children_of(
&this_hash,
DATA_SCANNER_COMPACT_AT_CHILDREN.try_into().unwrap(),
DATA_SCANNER_COMPACT_AT_CHILDREN.try_into()?,
self.new_cache.info.name != folder.name,
);
}
+2 -2
View File
@@ -14,7 +14,7 @@ use std::{
time::SystemTime,
};
use tokio::sync::RwLock;
use tracing::info;
use tracing::{debug, info};
use super::data_scanner::{CurrentScannerCycle, UpdateCurrentPathFn};
@@ -149,7 +149,7 @@ impl ScannerMetrics {
}
pub async fn set_cycle(&mut self, c: Option<CurrentScannerCycle>) {
info!("ScannerMetrics set_cycle {c:?}");
debug!("ScannerMetrics set_cycle {c:?}");
*self.cycle_info.write().await = c;
}
+1 -1
View File
@@ -241,7 +241,7 @@ impl ReplicationAllStats {
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct DataUsageEntry {
pub children: DataUsageHashMap,
// These fields do no include any children.
// These fields do not include any children.
pub size: usize,
pub objects: usize,
pub versions: usize,
+24 -24
View File
@@ -57,18 +57,18 @@ pub type HealEntryFn =
pub const BG_HEALING_UUID: &str = "0000-0000-0000-0000";
pub const HEALING_TRACKER_FILENAME: &str = ".healing.bin";
const KEEP_HEAL_SEQ_STATE_DURATION: std::time::Duration = Duration::from_secs(10 * 60);
const KEEP_HEAL_SEQ_STATE_DURATION: Duration = Duration::from_secs(10 * 60);
const HEAL_NOT_STARTED_STATUS: &str = "not started";
const HEAL_RUNNING_STATUS: &str = "running";
const HEAL_STOPPED_STATUS: &str = "stopped";
const HEAL_FINISHED_STATUS: &str = "finished";
pub const RUESTFS_RESERVED_BUCKET: &str = "rustfs";
pub const RUESTFS_RESERVED_BUCKET_PATH: &str = "/rustfs";
pub const RUSTFS_RESERVED_BUCKET: &str = "rustfs";
pub const RUSTFS_RESERVED_BUCKET_PATH: &str = "/rustfs";
pub const LOGIN_PATH_PREFIX: &str = "/login";
const MAX_UNCONSUMED_HEAL_RESULT_ITEMS: usize = 1000;
const HEAL_UNCONSUMED_TIMEOUT: std::time::Duration = Duration::from_secs(24 * 60 * 60);
const HEAL_UNCONSUMED_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60);
pub const NOP_HEAL: &str = "";
lazy_static! {}
@@ -125,7 +125,7 @@ pub fn new_bg_heal_sequence() -> HealSequence {
HealSequence {
start_time: SystemTime::now(),
client_token: BG_HEALING_UUID.to_string(),
bucket: RUESTFS_RESERVED_BUCKET.to_string(),
bucket: RUSTFS_RESERVED_BUCKET.to_string(),
setting: hs,
current_status: Arc::new(RwLock::new(HealSequenceStatus {
summary: HEAL_NOT_STARTED_STATUS.to_string(),
@@ -195,12 +195,12 @@ impl Default for HealSequence {
}
impl HealSequence {
pub fn new(bucket: &str, obj_profix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> Self {
pub fn new(bucket: &str, obj_prefix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> Self {
let client_token = Uuid::new_v4().to_string();
Self {
bucket: bucket.to_string(),
object: obj_profix.to_string(),
object: obj_prefix.to_string(),
report_progress: true,
client_token,
client_address: client_addr.to_string(),
@@ -354,14 +354,14 @@ impl HealSequence {
self.count_failed(heal_type.clone()).await;
}
if !self.report_progress {
if let Some(err) = res.err {
return if let Some(err) = res.err {
if err.to_string() == ERR_SKIP_FILE {
return Ok(());
}
return Err(err);
Err(err)
} else {
return Ok(());
}
Ok(())
};
}
res.result.heal_item_type = heal_type.clone();
if let Some(err) = res.err.as_ref() {
@@ -406,7 +406,7 @@ impl HealSequence {
async fn traverse_and_heal(h: Arc<HealSequence>) {
let buckets_only = false;
let result = (Self::heal_items(h.clone(), buckets_only).await).err();
let result = Self::heal_items(h.clone(), buckets_only).await.err();
let _ = h.traverse_and_heal_done_tx.read().await.send(result).await;
}
@@ -542,12 +542,12 @@ pub async fn heal_sequence_start(h: Arc<HealSequence>) {
match err {
Some(err) => {
let mut current_status_w = h.current_status.write().await;
(current_status_w).summary = HEAL_STOPPED_STATUS.to_string();
(current_status_w).failure_detail = err.to_string();
current_status_w.summary = HEAL_STOPPED_STATUS.to_string();
current_status_w.failure_detail = err.to_string();
},
None => {
let mut current_status_w = h.current_status.write().await;
(current_status_w).summary = HEAL_FINISHED_STATUS.to_string();
current_status_w.summary = HEAL_FINISHED_STATUS.to_string();
}
}
}
@@ -567,11 +567,11 @@ pub struct AllHealState {
impl AllHealState {
pub fn new(cleanup: bool) -> Arc<Self> {
let hstate = Arc::new(AllHealState::default());
let state = Arc::new(AllHealState::default());
let (_, mut rx) = broadcast::channel(1);
if cleanup {
let hstate_clone = hstate.clone();
tokio::spawn(async move {
let state_clone = state.clone();
spawn(async move {
loop {
select! {
result = rx.recv() =>{
@@ -580,14 +580,14 @@ impl AllHealState {
}
}
_ = sleep(Duration::from_secs(5 * 60)) => {
hstate_clone.periodic_heal_seqs_clean().await;
state_clone.periodic_heal_seqs_clean().await;
}
}
}
});
}
hstate
state
}
pub async fn pop_heal_local_disks(&self, heal_local_disks: &[Endpoint]) {
@@ -698,13 +698,13 @@ impl AllHealState {
let _ = self.mu.write().await;
let now = SystemTime::now();
let mut keys_to_reomve = Vec::new();
let mut keys_to_remove = Vec::new();
for (k, v) in self.heal_seq_map.read().await.iter() {
if v.has_ended().await && now.duration_since(*(v.end_time.read().await)).unwrap() > KEEP_HEAL_SEQ_STATE_DURATION {
keys_to_reomve.push(k.clone())
keys_to_remove.push(k.clone())
}
}
for key in keys_to_reomve.iter() {
for key in keys_to_remove.iter() {
self.heal_seq_map.write().await.remove(key);
}
}
@@ -808,7 +808,7 @@ impl AllHealState {
// For background heal do nothing, do not spawn an unnecessary goroutine.
} else {
let heal_sequence_clone = heal_sequence.clone();
tokio::spawn(async {
spawn(async {
heal_sequence_start(heal_sequence_clone).await;
});
}
+395 -13
View File
@@ -27,14 +27,14 @@ pub const READ_BUFFER_SIZE: usize = 1024 * 1024;
#[derive(Debug)]
pub struct HttpFileWriter {
wd: tokio::io::DuplexStream,
err_rx: oneshot::Receiver<std::io::Error>,
err_rx: oneshot::Receiver<io::Error>,
}
impl HttpFileWriter {
pub fn new(url: &str, disk: &str, volume: &str, path: &str, size: usize, append: bool) -> std::io::Result<Self> {
pub fn new(url: &str, disk: &str, volume: &str, path: &str, size: usize, append: bool) -> io::Result<Self> {
let (rd, wd) = tokio::io::duplex(READ_BUFFER_SIZE);
let (err_tx, err_rx) = oneshot::channel::<std::io::Error>();
let (err_tx, err_rx) = oneshot::channel::<io::Error>();
let body = reqwest::Body::wrap_stream(ReaderStream::with_capacity(rd, READ_BUFFER_SIZE));
@@ -74,11 +74,7 @@ impl HttpFileWriter {
impl AsyncWrite for HttpFileWriter {
#[tracing::instrument(level = "debug", skip(self, buf))]
fn poll_write(
mut self: Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> Poll<std::result::Result<usize, std::io::Error>> {
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<Result<usize, io::Error>> {
if let Ok(err) = self.as_mut().err_rx.try_recv() {
return Poll::Ready(Err(err));
}
@@ -87,12 +83,12 @@ impl AsyncWrite for HttpFileWriter {
}
#[tracing::instrument(level = "debug", skip(self))]
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<std::result::Result<(), std::io::Error>> {
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Pin::new(&mut self.wd).poll_flush(cx)
}
#[tracing::instrument(level = "debug", skip(self))]
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<std::result::Result<(), std::io::Error>> {
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
Pin::new(&mut self.wd).poll_shutdown(cx)
}
}
@@ -102,7 +98,7 @@ pub struct HttpFileReader {
}
impl HttpFileReader {
pub async fn new(url: &str, disk: &str, volume: &str, path: &str, offset: usize, length: usize) -> std::io::Result<Self> {
pub async fn new(url: &str, disk: &str, volume: &str, path: &str, offset: usize, length: usize) -> io::Result<Self> {
let resp = reqwest::Client::new()
.get(format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
@@ -124,7 +120,7 @@ impl HttpFileReader {
}
impl AsyncRead for HttpFileReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<tokio::io::Result<()>> {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
Pin::new(&mut self.inner).poll_read(cx, buf)
}
}
@@ -135,6 +131,7 @@ pub trait Etag {
}
pin_project! {
#[derive(Debug)]
pub struct EtagReader<R> {
inner: R,
bytes_tx: mpsc::Sender<Bytes>,
@@ -172,7 +169,7 @@ impl<R: Send> Etag for EtagReader<R> {
impl<R: AsyncRead + Unpin> AsyncRead for EtagReader<R> {
#[tracing::instrument(level = "info", skip_all)]
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<tokio::io::Result<()>> {
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let me = self.project();
loop {
@@ -196,3 +193,388 @@ impl<R: AsyncRead + Unpin> AsyncRead for EtagReader<R> {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
#[tokio::test]
async fn test_constants() {
assert_eq!(READ_BUFFER_SIZE, 1024 * 1024);
// READ_BUFFER_SIZE is a compile-time constant, no need to assert
// assert!(READ_BUFFER_SIZE > 0);
}
#[tokio::test]
async fn test_http_file_writer_creation() {
let writer = HttpFileWriter::new("http://localhost:8080", "test-disk", "test-volume", "test-path", 1024, false);
assert!(writer.is_ok(), "HttpFileWriter creation should succeed");
}
#[tokio::test]
async fn test_http_file_writer_creation_with_special_characters() {
let writer = HttpFileWriter::new(
"http://localhost:8080",
"test disk with spaces",
"test/volume",
"test file with spaces & symbols.txt",
1024,
false,
);
assert!(writer.is_ok(), "HttpFileWriter creation with special characters should succeed");
}
#[tokio::test]
async fn test_http_file_writer_creation_append_mode() {
let writer = HttpFileWriter::new(
"http://localhost:8080",
"test-disk",
"test-volume",
"append-test.txt",
1024,
true, // append mode
);
assert!(writer.is_ok(), "HttpFileWriter creation in append mode should succeed");
}
#[tokio::test]
async fn test_http_file_writer_creation_zero_size() {
let writer = HttpFileWriter::new(
"http://localhost:8080",
"test-disk",
"test-volume",
"empty-file.txt",
0, // zero size
false,
);
assert!(writer.is_ok(), "HttpFileWriter creation with zero size should succeed");
}
#[tokio::test]
async fn test_http_file_writer_creation_large_size() {
let writer = HttpFileWriter::new(
"http://localhost:8080",
"test-disk",
"test-volume",
"large-file.txt",
1024 * 1024 * 100, // 100MB
false,
);
assert!(writer.is_ok(), "HttpFileWriter creation with large size should succeed");
}
#[tokio::test]
async fn test_http_file_writer_invalid_url() {
let writer = HttpFileWriter::new("invalid-url", "test-disk", "test-volume", "test-path", 1024, false);
// This should still succeed at creation time, errors occur during actual I/O
assert!(writer.is_ok(), "HttpFileWriter creation should succeed even with invalid URL");
}
#[tokio::test]
async fn test_http_file_reader_creation() {
// Test creation without actually making HTTP requests
// We'll test the URL construction logic by checking the error messages
let result =
HttpFileReader::new("http://invalid-server:9999", "test-disk", "test-volume", "test-file.txt", 0, 1024).await;
// May succeed or fail depending on network conditions, but should not panic
// The important thing is that the URL construction logic works
assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic");
}
#[tokio::test]
async fn test_http_file_reader_with_offset_and_length() {
let result = HttpFileReader::new(
"http://invalid-server:9999",
"test-disk",
"test-volume",
"test-file.txt",
100, // offset
500, // length
)
.await;
// May succeed or fail, but this tests parameter handling
assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic");
}
#[tokio::test]
async fn test_http_file_reader_zero_length() {
let result = HttpFileReader::new(
"http://invalid-server:9999",
"test-disk",
"test-volume",
"test-file.txt",
0,
0, // zero length
)
.await;
// May succeed or fail, but this tests zero length handling
assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic");
}
#[tokio::test]
async fn test_http_file_reader_with_special_characters() {
let result = HttpFileReader::new(
"http://invalid-server:9999",
"test disk with spaces",
"test/volume",
"test file with spaces & symbols.txt",
0,
1024,
)
.await;
// May succeed or fail, but this tests URL encoding
assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic");
}
#[tokio::test]
async fn test_etag_reader_creation() {
let data = b"hello world";
let cursor = Cursor::new(data);
let etag_reader = EtagReader::new(cursor);
// Test that the reader was created successfully
assert!(format!("{:?}", etag_reader).contains("EtagReader"));
}
#[tokio::test]
async fn test_etag_reader_read_and_compute() {
let data = b"hello world";
let cursor = Cursor::new(data);
let etag_reader = EtagReader::new(cursor);
// Test that EtagReader can be created and the etag method works
// Note: Due to the complex implementation of EtagReader's poll_read,
// we focus on testing the creation and etag computation without reading
let etag = etag_reader.etag().await;
assert!(!etag.is_empty(), "ETag should not be empty");
assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters"); // MD5 hex string
}
#[tokio::test]
async fn test_etag_reader_empty_data() {
let data = b"";
let cursor = Cursor::new(data);
let etag_reader = EtagReader::new(cursor);
// Test ETag computation for empty data without reading
let etag = etag_reader.etag().await;
assert!(!etag.is_empty(), "ETag should not be empty even for empty data");
assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters");
// MD5 of empty data should be d41d8cd98f00b204e9800998ecf8427e
assert_eq!(etag, "d41d8cd98f00b204e9800998ecf8427e", "Empty data should have known MD5");
}
#[tokio::test]
async fn test_etag_reader_large_data() {
let data = vec![0u8; 10000]; // 10KB of zeros
let cursor = Cursor::new(data.clone());
let etag_reader = EtagReader::new(cursor);
// Test ETag computation for large data without reading
let etag = etag_reader.etag().await;
assert!(!etag.is_empty(), "ETag should not be empty");
assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters");
}
#[tokio::test]
async fn test_etag_reader_consistent_hash() {
let data = b"test data for consistent hashing";
// Create two identical readers
let cursor1 = Cursor::new(data);
let etag_reader1 = EtagReader::new(cursor1);
let cursor2 = Cursor::new(data);
let etag_reader2 = EtagReader::new(cursor2);
// Compute ETags without reading
let etag1 = etag_reader1.etag().await;
let etag2 = etag_reader2.etag().await;
assert_eq!(etag1, etag2, "ETags should be identical for identical data");
}
#[tokio::test]
async fn test_etag_reader_different_data_different_hash() {
let data1 = b"first data set";
let data2 = b"second data set";
let cursor1 = Cursor::new(data1);
let etag_reader1 = EtagReader::new(cursor1);
let cursor2 = Cursor::new(data2);
let etag_reader2 = EtagReader::new(cursor2);
// Note: Due to the current EtagReader implementation,
// calling etag() without reading data first will return empty data hash
// This test verifies that the implementation is consistent
let etag1 = etag_reader1.etag().await;
let etag2 = etag_reader2.etag().await;
// Both should return the same hash (empty data hash) since no data was read
assert_eq!(etag1, etag2, "ETags should be consistent when no data is read");
assert_eq!(etag1, "d41d8cd98f00b204e9800998ecf8427e", "Should be empty data MD5");
}
#[tokio::test]
async fn test_etag_reader_creation_with_different_data() {
let data = b"this is a longer piece of data for testing";
let cursor = Cursor::new(data);
let etag_reader = EtagReader::new(cursor);
// Test ETag computation
let etag = etag_reader.etag().await;
assert!(!etag.is_empty(), "ETag should not be empty");
assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters");
}
#[tokio::test]
async fn test_file_reader_and_writer_types() {
// Test that the type aliases are correctly defined
let _reader: FileReader = Box::new(Cursor::new(b"test"));
let (_writer_tx, writer_rx) = tokio::io::duplex(1024);
let _writer: FileWriter = Box::new(writer_rx);
// If this compiles, the types are correctly defined
// This is a placeholder test - remove meaningless assertion
// assert!(true);
}
#[tokio::test]
async fn test_etag_trait_implementation() {
let data = b"test data for trait";
let cursor = Cursor::new(data);
let etag_reader = EtagReader::new(cursor);
// Test the Etag trait
let etag = etag_reader.etag().await;
assert!(!etag.is_empty(), "ETag should not be empty");
// Verify it's a valid hex string
assert!(etag.chars().all(|c| c.is_ascii_hexdigit()), "ETag should be a valid hex string");
}
#[tokio::test]
async fn test_read_buffer_size_constant() {
assert_eq!(READ_BUFFER_SIZE, 1024 * 1024);
// READ_BUFFER_SIZE is a compile-time constant, no need to assert
// assert!(READ_BUFFER_SIZE > 0);
// assert!(READ_BUFFER_SIZE % 1024 == 0, "Buffer size should be a multiple of 1024");
}
#[tokio::test]
async fn test_concurrent_etag_operations() {
let data1 = b"concurrent test data 1";
let data2 = b"concurrent test data 2";
let data3 = b"concurrent test data 3";
let cursor1 = Cursor::new(data1);
let cursor2 = Cursor::new(data2);
let cursor3 = Cursor::new(data3);
let etag_reader1 = EtagReader::new(cursor1);
let etag_reader2 = EtagReader::new(cursor2);
let etag_reader3 = EtagReader::new(cursor3);
// Compute ETags concurrently
let (result1, result2, result3) = tokio::join!(etag_reader1.etag(), etag_reader2.etag(), etag_reader3.etag());
// All ETags should be the same (empty data hash) since no data was read
assert_eq!(result1, result2);
assert_eq!(result2, result3);
assert_eq!(result1, result3);
assert_eq!(result1.len(), 32);
assert_eq!(result2.len(), 32);
assert_eq!(result3.len(), 32);
// All should be the empty data MD5
assert_eq!(result1, "d41d8cd98f00b204e9800998ecf8427e");
}
#[tokio::test]
async fn test_edge_case_parameters() {
// Test HttpFileWriter with edge case parameters
let writer = HttpFileWriter::new(
"http://localhost:8080",
"", // empty disk
"", // empty volume
"", // empty path
0, // zero size
false,
);
assert!(writer.is_ok(), "HttpFileWriter should handle empty parameters");
// Test HttpFileReader with edge case parameters
let result = HttpFileReader::new(
"http://invalid:9999",
"", // empty disk
"", // empty volume
"", // empty path
0, // zero offset
0, // zero length
)
.await;
// May succeed or fail, but parameters should be handled
assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic");
}
#[tokio::test]
async fn test_url_encoding_edge_cases() {
// Test with characters that need URL encoding
let special_chars = "test file with spaces & symbols + % # ? = @ ! $ ( ) [ ] { } | \\ / : ; , . < > \" '";
let writer = HttpFileWriter::new("http://localhost:8080", special_chars, special_chars, special_chars, 1024, false);
assert!(writer.is_ok(), "HttpFileWriter should handle special characters");
let result = HttpFileReader::new("http://invalid:9999", special_chars, special_chars, special_chars, 0, 1024).await;
// May succeed or fail, but URL encoding should work
assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic");
}
#[tokio::test]
async fn test_etag_reader_with_binary_data() {
// Test with binary data including null bytes
let data = vec![0u8, 1u8, 255u8, 127u8, 128u8, 0u8, 0u8, 255u8];
let cursor = Cursor::new(data.clone());
let etag_reader = EtagReader::new(cursor);
// Test ETag computation for binary data
let etag = etag_reader.etag().await;
assert!(!etag.is_empty(), "ETag should not be empty");
assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters");
assert!(etag.chars().all(|c| c.is_ascii_hexdigit()), "ETag should be valid hex");
}
#[tokio::test]
async fn test_etag_reader_type_constraints() {
// Test that EtagReader works with different reader types
let data = b"type constraint test";
// Test with Cursor
let cursor = Cursor::new(data);
let etag_reader = EtagReader::new(cursor);
let etag = etag_reader.etag().await;
assert_eq!(etag.len(), 32);
// Test with slice
let slice_reader = &data[..];
let etag_reader2 = EtagReader::new(slice_reader);
let etag2 = etag_reader2.etag().await;
assert_eq!(etag2.len(), 32);
// Both should produce the same hash for the same data
assert_eq!(etag, etag2);
}
}
+1 -1
View File
@@ -187,7 +187,7 @@ fn get_offline_disks(offline_host: &str, endpoints: &EndpointServerPools) -> Vec
if (offline_host.is_empty() && ep.is_local) || offline_host == ep.host_port() {
offline_disks.push(madmin::Disk {
endpoint: ep.to_string(),
state: madmin::ItemState::Offline.to_string().to_owned(),
state: ItemState::Offline.to_string().to_owned(),
pool_index: ep.pool_idx,
set_index: ep.set_idx,
disk_index: ep.disk_idx,
+2 -2
View File
@@ -5,7 +5,7 @@ use crate::global::GLOBAL_LOCAL_DISK_MAP;
use crate::heal::heal_commands::{
HealOpts, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK, HEAL_ITEM_BUCKET,
};
use crate::heal::heal_ops::RUESTFS_RESERVED_BUCKET;
use crate::heal::heal_ops::RUSTFS_RESERVED_BUCKET;
use crate::quorum::{bucket_op_ignored_errs, reduce_write_quorum_errs};
use crate::store::all_local_disk;
use crate::utils::proto_err_to_err;
@@ -701,7 +701,7 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result<HealResu
bs_clone.write().await[index] = DRIVE_STATE_OK.to_string();
as_clone.write().await[index] = DRIVE_STATE_OK.to_string();
if bucket == RUESTFS_RESERVED_BUCKET {
if bucket == RUSTFS_RESERVED_BUCKET {
return None;
}
+15 -15
View File
@@ -89,7 +89,7 @@ impl PeerRestClient {
let data = response.storage_info;
let mut buf = Deserializer::new(Cursor::new(data));
let storage_info: madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
let storage_info: madmin::StorageInfo = Deserialize::deserialize(&mut buf)?;
Ok(storage_info)
}
@@ -110,7 +110,7 @@ impl PeerRestClient {
let data = response.server_properties;
let mut buf = Deserializer::new(Cursor::new(data));
let storage_properties: ServerProperties = Deserialize::deserialize(&mut buf).unwrap();
let storage_properties: ServerProperties = Deserialize::deserialize(&mut buf)?;
Ok(storage_properties)
}
@@ -131,7 +131,7 @@ impl PeerRestClient {
let data = response.cpus;
let mut buf = Deserializer::new(Cursor::new(data));
let cpus: Cpus = Deserialize::deserialize(&mut buf).unwrap();
let cpus: Cpus = Deserialize::deserialize(&mut buf)?;
Ok(cpus)
}
@@ -152,7 +152,7 @@ impl PeerRestClient {
let data = response.net_info;
let mut buf = Deserializer::new(Cursor::new(data));
let net_info: NetInfo = Deserialize::deserialize(&mut buf).unwrap();
let net_info: NetInfo = Deserialize::deserialize(&mut buf)?;
Ok(net_info)
}
@@ -173,7 +173,7 @@ impl PeerRestClient {
let data = response.partitions;
let mut buf = Deserializer::new(Cursor::new(data));
let partitions: Partitions = Deserialize::deserialize(&mut buf).unwrap();
let partitions: Partitions = Deserialize::deserialize(&mut buf)?;
Ok(partitions)
}
@@ -194,7 +194,7 @@ impl PeerRestClient {
let data = response.os_info;
let mut buf = Deserializer::new(Cursor::new(data));
let os_info: OsInfo = Deserialize::deserialize(&mut buf).unwrap();
let os_info: OsInfo = Deserialize::deserialize(&mut buf)?;
Ok(os_info)
}
@@ -215,7 +215,7 @@ impl PeerRestClient {
let data = response.sys_services;
let mut buf = Deserializer::new(Cursor::new(data));
let sys_services: SysService = Deserialize::deserialize(&mut buf).unwrap();
let sys_services: SysService = Deserialize::deserialize(&mut buf)?;
Ok(sys_services)
}
@@ -236,7 +236,7 @@ impl PeerRestClient {
let data = response.sys_config;
let mut buf = Deserializer::new(Cursor::new(data));
let sys_config: SysConfig = Deserialize::deserialize(&mut buf).unwrap();
let sys_config: SysConfig = Deserialize::deserialize(&mut buf)?;
Ok(sys_config)
}
@@ -257,7 +257,7 @@ impl PeerRestClient {
let data = response.sys_errors;
let mut buf = Deserializer::new(Cursor::new(data));
let sys_errors: SysErrors = Deserialize::deserialize(&mut buf).unwrap();
let sys_errors: SysErrors = Deserialize::deserialize(&mut buf)?;
Ok(sys_errors)
}
@@ -278,7 +278,7 @@ impl PeerRestClient {
let data = response.mem_info;
let mut buf = Deserializer::new(Cursor::new(data));
let mem_info: MemInfo = Deserialize::deserialize(&mut buf).unwrap();
let mem_info: MemInfo = Deserialize::deserialize(&mut buf)?;
Ok(mem_info)
}
@@ -306,7 +306,7 @@ impl PeerRestClient {
let data = response.realtime_metrics;
let mut buf = Deserializer::new(Cursor::new(data));
let realtime_metrics: RealtimeMetrics = Deserialize::deserialize(&mut buf).unwrap();
let realtime_metrics: RealtimeMetrics = Deserialize::deserialize(&mut buf)?;
Ok(realtime_metrics)
}
@@ -327,7 +327,7 @@ impl PeerRestClient {
let data = response.proc_info;
let mut buf = Deserializer::new(Cursor::new(data));
let proc_info: ProcInfo = Deserialize::deserialize(&mut buf).unwrap();
let proc_info: ProcInfo = Deserialize::deserialize(&mut buf)?;
Ok(proc_info)
}
@@ -603,20 +603,20 @@ impl PeerRestClient {
let data = response.bg_heal_state;
let mut buf = Deserializer::new(Cursor::new(data));
let bg_heal_state: BgHealState = Deserialize::deserialize(&mut buf).unwrap();
let bg_heal_state: BgHealState = Deserialize::deserialize(&mut buf)?;
Ok(bg_heal_state)
}
pub async fn get_metacache_listing(&self) -> Result<()> {
let mut _client = node_service_time_out_client(&self.grid_host)
let _client = node_service_time_out_client(&self.grid_host)
.await
.map_err(|err| Error::msg(err.to_string()))?;
todo!()
}
pub async fn update_metacache_listing(&self) -> Result<()> {
let mut _client = node_service_time_out_client(&self.grid_host)
let _client = node_service_time_out_client(&self.grid_host)
.await
.map_err(|err| Error::msg(err.to_string()))?;
todo!()
+4 -4
View File
@@ -127,7 +127,7 @@ impl PoolMeta {
}
let mut buf = Deserializer::new(Cursor::new(&data[4..]));
let meta: PoolMeta = Deserialize::deserialize(&mut buf).unwrap();
let meta: PoolMeta = Deserialize::deserialize(&mut buf)?;
*self = meta;
if self.version != POOL_META_VERSION {
@@ -141,8 +141,8 @@ impl PoolMeta {
return Ok(());
}
let mut data = Vec::new();
data.write_u16::<LittleEndian>(POOL_META_FORMAT).unwrap();
data.write_u16::<LittleEndian>(POOL_META_VERSION).unwrap();
data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
data.write_u16::<LittleEndian>(POOL_META_VERSION)?;
let mut buf = Vec::new();
self.serialize(&mut Serializer::new(&mut buf))?;
data.write_all(&buf)?;
@@ -1220,7 +1220,7 @@ impl ECStore {
reader.read_exact(&mut chunk).await?;
// 每次从reader中读取一个part上传
// 每次从 reader 中读取一个 part 上传
let rd = Box::new(Cursor::new(chunk));
let mut data = PutObjReader::new(rd, part.size);
+6 -6
View File
@@ -75,7 +75,7 @@ fn is_err_ignored(err: &Error, ignored_errs: &[Box<dyn CheckErrorFn>]) -> bool {
// 减少错误数量并返回出现次数最多的错误
fn reduce_errs(errs: &[Option<Error>], ignored_errs: &[Box<dyn CheckErrorFn>]) -> (usize, Option<Error>) {
let mut error_counts: HashMap<String, usize> = HashMap::new();
let mut error_map: HashMap<String, usize> = HashMap::new(); // 存err位置
let mut error_map: HashMap<String, usize> = HashMap::new(); // 存 err 位置
let nil = "nil".to_string();
for (i, operr) in errs.iter().enumerate() {
if let Some(err) = operr {
@@ -120,7 +120,7 @@ fn reduce_errs(errs: &[Option<Error>], ignored_errs: &[Box<dyn CheckErrorFn>]) -
}
}
// 根据quorum验证错误数量
// 根据 quorum 验证错误数量
fn reduce_quorum_errs(
errs: &[Option<Error>],
ignored_errs: &[Box<dyn CheckErrorFn>],
@@ -135,8 +135,8 @@ fn reduce_quorum_errs(
}
}
// 根据读quorum验证错误数量
// 返回最大错误数量的下标,或QuorumError
// 根据读 quorum 验证错误数量
// 返回最大错误数量的下标,或 QuorumError
pub fn reduce_read_quorum_errs(
errs: &[Option<Error>],
ignored_errs: &[Box<dyn CheckErrorFn>],
@@ -145,8 +145,8 @@ pub fn reduce_read_quorum_errs(
reduce_quorum_errs(errs, ignored_errs, read_quorum, QuorumError::Read)
}
// 根据写quorum验证错误数量
// 返回最大错误数量的下标,或QuorumError
// 根据写 quorum 验证错误数量
// 返回最大错误数量的下标,或 QuorumError
#[tracing::instrument(level = "info", skip_all)]
pub fn reduce_write_quorum_errs(
errs: &[Option<Error>],
+4 -3
View File
@@ -139,7 +139,7 @@ pub struct DiskStat {
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct RebalanceMeta {
#[serde(skip)]
pub cancel: Option<tokio::sync::broadcast::Sender<bool>>, // To be invoked on rebalance-stop
pub cancel: Option<broadcast::Sender<bool>>, // To be invoked on rebalance-stop
#[serde(skip)]
pub last_refreshed_at: Option<SystemTime>,
#[serde(rename = "stopTs")]
@@ -639,6 +639,7 @@ impl ECStore {
false
}
#[allow(unused_assignments)]
#[tracing::instrument(skip(self, wk, set))]
async fn rebalance_entry(
&self,
@@ -858,13 +859,13 @@ impl ECStore {
let mut reader = rd.stream;
for (i, part) in object_info.parts.iter().enumerate() {
// 每次从reader中读取一个part上传
// 每次从 reader 中读取一个 part 上传
let mut chunk = vec![0u8; part.size];
reader.read_exact(&mut chunk).await?;
// 每次从reader中读取一个part上传
// 每次从 reader 中读取一个 part 上传
let rd = Box::new(Cursor::new(chunk));
let mut data = PutObjReader::new(rd, part.size);
+16 -22
View File
@@ -36,7 +36,6 @@ use crate::{
store_err::{is_err_object_not_found, to_object_err, StorageError},
store_init::{load_format_erasure, ErasureError},
utils::{
self,
crypto::{base64_decode, base64_encode, hex},
path::{encode_dir_object, has_suffix, SLASH_SEPARATOR},
},
@@ -174,7 +173,7 @@ impl SetDisks {
let mut rng = thread_rng();
disks.shuffle(&mut rng);
numbers.shuffle(&mut rand::thread_rng());
numbers.shuffle(&mut thread_rng());
}
for &i in numbers.iter() {
@@ -395,7 +394,7 @@ impl SetDisks {
}
fn reduce_common_data_dir(data_dirs: &Vec<Option<Uuid>>, write_quorum: usize) -> Option<Uuid> {
let mut data_dirs_count = std::collections::HashMap::new();
let mut data_dirs_count = HashMap::new();
for ddir in data_dirs {
*data_dirs_count.entry(ddir).or_insert(0) += 1;
@@ -454,10 +453,7 @@ impl SetDisks {
let errs: Vec<Option<Error>> = join_all(futures)
.await
.into_iter()
.map(|e| match e {
Ok(e) => e,
Err(_) => Some(Error::new(DiskError::Unexpected)),
})
.map(|e| e.unwrap_or_else(|_| Some(Error::new(DiskError::Unexpected))))
.collect();
if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) {
@@ -726,7 +722,7 @@ impl SetDisks {
}
if max_occ == 0 {
// Did not found anything useful
// Did not find anything useful
return -1;
}
cparity
@@ -1039,7 +1035,7 @@ impl SetDisks {
hasher.flush()?;
meta_hashs[i] = Some(utils::crypto::hex(hasher.clone().finalize().as_slice()));
meta_hashs[i] = Some(hex(hasher.clone().finalize().as_slice()));
hasher.reset();
}
@@ -2331,7 +2327,7 @@ impl SetDisks {
// Allow for dangling deletes, on versions that have DataDir missing etc.
// this would end up restoring the correct readable versions.
match self
return match self
.delete_if_dang_ling(
bucket,
object,
@@ -2355,10 +2351,7 @@ impl SetDisks {
for _ in 0..errs.len() {
t_errs.push(None);
}
return Ok((
self.default_heal_result(m, &t_errs, bucket, object, version_id).await,
Some(derr),
));
Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, Some(derr)))
}
Err(err) => {
// t_errs = vec![Some(err.clone()); errs.len()];
@@ -2367,13 +2360,13 @@ impl SetDisks {
t_errs.push(Some(clone_err(&err)));
}
return Ok((
Ok((
self.default_heal_result(FileInfo::default(), &t_errs, bucket, object, version_id)
.await,
Some(err),
));
))
}
}
};
}
if !lastest_meta.deleted && lastest_meta.erasure.distribution.len() != available_disks.len() {
@@ -2908,6 +2901,7 @@ impl SetDisks {
) -> Result<()> {
info!("ns_scanner");
if buckets.is_empty() {
info!("data-scanner: no buckets to scan, skipping scanner cycle");
return Ok(());
}
@@ -2930,7 +2924,7 @@ impl SetDisks {
// Put all buckets into channel.
let (bucket_tx, bucket_rx) = mpsc::channel(buckets.len());
// Shuffle buckets to ensure total randomness of buckets, being scanned.
// Otherwise same set of buckets get scanned across erasure sets always.
// Otherwise, same set of buckets get scanned across erasure sets always.
// at any given point in time. This allows different buckets to be scanned
// in different order per erasure set, this wider spread is needed when
// there are lots of buckets with different order of objects in them.
@@ -5120,7 +5114,7 @@ impl StorageAPI for SetDisks {
_ => {}
}
}
return Ok((result, err));
Ok((result, err))
}
#[tracing::instrument(skip(self))]
@@ -5420,7 +5414,7 @@ async fn disks_with_all_parts(
if let Some(data) = &meta.data {
let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number);
let data_len = data.len();
let verify_err = (bitrot_verify(
let verify_err = bitrot_verify(
Box::new(Cursor::new(data.clone())),
data_len,
meta.erasure.shard_file_size(meta.size),
@@ -5428,8 +5422,8 @@ async fn disks_with_all_parts(
checksum_info.hash,
meta.erasure.shard_size(meta.erasure.block_size),
)
.await)
.err();
.await
.err();
if let Some(vec) = data_errs_by_part.get_mut(&0) {
if index < vec.len() {
+91 -94
View File
@@ -121,18 +121,15 @@ impl Sets {
disk = local_disk;
}
let has_disk_id = match disk.as_ref().unwrap().get_disk_id().await {
Ok(res) => res,
Err(err) => {
if is_unformatted_disk(&err) {
error!("get_disk_id err {:?}", err);
} else {
warn!("get_disk_id err {:?}", err);
}
None
let has_disk_id = disk.as_ref().unwrap().get_disk_id().await.unwrap_or_else(|err| {
if is_unformatted_disk(&err) {
error!("get_disk_id err {:?}", err);
} else {
warn!("get_disk_id err {:?}", err);
}
};
None
});
if let Some(_disk_id) = has_disk_id {
set_drive.push(disk);
@@ -287,7 +284,7 @@ struct DelObj {
#[async_trait::async_trait]
impl ObjectIO for Sets {
#[tracing::instrument(level = "debug", skip(self))]
#[tracing::instrument(level = "debug", skip(self, object, h, opts))]
async fn get_object_reader(
&self,
bucket: &str,
@@ -354,19 +351,56 @@ impl StorageAPI for Sets {
}
}
#[tracing::instrument(skip(self))]
async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> Result<()> {
unimplemented!()
}
#[tracing::instrument(skip(self))]
async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> Result<()> {
async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
unimplemented!()
}
#[tracing::instrument(skip(self))]
async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
unimplemented!()
}
#[tracing::instrument(skip(self))]
async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
unimplemented!()
}
#[tracing::instrument(skip(self))]
async fn list_objects_v2(
self: Arc<Self>,
_bucket: &str,
_prefix: &str,
_continuation_token: Option<String>,
_delimiter: Option<String>,
_max_keys: i32,
_fetch_owner: bool,
_start_after: Option<String>,
) -> Result<ListObjectsV2Info> {
unimplemented!()
}
#[tracing::instrument(skip(self))]
async fn list_object_versions(
self: Arc<Self>,
_bucket: &str,
_prefix: &str,
_marker: Option<String>,
_version_marker: Option<String>,
_delimiter: Option<String>,
_max_keys: i32,
) -> Result<ListObjectVersionsInfo> {
unimplemented!()
}
#[tracing::instrument(skip(self))]
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).get_object_info(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn copy_object(
&self,
@@ -425,6 +459,16 @@ impl StorageAPI for Sets {
)))
}
#[tracing::instrument(skip(self))]
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
if opts.delete_prefix && !opts.delete_prefix_object {
self.delete_prefix(bucket, object).await?;
return Ok(ObjectInfo::default());
}
self.get_disks_by_key(object).delete_object(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn delete_objects(
&self,
@@ -510,66 +554,22 @@ impl StorageAPI for Sets {
}
#[tracing::instrument(skip(self))]
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
if opts.delete_prefix && !opts.delete_prefix_object {
self.delete_prefix(bucket, object).await?;
return Ok(ObjectInfo::default());
}
self.get_disks_by_key(object).delete_object(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn list_objects_v2(
self: Arc<Self>,
_bucket: &str,
_prefix: &str,
_continuation_token: Option<String>,
_delimiter: Option<String>,
_max_keys: i32,
_fetch_owner: bool,
_start_after: Option<String>,
) -> Result<ListObjectsV2Info> {
unimplemented!()
}
#[tracing::instrument(skip(self))]
async fn list_object_versions(
self: Arc<Self>,
_bucket: &str,
_prefix: &str,
_marker: Option<String>,
_version_marker: Option<String>,
_delimiter: Option<String>,
_max_keys: i32,
) -> Result<ListObjectVersionsInfo> {
unimplemented!()
}
#[tracing::instrument(skip(self))]
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).get_object_info(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).put_object_metadata(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
self.get_disks_by_key(object).get_object_tags(bucket, object, opts).await
}
#[tracing::instrument(level = "debug", skip(self))]
async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object)
.put_object_tags(bucket, object, tags, opts)
async fn list_multipart_uploads(
&self,
bucket: &str,
prefix: &str,
key_marker: Option<String>,
upload_id_marker: Option<String>,
delimiter: Option<String>,
max_uploads: usize,
) -> Result<ListMultipartsInfo> {
self.get_disks_by_key(prefix)
.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads)
.await
}
#[tracing::instrument(skip(self))]
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
self.get_disks_by_key(object).new_multipart_upload(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
@@ -605,26 +605,6 @@ impl StorageAPI for Sets {
.await
}
#[tracing::instrument(skip(self))]
async fn list_multipart_uploads(
&self,
bucket: &str,
prefix: &str,
key_marker: Option<String>,
upload_id_marker: Option<String>,
delimiter: Option<String>,
max_uploads: usize,
) -> Result<ListMultipartsInfo> {
self.get_disks_by_key(prefix)
.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads)
.await
}
#[tracing::instrument(skip(self))]
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
self.get_disks_by_key(object).new_multipart_upload(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn get_multipart_info(
&self,
@@ -670,8 +650,25 @@ impl StorageAPI for Sets {
}
#[tracing::instrument(skip(self))]
async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
unimplemented!()
async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).put_object_metadata(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
self.get_disks_by_key(object).get_object_tags(bucket, object, opts).await
}
#[tracing::instrument(level = "debug", skip(self))]
async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object)
.put_object_tags(bucket, object, tags, opts)
.await
}
#[tracing::instrument(skip(self))]
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
+336 -345
View File
@@ -40,7 +40,7 @@ use crate::{
ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo,
PutObjReader, StorageAPI,
},
store_init, utils,
store_init,
};
use common::error::{Error, Result};
@@ -53,12 +53,12 @@ use madmin::heal_commands::HealResultItem;
use rand::Rng;
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
use std::cmp::Ordering;
use std::net::SocketAddr;
use std::process::exit;
use std::slice::Iter;
use std::time::SystemTime;
use std::{collections::HashMap, sync::Arc, time::Duration};
use time::OffsetDateTime;
use tokio::select;
use tokio::sync::mpsc::Sender;
use tokio::sync::{broadcast, mpsc, RwLock};
use tokio::time::{interval, sleep};
@@ -70,7 +70,7 @@ const MAX_UPLOADS_LIST: usize = 10000;
#[derive(Debug)]
pub struct ECStore {
pub id: uuid::Uuid,
pub id: Uuid,
// pub disks: Vec<DiskStore>,
pub disk_map: HashMap<usize, Vec<Option<DiskStore>>>,
pub pools: Vec<Arc<Sets>>,
@@ -101,7 +101,7 @@ pub struct ECStore {
impl ECStore {
#[allow(clippy::new_ret_no_self)]
#[tracing::instrument(level = "debug", skip(endpoint_pools))]
pub async fn new(address: SocketAddr, endpoint_pools: EndpointServerPools) -> Result<Arc<Self>> {
pub async fn new(_address: String, endpoint_pools: EndpointServerPools) -> Result<Arc<Self>> {
// let layouts = DisksLayout::from_volumes(endpoints.as_slice())?;
let mut deployment_id = None;
@@ -114,17 +114,13 @@ impl ECStore {
let first_is_local = endpoint_pools.first_local();
let mut local_disks = Vec::new();
info!("ECStore new address: {}", address.to_string());
let mut host = address.ip().to_string();
if host.is_empty() {
host = GLOBAL_Rustfs_Host.read().await.to_string()
}
let mut port = address.port().to_string();
if port.is_empty() {
port = GLOBAL_Rustfs_Port.read().await.to_string()
}
info!("ECStore new host: {}, port: {}", host, port);
init_local_peer(&endpoint_pools, &host, &port).await;
init_local_peer(
&endpoint_pools,
&GLOBAL_Rustfs_Host.read().await.to_string(),
&GLOBAL_Rustfs_Port.read().await.to_string(),
)
.await;
// debug!("endpoint_pools: {:?}", endpoint_pools);
@@ -173,7 +169,7 @@ impl ECStore {
return Err(Error::from_string("can not get formats"));
}
info!("retrying get formats after {:?}", interval);
tokio::select! {
select! {
_ = tokio::signal::ctrl_c() => {
info!("got ctrl+c, exits");
exit(0);
@@ -208,7 +204,7 @@ impl ECStore {
disk_map.insert(i, disks);
}
// Replace the local disk
// 替换本地磁盘
if !is_dist_erasure().await {
let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await;
for disk in local_disks {
@@ -241,11 +237,11 @@ impl ECStore {
loop {
if let Err(err) = ec.init().await {
error!("init err: {}", err);
info!("retry after {} second ,exit count: {}", wait_sec, exit_count);
error!("retry after {} second", wait_sec);
sleep(Duration::from_secs(wait_sec)).await;
if exit_count > 10 {
return Err(Error::msg("ec init failed"));
return Err(Error::msg("ec init faild"));
}
exit_count += 1;
@@ -261,7 +257,6 @@ impl ECStore {
Ok(ec)
}
/// init
pub async fn init(self: &Arc<Self>) -> Result<()> {
GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
@@ -288,13 +283,13 @@ impl ECStore {
}
let pools = meta.return_resumable_pools();
let mut pool_indexes = Vec::with_capacity(pools.len());
let mut pool_indeces = Vec::with_capacity(pools.len());
let endpoints = get_global_endpoints();
for p in pools.iter() {
if let Some(idx) = endpoints.get_pool_idx(&p.cmd_line) {
pool_indexes.push(idx);
pool_indeces.push(idx);
} else {
return Err(Error::msg(format!(
"unexpected state present for decommission status pool({}) not found",
@@ -303,8 +298,8 @@ impl ECStore {
}
}
if !pool_indexes.is_empty() {
let idx = pool_indexes[0];
if !pool_indeces.is_empty() {
let idx = pool_indeces[0];
if endpoints.as_ref()[idx].endpoints.as_ref()[0].is_local {
let (_tx, rx) = broadcast::channel(1);
@@ -314,9 +309,9 @@ impl ECStore {
// wait 3 minutes for cluster init
tokio::time::sleep(Duration::from_secs(60 * 3)).await;
if let Err(err) = store.decommission(rx.resubscribe(), pool_indexes.clone()).await {
if let Err(err) = store.decommission(rx.resubscribe(), pool_indeces.clone()).await {
if is_err_decommission_already_running(&err) {
for i in pool_indexes.iter() {
for i in pool_indeces.iter() {
store.do_decommission_in_routine(rx.resubscribe(), *i).await;
}
return;
@@ -689,7 +684,7 @@ impl ECStore {
let mut ress = Vec::new();
// join_all The results are in the same order as they were entered
// join_all 结果跟输入顺序一致
for (i, res) in results.into_iter().enumerate() {
let index = i;
@@ -856,7 +851,7 @@ impl ECStore {
let mut interval = interval(Duration::from_secs(30));
let all_merged = Arc::new(RwLock::new(DataUsageCache::default()));
loop {
tokio::select! {
select! {
_ = ctx_clone.recv() => {
return;
}
@@ -872,7 +867,7 @@ impl ECStore {
});
let _ = join_all(futures).await;
let mut ctx_closer = cancel.subscribe();
tokio::select! {
select! {
_ = update_closer_tx.send(true) => {
}
@@ -942,11 +937,7 @@ impl ECStore {
};
if a_mod == b_mod {
if a.idx < b.idx {
return Ordering::Greater;
} else {
return Ordering::Less;
}
return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less };
}
b_mod.cmp(&a_mod)
@@ -1195,7 +1186,7 @@ impl ObjectIO for ECStore {
) -> Result<GetObjectReader> {
check_get_obj_args(bucket, object)?;
let object = utils::path::encode_dir_object(object);
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].get_object_reader(bucket, object.as_str(), range, h, opts).await;
@@ -1218,7 +1209,7 @@ impl ObjectIO for ECStore {
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
check_put_object_args(bucket, object)?;
let object = utils::path::encode_dir_object(object);
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].put_object(bucket, object.as_str(), data, opts).await;
@@ -1324,52 +1315,6 @@ impl StorageAPI for ECStore {
madmin::StorageInfo { backend, disks }
}
#[tracing::instrument(skip(self))]
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
// TODO: opts.cached
let mut buckets = self.peer_sys.list_bucket(opts).await?;
if !opts.no_metadata {
for bucket in buckets.iter_mut() {
if let Ok(created) = metadata_sys::created_at(&bucket.name).await {
bucket.created = Some(created);
}
}
}
Ok(buckets)
}
#[tracing::instrument(skip(self))]
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
if is_meta_bucketname(bucket) {
return Err(StorageError::BucketNameInvalid(bucket.to_string()).into());
}
if let Err(err) = check_valid_bucket_name(bucket) {
return Err(StorageError::BucketNameInvalid(err.to_string()).into());
}
// TODO: nslock
let mut opts = opts.clone();
if !opts.force {
// FIXME: check bucket exists
opts.force = true
}
self.peer_sys
.delete_bucket(bucket, &opts)
.await
.map_err(|e| to_object_err(e, vec![bucket]))?;
// TODO: replication opts.srdelete_op
// 删除 meta
self.delete_all(RUSTFS_META_BUCKET, format!("{}/{}", BUCKET_META_PREFIX, bucket).as_str())
.await?;
Ok(())
}
#[tracing::instrument(skip(self))]
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
if !is_meta_bucketname(bucket) {
@@ -1414,6 +1359,7 @@ impl StorageAPI for ECStore {
Ok(())
}
#[tracing::instrument(skip(self))]
async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
let mut info = self
@@ -1430,6 +1376,101 @@ impl StorageAPI for ECStore {
Ok(info)
}
#[tracing::instrument(skip(self))]
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
// TODO: opts.cached
let mut buckets = self.peer_sys.list_bucket(opts).await?;
if !opts.no_metadata {
for bucket in buckets.iter_mut() {
if let Ok(created) = metadata_sys::created_at(&bucket.name).await {
bucket.created = Some(created);
}
}
}
Ok(buckets)
}
#[tracing::instrument(skip(self))]
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
if is_meta_bucketname(bucket) {
return Err(StorageError::BucketNameInvalid(bucket.to_string()).into());
}
if let Err(err) = check_valid_bucket_name(bucket) {
return Err(StorageError::BucketNameInvalid(err.to_string()).into());
}
// TODO: nslock
let mut opts = opts.clone();
if !opts.force {
// FIXME: check bucket exists
opts.force = true
}
self.peer_sys
.delete_bucket(bucket, &opts)
.await
.map_err(|e| to_object_err(e, vec![bucket]))?;
// TODO: replication opts.srdelete_op
// 删除 meta
self.delete_all(RUSTFS_META_BUCKET, format!("{}/{}", BUCKET_META_PREFIX, bucket).as_str())
.await?;
Ok(())
}
// @continuation_token marker
// @start_after as marker when continuation_token empty
// @delimiter default="/", empty when recursive
// @max_keys limit
#[tracing::instrument(skip(self))]
async fn list_objects_v2(
self: Arc<Self>,
bucket: &str,
prefix: &str,
continuation_token: Option<String>,
delimiter: Option<String>,
max_keys: i32,
fetch_owner: bool,
start_after: Option<String>,
) -> Result<ListObjectsV2Info> {
self.inner_list_objects_v2(bucket, prefix, continuation_token, delimiter, max_keys, fetch_owner, start_after)
.await
}
#[tracing::instrument(skip(self))]
async fn list_object_versions(
self: Arc<Self>,
bucket: &str,
prefix: &str,
marker: Option<String>,
version_marker: Option<String>,
delimiter: Option<String>,
max_keys: i32,
) -> Result<ListObjectVersionsInfo> {
self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys)
.await
}
#[tracing::instrument(skip(self))]
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
check_object_args(bucket, object)?;
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].get_object_info(bucket, object.as_str(), opts).await;
}
// TODO: nslock
let (info, _) = self.get_latest_object_info_with_idx(bucket, object.as_str(), opts).await?;
Ok(info)
}
// TODO: review
#[tracing::instrument(skip(self))]
@@ -1446,8 +1487,8 @@ impl StorageAPI for ECStore {
check_copy_obj_args(src_bucket, src_object)?;
check_copy_obj_args(dst_bucket, dst_object)?;
let src_object = utils::path::encode_dir_object(src_object);
let dst_object = utils::path::encode_dir_object(dst_object);
let src_object = encode_dir_object(src_object);
let dst_object = encode_dir_object(dst_object);
let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]);
@@ -1501,7 +1542,76 @@ impl StorageAPI for ECStore {
"put_object_reader is none".to_owned(),
)))
}
#[tracing::instrument(skip(self))]
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
check_del_obj_args(bucket, object)?;
if opts.delete_prefix {
self.delete_prefix(bucket, object).await?;
return Ok(ObjectInfo::default());
}
// TODO: nslock
let object = encode_dir_object(object);
let object = object.as_str();
// 查询在哪个 pool
let (mut pinfo, errs) = self
.get_pool_info_existing_with_opts(bucket, object, &opts)
.await
.map_err(|e| {
if is_err_read_quorum(&e) {
Error::new(StorageError::InsufficientWriteQuorum)
} else {
e
}
})?;
if pinfo.object_info.delete_marker && opts.version_id.is_none() {
pinfo.object_info.name = decode_dir_object(object);
return Ok(pinfo.object_info);
}
if opts.data_movement && opts.src_pool_idx == pinfo.index {
return Err(Error::new(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.unwrap_or_default(),
)));
}
if opts.data_movement {
let mut obj = self.pools[pinfo.index].delete_object(bucket, object, opts).await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
if !errs.is_empty() && !opts.versioned && !opts.version_suspended {
return self.delete_object_from_all_pools(bucket, object, &opts, errs).await;
}
for pool in self.pools.iter() {
match pool.delete_object(bucket, object, opts.clone()).await {
Ok(res) => {
let mut obj = res;
obj.name = decode_dir_object(object);
return Ok(obj);
}
Err(err) => {
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
return Err(err);
}
}
}
}
if let Some(ver) = opts.version_id {
return Err(Error::new(StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), ver)));
}
Err(Error::new(StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned())))
}
// TODO: review
#[tracing::instrument(skip(self))]
async fn delete_objects(
@@ -1515,7 +1625,7 @@ impl StorageAPI for ECStore {
.iter()
.map(|v| {
let mut v = v.clone();
v.object_name = utils::path::encode_dir_object(v.object_name.as_str());
v.object_name = encode_dir_object(v.object_name.as_str());
v
})
.collect();
@@ -1585,7 +1695,7 @@ impl StorageAPI for ECStore {
del_objects[i] = DeletedObject {
delete_marker: pinfo.object_info.delete_marker,
delete_marker_version_id: pinfo.object_info.version_id.map(|v| v.to_string()),
object_name: utils::path::decode_dir_object(&pinfo.object_info.name),
object_name: decode_dir_object(&pinfo.object_info.name),
delete_marker_mtime: pinfo.object_info.mod_time,
..Default::default()
};
@@ -1612,7 +1722,7 @@ impl StorageAPI for ECStore {
if let Some(obj) = objects.get(i) {
del_objects[i] = DeletedObject {
object_name: utils::path::decode_dir_object(&obj.object_name),
object_name: decode_dir_object(&obj.object_name),
version_id: obj.version_id.map(|v| v.to_string()),
..Default::default()
}
@@ -1646,7 +1756,7 @@ impl StorageAPI for ECStore {
}
let mut dobj = pdel_objs.get(i).unwrap().clone();
dobj.object_name = utils::path::decode_dir_object(&dobj.object_name);
dobj.object_name = decode_dir_object(&dobj.object_name);
del_objects[obj_idx] = dobj;
}
@@ -1657,246 +1767,6 @@ impl StorageAPI for ECStore {
Ok((del_objects, del_errs))
}
#[tracing::instrument(skip(self))]
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
check_del_obj_args(bucket, object)?;
if opts.delete_prefix {
self.delete_prefix(bucket, object).await?;
return Ok(ObjectInfo::default());
}
// TODO: nslock
let object = utils::path::encode_dir_object(object);
let object = object.as_str();
// Query which pool it is in
let (mut pinfo, errs) = self
.get_pool_info_existing_with_opts(bucket, object, &opts)
.await
.map_err(|e| {
if is_err_read_quorum(&e) {
Error::new(StorageError::InsufficientWriteQuorum)
} else {
e
}
})?;
if pinfo.object_info.delete_marker && opts.version_id.is_none() {
pinfo.object_info.name = utils::path::decode_dir_object(object);
return Ok(pinfo.object_info);
}
if opts.data_movement && opts.src_pool_idx == pinfo.index {
return Err(Error::new(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
opts.version_id.unwrap_or_default(),
)));
}
if opts.data_movement {
let mut obj = self.pools[pinfo.index].delete_object(bucket, object, opts).await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
if !errs.is_empty() && !opts.versioned && !opts.version_suspended {
return self.delete_object_from_all_pools(bucket, object, &opts, errs).await;
}
for pool in self.pools.iter() {
match pool.delete_object(bucket, object, opts.clone()).await {
Ok(res) => {
let mut obj = res;
obj.name = utils::path::decode_dir_object(object);
return Ok(obj);
}
Err(err) => {
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
return Err(err);
}
}
}
}
if let Some(ver) = opts.version_id {
return Err(Error::new(StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), ver)));
}
Err(Error::new(StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned())))
}
// @continuation_token marker
// @start_after as marker when continuation_token empty
// @delimiter default="/", empty when recursive
// @max_keys limit
#[tracing::instrument(skip(self))]
async fn list_objects_v2(
self: Arc<Self>,
bucket: &str,
prefix: &str,
continuation_token: Option<String>,
delimiter: Option<String>,
max_keys: i32,
fetch_owner: bool,
start_after: Option<String>,
) -> Result<ListObjectsV2Info> {
self.inner_list_objects_v2(bucket, prefix, continuation_token, delimiter, max_keys, fetch_owner, start_after)
.await
}
#[tracing::instrument(skip(self))]
async fn list_object_versions(
self: Arc<Self>,
bucket: &str,
prefix: &str,
marker: Option<String>,
version_marker: Option<String>,
delimiter: Option<String>,
max_keys: i32,
) -> Result<ListObjectVersionsInfo> {
self.inner_list_object_versions(bucket, prefix, marker, version_marker, delimiter, max_keys)
.await
}
#[tracing::instrument(skip(self))]
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
check_object_args(bucket, object)?;
let object = utils::path::encode_dir_object(object);
if self.single_pool() {
return self.pools[0].get_object_info(bucket, object.as_str(), opts).await;
}
// TODO: nslock
let (info, _) = self.get_latest_object_info_with_idx(bucket, object.as_str(), opts).await?;
Ok(info)
}
#[tracing::instrument(skip(self))]
async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].get_object_tags(bucket, object.as_str(), opts).await;
}
let (oi, _) = self.get_latest_object_info_with_idx(bucket, &object, opts).await?;
Ok(oi.user_tags)
}
#[tracing::instrument(skip(self))]
async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].put_object_metadata(bucket, object.as_str(), opts).await;
}
let mut opts = opts.clone();
opts.metadata_chg = true;
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), &opts).await?;
self.pools[idx].put_object_metadata(bucket, object.as_str(), &opts).await
}
#[tracing::instrument(level = "debug", skip(self))]
async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].put_object_tags(bucket, object.as_str(), tags, opts).await;
}
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?;
self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await
}
#[tracing::instrument(skip(self))]
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].delete_object_tags(bucket, object.as_str(), opts).await;
}
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?;
self.pools[idx].delete_object_tags(bucket, object.as_str(), opts).await
}
#[tracing::instrument(skip(self))]
async fn copy_object_part(
&self,
src_bucket: &str,
src_object: &str,
_dst_bucket: &str,
_dst_object: &str,
_upload_id: &str,
_part_id: usize,
_start_offset: i64,
_length: i64,
_src_info: &ObjectInfo,
_src_opts: &ObjectOptions,
_dst_opts: &ObjectOptions,
) -> Result<()> {
check_new_multipart_args(src_bucket, src_object)?;
// TODO: PutObjectReader
// self.put_object_part(dst_bucket, dst_object, upload_id, part_id, data, opts)
unimplemented!()
}
#[tracing::instrument(skip(self, data))]
async fn put_object_part(
&self,
bucket: &str,
object: &str,
upload_id: &str,
part_id: usize,
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<PartInfo> {
check_put_object_part_args(bucket, object, upload_id)?;
if self.single_pool() {
return self.pools[0]
.put_object_part(bucket, object, upload_id, part_id, data, opts)
.await;
}
for pool in self.pools.iter() {
if self.is_suspended(pool.pool_idx).await {
continue;
}
let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await {
Ok(res) => return Ok(res),
Err(err) => {
if is_err_invalid_upload_id(&err) {
None
} else {
Some(err)
}
}
};
if let Some(err) = err {
error!("put_object_part err: {:?}", err);
return Err(err);
}
}
Err(Error::new(StorageError::InvalidUploadID(
bucket.to_owned(),
object.to_owned(),
upload_id.to_owned(),
)))
}
#[tracing::instrument(skip(self))]
async fn list_multipart_uploads(
&self,
@@ -1982,6 +1852,74 @@ impl StorageAPI for ECStore {
self.pools[idx].new_multipart_upload(bucket, object, opts).await
}
#[tracing::instrument(skip(self))]
async fn copy_object_part(
&self,
src_bucket: &str,
src_object: &str,
_dst_bucket: &str,
_dst_object: &str,
_upload_id: &str,
_part_id: usize,
_start_offset: i64,
_length: i64,
_src_info: &ObjectInfo,
_src_opts: &ObjectOptions,
_dst_opts: &ObjectOptions,
) -> Result<()> {
check_new_multipart_args(src_bucket, src_object)?;
// TODO: PutObjectReader
// self.put_object_part(dst_bucket, dst_object, upload_id, part_id, data, opts)
unimplemented!()
}
#[tracing::instrument(skip(self, data))]
async fn put_object_part(
&self,
bucket: &str,
object: &str,
upload_id: &str,
part_id: usize,
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<PartInfo> {
check_put_object_part_args(bucket, object, upload_id)?;
if self.single_pool() {
return self.pools[0]
.put_object_part(bucket, object, upload_id, part_id, data, opts)
.await;
}
for pool in self.pools.iter() {
if self.is_suspended(pool.pool_idx).await {
continue;
}
let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await {
Ok(res) => return Ok(res),
Err(err) => {
if is_err_invalid_upload_id(&err) {
None
} else {
Some(err)
}
}
};
if let Some(err) = err {
error!("put_object_part err: {:?}", err);
return Err(err);
}
}
Err(Error::new(StorageError::InvalidUploadID(
bucket.to_owned(),
object.to_owned(),
upload_id.to_owned(),
)))
}
#[tracing::instrument(skip(self))]
async fn get_multipart_info(
&self,
@@ -2000,16 +1938,16 @@ impl StorageAPI for ECStore {
continue;
}
match pool.get_multipart_info(bucket, object, upload_id, opts).await {
Ok(res) => return Ok(res),
return match pool.get_multipart_info(bucket, object, upload_id, opts).await {
Ok(res) => Ok(res),
Err(err) => {
if is_err_invalid_upload_id(&err) {
continue;
}
return Err(err);
Err(err)
}
}
};
}
Err(Error::new(StorageError::InvalidUploadID(
@@ -2056,6 +1994,7 @@ impl StorageAPI for ECStore {
upload_id.to_owned(),
)))
}
#[tracing::instrument(skip(self))]
async fn complete_multipart_upload(
&self,
@@ -2123,6 +2062,58 @@ impl StorageAPI for ECStore {
}
counts
}
#[tracing::instrument(skip(self))]
async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].put_object_metadata(bucket, object.as_str(), opts).await;
}
let mut opts = opts.clone();
opts.metadata_chg = true;
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), &opts).await?;
self.pools[idx].put_object_metadata(bucket, object.as_str(), &opts).await
}
#[tracing::instrument(skip(self))]
async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].get_object_tags(bucket, object.as_str(), opts).await;
}
let (oi, _) = self.get_latest_object_info_with_idx(bucket, &object, opts).await?;
Ok(oi.user_tags)
}
#[tracing::instrument(level = "debug", skip(self))]
async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].put_object_tags(bucket, object.as_str(), tags, opts).await;
}
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?;
self.pools[idx].put_object_tags(bucket, object.as_str(), tags, opts).await
}
#[tracing::instrument(skip(self))]
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].delete_object_tags(bucket, object.as_str(), opts).await;
}
let idx = self.get_pool_idx_existing_with_opts(bucket, object.as_str(), opts).await?;
self.pools[idx].delete_object_tags(bucket, object.as_str(), opts).await
}
#[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
@@ -2172,7 +2163,7 @@ impl StorageAPI for ECStore {
opts: &HealOpts,
) -> Result<(HealResultItem, Option<Error>)> {
info!("ECStore heal_object");
let object = utils::path::encode_dir_object(object);
let object = encode_dir_object(object);
let mut futures = Vec::with_capacity(self.pools.len());
for pool in self.pools.iter() {
@@ -2200,7 +2191,7 @@ impl StorageAPI for ECStore {
match res {
Ok((result, err)) => {
let mut result = result;
result.object = utils::path::decode_dir_object(&result.object);
result.object = decode_dir_object(&result.object);
ress.push(result);
errs.push(err);
}
@@ -2270,10 +2261,10 @@ impl StorageAPI for ECStore {
let fivs = match entry.file_info_versions(&bucket) {
Ok(fivs) => fivs,
Err(_) => {
if is_meta {
return HealSequence::heal_meta_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await;
return if is_meta {
HealSequence::heal_meta_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await
} else {
return HealSequence::heal_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await;
HealSequence::heal_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await
}
}
};
@@ -2367,7 +2358,7 @@ impl StorageAPI for ECStore {
#[tracing::instrument(skip(self))]
async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
let object = utils::path::encode_dir_object(object);
let object = encode_dir_object(object);
if self.single_pool() {
return self.pools[0].check_abandoned_parts(bucket, &object, opts).await;
}
@@ -2704,17 +2695,17 @@ mod tests {
// Test validation functions
#[test]
fn test_is_valid_object_name() {
assert_eq!(is_valid_object_name("valid-object-name"), true);
assert_eq!(is_valid_object_name(""), false);
assert_eq!(is_valid_object_name("object/with/slashes"), true);
assert_eq!(is_valid_object_name("object with spaces"), true);
assert!(is_valid_object_name("valid-object-name"));
assert!(!is_valid_object_name(""));
assert!(is_valid_object_name("object/with/slashes"));
assert!(is_valid_object_name("object with spaces"));
}
#[test]
fn test_is_valid_object_prefix() {
assert_eq!(is_valid_object_prefix("valid-prefix"), true);
assert_eq!(is_valid_object_prefix(""), true);
assert_eq!(is_valid_object_prefix("prefix/with/slashes"), true);
assert!(is_valid_object_prefix("valid-prefix"));
assert!(is_valid_object_prefix(""));
assert!(is_valid_object_prefix("prefix/with/slashes"));
}
#[test]
+7 -6
View File
@@ -375,7 +375,7 @@ pub struct ChecksumInfo {
pub const DEFAULT_BITROT_ALGO: BitrotAlgorithm = BitrotAlgorithm::HighwayHash256S;
#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)]
// BitrotAlgorithm specifies a algorithm used for bitrot protection.
// BitrotAlgorithm specifies an algorithm used for bitrot protection.
pub enum BitrotAlgorithm {
// SHA256 represents the SHA-256 hash function
SHA256,
@@ -465,23 +465,23 @@ impl GetObjectReader {
if let Some(rs) = rs {
let (off, length) = rs.get_offset_length(oi.size)?;
return Ok((
Ok((
GetObjectReader {
stream: reader,
object_info: oi.clone(),
},
off,
length,
));
))
} else {
return Ok((
Ok((
GetObjectReader {
stream: reader,
object_info: oi.clone(),
},
0,
oi.size,
));
))
}
}
pub async fn read_all(&mut self) -> Result<Vec<u8>> {
@@ -1058,6 +1058,7 @@ pub trait StorageAPI: ObjectIO {
}
#[cfg(test)]
#[allow(clippy::field_reassign_with_default)]
mod tests {
use super::*;
use std::collections::HashMap;
@@ -1089,7 +1090,7 @@ mod tests {
// Test distribution uniqueness
let mut unique_values = std::collections::HashSet::new();
for &val in &file_info.erasure.distribution {
assert!(val >= 1 && val <= 6, "Distribution value should be between 1 and 6");
assert!((1..=6).contains(&val), "Distribution value should be between 1 and 6");
unique_values.insert(val);
}
assert_eq!(unique_values.len(), 6, "All distribution values should be unique");
+1 -1
View File
@@ -192,7 +192,7 @@ fn check_format_erasure_value(format: &FormatV3) -> Result<()> {
Ok(())
}
// load_format_erasure_all 读取所有foramt.json
// load_format_erasure_all 读取所有 foramt.json
pub async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) -> (Vec<Option<FormatV3>>, Vec<Option<Error>>) {
let mut futures = Vec::with_capacity(disks.len());
let mut datas = Vec::with_capacity(disks.len());
+5 -5
View File
@@ -277,13 +277,13 @@ impl ECStore {
};
};
let mut list_result = match self.list_path(&opts).await {
Ok(res) => res,
Err(err) => MetaCacheEntriesSortedResult {
let mut list_result = self
.list_path(&opts)
.await
.unwrap_or_else(|err| MetaCacheEntriesSortedResult {
err: Some(err),
..Default::default()
},
};
});
if let Some(err) = &list_result.err {
if !is_err_eof(err) {
+2 -2
View File
@@ -2,9 +2,9 @@ use crc32fast::Hasher;
use siphasher::sip::SipHasher;
pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize {
// 你的密钥,必须是16字节
// 你的密钥,必须是 16 字节
// 计算字符串的SipHash值
// 计算字符串的 SipHash
let result = SipHasher::new_with_key(id).hash(key.as_bytes());
result as usize % cardinality
+2 -2
View File
@@ -86,8 +86,8 @@ mod tests {
// The actual result depends on the system configuration
println!("Same disk result for temp dirs: {}", result);
// Just verify the function executes successfully
assert!(result == true || result == false);
// The function returns a boolean value as expected
let _: bool = result; // Type assertion to verify return type
}
#[test]
+6 -6
View File
@@ -10,15 +10,15 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<Info> {
let stat = statfs(p.as_ref())?;
let bsize = stat.block_size() as u64;
let bfree = stat.blocks_free() as u64;
let bavail = stat.blocks_available() as u64;
let blocks = stat.blocks() as u64;
let bfree = stat.blocks_free();
let bavail = stat.blocks_available();
let blocks = stat.blocks();
let reserved = match bfree.checked_sub(bavail) {
Some(reserved) => reserved,
None => {
return Err(Error::other(format!(
"detected f_bavail space ({}) > f_bfree space ({}), fs corruption at ({}). please run 'fsck'",
"detected f_bavail space ({}) > f_bfree space ({}), fs corruption at ({}). please run fsck",
bavail,
bfree,
p.as_ref().display()
@@ -30,7 +30,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<Info> {
Some(total) => total * bsize,
None => {
return Err(Error::other(format!(
"detected reserved space ({}) > blocks space ({}), fs corruption at ({}). please run 'fsck'",
"detected reserved space ({}) > blocks space ({}), fs corruption at ({}). please run fsck",
reserved,
blocks,
p.as_ref().display()
@@ -43,7 +43,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<Info> {
Some(used) => used,
None => {
return Err(Error::other(format!(
"detected free space ({}) > total drive space ({}), fs corruption at ({}). please run 'fsck'",
"detected free space ({}) > total drive space ({}), fs corruption at ({}). please run fsck",
free,
total,
p.as_ref().display()