gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-17 18:27:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

update filereader/writer todo

Browse Source
This commit is contained in:
weisd
2025-06-06 18:04:51 +08:00
parent 15a3012d05
commit b51ee48699
22 changed files with 178 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/disk/error.rs
+4 -4
View File
@@ -253,11 +253,11 @@ impl From<protos::proto_gen::node_service::Error> for DiskError {
}
}
impl Into<protos::proto_gen::node_service::Error> for DiskError {
fn into(self) -> protos::proto_gen::node_service::Error {
impl From<DiskError> for protos::proto_gen::node_service::Error {
fn from(e: DiskError) -> Self {
protos::proto_gen::node_service::Error {
code: self.to_u32(),
error_info: self.to_string(),
code: e.to_u32(),
error_info: e.to_string(),
}
}
}
ecstore/src/disk/error_reduce.rs
+2 -6
View File
@@ -29,11 +29,7 @@ pub fn reduce_read_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error],
pub fn reduce_quorum_errs(errors: &[Option<Error>], ignored_errs: &[Error], quorun: usize, quorun_err: Error) -> Option<Error> {
let (max_count, err) = reduce_errs(errors, ignored_errs);
if max_count >= quorun {
err
} else {
Some(quorun_err)
}
if max_count >= quorun { err } else { Some(quorun_err) }
}
pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize, Option<Error>) {
@@ -59,7 +55,7 @@ pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize,
match (e1.to_string().as_str(), e2.to_string().as_str()) {
("nil", _) => std::cmp::Ordering::Greater,
(_, "nil") => std::cmp::Ordering::Less,
(a, b) => a.cmp(&b),
(a, b) => a.cmp(b),
}
} else {
count_cmp
ecstore/src/disk/local.rs
+9 -10
View File
@@ -10,7 +10,6 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
use crate::bucket::metadata_sys::{self};
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::disk::STORAGE_FORMAT_FILE;
use crate::disk::error::FileAccessDeniedWithContext;
use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error};
use crate::disk::fs::{
@@ -19,8 +18,9 @@ use crate::disk::fs::{
use crate::disk::os::{check_path_length, is_empty_dir};
use crate::disk::{
CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND,
conv_part_err_to_int,
FileReader, conv_part_err_to_int,
};
use crate::disk::{FileWriter, STORAGE_FORMAT_FILE};
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
use crate::heal::data_scanner::{
ScannerItem, ShouldSleepFn, SizeSummary, lc_has_active_rules, rep_has_active_rules, scan_data_folder,
@@ -30,7 +30,6 @@ use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE};
use crate::heal::heal_commands::{HealScanMode, HealingTracker};
use crate::heal::heal_ops::HEALING_TRACKER_FILENAME;
use crate::io::FileWriter;
use crate::new_object_layer_fn;
use crate::store_api::{ObjectInfo, StorageAPI};
use crate::utils::os::get_info;
@@ -45,7 +44,7 @@ use rustfs_filemeta::{
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, Opts, RawFileInfo, UpdateFn, get_file_info,
read_xl_meta_no_data,
};
use rustfs_rio::{Reader, bitrot_verify};
use rustfs_rio::bitrot_verify;
use rustfs_utils::HashAlgorithm;
use std::collections::{HashMap, HashSet};
use std::fmt::Debug;
@@ -1299,7 +1298,7 @@ impl DiskAPI for LocalDisk {
}
}
remove_std(&dst_file_path).map_err(|e| to_file_error(e))?;
remove_std(&dst_file_path).map_err(to_file_error)?;
}
rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await?;
@@ -1356,11 +1355,11 @@ impl DiskAPI for LocalDisk {
if let Some(meta) = meta_op {
if !meta.is_dir() {
return Err(DiskError::FileAccessDenied.into());
return Err(DiskError::FileAccessDenied);
}
}
remove(&dst_file_path).await.map_err(|e| to_file_error(e))?;
remove(&dst_file_path).await.map_err(to_file_error)?;
}
rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await?;
@@ -1425,7 +1424,7 @@ impl DiskAPI for LocalDisk {
// TODO: io verifier
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn Reader>> {
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
// warn!("disk read_file: volume: {}, path: {}", volume, path);
let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) {
@@ -1443,7 +1442,7 @@ impl DiskAPI for LocalDisk {
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn Reader>> {
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
// warn!(
// "disk read_file_stream: volume: {}, path: {}, offset: {}, length: {}",
// volume, path, offset, length
@@ -1615,7 +1614,7 @@ impl DiskAPI for LocalDisk {
let e: DiskError = to_file_error(e).into();
if e != DiskError::FileNotFound {
return Err(e.into());
return Err(e);
}
None
ecstore/src/disk/mod.rs
+15 -13
View File
@@ -17,13 +17,10 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
use crate::{
heal::{
data_scanner::ShouldSleepFn,
data_usage_cache::{DataUsageCache, DataUsageEntry},
heal_commands::{HealScanMode, HealingTracker},
},
io::FileWriter,
use crate::heal::{
data_scanner::ShouldSleepFn,
data_usage_cache::{DataUsageCache, DataUsageEntry},
heal_commands::{HealScanMode, HealingTracker},
};
use endpoint::Endpoint;
use error::DiskError;
@@ -32,16 +29,21 @@ use local::LocalDisk;
use madmin::info_commands::DiskMetrics;
use remote::RemoteDisk;
use rustfs_filemeta::{FileInfo, RawFileInfo};
use rustfs_rio::Reader;
use serde::{Deserialize, Serialize};
use std::{fmt::Debug, path::PathBuf, sync::Arc};
use time::OffsetDateTime;
use tokio::{io::AsyncWrite, sync::mpsc::Sender};
use tokio::{
io::{AsyncRead, AsyncWrite},
sync::mpsc::Sender,
};
use tracing::warn;
use uuid::Uuid;
pub type DiskStore = Arc<Disk>;
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
pub type FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>;
#[derive(Debug)]
pub enum Disk {
Local(Box<LocalDisk>),
@@ -277,7 +279,7 @@ impl DiskAPI for Disk {
}
#[tracing::instrument(skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn Reader>> {
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
match self {
Disk::Local(local_disk) => local_disk.read_file(volume, path).await,
Disk::Remote(remote_disk) => remote_disk.read_file(volume, path).await,
@@ -285,7 +287,7 @@ impl DiskAPI for Disk {
}
#[tracing::instrument(skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn Reader>> {
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
match self {
Disk::Local(local_disk) => local_disk.read_file_stream(volume, path, offset, length).await,
Disk::Remote(remote_disk) => remote_disk.read_file_stream(volume, path, offset, length).await,
@@ -485,8 +487,8 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
// File operations.
// 读目录下的所有文件、目录
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>>;
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn Reader>>;
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn Reader>>;
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader>;
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader>;
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter>;
// ReadFileStream
ecstore/src/disk/os.rs
+1 -1
View File
@@ -108,7 +108,7 @@ pub async fn rename_all(
) -> Result<()> {
reliable_rename(src_file_path, dst_file_path.as_ref(), base_dir)
.await
.map_err(|e| to_file_error(e))?;
.map_err(to_file_error)?;
Ok(())
}
ecstore/src/disk/remote.rs
+37 -26
View File
@@ -13,30 +13,33 @@ use protos::{
};
use rmp_serde::Serializer;
use rustfs_filemeta::{FileInfo, MetaCacheEntry, MetacacheWriter, RawFileInfo};
use rustfs_rio::{HttpReader, Reader};
use rustfs_rio::{HttpReader, HttpWriter};
use serde::Serialize;
use tokio::{
io::AsyncWrite,
sync::mpsc::{self, Sender},
};
use tokio_stream::{wrappers::ReceiverStream, StreamExt};
use tokio_stream::{StreamExt, wrappers::ReceiverStream};
use tonic::Request;
use tracing::info;
use uuid::Uuid;
use super::error::{Error, Result};
use super::{
endpoint::Endpoint, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions,
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
endpoint::Endpoint,
};
use crate::heal::{
data_scanner::ShouldSleepFn,
data_usage_cache::{DataUsageCache, DataUsageEntry},
heal_commands::{HealScanMode, HealingTracker},
use crate::{
disk::{FileReader, FileWriter},
heal::{
data_scanner::ShouldSleepFn,
data_usage_cache::{DataUsageCache, DataUsageEntry},
heal_commands::{HealScanMode, HealingTracker},
},
};
use crate::io::{FileWriter, HttpFileWriter};
use protos::proto_gen::node_service::RenamePartRequst;
#[derive(Debug)]
@@ -552,7 +555,7 @@ impl DiskAPI for RemoteDisk {
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file(&self, volume: &str, path: &str) -> Result<Box<dyn Reader>> {
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
info!("read_file {}/{}", volume, path);
let url = format!(
@@ -569,7 +572,7 @@ impl DiskAPI for RemoteDisk {
}
#[tracing::instrument(level = "debug", skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<Box<dyn Reader>> {
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
info!("read_file_stream {}/{}/{}", self.endpoint.to_string(), volume, path);
let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
@@ -587,27 +590,35 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))]
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
info!("append_file {}/{}", volume, path);
Ok(Box::new(HttpFileWriter::new(
self.endpoint.grid_host().as_str(),
self.endpoint.to_string().as_str(),
volume,
path,
0,
let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
self.endpoint.grid_host(),
urlencoding::encode(self.endpoint.to_string().as_str()),
urlencoding::encode(volume),
urlencoding::encode(path),
true,
)?))
0
);
Ok(Box::new(HttpWriter::new(url, Method::PUT, HeaderMap::new()).await?))
}
#[tracing::instrument(level = "debug", skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter> {
info!("create_file {}/{}/{}", self.endpoint.to_string(), volume, path);
Ok(Box::new(HttpFileWriter::new(
self.endpoint.grid_host().as_str(),
self.endpoint.to_string().as_str(),
volume,
path,
file_size,
let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
self.endpoint.grid_host(),
urlencoding::encode(self.endpoint.to_string().as_str()),
urlencoding::encode(volume),
urlencoding::encode(path),
false,
)?))
file_size
);
Ok(Box::new(HttpWriter::new(url, Method::PUT, HeaderMap::new()).await?))
}
#[tracing::instrument(level = "debug", skip(self))]