gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-29 01:29:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge main

Browse Source
This commit is contained in:
weisd
2024-09-13 13:33:23 +08:00
parent 7f35685b35 ff52b376fe
commit 2dd46203ee
38 changed files with 6523 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/disk/local.rs
+57 -24
View File
@@ -3,7 +3,7 @@ use super::{
DeleteOptions, DiskAPI, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp,
ReadOptions, RenameDataResp, VolumeInfo, WalkDirOptions,
};
use crate::disk::STORAGE_FORMAT_FILE;
use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
use crate::{
error::{Error, Result},
file_meta::FileMeta,
@@ -19,18 +19,29 @@ use std::{
use time::OffsetDateTime;
use tokio::fs::{self, File};
use tokio::io::ErrorKind;
use tracing::{debug, error, warn};
use tokio::sync::Mutex;
use tracing::{debug, warn};
use uuid::Uuid;
#[derive(Debug)]
pub struct FormatInfo {
pub id: Option<Uuid>,
pub _data: Vec<u8>,
pub _file_info: Option<Metadata>,
pub _last_check: Option<OffsetDateTime>,
}
impl FormatInfo {}
#[derive(Debug)]
pub struct LocalDisk {
pub root: PathBuf,
pub id: Uuid,
pub _format_data: Vec<u8>,
pub _format_meta: Option<Metadata>,
pub _format_path: PathBuf,
// pub format_legacy: bool, // drop
pub _format_last_check: Option<OffsetDateTime>,
pub format_info: Mutex<FormatInfo>,
// pub id: Mutex<Option<Uuid>>,
// pub format_data: Mutex<Vec<u8>>,
// pub format_file_info: Mutex<Option<Metadata>>,
// pub format_last_check: Mutex<Option<OffsetDateTime>>,
}
impl LocalDisk {
@@ -48,7 +59,7 @@ impl LocalDisk {
let (format_data, format_meta) = read_file_exists(&format_path).await?;
let mut id = Uuid::nil();
let mut id = None;
// let mut format_legacy = false;
let mut format_last_check = None;
@@ -61,19 +72,26 @@ impl LocalDisk {
return Err(Error::from(DiskError::InconsistentDisk));
}
id = fm.erasure.this;
id = Some(fm.erasure.this);
// format_legacy = fm.erasure.distribution_algo == DistributionAlgoVersion::V1;
format_last_check = Some(OffsetDateTime::now_utc());
}
let format_info = FormatInfo {
id,
_data: format_data,
_file_info: format_meta,
_last_check: format_last_check,
};
let disk = Self {
root,
id,
_format_meta: format_meta,
_format_data: format_data,
_format_path: format_path,
// format_legacy,
_format_last_check: format_last_check,
format_info: Mutex::new(format_info),
// // format_legacy,
// format_file_info: Mutex::new(format_meta),
// format_data: Mutex::new(format_data),
// format_last_check: Mutex::new(format_last_check),
};
disk.make_meta_volumes().await?;
@@ -204,7 +222,7 @@ impl LocalDisk {
} else {
if delete_path.is_dir() {
if let Err(err) = fs::remove_dir(&delete_path).await {
error!("remove_dir err {:?} when {:?}", &err, &delete_path);
debug!("remove_dir err {:?} when {:?}", &err, &delete_path);
match err.kind() {
ErrorKind::NotFound => (),
// ErrorKind::DirectoryNotEmpty => (),
@@ -218,7 +236,7 @@ impl LocalDisk {
}
} else {
if let Err(err) = fs::remove_file(&delete_path).await {
error!("remove_file err {:?} when {:?}", &err, &delete_path);
debug!("remove_file err {:?} when {:?}", &err, &delete_path);
match err.kind() {
ErrorKind::NotFound => (),
_ => {
@@ -413,9 +431,24 @@ impl DiskAPI for LocalDisk {
fn is_local(&self) -> bool {
true
}
async fn close(&self) -> Result<()> {
Ok(())
}
fn path(&self) -> PathBuf {
self.root.clone()
}
fn id(&self) -> Uuid {
self.id
async fn get_disk_id(&self) -> Option<Uuid> {
// TODO: check format file
let format_info = self.format_info.lock().await;
format_info.id.clone()
// TODO: 判断源文件id,是否有效
}
async fn set_disk_id(&self, _id: Option<Uuid>) -> Result<()> {
// 本地不需要设置
Ok(())
}
#[must_use]
@@ -517,7 +550,8 @@ impl DiskAPI for LocalDisk {
let file = File::create(&fpath).await?;
Ok(FileWriter::new(file))
Ok(FileWriter::Local(LocalFileWriter::new(file)))
// Ok(FileWriter::new(file))
// let mut writer = BufWriter::new(file);
@@ -543,7 +577,8 @@ impl DiskAPI for LocalDisk {
.open(&p)
.await?;
Ok(FileWriter::new(file))
Ok(FileWriter::Local(LocalFileWriter::new(file)))
// Ok(FileWriter::new(file))
// let mut writer = BufWriter::new(file);
@@ -560,7 +595,7 @@ impl DiskAPI for LocalDisk {
debug!("read_file {:?}", &p);
let file = File::options().read(true).open(&p).await?;
Ok(FileReader::new(file))
Ok(FileReader::Local(LocalFileReader::new(file)))
// file.seek(SeekFrom::Start(offset as u64)).await?;
@@ -756,9 +791,7 @@ impl DiskAPI for LocalDisk {
.await?;
}
Ok(RenameDataResp {
old_data_dir: old_data_dir,
})
Ok(RenameDataResp { old_data_dir })
}
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
ecstore/src/disk/mod.rs
+179 -47
View File
@@ -2,6 +2,7 @@ pub mod endpoint;
pub mod error;
pub mod format;
mod local;
mod remote;
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
@@ -12,18 +13,21 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
const STORAGE_FORMAT_FILE: &str = "xl.meta";
use crate::{
erasure::ReadAt,
erasure::{ReadAt, Write},
error::{Error, Result},
file_meta::FileMeta,
store_api::{FileInfo, RawFileInfo},
};
use bytes::Bytes;
use std::{fmt::Debug, io::SeekFrom, pin::Pin, sync::Arc};
use protos::proto_gen::node_service::{node_service_client::NodeServiceClient, ReadAtRequest, WriteRequest};
use serde::{Deserialize, Serialize};
use std::{fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc};
use time::OffsetDateTime;
use tokio::{
fs::File,
io::{AsyncReadExt, AsyncSeekExt, AsyncWrite},
io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt},
};
use tonic::{transport::Channel, Request};
use uuid::Uuid;
pub type DiskStore = Arc<Box<dyn DiskAPI>>;
@@ -33,15 +37,18 @@ pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result<DiskS
let s = local::LocalDisk::new(ep, opt.cleanup).await?;
Ok(Arc::new(Box::new(s)))
} else {
let _ = opt.health_check;
unimplemented!()
let remote_disk = remote::RemoteDisk::new(ep, opt).await?;
Ok(Arc::new(Box::new(remote_disk)))
}
}
#[async_trait::async_trait]
pub trait DiskAPI: Debug + Send + Sync + 'static {
fn is_local(&self) -> bool;
fn id(&self) -> Uuid;
fn path(&self) -> PathBuf;
async fn close(&self) -> Result<()>;
async fn get_disk_id(&self) -> Option<Uuid>;
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes>;
@@ -88,7 +95,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
}
#[derive(Debug, Default, Clone)]
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct FileInfoVersions {
// Name of the volume.
pub volume: String,
@@ -104,7 +111,7 @@ pub struct FileInfoVersions {
pub free_versions: Vec<FileInfo>,
}
#[derive(Debug, Default, Clone)]
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct WalkDirOptions {
// Bucket to scanner
pub bucket: String,
@@ -131,7 +138,7 @@ pub struct WalkDirOptions {
pub disk_id: String,
}
#[derive(Debug, Default)]
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct MetaCacheEntry {
// name is the full name of the object including prefixes
pub name: String,
@@ -206,17 +213,18 @@ pub struct DiskOption {
pub health_check: bool,
}
#[derive(Serialize, Deserialize)]
pub struct RenameDataResp {
pub old_data_dir: Option<Uuid>,
}
#[derive(Debug, Clone, Default)]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DeleteOptions {
pub recursive: bool,
pub immediate: bool,
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReadMultipleReq {
pub bucket: String,
pub prefix: String,
@@ -227,7 +235,7 @@ pub struct ReadMultipleReq {
pub max_results: usize,
}
#[derive(Debug, Clone, Default)]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ReadMultipleResp {
pub bucket: String,
pub prefix: String,
@@ -252,65 +260,154 @@ pub struct ReadMultipleResp {
// }
// }
#[derive(Debug, Deserialize, Serialize)]
pub struct VolumeInfo {
pub name: String,
pub created: Option<OffsetDateTime>,
}
#[derive(Deserialize, Serialize)]
pub struct ReadOptions {
pub read_data: bool,
pub healing: bool,
}
pub struct FileWriter {
pub inner: Pin<Box<dyn AsyncWrite + Send + Sync + 'static>>,
// pub struct FileWriter {
// pub inner: Pin<Box<dyn AsyncWrite + Send + Sync + 'static>>,
// }
// impl AsyncWrite for FileWriter {
// fn poll_write(
// mut self: Pin<&mut Self>,
// cx: &mut std::task::Context<'_>,
// buf: &[u8],
// ) -> std::task::Poll<std::result::Result<usize, std::io::Error>> {
// Pin::new(&mut self.inner).poll_write(cx, buf)
// }
// fn poll_flush(
// mut self: Pin<&mut Self>,
// cx: &mut std::task::Context<'_>,
// ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
// Pin::new(&mut self.inner).poll_flush(cx)
// }
// fn poll_shutdown(
// mut self: Pin<&mut Self>,
// cx: &mut std::task::Context<'_>,
// ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
// Pin::new(&mut self.inner).poll_shutdown(cx)
// }
// }
// impl FileWriter {
// pub fn new<W>(inner: W) -> Self
// where
// W: AsyncWrite + Send + Sync + 'static,
// {
// Self { inner: Box::pin(inner) }
// }
// }
pub enum FileWriter {
Local(LocalFileWriter),
Remote(RemoteFileWriter),
}
impl AsyncWrite for FileWriter {
fn poll_write(
mut self: Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<std::result::Result<usize, std::io::Error>> {
Pin::new(&mut self.inner).poll_write(cx, buf)
}
fn poll_flush(
mut self: Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
Pin::new(&mut self.inner).poll_flush(cx)
}
fn poll_shutdown(
mut self: Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
Pin::new(&mut self.inner).poll_shutdown(cx)
#[async_trait::async_trait]
impl Write for FileWriter {
async fn write(&mut self, buf: &[u8]) -> Result<()> {
match self {
Self::Local(local_file_writer) => local_file_writer.write(buf).await,
Self::Remote(remote_file_writer) => remote_file_writer.write(buf).await,
}
}
}
impl FileWriter {
pub fn new<W>(inner: W) -> Self
where
W: AsyncWrite + Send + Sync + 'static,
{
Self { inner: Box::pin(inner) }
}
}
#[derive(Debug)]
pub struct FileReader {
pub struct LocalFileWriter {
pub inner: File,
}
impl FileReader {
impl LocalFileWriter {
pub fn new(inner: File) -> Self {
Self { inner }
}
}
#[async_trait::async_trait]
impl Write for LocalFileWriter {
async fn write(&mut self, buf: &[u8]) -> Result<()> {
self.inner.write(buf).await?;
self.inner.flush().await?;
Ok(())
}
}
pub struct RemoteFileWriter {
pub root: PathBuf,
pub volume: String,
pub path: String,
pub is_append: bool,
client: NodeServiceClient<Channel>,
}
impl RemoteFileWriter {
pub fn new(root: PathBuf, volume: String, path: String, is_append: bool, client: NodeServiceClient<Channel>) -> Self {
Self {
root,
volume,
path,
is_append,
client,
}
}
}
#[async_trait::async_trait]
impl Write for RemoteFileWriter {
async fn write(&mut self, buf: &[u8]) -> Result<()> {
let request = Request::new(WriteRequest {
disk: self.root.to_string_lossy().to_string(),
volume: self.volume.to_string(),
path: self.path.to_string(),
is_append: self.is_append,
data: buf.to_vec(),
});
let _response = self.client.write(request).await?.into_inner();
Ok(())
}
}
#[derive(Debug)]
pub enum FileReader {
Local(LocalFileReader),
Remote(RemoteFileReader),
}
#[async_trait::async_trait]
impl ReadAt for FileReader {
async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec<u8>, usize)> {
match self {
Self::Local(local_file_writer) => local_file_writer.read_at(offset, length).await,
Self::Remote(remote_file_writer) => remote_file_writer.read_at(offset, length).await,
}
}
}
#[derive(Debug)]
pub struct LocalFileReader {
pub inner: File,
}
impl LocalFileReader {
pub fn new(inner: File) -> Self {
Self { inner }
}
}
#[async_trait::async_trait]
impl ReadAt for LocalFileReader {
async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec<u8>, usize)> {
self.inner.seek(SeekFrom::Start(offset as u64)).await?;
@@ -323,3 +420,38 @@ impl ReadAt for FileReader {
Ok((buffer, bytes_read))
}
}
#[derive(Debug)]
pub struct RemoteFileReader {
pub root: PathBuf,
pub volume: String,
pub path: String,
client: NodeServiceClient<Channel>,
}
impl RemoteFileReader {
pub fn new(root: PathBuf, volume: String, path: String, client: NodeServiceClient<Channel>) -> Self {
Self {
root,
volume,
path,
client,
}
}
}
#[async_trait::async_trait]
impl ReadAt for RemoteFileReader {
async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec<u8>, usize)> {
let request = Request::new(ReadAtRequest {
disk: self.root.to_string_lossy().to_string(),
volume: self.volume.to_string(),
path: self.path.to_string(),
offset: offset.try_into().unwrap(),
length: length.try_into().unwrap(),
});
let response = self.client.read_at(request).await?.into_inner();
Ok((response.data, response.read_size.try_into().unwrap()))
}
}
ecstore/src/disk/remote.rs
+529
View File
@@ -0,0 +1,529 @@
use std::{path::PathBuf, sync::Arc, time::Duration};
use bytes::Bytes;
use futures::lock::Mutex;
use protos::{
node_service_time_out_client,
proto_gen::node_service::{
node_service_client::NodeServiceClient, DeleteRequest, DeleteVersionsRequest, DeleteVolumeRequest, ListDirRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest,
ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, WalkDirRequest, WriteAllRequest,
WriteMetadataRequest,
},
DEFAULT_GRPC_SERVER_MESSAGE_LEN,
};
use tokio::{fs, sync::RwLock};
use tonic::{
transport::{Channel, Endpoint as tonic_Endpoint},
Request,
};
use tower::timeout::Timeout;
use tracing::info;
use uuid::Uuid;
use crate::{
disk::error::DiskError,
error::{Error, Result},
store_api::{FileInfo, RawFileInfo},
};
use super::{
endpoint::Endpoint, DeleteOptions, DiskAPI, DiskOption, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, VolumeInfo,
WalkDirOptions,
};
#[derive(Debug)]
pub struct RemoteDisk {
id: Mutex<Option<Uuid>>,
channel: Arc<RwLock<Option<Channel>>>,
url: url::Url,
pub root: PathBuf,
}
impl RemoteDisk {
pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result<Self> {
let root = fs::canonicalize(ep.url.path()).await?;
Ok(Self {
channel: Arc::new(RwLock::new(None)),
url: ep.url.clone(),
root,
id: Mutex::new(None),
})
}
#[allow(dead_code)]
async fn get_client(&self) -> Result<NodeServiceClient<Timeout<Channel>>> {
let channel_clone = self.channel.clone();
let channel = {
let read_lock = channel_clone.read().await;
if let Some(ref channel) = *read_lock {
channel.clone()
} else {
let addr = format!("{}://{}:{}", self.url.scheme(), self.url.host_str().unwrap(), self.url.port().unwrap());
info!("disk url: {}", addr);
let connector = tonic_Endpoint::from_shared(addr.clone())?;
let new_channel = connector.connect().await.map_err(|_err| DiskError::DiskNotFound)?;
info!("get channel success");
*self.channel.write().await = Some(new_channel.clone());
new_channel
}
};
Ok(node_service_time_out_client(
channel,
Duration::new(30, 0), // TODO: use config setting
DEFAULT_GRPC_SERVER_MESSAGE_LEN,
// grpc_enable_gzip,
false, // TODO: use config setting
))
}
async fn get_client_v2(&self) -> Result<NodeServiceClient<tonic::transport::Channel>> {
// Ok(NodeServiceClient::connect("http://220.181.1.138:9000").await?)
let addr = format!("{}://{}:{}", self.url.scheme(), self.url.host_str().unwrap(), self.url.port().unwrap());
Ok(NodeServiceClient::connect(addr).await?)
}
}
// TODO: all api need to handle errors
#[async_trait::async_trait]
impl DiskAPI for RemoteDisk {
fn is_local(&self) -> bool {
false
}
async fn close(&self) -> Result<()> {
Ok(())
}
fn path(&self) -> PathBuf {
self.root.clone()
}
async fn get_disk_id(&self) -> Option<Uuid> {
self.id.lock().await.clone()
}
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
let mut lock = self.id.lock().await;
*lock = id;
Ok(())
}
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes> {
info!("read_all");
let mut client = self.get_client_v2().await?;
let request = Request::new(ReadAllRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
path: path.to_string(),
});
let response = client.read_all(request).await?.into_inner();
info!("read_all success");
if !response.success {
return Err(DiskError::FileNotFound.into());
}
Ok(Bytes::from(response.data))
}
async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
info!("write_all");
let mut client = self.get_client_v2().await?;
let request = Request::new(WriteAllRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
path: path.to_string(),
data,
});
let response = client.write_all(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
info!("delete");
let options = serde_json::to_string(&opt)?;
let mut client = self.get_client_v2().await?;
let request = Request::new(DeleteRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
path: path.to_string(),
options,
});
let response = client.delete(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
info!("rename_file");
let mut client = self.get_client_v2().await?;
let request = Request::new(RenameFileRequst {
disk: self.root.to_string_lossy().to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_file(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result<FileWriter> {
info!("create_file");
Ok(FileWriter::Remote(RemoteFileWriter::new(
self.root.clone(),
volume.to_string(),
path.to_string(),
false,
self.get_client_v2().await?,
)))
}
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
info!("append_file");
Ok(FileWriter::Remote(RemoteFileWriter::new(
self.root.clone(),
volume.to_string(),
path.to_string(),
true,
self.get_client_v2().await?,
)))
}
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
info!("read_file");
Ok(FileReader::Remote(RemoteFileReader::new(
self.root.clone(),
volume.to_string(),
path.to_string(),
self.get_client_v2().await?,
)))
}
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
info!("list_dir");
let mut client = self.get_client_v2().await?;
let request = Request::new(ListDirRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
});
let response = client.list_dir(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(response.volumes)
}
async fn walk_dir(&self, opts: WalkDirOptions) -> Result<Vec<MetaCacheEntry>> {
info!("walk_dir");
let walk_dir_options = serde_json::to_string(&opts)?;
let mut client = self.get_client_v2().await?;
let request = Request::new(WalkDirRequest {
disk: self.root.to_string_lossy().to_string(),
walk_dir_options,
});
let response = client.walk_dir(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
let entries = response
.meta_cache_entry
.into_iter()
.filter_map(|json_str| serde_json::from_str::<MetaCacheEntry>(&json_str).ok())
.collect();
Ok(entries)
}
async fn rename_data(
&self,
src_volume: &str,
src_path: &str,
fi: FileInfo,
dst_volume: &str,
dst_path: &str,
) -> Result<RenameDataResp> {
info!("rename_data");
let file_info = serde_json::to_string(&fi)?;
let mut client = self.get_client_v2().await?;
let request = Request::new(RenameDataRequest {
disk: self.root.to_string_lossy().to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
file_info,
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
});
let response = client.rename_data(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
Ok(rename_data_resp)
}
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
info!("make_volumes");
let mut client = self.get_client_v2().await?;
let request = Request::new(MakeVolumesRequest {
disk: self.root.to_string_lossy().to_string(),
volumes: volumes.iter().map(|s| (*s).to_string()).collect(),
});
let response = client.make_volumes(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
async fn make_volume(&self, volume: &str) -> Result<()> {
info!("make_volume");
let mut client = self.get_client_v2().await?;
let request = Request::new(MakeVolumeRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
});
let response = client.make_volume(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
info!("list_volumes");
let mut client = self.get_client_v2().await?;
let request = Request::new(ListVolumesRequest {
disk: self.root.to_string_lossy().to_string(),
});
let response = client.list_volumes(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
let infos = response
.volume_infos
.into_iter()
.filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
.collect();
Ok(infos)
}
async fn stat_volume(&self, volume: &str) -> Result<VolumeInfo> {
info!("stat_volume");
let mut client = self.get_client_v2().await?;
let request = Request::new(StatVolumeRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
});
let response = client.stat_volume(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
let volume_info = serde_json::from_str::<VolumeInfo>(&response.volume_info)?;
Ok(volume_info)
}
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
info!("write_metadata");
let file_info = serde_json::to_string(&fi)?;
let mut client = self.get_client_v2().await?;
let request = Request::new(WriteMetadataRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
path: path.to_string(),
file_info,
});
let response = client.write_metadata(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
async fn read_version(
&self,
_org_volume: &str,
volume: &str,
path: &str,
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
info!("read_version");
let opts = serde_json::to_string(opts)?;
let mut client = self.get_client_v2().await?;
let request = Request::new(ReadVersionRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
path: path.to_string(),
version_id: version_id.to_string(),
opts,
});
let response = client.read_version(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
Ok(file_info)
}
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
info!("read_xl");
let mut client = self.get_client_v2().await?;
let request = Request::new(ReadXlRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
path: path.to_string(),
read_data,
});
let response = client.read_xl(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
Ok(raw_file_info)
}
async fn delete_versions(
&self,
volume: &str,
versions: Vec<FileInfoVersions>,
opts: DeleteOptions,
) -> Result<Vec<Option<Error>>> {
info!("delete_versions");
let opts = serde_json::to_string(&opts)?;
let mut versions_str = Vec::with_capacity(versions.len());
for file_info_versions in versions.iter() {
versions_str.push(serde_json::to_string(file_info_versions)?);
}
let mut client = self.get_client_v2().await?;
let request = Request::new(DeleteVersionsRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
versions: versions_str,
opts,
});
let response = client.delete_versions(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(format!(
"delete versions remote err: {}",
response.error_info.unwrap_or("None".to_string())
)));
}
let errors = response
.errors
.iter()
.map(|error| {
if error.is_empty() {
None
} else {
Some(Error::from_string(error))
}
})
.collect();
Ok(errors)
}
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
info!("read_multiple");
let read_multiple_req = serde_json::to_string(&req)?;
let mut client = self.get_client_v2().await?;
let request = Request::new(ReadMultipleRequest {
disk: self.root.to_string_lossy().to_string(),
read_multiple_req,
});
let response = client.read_multiple(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
let read_multiple_resps = response
.read_multiple_resps
.into_iter()
.filter_map(|json_str| serde_json::from_str::<ReadMultipleResp>(&json_str).ok())
.collect();
Ok(read_multiple_resps)
}
async fn delete_volume(&self, volume: &str) -> Result<()> {
info!("delete_volume");
let mut client = self.get_client_v2().await?;
let request = Request::new(DeleteVolumeRequest {
disk: self.root.to_string_lossy().to_string(),
volume: volume.to_string(),
});
let response = client.delete_volume(request).await?.into_inner();
if !response.success {
return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
}
Ok(())
}
}
ecstore/src/endpoints.rs
+5
View File
@@ -408,6 +408,11 @@ impl AsMut<Vec<PoolEndpoints>> for EndpointServerPools {
}
impl EndpointServerPools {
pub fn from_volumes(server_addr: &str, endpoints: Vec<String>) -> Result<(EndpointServerPools, SetupType)> {
let layouts = DisksLayout::try_from(endpoints.as_slice())?;
Self::create_server_endpoints(server_addr, &layouts)
}
/// validates and creates new endpoints from input args, supports
/// both ellipses and without ellipses transparently.
pub fn create_server_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result<(EndpointServerPools, SetupType)> {
ecstore/src/erasure.rs
+11 -6
View File
@@ -3,7 +3,7 @@ use bytes::Bytes;
use futures::future::join_all;
use futures::{Stream, StreamExt};
use reed_solomon_erasure::galois_8::ReedSolomon;
use tokio::io::AsyncWrite;
use std::fmt::Debug;
use tokio::io::AsyncWriteExt;
use tokio::io::DuplexStream;
use tracing::debug;
@@ -13,7 +13,7 @@ use uuid::Uuid;
use crate::chunk_stream::ChunkedStream;
use crate::disk::error::DiskError;
use crate::disk::FileReader;
use crate::disk::{FileReader, FileWriter};
pub struct Erasure {
data_shards: usize,
@@ -43,17 +43,16 @@ impl Erasure {
}
}
pub async fn encode<S, W>(
pub async fn encode<S>(
&self,
body: S,
writers: &mut [W],
writers: &mut [FileWriter],
// block_size: usize,
total_size: usize,
_write_quorum: usize,
) -> Result<usize>
where
S: Stream<Item = Result<Bytes, StdError>> + Send + Sync + 'static,
W: AsyncWrite + Unpin,
{
let mut stream = ChunkedStream::new(body, total_size, self.block_size, false);
let mut total: usize = 0;
@@ -85,7 +84,7 @@ impl Erasure {
let mut errs = Vec::new();
for (i, w) in writers.iter_mut().enumerate() {
match w.write_all(blocks[i].as_ref()).await {
match w.write(blocks[i].as_ref()).await {
Ok(_) => errs.push(None),
Err(e) => errs.push(Some(e)),
}
@@ -318,6 +317,12 @@ impl Erasure {
}
}
#[async_trait::async_trait]
pub trait Write {
async fn write(&mut self, buf: &[u8]) -> Result<()>;
}
#[async_trait::async_trait]
pub trait ReadAt {
async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec<u8>, usize)>;
}
ecstore/src/lib.rs
+4 -4
View File
@@ -1,12 +1,12 @@
mod bucket_meta;
mod chunk_stream;
pub mod disk;
mod disks_layout;
mod endpoints;
mod erasure;
pub mod disks_layout;
pub mod endpoints;
pub mod erasure;
pub mod error;
mod file_meta;
mod peer;
pub mod peer;
pub mod set_disk;
mod sets;
pub mod store;
ecstore/src/peer.rs
+140 -48
View File
@@ -1,11 +1,19 @@
use async_trait::async_trait;
use futures::future::join_all;
use protos::proto_gen::node_service::node_service_client::NodeServiceClient;
use protos::proto_gen::node_service::{DeleteBucketRequest, GetBucketInfoRequest, ListBucketRequest, MakeBucketRequest};
use protos::{node_service_time_out_client, DEFAULT_GRPC_SERVER_MESSAGE_LEN};
use regex::Regex;
use std::{collections::HashMap, fmt::Debug, sync::Arc};
use tracing::warn;
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
use tokio::sync::RwLock;
use tonic::transport::{Channel, Endpoint};
use tonic::Request;
use tower::timeout::Timeout;
use tracing::{info, warn};
use crate::store::all_local_disk;
use crate::{
disk::{self, error::DiskError, DiskStore, VolumeInfo},
disk::{self, error::DiskError, VolumeInfo},
endpoints::{EndpointServerPools, Node},
error::{Error, Result},
store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
@@ -19,7 +27,7 @@ pub trait PeerS3Client: Debug + Sync + Send + 'static {
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>>;
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>;
async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo>;
fn get_pools(&self) -> Vec<usize>;
fn get_pools(&self) -> Option<Vec<usize>>;
}
#[derive(Debug)]
@@ -29,24 +37,23 @@ pub struct S3PeerSys {
}
impl S3PeerSys {
pub fn new(eps: &EndpointServerPools, local_disks: Vec<DiskStore>) -> Self {
pub fn new(eps: &EndpointServerPools) -> Self {
Self {
clients: Self::new_clients(eps, local_disks),
clients: Self::new_clients(eps),
pools_count: eps.as_ref().len(),
}
}
fn new_clients(eps: &EndpointServerPools, local_disks: Vec<DiskStore>) -> Vec<Client> {
fn new_clients(eps: &EndpointServerPools) -> Vec<Client> {
let nodes = eps.get_nodes();
let v: Vec<Client> = nodes
.iter()
.map(|e| {
if e.is_local {
let cli: Box<dyn PeerS3Client> =
Box::new(LocalPeerS3Client::new(local_disks.clone(), e.clone(), e.pools.clone()));
let cli: Box<dyn PeerS3Client> = Box::new(LocalPeerS3Client::new(Some(e.clone()), Some(e.pools.clone())));
Arc::new(cli)
} else {
let cli: Box<dyn PeerS3Client> = Box::new(RemotePeerS3Client::new(e.clone(), e.pools.clone()));
let cli: Box<dyn PeerS3Client> = Box::new(RemotePeerS3Client::new(Some(e.clone()), Some(e.pools.clone())));
Arc::new(cli)
}
})
@@ -56,9 +63,8 @@ impl S3PeerSys {
}
}
#[async_trait]
impl PeerS3Client for S3PeerSys {
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
impl S3PeerSys {
pub async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
let mut futures = Vec::with_capacity(self.clients.len());
for cli in self.clients.iter() {
futures.push(cli.make_bucket(bucket, opts));
@@ -83,7 +89,7 @@ impl PeerS3Client for S3PeerSys {
for (j, cli) in self.clients.iter().enumerate() {
let pools = cli.get_pools();
let idx = i;
if pools.contains(&idx) {
if pools.unwrap_or_default().contains(&idx) {
per_pool_errs.push(errors[j].as_ref());
}
@@ -95,7 +101,7 @@ impl PeerS3Client for S3PeerSys {
Ok(())
}
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
pub async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
let mut futures = Vec::with_capacity(self.clients.len());
for cli in self.clients.iter() {
futures.push(cli.list_bucket(opts));
@@ -165,7 +171,7 @@ impl PeerS3Client for S3PeerSys {
Ok(())
}
async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
pub async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
let mut futures = Vec::with_capacity(self.clients.len());
for cli in self.clients.iter() {
futures.push(cli.get_bucket_info(bucket, opts));
@@ -193,7 +199,7 @@ impl PeerS3Client for S3PeerSys {
for (j, cli) in self.clients.iter().enumerate() {
let pools = cli.get_pools();
let idx = i;
if pools.contains(&idx) {
if pools.unwrap_or_default().contains(&idx) {
per_pool_errs.push(errors[j].as_ref());
}
@@ -206,22 +212,22 @@ impl PeerS3Client for S3PeerSys {
.ok_or(Error::new(DiskError::VolumeNotFound))
}
fn get_pools(&self) -> Vec<usize> {
pub fn get_pools(&self) -> Option<Vec<usize>> {
unimplemented!()
}
}
#[derive(Debug)]
pub struct LocalPeerS3Client {
pub local_disks: Vec<DiskStore>,
// pub local_disks: Vec<DiskStore>,
// pub node: Node,
pub pools: Vec<usize>,
pub pools: Option<Vec<usize>>,
}
impl LocalPeerS3Client {
fn new(local_disks: Vec<DiskStore>, _node: Node, pools: Vec<usize>) -> Self {
pub fn new(_node: Option<Node>, pools: Option<Vec<usize>>) -> Self {
Self {
local_disks,
// local_disks,
// node,
pools,
}
@@ -230,12 +236,14 @@ impl LocalPeerS3Client {
#[async_trait]
impl PeerS3Client for LocalPeerS3Client {
fn get_pools(&self) -> Vec<usize> {
fn get_pools(&self) -> Option<Vec<usize>> {
self.pools.clone()
}
async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
let mut futures = Vec::with_capacity(self.local_disks.len());
for disk in self.local_disks.iter() {
let local_disks = all_local_disk().await;
let mut futures = Vec::with_capacity(local_disks.len());
for disk in local_disks.iter() {
futures.push(disk.list_volumes());
}
@@ -280,8 +288,9 @@ impl PeerS3Client for LocalPeerS3Client {
Ok(buckets)
}
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
let mut futures = Vec::with_capacity(self.local_disks.len());
for disk in self.local_disks.iter() {
let local_disks = all_local_disk().await;
let mut futures = Vec::with_capacity(local_disks.len());
for disk in local_disks.iter() {
futures.push(async move {
match disk.make_volume(bucket).await {
Ok(_) => Ok(()),
@@ -313,15 +322,16 @@ impl PeerS3Client for LocalPeerS3Client {
}
async fn get_bucket_info(&self, bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
let mut futures = Vec::with_capacity(self.local_disks.len());
for disk in self.local_disks.iter() {
let local_disks = all_local_disk().await;
let mut futures = Vec::with_capacity(local_disks.len());
for disk in local_disks.iter() {
futures.push(disk.stat_volume(bucket));
}
let results = join_all(futures).await;
let mut ress = Vec::with_capacity(self.local_disks.len());
let mut errs = Vec::with_capacity(self.local_disks.len());
let mut ress = Vec::with_capacity(local_disks.len());
let mut errs = Vec::with_capacity(local_disks.len());
for res in results {
match res {
@@ -351,9 +361,10 @@ impl PeerS3Client for LocalPeerS3Client {
}
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
let mut futures = Vec::with_capacity(self.local_disks.len());
let local_disks = all_local_disk().await;
let mut futures = Vec::with_capacity(local_disks.len());
for disk in self.local_disks.iter() {
for disk in local_disks.iter() {
futures.push(disk.delete_volume(bucket));
}
@@ -398,34 +409,115 @@ impl PeerS3Client for LocalPeerS3Client {
#[derive(Debug)]
pub struct RemotePeerS3Client {
// pub node: Node,
// pub pools: Vec<usize>,
pub node: Option<Node>,
pub pools: Option<Vec<usize>>,
connector: Endpoint,
channel: Arc<RwLock<Option<Channel>>>,
}
impl RemotePeerS3Client {
fn new(_node: Node, _pools: Vec<usize>) -> Self {
// Self { node, pools }
Self {}
fn new(node: Option<Node>, pools: Option<Vec<usize>>) -> Self {
let connector =
Endpoint::from_shared(format!("{}", node.as_ref().map(|v| { v.url.to_string() }).unwrap_or_default())).unwrap();
Self {
node,
pools,
connector,
channel: Arc::new(RwLock::new(None)),
}
}
#[allow(dead_code)]
async fn get_client(&self) -> Result<NodeServiceClient<Timeout<Channel>>> {
let channel_clone = self.channel.clone();
let channel = {
let read_lock = channel_clone.read().await;
if let Some(ref channel) = *read_lock {
channel.clone()
} else {
let new_channel = self.connector.connect().await?;
info!("get channel success");
*self.channel.write().await = Some(new_channel.clone());
new_channel
}
};
Ok(node_service_time_out_client(
channel,
Duration::new(30, 0), // TODO: use config setting
DEFAULT_GRPC_SERVER_MESSAGE_LEN,
// grpc_enable_gzip,
false, // TODO: use config setting
))
}
async fn get_client_v2(&self) -> Result<NodeServiceClient<tonic::transport::Channel>> {
// Ok(NodeServiceClient::connect("http://220.181.1.138:9000").await?)
// let addr = format!("{}://{}:{}", self.url.scheme(), self.url.host_str().unwrap(), self.url.port().unwrap());
let addr = format!("{}", self.node.as_ref().map(|v| { v.url.to_string() }).unwrap_or_default());
Ok(NodeServiceClient::connect(addr).await?)
}
}
#[async_trait]
impl PeerS3Client for RemotePeerS3Client {
fn get_pools(&self) -> Vec<usize> {
unimplemented!()
fn get_pools(&self) -> Option<Vec<usize>> {
self.pools.clone()
}
async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
unimplemented!()
async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
let options = serde_json::to_string(opts)?;
let mut client = self.get_client_v2().await?;
let request = Request::new(ListBucketRequest { options });
let response = client.list_bucket(request).await?.into_inner();
let bucket_infos = response
.bucket_infos
.into_iter()
.filter_map(|json_str| serde_json::from_str::<BucketInfo>(&json_str).ok())
.collect();
Ok(bucket_infos)
}
async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> Result<()> {
unimplemented!()
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
let options = serde_json::to_string(opts)?;
let mut client = self.get_client_v2().await?;
let request = Request::new(MakeBucketRequest {
name: bucket.to_string(),
options,
});
let response = client.make_bucket(request).await?.into_inner();
// TODO: deal with error
if !response.success {
warn!("make bucket error: {:?}", response.error_info);
}
Ok(())
}
async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
unimplemented!()
async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
let options = serde_json::to_string(opts)?;
let mut client = self.get_client_v2().await?;
let request = Request::new(GetBucketInfoRequest {
bucket: bucket.to_string(),
options,
});
let response = client.get_bucket_info(request).await?.into_inner();
let bucket_info = serde_json::from_str::<BucketInfo>(&response.bucket_info)?;
Ok(bucket_info)
}
async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
unimplemented!()
async fn delete_bucket(&self, bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
let mut client = self.get_client_v2().await?;
let request = Request::new(DeleteBucketRequest {
bucket: bucket.to_string(),
});
let _response = client.delete_bucket(request).await?.into_inner();
Ok(())
}
}
ecstore/src/sets.rs
+26 -4
View File
@@ -12,6 +12,7 @@ use crate::{
endpoints::PoolEndpoints,
error::{Error, Result},
set_disk::SetDisks,
store::{GLOBAL_IsDistErasure, GLOBAL_LOCAL_DISK_SET_DRIVES},
store_api::{
BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo,
@@ -36,7 +37,7 @@ pub struct Sets {
}
impl Sets {
pub fn new(
pub async fn new(
disks: Vec<Option<DiskStore>>,
endpoints: &PoolEndpoints,
fm: &FormatV3,
@@ -52,11 +53,32 @@ impl Sets {
let mut set_drive = Vec::with_capacity(set_drive_count);
for j in 0..set_drive_count {
let idx = i * set_drive_count + j;
if disks[idx].is_none() {
let mut disk = disks[idx].clone();
if disk.is_none() {
set_drive.push(None);
} else {
let disk = disks[idx].clone();
continue;
}
if disk.as_ref().unwrap().is_local() && *GLOBAL_IsDistErasure.read().await {
let local_disk = {
let local_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.read().await;
local_set_drives[pool_idx][i][j].clone()
};
if local_disk.is_none() {
set_drive.push(None);
continue;
}
let _ = disk.as_ref().unwrap().close().await;
disk = local_disk;
}
if let Some(_disk_id) = disk.as_ref().unwrap().get_disk_id().await {
set_drive.push(disk);
} else {
set_drive.push(None);
}
}
ecstore/src/store.rs
+174 -22
View File
@@ -1,10 +1,11 @@
use crate::{
bucket_meta::BucketMetadata,
disk::{error::DiskError, DeleteOptions, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
disks_layout::DisksLayout,
endpoints::EndpointServerPools,
disk::{
error::DiskError, new_disk, DeleteOptions, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET,
},
endpoints::{EndpointServerPools, SetupType},
error::{Error, Result},
peer::{PeerS3Client, S3PeerSys},
peer::S3PeerSys,
sets::Sets,
store_api::{
BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
@@ -13,14 +14,137 @@ use crate::{
},
store_init, utils,
};
use backon::{ExponentialBuilder, Retryable};
use futures::future::join_all;
use http::HeaderMap;
use s3s::{dto::StreamingBlob, Body};
use std::collections::{HashMap, HashSet};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
time::Duration,
};
use time::OffsetDateTime;
use tracing::{debug, warn};
use tokio::{fs, sync::RwLock};
use tracing::{debug, info, warn};
use uuid::Uuid;
use lazy_static::lazy_static;
lazy_static! {
pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
pub static ref GLOBAL_IsDistErasure: RwLock<bool> = RwLock::new(false);
pub static ref GLOBAL_IsErasureSD: RwLock<bool> = RwLock::new(false);
}
pub async fn update_erasure_type(setup_type: SetupType) {
let mut is_erasure = GLOBAL_IsErasure.write().await;
*is_erasure = setup_type == SetupType::Erasure;
let mut is_dist_erasure = GLOBAL_IsDistErasure.write().await;
*is_dist_erasure = setup_type == SetupType::DistErasure;
if *is_dist_erasure {
*is_erasure = true
}
let mut is_erasure_sd = GLOBAL_IsErasureSD.write().await;
*is_erasure_sd = setup_type == SetupType::ErasureSD;
}
lazy_static! {
pub static ref GLOBAL_LOCAL_DISK_MAP: Arc<RwLock<HashMap<String, Option<DiskStore>>>> = Arc::new(RwLock::new(HashMap::new()));
pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<Vec<Vec<Vec<Option<DiskStore>>>>>> =
Arc::new(RwLock::new(Vec::new()));
}
pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
let disk_path = match fs::canonicalize(disk_path).await {
Ok(disk_path) => disk_path,
Err(_) => return None,
};
let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
let path = disk_path.to_string_lossy().to_string();
if disk_map.contains_key(&path) {
let a = disk_map[&path].as_ref().cloned();
return a;
}
None
}
pub async fn all_local_disk_path() -> Vec<String> {
let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
disk_map.keys().map(|v| v.clone()).collect()
}
pub async fn all_local_disk() -> Vec<DiskStore> {
let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
disk_map
.values()
.filter(|v| v.is_some())
.map(|v| v.as_ref().unwrap().clone())
.collect()
}
// init_local_disks 初始化本地磁盘，server启动前必须初始化成功
pub async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> {
let opt = &DiskOption {
cleanup: true,
health_check: true,
};
let mut global_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.write().await;
for pool_eps in endpoint_pools.as_ref().iter() {
let mut set_count_drives = Vec::with_capacity(pool_eps.set_count);
for _ in 0..pool_eps.set_count {
set_count_drives.push(vec![None; pool_eps.drives_per_set]);
}
global_set_drives.push(set_count_drives);
}
let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await;
for pool_eps in endpoint_pools.as_ref().iter() {
let mut set_drives = HashMap::new();
for ep in pool_eps.endpoints.as_ref().iter() {
if !ep.is_local {
continue;
}
let disk = new_disk(ep, opt).await?;
let path = disk.path().to_string_lossy().to_string();
global_local_disk_map.insert(path, Some(disk.clone()));
set_drives.insert(ep.disk_idx, Some(disk.clone()));
if ep.pool_idx.is_some() && ep.set_idx.is_some() && ep.disk_idx.is_some() {
global_set_drives[ep.pool_idx.unwrap()][ep.set_idx.unwrap()][ep.disk_idx.unwrap()] = Some(disk.clone());
}
}
}
Ok(())
}
lazy_static! {
pub static ref GLOBAL_OBJECT_API: Arc<RwLock<Option<ECStore>>> = Arc::new(RwLock::new(None));
pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
}
pub fn new_object_layer_fn() -> Arc<RwLock<Option<ECStore>>> {
GLOBAL_OBJECT_API.clone()
}
async fn set_object_layer(o: ECStore) {
let mut global_object_api = GLOBAL_OBJECT_API.write().await;
*global_object_api = Some(o);
}
#[derive(Debug)]
pub struct ECStore {
pub id: uuid::Uuid,
@@ -28,16 +152,16 @@ pub struct ECStore {
pub disk_map: HashMap<usize, Vec<Option<DiskStore>>>,
pub pools: Vec<Sets>,
pub peer_sys: S3PeerSys,
pub local_disks: Vec<DiskStore>,
// pub local_disks: Vec<DiskStore>,
}
impl ECStore {
pub async fn new(address: String, endpoints: Vec<String>) -> Result<Self> {
let layouts = DisksLayout::try_from(endpoints.as_slice())?;
pub async fn new(_address: String, endpoint_pools: EndpointServerPools) -> Result<()> {
// let layouts = DisksLayout::try_from(endpoints.as_slice())?;
let mut deployment_id = None;
let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?;
// let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?;
let mut pools = Vec::with_capacity(endpoint_pools.as_ref().len());
let mut disk_map = HashMap::with_capacity(endpoint_pools.as_ref().len());
@@ -46,6 +170,8 @@ impl ECStore {
let mut local_disks = Vec::new();
info!("endpoint_pools: {:?}", endpoint_pools);
for (i, pool_eps) in endpoint_pools.as_ref().iter().enumerate() {
// TODO: read from config parseStorageClass
let partiy_count = store_init::default_partiy_count(pool_eps.drives_per_set);
@@ -61,13 +187,21 @@ impl ECStore {
DiskError::check_disk_fatal_errs(&errs)?;
let fm = store_init::do_init_format_file(
first_is_local,
&disks,
pool_eps.set_count,
pool_eps.drives_per_set,
deployment_id,
)
let fm = (|| async {
store_init::connect_load_init_formats(
first_is_local,
&disks,
pool_eps.set_count,
pool_eps.drives_per_set,
deployment_id,
)
.await
})
.retry(ExponentialBuilder::default().with_max_times(usize::MAX))
.sleep(tokio::time::sleep)
.notify(|err, dur: Duration| {
info!("retrying get formats {:?} after {:?}", err, dur);
})
.await?;
if deployment_id.is_none() {
@@ -88,24 +222,42 @@ impl ECStore {
}
}
let sets = Sets::new(disks.clone(), pool_eps, &fm, i, partiy_count)?;
let sets = Sets::new(disks.clone(), pool_eps, &fm, i, partiy_count).await?;
pools.push(sets);
disk_map.insert(i, disks);
}
let peer_sys = S3PeerSys::new(&endpoint_pools, local_disks.clone());
// 替换本地磁盘
if !*GLOBAL_IsDistErasure.read().await {
let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await;
for disk in local_disks {
let path = disk.path().to_string_lossy().to_string();
global_local_disk_map.insert(path, Some(disk.clone()));
}
}
Ok(ECStore {
let peer_sys = S3PeerSys::new(&endpoint_pools);
let ec = ECStore {
id: deployment_id.unwrap(),
disk_map,
pools,
local_disks,
peer_sys,
})
};
set_object_layer(ec).await;
Ok(())
}
pub fn init_local_disks() {}
// pub fn local_disks(&self) -> Vec<DiskStore> {
// self.local_disks.clone()
// }
fn single_pool(&self) -> bool {
self.pools.len() == 1
}
ecstore/src/store_api.rs
+4 -1
View File
@@ -194,6 +194,7 @@ pub struct ObjectPartInfo {
// }
// }
#[derive(Serialize, Deserialize)]
pub struct RawFileInfo {
pub buf: Vec<u8>,
}
@@ -239,6 +240,7 @@ pub enum BitrotAlgorithm {
BLAKE2b512,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct MakeBucketOptions {
pub force_create: bool,
}
@@ -380,9 +382,10 @@ pub struct ObjectOptions {
// }
// }
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct BucketOptions {}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BucketInfo {
pub name: String,
pub created: Option<OffsetDateTime>,
ecstore/src/store_init.rs
+40 -21
View File
@@ -1,7 +1,9 @@
use crate::{
disk::error::DiskError,
disk::format::{FormatErasureVersion, FormatMetaVersion, FormatV3},
disk::{new_disk, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET},
disk::{
error::DiskError,
format::{FormatErasureVersion, FormatMetaVersion, FormatV3},
new_disk, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET,
},
endpoints::Endpoints,
error::{Error, Result},
};
@@ -10,6 +12,7 @@ use std::{
collections::{hash_map::Entry, HashMap},
fmt::Debug,
};
use tracing::warn;
use uuid::Uuid;
@@ -40,30 +43,34 @@ pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec<Option<DiskSt
(res, errors)
}
pub async fn do_init_format_file(
pub async fn connect_load_init_formats(
first_disk: bool,
disks: &[Option<DiskStore>],
set_count: usize,
set_drive_count: usize,
deployment_id: Option<Uuid>,
) -> Result<FormatV3, Error> {
let (formats, errs) = read_format_file_all(disks, false).await;
warn!("connect_load_init_formats id: {:?}, first_disk: {}", deployment_id, first_disk);
let (formats, errs) = load_format_erasure_all(disks, false).await;
DiskError::check_disk_fatal_errs(&errs)?;
warn!("load_format_erasure_all errs {:?}", &errs);
check_format_erasure_values(&formats, set_drive_count)?;
if first_disk && DiskError::should_init_erasure_disks(&errs) {
// UnformattedDisk, not format file create
// new format and save
let fms = init_format_files(disks, set_count, set_drive_count, deployment_id);
let fms = init_format_erasure(disks, set_count, set_drive_count, deployment_id);
let _errs = save_format_file_all(disks, &fms).await;
// TODO: check quorum
// reduceWriteQuorumErrs(&errs)?;
let fm = get_format_file_in_quorum(&fms)?;
let fm = get_format_erasure_in_quorum(&fms)?;
return Ok(fm);
}
@@ -77,12 +84,12 @@ pub async fn do_init_format_file(
return Err(Error::new(ErasureError::FirstDiskWait));
}
let fm = get_format_file_in_quorum(&formats)?;
let fm = get_format_erasure_in_quorum(&formats)?;
Ok(fm)
}
fn init_format_files(
fn init_format_erasure(
disks: &[Option<DiskStore>],
set_count: usize,
set_drive_count: usize,
@@ -106,7 +113,7 @@ fn init_format_files(
fms
}
fn get_format_file_in_quorum(formats: &[Option<FormatV3>]) -> Result<FormatV3> {
fn get_format_erasure_in_quorum(formats: &[Option<FormatV3>]) -> Result<FormatV3> {
let mut countmap = HashMap::new();
for f in formats.iter() {
@@ -124,8 +131,15 @@ fn get_format_file_in_quorum(formats: &[Option<FormatV3>]) -> Result<FormatV3> {
let (max_drives, max_count) = countmap.iter().max_by_key(|&(_, c)| c).unwrap_or((&0, &0));
if *max_drives == 0 || *max_count < formats.len() / 2 {
warn!("*max_drives == 0 || *max_count < formats.len() / 2");
warn!("get_format_erasure_in_quorum fi: {:?}", &formats);
if *max_drives == 0 || *max_count <= formats.len() / 2 {
warn!(
"*max_drives == 0 || *max_count < formats.len() / 2, {} || {}<{}",
max_drives,
max_count,
formats.len() / 2
);
return Err(Error::new(ErasureError::ErasureReadQuorum));
}
@@ -184,21 +198,26 @@ pub fn default_partiy_count(drive: usize) -> usize {
_ => 4,
}
}
// read_format_file_all 读取所有foramt.json
async fn read_format_file_all(disks: &[Option<DiskStore>], heal: bool) -> (Vec<Option<FormatV3>>, Vec<Option<Error>>) {
// load_format_erasure_all 读取所有foramt.json
async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) -> (Vec<Option<FormatV3>>, Vec<Option<Error>>) {
let mut futures = Vec::with_capacity(disks.len());
for ep in disks.iter() {
futures.push(read_format_file(ep, heal));
for disk in disks.iter() {
futures.push(read_format_file(disk, heal));
}
let mut datas = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
let results = join_all(futures).await;
let mut i = 0;
for result in results {
match result {
Ok(s) => {
if !heal {
let _ = disks[i].as_ref().unwrap().set_disk_id(Some(s.erasure.this.clone())).await;
}
datas.push(Some(s));
errors.push(None);
}
@@ -207,6 +226,8 @@ async fn read_format_file_all(disks: &[Option<DiskStore>], heal: bool) -> (Vec<O
errors.push(Some(e));
}
}
i += 1;
}
(datas, errors)
@@ -238,8 +259,8 @@ async fn read_format_file(disk: &Option<DiskStore>, _heal: bool) -> Result<Forma
async fn save_format_file_all(disks: &[Option<DiskStore>], formats: &[Option<FormatV3>]) -> Vec<Option<Error>> {
let mut futures = Vec::with_capacity(disks.len());
for (i, ep) in disks.iter().enumerate() {
futures.push(save_format_file(ep, &formats[i]));
for (i, disk) in disks.iter().enumerate() {
futures.push(save_format_file(disk, &formats[i]));
}
let mut errors = Vec::with_capacity(disks.len());
@@ -277,9 +298,7 @@ async fn save_format_file(disk: &Option<DiskStore>, format: &Option<FormatV3>) -
disk.rename_file(RUSTFS_META_BUCKET, tmpfile.as_str(), RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
.await?;
// let mut disk = disk;
// disk.set_disk_id(format.erasure.this);
disk.set_disk_id(Some(format.erasure.this)).await?;
Ok(())
}