pub mod endpoint; pub mod error; pub mod format; mod local; pub mod os; mod remote; pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys"; pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart"; pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp"; pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash"; pub const BUCKET_META_PREFIX: &str = "buckets"; pub const FORMAT_CONFIG_FILE: &str = "format.json"; const STORAGE_FORMAT_FILE: &str = "xl.meta"; use crate::{ erasure::{ReadAt, Write}, error::{Error, Result}, file_meta::FileMeta, store_api::{FileInfo, RawFileInfo}, }; use bytes::Bytes; use endpoint::Endpoint; use protos::proto_gen::node_service::{node_service_client::NodeServiceClient, ReadAtRequest, WriteRequest}; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc}; use time::OffsetDateTime; use tokio::{ fs::File, io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, }; use tonic::{transport::Channel, Request}; use uuid::Uuid; pub type DiskStore = Arc>; pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result { if ep.is_local { let s = local::LocalDisk::new(ep, opt.cleanup).await?; Ok(Arc::new(Box::new(s))) } else { let remote_disk = remote::RemoteDisk::new(ep, opt).await?; Ok(Arc::new(Box::new(remote_disk))) } } #[async_trait::async_trait] pub trait DiskAPI: Debug + Send + Sync + 'static { fn to_string(&self) -> String; async fn is_online(&self) -> bool; fn is_local(&self) -> bool; // LastConn fn host_name(&self) -> String; fn endpoint(&self) -> Endpoint; async fn close(&self) -> Result<()>; async fn get_disk_id(&self) -> Result>; async fn set_disk_id(&self, id: Option) -> Result<()>; fn path(&self) -> PathBuf; fn get_disk_location(&self) -> DiskLocation; // Healing // DiskInfo // NSScanner // Volume operations. async fn make_volume(&self, volume: &str) -> Result<()>; async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>; async fn list_volumes(&self) -> Result>; async fn stat_volume(&self, volume: &str) -> Result; async fn delete_volume(&self, volume: &str) -> Result<()>; // 并发边读边写 TODO: wr io.Writer async fn walk_dir(&self, opts: WalkDirOptions) -> Result>; // Metadata operations async fn delete_version( &self, volume: &str, path: &str, fi: FileInfo, force_del_marker: bool, opts: DeleteOptions, ) -> Result; async fn delete_versions( &self, volume: &str, versions: Vec, opts: DeleteOptions, ) -> Result>>; async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()>; async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>; async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) -> Result<()>; async fn read_version( &self, org_volume: &str, volume: &str, path: &str, version_id: &str, opts: &ReadOptions, ) -> Result; async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result; async fn rename_data( &self, src_volume: &str, src_path: &str, file_info: FileInfo, dst_volume: &str, dst_path: &str, ) -> Result; // File operations. // 读目录下的所有文件、目录 async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result>; async fn read_file(&self, volume: &str, path: &str) -> Result; async fn append_file(&self, volume: &str, path: &str) -> Result; async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result; // ReadFileStream async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>; async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()>; // CheckParts async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; // VerifyFile // StatInfoFile // ReadParts async fn read_multiple(&self, req: ReadMultipleReq) -> Result>; // CleanAbandonedData async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result; } pub struct UpdateMetadataOpts { pub no_persistence: bool, } pub struct DiskLocation { pub pool_idx: Option, pub set_idx: Option, pub disk_idx: Option, } impl DiskLocation { pub fn valid(&self) -> bool { self.pool_idx.is_some() && self.set_idx.is_some() && self.disk_idx.is_some() } } #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct FileInfoVersions { // Name of the volume. pub volume: String, // Name of the file. pub name: String, // Represents the latest mod time of the // latest version. pub latest_mod_time: Option, pub versions: Vec, pub free_versions: Vec, } #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct WalkDirOptions { // Bucket to scanner pub bucket: String, // Directory inside the bucket. pub base_dir: String, // Do a full recursive scan. pub recursive: bool, // ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found. pub report_notfound: bool, // FilterPrefix will only return results with given prefix within folder. // Should never contain a slash. pub filter_prefix: String, // ForwardTo will forward to the given object path. pub forward_to: String, // Limit the number of returned objects if > 0. pub limit: i32, // DiskID contains the disk ID of the disk. // Leave empty to not check disk ID. pub disk_id: String, } #[derive(Debug, Default, Serialize, Deserialize)] pub struct MetaCacheEntry { // name is the full name of the object including prefixes pub name: String, // Metadata. If none is present it is not an object but only a prefix. // Entries without metadata will only be present in non-recursive scans. pub metadata: Vec, // cached contains the metadata if decoded. cached: Option, // Indicates the entry can be reused and only one reference to metadata is expected. _reusable: bool, } impl MetaCacheEntry { pub fn marshal_msg(&self) -> Result> { let mut wr = Vec::new(); rmp::encode::write_bool(&mut wr, true)?; rmp::encode::write_str(&mut wr, &self.name)?; rmp::encode::write_bin(&mut wr, &self.metadata)?; Ok(wr) } pub fn is_dir(&self) -> bool { self.metadata.is_empty() && self.name.ends_with("/") } pub fn is_object(&self) -> bool { !self.metadata.is_empty() } #[tracing::instrument(level = "debug", skip(self))] pub fn to_fileinfo(&self, bucket: &str) -> Result> { if self.is_dir() { return Ok(Some(FileInfo { volume: bucket.to_owned(), name: self.name.clone(), ..Default::default() })); } if self.cached.is_some() { let fm = self.cached.as_ref().unwrap(); if fm.versions.is_empty() { return Ok(Some(FileInfo { volume: bucket.to_owned(), name: self.name.clone(), deleted: true, is_latest: true, mod_time: Some(OffsetDateTime::UNIX_EPOCH), ..Default::default() })); } let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?; return Ok(Some(fi)); } let mut fm = FileMeta::new(); fm.unmarshal_msg(&self.metadata)?; let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?; return Ok(Some(fi)); } } #[derive(Debug, Default)] pub struct DiskOption { pub cleanup: bool, pub health_check: bool, } #[derive(Debug, Default, Serialize, Deserialize)] pub struct RenameDataResp { pub old_data_dir: Option, pub sign: Option>, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct DeleteOptions { pub recursive: bool, pub immediate: bool, pub undo_write: bool, pub old_data_dir: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReadMultipleReq { pub bucket: String, pub prefix: String, pub files: Vec, pub max_size: usize, pub metadata_only: bool, pub abort404: bool, pub max_results: usize, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ReadMultipleResp { pub bucket: String, pub prefix: String, pub file: String, pub exists: bool, pub error: String, pub data: Vec, pub mod_time: Option, } // impl Default for ReadMultipleResp { // fn default() -> Self { // Self { // bucket: String::new(), // prefix: String::new(), // file: String::new(), // exists: false, // error: String::new(), // data: Vec::new(), // mod_time: OffsetDateTime::UNIX_EPOCH, // } // } // } #[derive(Debug, Deserialize, Serialize)] pub struct VolumeInfo { pub name: String, pub created: Option, } #[derive(Deserialize, Serialize)] pub struct ReadOptions { pub read_data: bool, pub healing: bool, } // pub struct FileWriter { // pub inner: Pin>, // } // impl AsyncWrite for FileWriter { // fn poll_write( // mut self: Pin<&mut Self>, // cx: &mut std::task::Context<'_>, // buf: &[u8], // ) -> std::task::Poll> { // Pin::new(&mut self.inner).poll_write(cx, buf) // } // fn poll_flush( // mut self: Pin<&mut Self>, // cx: &mut std::task::Context<'_>, // ) -> std::task::Poll> { // Pin::new(&mut self.inner).poll_flush(cx) // } // fn poll_shutdown( // mut self: Pin<&mut Self>, // cx: &mut std::task::Context<'_>, // ) -> std::task::Poll> { // Pin::new(&mut self.inner).poll_shutdown(cx) // } // } // impl FileWriter { // pub fn new(inner: W) -> Self // where // W: AsyncWrite + Send + Sync + 'static, // { // Self { inner: Box::pin(inner) } // } // } pub enum FileWriter { Local(LocalFileWriter), Remote(RemoteFileWriter), } #[async_trait::async_trait] impl Write for FileWriter { async fn write(&mut self, buf: &[u8]) -> Result<()> { match self { Self::Local(local_file_writer) => local_file_writer.write(buf).await, Self::Remote(remote_file_writer) => remote_file_writer.write(buf).await, } } } pub struct LocalFileWriter { pub inner: File, } impl LocalFileWriter { pub fn new(inner: File) -> Self { Self { inner } } } #[async_trait::async_trait] impl Write for LocalFileWriter { async fn write(&mut self, buf: &[u8]) -> Result<()> { self.inner.write(buf).await?; self.inner.flush().await?; Ok(()) } } pub struct RemoteFileWriter { pub root: PathBuf, pub volume: String, pub path: String, pub is_append: bool, client: NodeServiceClient, } impl RemoteFileWriter { pub fn new(root: PathBuf, volume: String, path: String, is_append: bool, client: NodeServiceClient) -> Self { Self { root, volume, path, is_append, client, } } } #[async_trait::async_trait] impl Write for RemoteFileWriter { async fn write(&mut self, buf: &[u8]) -> Result<()> { let request = Request::new(WriteRequest { disk: self.root.to_string_lossy().to_string(), volume: self.volume.to_string(), path: self.path.to_string(), is_append: self.is_append, data: buf.to_vec(), }); let _response = self.client.write(request).await?.into_inner(); Ok(()) } } #[derive(Debug)] pub enum FileReader { Local(LocalFileReader), Remote(RemoteFileReader), } #[async_trait::async_trait] impl ReadAt for FileReader { async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { match self { Self::Local(local_file_writer) => local_file_writer.read_at(offset, length).await, Self::Remote(remote_file_writer) => remote_file_writer.read_at(offset, length).await, } } } #[derive(Debug)] pub struct LocalFileReader { pub inner: File, } impl LocalFileReader { pub fn new(inner: File) -> Self { Self { inner } } } #[async_trait::async_trait] impl ReadAt for LocalFileReader { async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { self.inner.seek(SeekFrom::Start(offset as u64)).await?; let mut buffer = vec![0; length]; let bytes_read = self.inner.read(&mut buffer).await?; buffer.truncate(bytes_read); Ok((buffer, bytes_read)) } } #[derive(Debug)] pub struct RemoteFileReader { pub root: PathBuf, pub volume: String, pub path: String, client: NodeServiceClient, } impl RemoteFileReader { pub fn new(root: PathBuf, volume: String, path: String, client: NodeServiceClient) -> Self { Self { root, volume, path, client, } } } #[async_trait::async_trait] impl ReadAt for RemoteFileReader { async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { let request = Request::new(ReadAtRequest { disk: self.root.to_string_lossy().to_string(), volume: self.volume.to_string(), path: self.path.to_string(), offset: offset.try_into().unwrap(), length: length.try_into().unwrap(), }); let response = self.client.read_at(request).await?.into_inner(); Ok((response.data, response.read_size.try_into().unwrap())) } }