gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge getdiskid

Browse Source
This commit is contained in:
weisd
2024-09-23 17:27:10 +08:00
parent e1071574e6 83590cb742
commit c95cca5efc
10 changed files with 339 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/disk/format.rs
+49
View File
@@ -182,6 +182,55 @@ impl FormatV3 {
Err(Error::msg(format!("disk id not found {}", disk_id)))
}
pub fn check_other(&self, other: &FormatV3) -> Result<()> {
let mut tmp = other.clone();
let this = tmp.erasure.this;
tmp.erasure.this = Uuid::nil();
if self.erasure.sets.len() != other.erasure.sets.len() {
return Err(Error::from_string(format!(
"Expected number of sets {}, got {}",
self.erasure.sets.len(),
other.erasure.sets.len()
)));
}
for i in 0..self.erasure.sets.len() {
if self.erasure.sets[i].len() != other.erasure.sets[i].len() {
return Err(Error::from_string(format!(
"Each set should be of same size, expected {}, got {}",
self.erasure.sets[i].len(),
other.erasure.sets[i].len()
)));
}
for j in 0..self.erasure.sets[i].len() {
if self.erasure.sets[i][j] != self.erasure.sets[i][j] {
return Err(Error::from_string(format!(
"UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)",
i,
j,
self.erasure.sets[i][j].to_string(),
other.erasure.sets[i][j].to_string(),
)));
}
}
}
for i in 0..tmp.erasure.sets.len() {
for j in 0..tmp.erasure.sets[i].len() {
if this == tmp.erasure.sets[i][j] {
return Ok(());
}
}
}
Err(Error::msg(format!(
"DriveID {:?} not found in any drive sets {:?}",
this, other.erasure.sets
)))
}
}
#[cfg(test)]
ecstore/src/disk/local.rs
+94 -14
View File
@@ -1,7 +1,7 @@
use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
use super::{
DeleteOptions, DiskAPI, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp,
ReadOptions, RenameDataResp, VolumeInfo, WalkDirOptions,
DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq,
ReadMultipleResp, ReadOptions, RenameDataResp, VolumeInfo, WalkDirOptions,
};
use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
use crate::{
@@ -26,18 +26,27 @@ use uuid::Uuid;
#[derive(Debug)]
pub struct FormatInfo {
pub id: Option<Uuid>,
pub _data: Vec<u8>,
pub _file_info: Option<Metadata>,
pub _last_check: Option<OffsetDateTime>,
pub data: Vec<u8>,
pub file_info: Option<Metadata>,
pub last_check: Option<OffsetDateTime>,
}
impl FormatInfo {}
impl FormatInfo {
pub fn last_check_valid(&self) -> bool {
let now = OffsetDateTime::now_utc();
self.file_info.is_some()
&& self.id.is_some()
&& self.last_check.is_some()
&& (now.unix_timestamp() - self.last_check.unwrap().unix_timestamp() <= 1)
}
}
#[derive(Debug)]
pub struct LocalDisk {
pub root: PathBuf,
pub _format_path: PathBuf,
pub format_path: PathBuf,
pub format_info: Mutex<FormatInfo>,
pub endpoint: Endpoint,
// pub id: Mutex<Option<Uuid>>,
// pub format_data: Mutex<Vec<u8>>,
// pub format_file_info: Mutex<Option<Metadata>>,
@@ -79,14 +88,15 @@ impl LocalDisk {
let format_info = FormatInfo {
id,
_data: format_data,
_file_info: format_meta,
_last_check: format_last_check,
data: format_data,
file_info: format_meta,
last_check: format_last_check,
};
let disk = Self {
root,
_format_path: format_path,
endpoint: ep.clone(),
format_path: format_path,
format_info: Mutex::new(format_info),
// // format_legacy,
// format_file_info: Mutex::new(format_meta),
@@ -99,6 +109,17 @@ impl LocalDisk {
Ok(disk)
}
async fn check_format_json(&self) -> Result<Metadata> {
let md = fs::metadata(&self.format_path).await.map_err(|e| match e.kind() {
ErrorKind::NotFound => DiskError::DiskNotFound,
ErrorKind::PermissionDenied => DiskError::FileAccessDenied,
_ => {
warn!("check_format_json err {:?}", e);
DiskError::CorruptedBackend
}
})?;
Ok(md)
}
async fn make_meta_volumes(&self) -> Result<()> {
let buckets = format!("{}/{}", super::RUSTFS_META_BUCKET, super::BUCKET_META_PREFIX);
let multipart = format!("{}/{}", super::RUSTFS_META_BUCKET, "multipart");
@@ -433,6 +454,9 @@ impl DiskAPI for LocalDisk {
fn is_local(&self) -> bool {
true
}
async fn is_online(&self) -> bool {
true
}
async fn close(&self) -> Result<()> {
Ok(())
}
@@ -440,12 +464,68 @@ impl DiskAPI for LocalDisk {
self.root.clone()
}
async fn get_disk_id(&self) -> Option<Uuid> {
fn get_location(&self) -> DiskLocation {
DiskLocation {
pool_idx: self.endpoint.pool_idx,
set_idx: self.endpoint.set_idx,
disk_idx: self.endpoint.pool_idx,
}
}
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
warn!("local get_disk_id");
// TODO: check format file
let format_info = self.format_info.lock().await;
let mut format_info = self.format_info.lock().await;
format_info.id.clone()
let id = format_info.id.clone();
if format_info.last_check_valid() {
return Ok(id);
}
let file_meta = self.check_format_json().await?;
if let Some(file_info) = &format_info.file_info {
if utils::fs::same_file(&file_meta, file_info) {
format_info.last_check = Some(OffsetDateTime::now_utc());
return Ok(id);
}
}
let b = fs::read(&self.format_path).await.map_err(|e| match e.kind() {
ErrorKind::NotFound => DiskError::DiskNotFound,
ErrorKind::PermissionDenied => DiskError::FileAccessDenied,
_ => {
warn!("check_format_json err {:?}", e);
DiskError::CorruptedBackend
}
})?;
let fm = FormatV3::try_from(b.as_slice()).map_err(|e| {
warn!("decode format.json err {:?}", e);
DiskError::CorruptedBackend
})?;
let (m, n) = fm.find_disk_index_by_disk_id(fm.erasure.this)?;
let disk_id = fm.erasure.this;
match (self.endpoint.set_idx, self.endpoint.disk_idx) {
(Some(set_idx), Some(disk_idx)) => {
if m != set_idx || n != disk_idx {
return Err(Error::new(DiskError::InconsistentDisk));
}
}
_ => return Err(Error::new(DiskError::InconsistentDisk)),
}
format_info.id = Some(disk_id);
format_info.file_info = Some(file_meta);
format_info.data = b;
format_info.last_check = Some(OffsetDateTime::now_utc());
Ok(Some(disk_id))
// TODO: 判断源文件id,是否有效
}
ecstore/src/disk/mod.rs
+16 -1
View File
@@ -45,10 +45,12 @@ pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result<DiskS
#[async_trait::async_trait]
pub trait DiskAPI: Debug + Send + Sync + 'static {
fn is_local(&self) -> bool;
async fn is_online(&self) -> bool;
fn path(&self) -> PathBuf;
async fn close(&self) -> Result<()>;
async fn get_disk_id(&self) -> Option<Uuid>;
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
fn get_location(&self) -> DiskLocation;
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes>;
@@ -103,6 +105,18 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
}
pub struct DiskLocation {
pub pool_idx: Option<usize>,
pub set_idx: Option<usize>,
pub disk_idx: Option<usize>,
}
impl DiskLocation {
pub fn valid(&self) -> bool {
self.pool_idx.is_some() && self.set_idx.is_some() && self.disk_idx.is_some()
}
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct FileInfoVersions {
// Name of the volume.
@@ -216,6 +230,7 @@ impl MetaCacheEntry {
}
}
#[derive(Debug, Default)]
pub struct DiskOption {
pub cleanup: bool,
pub health_check: bool,
ecstore/src/disk/remote.rs
+22 -5
View File
@@ -28,9 +28,9 @@ use crate::{
};
use super::{
endpoint::Endpoint, DeleteOptions, DiskAPI, DiskOption, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, VolumeInfo,
WalkDirOptions,
endpoint::Endpoint, DeleteOptions, DiskAPI, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter,
MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp,
VolumeInfo, WalkDirOptions,
};
#[derive(Debug)]
@@ -39,6 +39,7 @@ pub struct RemoteDisk {
channel: Arc<RwLock<Option<Channel>>>,
url: url::Url,
pub root: PathBuf,
endpoint: Endpoint,
}
impl RemoteDisk {
@@ -50,6 +51,7 @@ impl RemoteDisk {
url: ep.url.clone(),
root,
id: Mutex::new(None),
endpoint: ep.clone(),
})
}
@@ -98,6 +100,13 @@ impl DiskAPI for RemoteDisk {
fn is_local(&self) -> bool {
false
}
async fn is_online(&self) -> bool {
// TODO: 连接状态
if let Ok(_) = self.get_client_v2().await {
return true;
}
false
}
async fn close(&self) -> Result<()> {
Ok(())
}
@@ -105,8 +114,16 @@ impl DiskAPI for RemoteDisk {
self.root.clone()
}
async fn get_disk_id(&self) -> Option<Uuid> {
self.id.lock().await.clone()
fn get_location(&self) -> DiskLocation {
DiskLocation {
pool_idx: self.endpoint.pool_idx,
set_idx: self.endpoint.set_idx,
disk_idx: self.endpoint.pool_idx,
}
}
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
Ok(self.id.lock().await.clone())
}
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
let mut lock = self.id.lock().await;
ecstore/src/erasure.rs
+5 -2
View File
@@ -47,7 +47,7 @@ impl Erasure {
pub async fn encode<S>(
&self,
body: S,
writers: &mut [FileWriter],
writers: &mut [Option<FileWriter>],
// block_size: usize,
total_size: usize,
write_quorum: usize,
@@ -85,7 +85,10 @@ impl Erasure {
let mut errs = Vec::new();
for (i, w) in writers.iter_mut().enumerate() {
match w.write(blocks[i].as_ref()).await {
if w.is_none() {
continue;
}
match w.as_mut().unwrap().write(blocks[i].as_ref()).await {
Ok(_) => errs.push(None),
Err(e) => errs.push(Some(e)),
}
ecstore/src/sets.rs
+64 -15
View File
@@ -1,9 +1,4 @@
use std::collections::HashMap;
use futures::future::join_all;
use http::HeaderMap;
use tracing::warn;
use uuid::Uuid;
use std::{collections::HashMap, sync::Arc, time::Duration};
use crate::{
disk::{
@@ -21,13 +16,19 @@ use crate::{
},
utils::hash,
};
use futures::future::join_all;
use http::HeaderMap;
use tokio::sync::RwLock;
use tokio_util::sync::CancellationToken;
use tracing::{debug, warn};
use uuid::Uuid;
#[derive(Debug)]
pub struct Sets {
pub id: Uuid,
// pub sets: Vec<Objects>,
// pub disk_set: Vec<Vec<Option<DiskStore>>>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub disk_set: Vec<SetDisks>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub disk_set: Vec<Arc<SetDisks>>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub pool_idx: usize,
pub endpoints: PoolEndpoints,
pub format: FormatV3,
@@ -35,6 +36,7 @@ pub struct Sets {
pub set_count: usize,
pub set_drive_count: usize,
pub distribution_algo: DistributionAlgoVersion,
ctx: CancellationToken,
}
impl Sets {
@@ -44,7 +46,7 @@ impl Sets {
fm: &FormatV3,
pool_idx: usize,
partiy_count: usize,
) -> Result<Self> {
) -> Result<Arc<Self>> {
let set_count = fm.erasure.sets.len();
let set_drive_count = fm.erasure.sets[0].len();
@@ -52,9 +54,14 @@ impl Sets {
for i in 0..set_count {
let mut set_drive = Vec::with_capacity(set_drive_count);
let mut set_endpoints = Vec::with_capacity(set_drive_count);
for j in 0..set_drive_count {
let idx = i * set_drive_count + j;
let mut disk = disks[idx].clone();
let endpoint = endpoints.endpoints.as_ref().get(idx).cloned();
set_endpoints.push(endpoint);
if disk.is_none() {
warn!("sets new set_drive {}-{} is none", i, j);
set_drive.push(None);
@@ -78,7 +85,7 @@ impl Sets {
disk = local_disk;
}
if let Some(_disk_id) = disk.as_ref().unwrap().get_disk_id().await {
if let Some(_disk_id) = disk.as_ref().unwrap().get_disk_id().await? {
set_drive.push(disk);
} else {
warn!("sets new set_drive {}-{} get_disk_id is none", i, j);
@@ -89,17 +96,19 @@ impl Sets {
warn!("sets new set_drive {:?}", &set_drive);
let set_disks = SetDisks {
disks: set_drive,
disks: RwLock::new(set_drive),
set_drive_count,
parity_count: partiy_count,
set_index: i,
pool_index: pool_idx,
set_endpoints,
format: fm.clone(),
};
disk_set.push(set_disks);
disk_set.push(Arc::new(set_disks));
}
let sets = Self {
let sets = Arc::new(Self {
id: fm.id,
// sets: todo!(),
disk_set,
@@ -110,15 +119,55 @@ impl Sets {
set_count,
set_drive_count,
distribution_algo: fm.erasure.distribution_algo.clone(),
};
ctx: CancellationToken::new(),
});
let asets = sets.clone();
tokio::spawn(async move { asets.monitor_and_connect_endpoints().await });
Ok(sets)
}
pub fn get_disks(&self, set_idx: usize) -> SetDisks {
pub async fn monitor_and_connect_endpoints(&self) {
tokio::time::sleep(Duration::from_secs(5)).await;
self.connect_disks().await;
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(15 * 3));
let cloned_token = self.ctx.clone();
loop {
tokio::select! {
_= interval.tick()=>{
debug!("tick...");
self.connect_disks().await;
interval.reset();
},
_ = cloned_token.cancelled() => {
warn!("ctx cancelled");
break;
}
}
}
warn!("monitor_and_connect_endpoints exit");
}
async fn connect_disks(&self) {
debug!("start connect_disks ...");
for set in self.disk_set.iter() {
set.connect_disks().await;
}
debug!("done connect_disks ...");
}
pub fn get_disks(&self, set_idx: usize) -> Arc<SetDisks> {
self.disk_set[set_idx].clone()
}
pub fn get_disks_by_key(&self, key: &str) -> SetDisks {
pub fn get_disks_by_key(&self, key: &str) -> Arc<SetDisks> {
self.get_disks(self.get_hashed_set_index(key))
}
ecstore/src/store.rs
+51 -61
View File
@@ -1,8 +1,6 @@
use crate::{
bucket_meta::BucketMetadata,
disk::{
error::DiskError, new_disk, DeleteOptions, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET,
},
disk::{error::DiskError, new_disk, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
endpoints::{EndpointServerPools, SetupType},
error::{Error, Result},
peer::S3PeerSys,
@@ -26,7 +24,7 @@ use std::{
};
use time::OffsetDateTime;
use tokio::{fs, sync::RwLock};
use tracing::{debug, info, warn};
use tracing::{debug, info};
use uuid::Uuid;
use lazy_static::lazy_static;
@@ -151,7 +149,7 @@ pub struct ECStore {
pub id: uuid::Uuid,
// pub disks: Vec<DiskStore>,
pub disk_map: HashMap<usize, Vec<Option<DiskStore>>>,
pub pools: Vec<Sets>,
pub pools: Vec<Arc<Sets>>,
pub peer_sys: S3PeerSys,
// pub local_disks: Vec<DiskStore>,
}
@@ -226,7 +224,6 @@ impl ECStore {
}
let sets = Sets::new(disks.clone(), pool_eps, &fm, i, partiy_count).await?;
pools.push(sets);
disk_map.insert(i, disks);
@@ -288,62 +285,54 @@ impl ECStore {
for sets in self.pools.iter() {
for set in sets.disk_set.iter() {
for disk in set.disks.iter() {
if disk.is_none() {
continue;
}
let disk = disk.as_ref().unwrap();
let opts = opts.clone();
// let mut wr = &mut wr;
futures.push(disk.walk_dir(opts));
// tokio::spawn(async move { disk.walk_dir(opts, wr).await });
}
futures.push(set.walk_dir(&opts));
}
}
let results = join_all(futures).await;
let mut errs = Vec::new();
// let mut errs = Vec::new();
let mut ress = Vec::new();
let mut uniq = HashSet::new();
for res in results {
match res {
Ok(entrys) => {
for entry in entrys {
if !uniq.contains(&entry.name) {
uniq.insert(entry.name.clone());
// TODO: 过滤
if opts.limit > 0 && ress.len() as i32 >= opts.limit {
return Ok(ress);
}
for (disks_ress, _disks_errs) in results {
for (_i, disks_res) in disks_ress.iter().enumerate() {
if disks_res.is_none() {
// TODO handle errs
continue;
}
let entrys = disks_res.as_ref().unwrap();
if entry.is_object() {
let fi = entry.to_fileinfo(&opts.bucket)?;
if fi.is_some() {
ress.push(fi.unwrap().into_object_info(&opts.bucket, &entry.name, false));
}
continue;
}
for entry in entrys {
if !uniq.contains(&entry.name) {
uniq.insert(entry.name.clone());
// TODO: 过滤
if opts.limit > 0 && ress.len() as i32 >= opts.limit {
return Ok(ress);
}
if entry.is_dir() {
ress.push(ObjectInfo {
is_dir: true,
bucket: opts.bucket.clone(),
name: entry.name,
..Default::default()
});
if entry.is_object() {
let fi = entry.to_fileinfo(&opts.bucket)?;
if fi.is_some() {
ress.push(fi.unwrap().into_object_info(&opts.bucket, &entry.name, false));
}
continue;
}
if entry.is_dir() {
ress.push(ObjectInfo {
is_dir: true,
bucket: opts.bucket.clone(),
name: entry.name.clone(),
..Default::default()
});
}
}
errs.push(None);
}
Err(e) => errs.push(Some(e)),
}
}
warn!("list_merged errs {:?}", errs);
// warn!("list_merged errs {:?}", errs);
Ok(ress)
}
@@ -352,22 +341,23 @@ impl ECStore {
let mut futures = Vec::new();
for sets in self.pools.iter() {
for set in sets.disk_set.iter() {
for disk in set.disks.iter() {
if disk.is_none() {
continue;
}
let disk = disk.as_ref().unwrap();
futures.push(disk.delete(
bucket,
prefix,
DeleteOptions {
recursive: true,
immediate: false,
..Default::default()
},
));
}
futures.push(set.delete_all(bucket, prefix));
// let disks = set.disks.read().await;
// let dd = disks.clone();
// for disk in dd {
// if disk.is_none() {
// continue;
// }
// // let disk = disk.as_ref().unwrap().clone();
// // futures.push(disk.delete(
// // bucket,
// // prefix,
// // DeleteOptions {
// // recursive: true,
// // immediate: false,
// // },
// // ));
// }
}
}
let results = join_all(futures).await;
ecstore/src/store_init.rs
+13 -12
View File
@@ -65,8 +65,9 @@ pub async fn connect_load_init_formats(
// new format and save
let fms = init_format_erasure(disks, set_count, set_drive_count, deployment_id);
let _errs = save_format_file_all(disks, &fms).await;
let errs = save_format_file_all(disks, &fms).await;
warn!("save_format_file_all errs {:?}", &errs);
// TODO: check quorum
// reduceWriteQuorumErrs(&errs)?;
@@ -192,14 +193,19 @@ fn check_format_erasure_value(format: &FormatV3) -> Result<()> {
// load_format_erasure_all 读取所有foramt.json
async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) -> (Vec<Option<FormatV3>>, Vec<Option<Error>>) {
let mut futures = Vec::with_capacity(disks.len());
for disk in disks.iter() {
futures.push(load_format_erasure(disk, heal));
}
let mut datas = Vec::with_capacity(disks.len());
let mut errors = Vec::with_capacity(disks.len());
for disk in disks.iter() {
if disk.is_none() {
datas.push(None);
errors.push(Some(Error::new(DiskError::DiskNotFound)));
}
let disk = disk.as_ref().unwrap();
futures.push(load_format_erasure(disk, heal));
}
let results = join_all(futures).await;
let mut i = 0;
for result in results {
@@ -224,12 +230,7 @@ async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) -> (Ve
(datas, errors)
}
async fn load_format_erasure(disk: &Option<DiskStore>, _heal: bool) -> Result<FormatV3, Error> {
if disk.is_none() {
return Err(Error::new(DiskError::DiskNotFound));
}
let disk = disk.as_ref().unwrap();
pub async fn load_format_erasure(disk: &DiskStore, _heal: bool) -> Result<FormatV3, Error> {
let data = disk
.read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
.await
ecstore/src/utils/fs.rs
+24
View File
@@ -0,0 +1,24 @@
use std::{fs::Metadata, os::unix::fs::MetadataExt};
pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
if f1.dev() != f2.dev() {
return false;
}
if f1.ino() != f2.ino() {
return false;
}
if f1.size() != f2.size() {
return false;
}
if f1.permissions() != f2.permissions() {
return false;
}
if f1.mtime() != f2.mtime() {
return false;
}
true
}
ecstore/src/utils/mod.rs
+1
View File
@@ -1,5 +1,6 @@
pub mod crypto;
pub mod ellipses;
pub mod fs;
pub mod hash;
pub mod net;
pub mod path;