heal object

Signed-off-by: junxiang Mu <1948535941@qq.com>

ecstore/src/disk/endpoint.rs

@@ -16,7 +16,7 @@ pub enum EndpointType {
 }
 
 /// any type of endpoint.
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Eq, Clone, Hash)]
 pub struct Endpoint {
     pub url: url::Url,
     pub is_local: bool,

ecstore/src/disk/local.rs

@@ -4,9 +4,9 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
 use super::{
     os, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions, FileReader, FileWriter,
     Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo,
-    WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET,
+    WalkDirOptions,
 };
-use crate::cache_value::cache::Cache;
+use crate::cache_value::cache::{Cache, Opts};
 use crate::disk::error::{
     convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir,
     map_err_not_exists, os_err_to_file_err,
@@ -15,9 +15,7 @@ use crate::disk::os::check_path_length;
 use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
 use crate::error::{Error, Result};
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
-use crate::heal::heal_commands::HealingTracker;
-use crate::heal::heal_ops::HEALING_TRACKER_FILENAME;
-use crate::utils::fs::{lstat, read_file, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY};
+use crate::utils::fs::{lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY};
 use crate::utils::path::{clean, has_suffix, SLASH_SEPARATOR};
 use crate::utils::stat_linux::get_info;
 use crate::{
@@ -26,7 +24,11 @@ use crate::{
     utils,
 };
 use path_absolutize::Absolutize;
+use tokio::runtime::Runtime;
 use std::fmt::Debug;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
 use std::{
     fs::Metadata,
     path::{Path, PathBuf},
@@ -61,7 +63,13 @@ pub struct LocalDisk {
     pub format_path: PathBuf,
     pub format_info: RwLock<FormatInfo>,
     pub endpoint: Endpoint,
-    disk_info_cache: Cache<DiskInfo>,
+    pub disk_info_cache: Arc<Cache<DiskInfo>>,
+    pub scanning: AtomicU32,
+    pub rotational: bool,
+    pub fstype: String,
+    pub major: u64,
+    pub minor: u64,
+    pub nrrequests: u64,
     // pub id: Mutex<Option<Uuid>>,
     // pub format_data: Mutex<Vec<u8>>,
     // pub format_file_info: Mutex<Option<Metadata>>,
@@ -118,46 +126,77 @@ impl LocalDisk {
             file_info: format_meta,
             last_check: format_last_check,
         };
-
-        let derive_path = root.to_string_lossy().to_string();
-        let disk_id = id.map_or("".to_string(), |id| id.to_string());
-        let update_fn = || async move {
-            if let Ok((info, root)) = get_disk_info(&derive_path).await {
-                let mut disk_info = DiskInfo {
-                    total: info.total,
-                    free: info.free,
-                    used: info.used,
-                    used_inodes: info.files - info.ffree,
-                    free_inodes: info.ffree,
-                    major: info.major,
-                    minor: info.minor,
-                    fs_type: info.fstype,
-                    root_disk: root,
-                    id: disk_id,
-                    ..Default::default()
-                };
-                if root {
-                    return Err(Error::new(DiskError::DriveIsRoot));
+        let root_clone = root.clone();
+        let disk_id = Arc::new(id.map_or("".to_string(), |id| id.to_string()));
+        let update_fn = move || {
+            let rt = Runtime::new().unwrap();
+            rt.block_on(async {
+                match get_disk_info(root.clone()).await {
+                    Ok((info, root)) => {
+                        let disk_info = DiskInfo {
+                            total: info.total,
+                            free: info.free,
+                            used: info.used,
+                            used_inodes: info.files - info.ffree,
+                            free_inodes: info.ffree,
+                            major: info.major,
+                            minor: info.minor,
+                            fs_type: info.fstype,
+                            root_disk: root,
+                            id: disk_id.to_string(),
+                            ..Default::default()
+                        };
+                        if root {
+                            return Err(Error::new(DiskError::DriveIsRoot));
+                        }
+        
+                        // disk_info.healing =
+                        Ok(disk_info)
+                    },
+                    Err(err) => {
+                        Err(err)
+                    }
                 }
-
-                // disk_info.healing =
-            } else {
-                return Ok(DiskInfo::default());
-            }
+            })
         };
-
+        
+        let cache = Cache::new(Box::new(update_fn), Duration::from_secs(1), Opts::default());
+        
         // TODO: DIRECT suport
         // TODD: DiskInfo
-        let disk = Self {
-            root,
+        let mut disk = Self {
+            root: root_clone.clone(),
             endpoint: ep.clone(),
             format_path,
             format_info: RwLock::new(format_info),
+            disk_info_cache: Arc::new(cache),
+            scanning: AtomicU32::new(0),
+            rotational: Default::default(),
+            fstype: Default::default(),
+            minor: Default::default(),
+            major: Default::default(),
+            nrrequests: Default::default(),
             // // format_legacy,
             // format_file_info: Mutex::new(format_meta),
             // format_data: Mutex::new(format_data),
             // format_last_check: Mutex::new(format_last_check),
         };
+        let (info, root) = get_disk_info(root_clone).await?;
+        disk.major = info.major;
+        disk.minor = info.minor;
+        disk.fstype = info.fstype;
+
+        if root {
+            return Err(Error::new(DiskError::DriveIsRoot));
+        }
+
+        if info.nrrequests > 0 {
+            disk.nrrequests = info.nrrequests;
+        }
+
+        if info.rotational {
+            disk.rotational = true;
+        }
 
         disk.make_meta_volumes().await?;
 
@@ -607,6 +646,7 @@ impl LocalDisk {
         Ok(f)
     }
 
+    #[allow(dead_code)]
     fn get_metrics(&self) -> DiskMetrics {
         DiskMetrics::default()
     }
@@ -1650,23 +1690,30 @@ impl DiskAPI for LocalDisk {
         Ok(())
     }
 
-    async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
-        let mut info = DiskInfo::default();
+    async fn disk_info(&self, _: &DiskInfoOptions) -> Result<DiskInfo> {
+        let mut info = Cache::get(self.disk_info_cache.clone()).await?;
+        // TODO: nr_requests, rotational
+        info.nr_requests = self.nrrequests;
+        info.rotational = self.rotational;
+        info.mount_path = self.path().to_str().unwrap().to_string();
+        info.endpoint = self.endpoint.to_string();
+        info.scanning = self.scanning.load(Ordering::SeqCst) == 1;
 
         Ok(info)
     }
 }
 
-async fn get_disk_info(drive_path: &str) -> Result<(Info, bool)> {
-    check_path_length(drive_path)?;
+async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> {
+    let drive_path = drive_path.to_string_lossy().to_string();
+    check_path_length(&drive_path)?;
 
-    let disk_info = get_info(drive_path, false)?;
+    let disk_info = get_info(&drive_path, false)?;
     let root_drive = if !*GLOBAL_IsErasureSD.read().await {
         let root_disk_threshold = *GLOBAL_RootDiskThreshold.read().await;
         if root_disk_threshold > 0 {
             disk_info.total <= root_disk_threshold
         } else {
-            match is_root_disk(drive_path, SLASH_SEPARATOR) {
+            match is_root_disk(&drive_path, SLASH_SEPARATOR) {
                 Ok(result) => result,
                 Err(_) => false,
             }

ecstore/src/disk/mod.rs

@@ -14,7 +14,10 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
 const STORAGE_FORMAT_FILE: &str = "xl.meta";
 
 use crate::{
-    erasure::{ReadAt, Write}, error::{Error, Result}, file_meta::FileMeta, heal::heal_commands::HealingTracker, store_api::{FileInfo, RawFileInfo}
+    erasure::{ReadAt, Write},
+    error::{Error, Result},
+    file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion},
+    store_api::{FileInfo, RawFileInfo},
 };
 
 use endpoint::Endpoint;
@@ -23,7 +26,7 @@ use protos::proto_gen::node_service::{
     node_service_client::NodeServiceClient, ReadAtRequest, ReadAtResponse, WriteRequest, WriteResponse,
 };
 use serde::{Deserialize, Serialize};
-use std::{collections::HashMap, fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc, usize};
+use std::{cmp::Ordering, collections::HashMap, fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc, usize};
 use time::OffsetDateTime;
 use tokio::{
     fs::File,
@@ -247,7 +250,17 @@ pub struct WalkDirOptions {
     pub disk_id: String,
 }
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default)]
+pub struct MetadataResolutionParams {
+    pub dir_quorum: usize,
+    pub obj_quorum: usize,
+    pub requested_versions: usize,
+    pub bucket: String,
+    pub strict: bool,
+    pub candidates: Vec<Vec<FileMetaShallowVersion>>,
+}
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct MetaCacheEntry {
     // name is the full name of the object including prefixes
     pub name: String,
@@ -335,6 +348,184 @@ impl MetaCacheEntry {
 
         Ok(fm.into_file_info_versions(bucket, self.name.as_str(), false)?)
     }
+
+    pub fn matches(&self, other: &MetaCacheEntry, strict: bool) -> Result<(Option<MetaCacheEntry>, bool)> {
+        let mut prefer = None;
+        if self.name != other.name {
+            if self.name < other.name {
+                return Ok((Some(self.clone()), false));
+            }
+            return Ok((Some(other.clone()), false));
+        }
+
+        if other.is_dir() || self.is_dir() {
+            if self.is_dir() {
+                return Ok((Some(self.clone()), other.is_dir()));
+            }
+
+            return Ok((Some(other.clone()), other.is_dir() == self.is_dir()));
+        }
+        let self_vers = match &self.cached {
+            Some(file_meta) => file_meta.clone(),
+            None => FileMeta::load(&self.metadata)?,
+        };
+        let other_vers = match &other.cached {
+            Some(file_meta) => file_meta.clone(),
+            None => FileMeta::load(&other.metadata)?,
+        };
+
+        if self_vers.versions.len() != other_vers.versions.len() {
+            match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) {
+                Ordering::Greater => {
+                    return Ok((Some(self.clone()), false));
+                }
+                Ordering::Less => {
+                    return Ok((Some(self.clone()), false));
+                }
+                _ => {}
+            }
+
+            if self_vers.versions.len() > other_vers.versions.len() {
+                return Ok((Some(self.clone()), false));
+            }
+            return Ok((Some(self.clone()), false));
+        }
+
+        for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) {
+            if s_version.header != o_version.header {
+                if s_version.header.has_ec() != o_version.header.has_ec() {
+                    // One version has EC and the other doesn't - may have been written later.
+                    // Compare without considering EC.
+                    let (mut a, mut b) = (s_version.header.clone(), o_version.header.clone());
+                    (a.ec_n, a.ec_m, b.ec_n, b.ec_m) = (0, 0, 0, 0);
+                    if a == b {
+                        continue;
+                    }
+                }
+
+                if !strict && s_version.header.matches_not_strict(&o_version.header) {
+                    if prefer.is_none() {
+                        if s_version.header.sorts_before(&o_version.header) {
+                            prefer = Some(self.clone());
+                        } else {
+                            prefer = Some(other.clone());
+                        }
+                    }
+
+                    continue;
+                }
+
+                if prefer.is_some() {
+                    return Ok((prefer, false));
+                }
+
+                if s_version.header.sorts_before(&o_version.header) {
+                    return Ok((Some(self.clone()), false));
+                }
+
+                return Ok((Some(other.clone()), false));
+            }
+        }
+
+        if prefer.is_none() {
+            prefer = Some(self.clone());
+        }
+
+        Ok((prefer, true))
+    }
+}
+
+pub struct MetaCacheEntries(pub Vec<MetaCacheEntry>);
+
+impl MetaCacheEntries {
+    pub fn resolve(&self, mut params: MetadataResolutionParams) -> Result<Option<MetaCacheEntry>> {
+        if self.0.is_empty() {
+            return Ok(None);
+        }
+
+        let mut dir_exists = 0;
+        let mut selected = None;
+
+        params.candidates.clear();
+        let mut objs_agree = 0;
+        let mut objs_valid = 0;
+
+        for entry in self.0.iter() {
+            if entry.name.is_empty() {
+                continue;
+            }
+            if entry.is_dir() {
+                dir_exists += 1;
+                selected = Some(entry.clone());
+                continue;
+            }
+
+            objs_valid += 1;
+
+            match &entry.cached {
+                Some(file_meta) => {
+                    params.candidates.push(file_meta.versions.clone());
+                }
+                None => {
+                    params.candidates.push(FileMeta::load(&entry.metadata)?.versions);
+                }
+            }
+
+            if selected.is_none() {
+                selected = Some(entry.clone());
+                objs_agree = 1;
+                continue;
+            }
+
+            if let (Some(prefer), true) = entry.matches(selected.as_ref().unwrap(), params.strict)? {
+                selected = Some(prefer);
+                objs_agree += 1;
+                continue;
+            }
+        }
+
+        // Return dir entries, if enough...
+        if selected.is_some() && selected.as_ref().unwrap().is_dir() && dir_exists >= params.dir_quorum {
+            return Ok(selected);
+        }
+        // If we would never be able to reach read quorum.
+        if objs_valid < params.obj_quorum {
+            return Ok(None);
+        }
+        // If all objects agree.
+        if selected.is_some() && objs_agree == objs_valid {
+            return Ok(selected);
+        }
+        // If cached is nil we shall skip the entry.
+        if selected.is_none() || (selected.is_some() && selected.as_ref().unwrap().cached.is_none()) {
+            return Ok(None);
+        }
+        // Merge if we have disagreement.
+        // Create a new merged result.
+        selected = Some(MetaCacheEntry {
+            name: selected.as_ref().unwrap().name.clone(),
+            cached: Some(FileMeta {
+                meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver.clone(),
+                ..Default::default()
+            }),
+            _reusable: true,
+            ..Default::default()
+        });
+
+        selected.as_mut().unwrap().cached.as_mut().unwrap().versions =
+            merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, &params.candidates);
+        if selected.as_ref().unwrap().cached.as_ref().unwrap().versions.is_empty() {
+            return Ok(None);
+        }
+
+        selected.as_mut().unwrap().metadata = selected.as_ref().unwrap().cached.as_ref().unwrap().marshal_msg()?;
+
+        Ok(selected)
+    }
+
+    pub fn first_found(&self) -> (Option<MetaCacheEntry>, usize) {
+        (self.0.iter().find(|x| !x.name.is_empty()).cloned(), self.0.len())
+    }
 }
 
 #[derive(Debug, Default)]

ecstore/src/disk/remote.rs

@@ -20,7 +20,9 @@ use super::{
     RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
 };
 use crate::{
-    disk::error::DiskError, error::{Error, Result}, heal::heal_commands::HealingTracker, store_api::{FileInfo, RawFileInfo}
+    disk::error::DiskError,
+    error::{Error, Result},
+    store_api::{FileInfo, RawFileInfo},
 };
 use protos::proto_gen::node_service::RenamePartRequst;