diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 771c165ee..40c88b3e5 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -120,6 +120,15 @@ pub const MAX_PARTS_COUNT: usize = 10000; const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1); const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750); +mod heal; +mod list; +mod lock; +mod metadata; +mod multipart; +mod read; +mod replication; +mod write; + /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds) /// Defaults to 30 seconds if not set or invalid pub fn get_lock_acquire_timeout() -> Duration { @@ -233,708 +242,6 @@ impl SetDisks { // (filtered, online_count) // } - fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String { - match err { - LockResult::Timeout => { - format!("{mode} lock acquisition timed out on {bucket}/{object} (owner={})", self.locker_owner) - } - LockResult::Conflict { - current_owner, - current_mode, - } => format!("{mode} lock conflicted on {bucket}/{object}: held by {current_owner} as {current_mode:?}"), - LockResult::Acquired => format!("unexpected lock state while acquiring {mode} lock on {bucket}/{object}"), - } - } - - fn format_lock_error_from_error( - &self, - bucket: &str, - object: &str, - mode: &str, - err: &rustfs_lock::error::LockError, - ) -> String { - match err { - rustfs_lock::error::LockError::Timeout { .. } => { - format!( - "ns_loc: {mode} lock acquisition timed out on {bucket}/{object} (owner={})", - self.locker_owner - ) - } - rustfs_lock::error::LockError::AlreadyLocked { owner, .. } => { - format!("ns_loc: {mode} lock conflicted on {bucket}/{object}: held by {owner}") - } - _ => format!("ns_loc: {mode} lock acquisition failed on {bucket}/{object}: {}", err), - } - } - async fn get_disks_internal(&self) -> Vec> { - let rl = self.disks.read().await; - - rl.clone() - } - - pub async fn get_local_disks(&self) -> Vec> { - let rl = self.disks.read().await; - - let mut disks: Vec> = rl - .clone() - .into_iter() - .filter(|v| v.as_ref().is_some_and(|d| d.is_local())) - .collect(); - - let mut rng = rand::rng(); - - disks.shuffle(&mut rng); - - disks - } - - async fn get_online_disks(&self) -> Vec> { - let mut disks = self.get_disks_internal().await; - - // TODO: diskinfo filter online - - let mut new_disk = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { - if let Some(d) = disk - && d.is_online().await - { - new_disk.push(disk.clone()); - } - } - - let mut rng = rand::rng(); - - disks.shuffle(&mut rng); - - new_disk - // let disks = self.get_disks_internal().await; - // let (filtered, _) = self.filter_online_disks(disks).await; - // filtered.into_iter().filter(|disk| disk.is_some()).collect() - } - async fn get_online_local_disks(&self) -> Vec> { - let mut disks = self.get_online_disks().await; - - let mut rng = rand::rng(); - - disks.shuffle(&mut rng); - - disks - .into_iter() - .filter(|v| v.as_ref().is_some_and(|d| d.is_local())) - .collect() - } - - pub async fn get_online_disks_with_healing(&self, incl_healing: bool) -> (Vec, bool) { - let (disks, _, healing) = self.get_online_disks_with_healing_and_info(incl_healing).await; - (disks, healing > 0) - } - - pub async fn get_online_disks_with_healing_and_info(&self, incl_healing: bool) -> (Vec, Vec, usize) { - let mut disks = self.get_disks_internal().await; - - let mut infos = Vec::with_capacity(disks.len()); - - let mut futures = Vec::with_capacity(disks.len()); - let mut numbers: Vec = (0..disks.len()).collect(); - { - let mut rng = rand::rng(); - disks.shuffle(&mut rng); - - numbers.shuffle(&mut rng); - } - - for &i in numbers.iter() { - let disk = disks[i].clone(); - futures.push(async move { - if let Some(disk) = disk { - disk.disk_info(&DiskInfoOptions::default()).await - } else { - Err(DiskError::DiskNotFound) - } - }); - } - - // Use optimized batch processor for disk info retrieval - let processor = get_global_processors().metadata_processor(); - let results = processor.execute_batch(futures).await; - - for result in results { - match result { - Ok(res) => { - infos.push(res); - } - Err(err) => { - infos.push(DiskInfo { - error: err.to_string(), - ..Default::default() - }); - } - } - } - - let mut healing: usize = 0; - - let mut scanning_disks = Vec::new(); - let mut healing_disks = Vec::new(); - let mut scanning_infos = Vec::new(); - let mut healing_infos = Vec::new(); - - let mut new_disks = Vec::new(); - let mut new_infos = Vec::new(); - - for &i in numbers.iter() { - let (info, disk) = (infos[i].clone(), disks[i].clone()); - if !info.error.is_empty() || disk.is_none() { - continue; - } - - if info.healing { - healing += 1; - if incl_healing { - healing_disks.push(disk.unwrap()); - healing_infos.push(info); - } - - continue; - } - - if !info.healing { - new_disks.push(disk.unwrap()); - new_infos.push(info); - } else { - scanning_disks.push(disk.unwrap()); - scanning_infos.push(info); - } - } - - new_disks.extend(scanning_disks); - new_infos.extend(scanning_infos); - new_disks.extend(healing_disks); - new_infos.extend(healing_infos); - - (new_disks, new_infos, healing) - } - async fn _get_local_disks(&self) -> Vec> { - let mut disks = self.get_disks_internal().await; - - let mut rng = rand::rng(); - - disks.shuffle(&mut rng); - - disks - .into_iter() - .filter(|v| v.as_ref().is_some_and(|d| d.is_local())) - .collect() - } - fn default_read_quorum(&self) -> usize { - self.set_drive_count - self.default_parity_count - } - fn default_write_quorum(&self) -> usize { - let mut data_count = self.set_drive_count - self.default_parity_count; - if data_count == self.default_parity_count { - data_count += 1 - } - - data_count - } - - #[tracing::instrument(level = "debug", skip(disks, file_infos))] - #[allow(clippy::type_complexity)] - async fn rename_data( - disks: &[Option], - src_bucket: &str, - src_object: &str, - file_infos: &[FileInfo], - dst_bucket: &str, - dst_object: &str, - write_quorum: usize, - ) -> disk::error::Result<(Vec>, Option>, Option)> { - let mut futures = Vec::with_capacity(disks.len()); - - // let mut ress = Vec::with_capacity(disks.len()); - let mut errs = Vec::with_capacity(disks.len()); - - let src_bucket = Arc::new(src_bucket.to_string()); - let src_object = Arc::new(src_object.to_string()); - let dst_bucket = Arc::new(dst_bucket.to_string()); - let dst_object = Arc::new(dst_object.to_string()); - - for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() { - let mut file_info = file_info.clone(); - let disk = disk.clone(); - let src_bucket = src_bucket.clone(); - let src_object = src_object.clone(); - let dst_object = dst_object.clone(); - let dst_bucket = dst_bucket.clone(); - - futures.push(tokio::spawn(async move { - if file_info.erasure.index == 0 { - file_info.erasure.index = i + 1; - } - - if !file_info.is_valid() { - return Err(DiskError::FileCorrupt); - } - - if let Some(disk) = disk { - disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) - .await - } else { - Err(DiskError::DiskNotFound) - } - })); - } - - let mut disk_versions = vec![None; disks.len()]; - let mut data_dirs = vec![None; disks.len()]; - - let results = join_all(futures).await; - - for (idx, result) in results.iter().enumerate() { - match result.as_ref().map_err(|_| DiskError::Unexpected)? { - Ok(res) => { - data_dirs[idx] = res.old_data_dir; - disk_versions[idx].clone_from(&res.sign); - errs.push(None); - } - Err(e) => { - errs.push(Some(e.clone())); - } - } - } - - let mut futures = Vec::with_capacity(disks.len()); - if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - // TODO: add concurrency - for (i, err) in errs.iter().enumerate() { - if err.is_some() { - continue; - } - - if let Some(disk) = disks[i].as_ref() { - let fi = file_infos[i].clone(); - let old_data_dir = data_dirs[i]; - let disk = disk.clone(); - let src_bucket = src_bucket.clone(); - let src_object = src_object.clone(); - futures.push(tokio::spawn(async move { - let _ = disk - .delete_version( - &src_bucket, - &src_object, - fi, - false, - DeleteOptions { - undo_write: true, - old_data_dir, - ..Default::default() - }, - ) - .await - .map_err(|e| { - debug!("rename_data delete_version err {:?}", e); - e - }); - })); - } - } - - let _ = join_all(futures).await; - return Err(ret_err); - } - - let versions = None; - // TODO: reduceCommonVersions - - let data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum); - - // // TODO: reduce_common_data_dir - // if let Some(old_dir) = rename_ress - // .iter() - // .filter_map(|v| if v.is_some() { v.as_ref().unwrap().old_data_dir } else { None }) - // .map(|v| v.to_string()) - // .next() - // { - // let cm_errs = self.commit_rename_data_dir(&shuffle_disks, &bucket, &object, &old_dir).await; - // warn!("put_object commit_rename_data_dir:{:?}", &cm_errs); - // } - - // self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?; - - Ok((Self::eval_disks(disks, &errs), versions, data_dir)) - } - - fn reduce_common_data_dir(data_dirs: &Vec>, write_quorum: usize) -> Option { - let mut data_dirs_count = HashMap::new(); - - for ddir in data_dirs { - *data_dirs_count.entry(ddir).or_insert(0) += 1; - } - - let mut max = 0; - let mut data_dir = None; - for (ddir, count) in data_dirs_count { - if count > max { - max = count; - data_dir = *ddir; - } - } - - if max >= write_quorum { data_dir } else { None } - } - - #[allow(dead_code)] - #[tracing::instrument(level = "debug", skip(self, disks))] - async fn commit_rename_data_dir( - &self, - disks: &[Option], - bucket: &str, - object: &str, - data_dir: &str, - write_quorum: usize, - ) -> disk::error::Result<()> { - let file_path = Arc::new(format!("{object}/{data_dir}")); - let bucket = Arc::new(bucket.to_string()); - let futures = disks.iter().map(|disk| { - let file_path = file_path.clone(); - let bucket = bucket.clone(); - let disk = disk.clone(); - tokio::spawn(async move { - if let Some(disk) = disk { - (disk - .delete( - &bucket, - &file_path, - DeleteOptions { - recursive: true, - ..Default::default() - }, - ) - .await) - .err() - } else { - Some(DiskError::DiskNotFound) - } - }) - }); - let errs: Vec> = join_all(futures) - .await - .into_iter() - .map(|e| e.unwrap_or(Some(DiskError::Unexpected))) - .collect(); - - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - return Err(err); - } - - Ok(()) - } - - #[tracing::instrument(skip(self))] - async fn cleanup_multipart_path(&self, paths: &[String]) { - let disks = self.get_disks_internal().await; - - let mut errs = Vec::with_capacity(disks.len()); - - // Use improved simple batch processor instead of join_all for better performance - let processor = get_global_processors().write_processor(); - - let tasks: Vec<_> = disks - .iter() - .map(|disk| { - let disk = disk.clone(); - let paths = paths.to_vec(); - - async move { - if let Some(disk) = disk { - disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &paths).await - } else { - Err(DiskError::DiskNotFound) - } - } - }) - .collect(); - - let results = processor.execute_batch(tasks).await; - for result in results { - match result { - Ok(_) => { - errs.push(None); - } - Err(e) => { - errs.push(Some(e)); - } - } - } - - if errs.iter().any(|e| e.is_some()) { - warn!("cleanup_multipart_path errs {:?}", &errs); - } - } - - async fn read_parts( - disks: &[Option], - bucket: &str, - part_meta_paths: &[String], - part_numbers: &[usize], - read_quorum: usize, - ) -> disk::error::Result> { - let mut errs = Vec::with_capacity(disks.len()); - let mut object_parts = Vec::with_capacity(disks.len()); - - // Use batch processor for better performance - let processor = get_global_processors().read_processor(); - let bucket = bucket.to_string(); - let part_meta_paths = part_meta_paths.to_vec(); - - let tasks: Vec<_> = disks - .iter() - .map(|disk| { - let disk = disk.clone(); - let bucket = bucket.clone(); - let part_meta_paths = part_meta_paths.clone(); - - async move { - if let Some(disk) = disk { - disk.read_parts(&bucket, &part_meta_paths).await - } else { - Err(DiskError::DiskNotFound) - } - } - }) - .collect(); - - let results = processor.execute_batch(tasks).await; - for result in results { - match result { - Ok(res) => { - errs.push(None); - object_parts.push(res); - } - Err(e) => { - errs.push(Some(e)); - object_parts.push(vec![]); - } - } - } - - if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { - return Err(err); - } - - let mut ret = vec![ObjectPartInfo::default(); part_meta_paths.len()]; - - for (part_idx, part_info) in part_meta_paths.iter().enumerate() { - let mut part_meta_quorum = HashMap::new(); - let mut part_infos = Vec::new(); - for (j, parts) in object_parts.iter().enumerate() { - if parts.len() != part_meta_paths.len() { - *part_meta_quorum.entry(part_info.clone()).or_insert(0) += 1; - continue; - } - - if !parts[part_idx].etag.is_empty() { - *part_meta_quorum.entry(parts[part_idx].etag.clone()).or_insert(0) += 1; - part_infos.push(parts[part_idx].clone()); - continue; - } - - *part_meta_quorum.entry(part_info.clone()).or_insert(0) += 1; - } - - let mut max_quorum = 0; - let mut max_etag = None; - let mut max_part_meta = None; - for (etag, quorum) in part_meta_quorum.iter() { - if quorum > &max_quorum { - max_quorum = *quorum; - max_etag = Some(etag); - max_part_meta = Some(etag); - } - } - - let mut found = None; - for info in part_infos.iter() { - if let Some(etag) = max_etag - && info.etag == *etag - { - found = Some(info.clone()); - break; - } - - if let Some(part_meta) = max_part_meta - && info.etag.is_empty() - && part_meta.ends_with(format!("part.{0}.meta", info.number).as_str()) - { - found = Some(info.clone()); - break; - } - } - - if let (Some(found), Some(max_etag)) = (found, max_etag) - && !found.etag.is_empty() - && part_meta_quorum.get(max_etag).unwrap_or(&0) >= &read_quorum - { - ret[part_idx] = found.clone(); - } else { - ret[part_idx] = ObjectPartInfo { - number: part_numbers[part_idx], - error: Some(format!("part.{} not found", part_numbers[part_idx])), - ..Default::default() - }; - } - } - - Ok(ret) - } - - async fn list_parts(disks: &[Option], part_path: &str, read_quorum: usize) -> disk::error::Result> { - let mut futures = Vec::with_capacity(disks.len()); - for (i, disk) in disks.iter().enumerate() { - futures.push(async move { - if let Some(disk) = disk { - disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1) - .await - } else { - Err(DiskError::DiskNotFound) - } - }); - } - - let mut errs = Vec::with_capacity(disks.len()); - let mut object_parts = Vec::with_capacity(disks.len()); - - let results = join_all(futures).await; - for result in results { - match result { - Ok(res) => { - errs.push(None); - object_parts.push(res); - } - Err(e) => { - errs.push(Some(e)); - object_parts.push(vec![]); - } - } - } - - if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { - return Err(err); - } - - let mut part_quorum_map: HashMap = HashMap::new(); - - for drive_parts in object_parts { - let mut parts_with_meta_count: HashMap = HashMap::new(); - - // part files can be either part.N or part.N.meta - for part_path in drive_parts { - if let Some(num_str) = part_path.strip_prefix("part.") { - if let Some(meta_idx) = num_str.find(".meta") { - if let Ok(part_num) = num_str[..meta_idx].parse::() { - *parts_with_meta_count.entry(part_num).or_insert(0) += 1; - } - } else if let Ok(part_num) = num_str.parse::() { - *parts_with_meta_count.entry(part_num).or_insert(0) += 1; - } - } - } - - // Include only part.N.meta files with corresponding part.N - for (&part_num, &cnt) in &parts_with_meta_count { - if cnt >= 2 { - *part_quorum_map.entry(part_num).or_insert(0) += 1; - } - } - } - - let mut part_numbers = Vec::with_capacity(part_quorum_map.len()); - for (part_num, count) in part_quorum_map { - if count >= read_quorum { - part_numbers.push(part_num); - } - } - - part_numbers.sort(); - - Ok(part_numbers) - } - - #[tracing::instrument(skip(disks, meta))] - #[allow(clippy::too_many_arguments)] - async fn rename_part( - &self, - disks: &[Option], - src_bucket: &str, - src_object: &str, - dst_bucket: &str, - dst_object: &str, - meta: Bytes, - write_quorum: usize, - ) -> disk::error::Result>> { - let src_bucket = Arc::new(src_bucket.to_string()); - let src_object = Arc::new(src_object.to_string()); - let dst_bucket = Arc::new(dst_bucket.to_string()); - let dst_object = Arc::new(dst_object.to_string()); - - let mut errs = Vec::with_capacity(disks.len()); - - let futures = disks.iter().map(|disk| { - let disk = disk.clone(); - let meta = meta.clone(); - let src_bucket = src_bucket.clone(); - let src_object = src_object.clone(); - let dst_bucket = dst_bucket.clone(); - let dst_object = dst_object.clone(); - tokio::spawn(async move { - if let Some(disk) = disk { - disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta) - .await - } else { - Err(DiskError::DiskNotFound) - } - }) - }); - - let results = join_all(futures).await; - for result in results { - match result? { - Ok(_) => { - errs.push(None); - } - Err(e) => { - errs.push(Some(e)); - } - } - } - - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - warn!("rename_part errs {:?}", &errs); - self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")]) - .await; - return Err(err); - } - - let disks = Self::eval_disks(disks, &errs); - Ok(disks) - } - - fn eval_disks(disks: &[Option], errs: &[Option]) -> Vec> { - if disks.len() != errs.len() { - return Vec::new(); - } - - let mut online_disks = vec![None; disks.len()]; - - for (i, err_op) in errs.iter().enumerate() { - if err_op.is_none() { - online_disks[i].clone_from(&disks[i]); - } - } - - online_disks - } // async fn write_all(disks: &[Option], bucket: &str, object: &str, buff: Vec) -> Vec> { // let mut futures = Vec::with_capacity(disks.len()); @@ -964,953 +271,11 @@ impl SetDisks { // errors // } - #[tracing::instrument(skip(disks, files))] - async fn write_unique_file_info( - disks: &[Option], - org_bucket: &str, - bucket: &str, - prefix: &str, - files: &[FileInfo], - write_quorum: usize, - ) -> disk::error::Result<()> { - let mut futures = Vec::with_capacity(disks.len()); - let mut errs = Vec::with_capacity(disks.len()); - - for (i, disk) in disks.iter().enumerate() { - let mut file_info = files[i].clone(); - file_info.erasure.index = i + 1; - futures.push(async move { - if let Some(disk) = disk { - disk.write_metadata(org_bucket, bucket, prefix, file_info).await - } else { - Err(DiskError::DiskNotFound) - } - }); - } - - let results = join_all(futures).await; - for result in results { - match result { - Ok(_) => { - errs.push(None); - } - Err(e) => { - errs.push(Some(e)); - } - } - } - - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - // TODO: add concurrency - for (i, err) in errs.iter().enumerate() { - if err.is_some() { - continue; - } - - if let Some(disk) = disks[i].as_ref() { - let _ = disk - .delete( - bucket, - &path_join_buf(&[prefix, STORAGE_FORMAT_FILE]), - DeleteOptions { - recursive: true, - ..Default::default() - }, - ) - .await - .map_err(|e| { - warn!("write meta revert err {:?}", e); - e - }); - } - } - - return Err(err); - } - Ok(()) - } - - fn get_upload_id_dir(bucket: &str, object: &str, upload_id: &str) -> String { - let upload_uuid = base64_simd::URL_SAFE_NO_PAD - .decode_to_vec(upload_id.as_bytes()) - .and_then(|v| { - String::from_utf8(v).map_or(Ok(upload_id.to_owned()), |v| { - let parts: Vec<_> = v.splitn(2, '.').collect(); - if parts.len() == 2 { - Ok(parts[1].to_string()) - } else { - Ok(upload_id.to_string()) - } - }) - }) - .unwrap_or_default(); - - format!("{}/{}", Self::get_multipart_sha_dir(bucket, object), upload_uuid) - } - - fn get_multipart_sha_dir(bucket: &str, object: &str) -> String { - let path = format!("{bucket}/{object}"); - let mut hasher = Sha256::new(); - hasher.update(path); - hex(hasher.finalize()) - } - - fn common_parity(parities: &[i32], default_parity_count: i32) -> i32 { - let n = parities.len() as i32; - - let mut occ_map: HashMap = HashMap::new(); - for &p in parities { - *occ_map.entry(p).or_insert(0) += 1; - } - - let mut max_occ = 0; - let mut cparity = 0; - for (&parity, &occ) in &occ_map { - if parity == -1 { - // Ignore non defined parity - continue; - } - - let mut read_quorum = n - parity; - if default_parity_count > 0 && parity == 0 { - // In this case, parity == 0 implies that this object version is a - // delete marker - read_quorum = n / 2 + 1; - } - if occ < read_quorum { - // Ignore this parity since we don't have enough shards for read quorum - continue; - } - - if occ > max_occ { - max_occ = occ; - cparity = parity; - } - } - - if max_occ == 0 { - // Did not find anything useful - return -1; - } - cparity - } - - fn list_object_modtimes(parts_metadata: &[FileInfo], errs: &[Option]) -> Vec> { - let mut times = vec![None; parts_metadata.len()]; - - for (i, metadata) in parts_metadata.iter().enumerate() { - if errs[i].is_some() { - continue; - } - - times[i] = metadata.mod_time - } - - times - } - - fn common_time(times: &[Option], quorum: usize) -> Option { - let (time, count) = Self::common_time_and_occurrence(times); - if count >= quorum { time } else { None } - } - - fn common_time_and_occurrence(times: &[Option]) -> (Option, usize) { - let mut time_occurrence_map = HashMap::new(); - - // Ignore the uuid sentinel and count the rest. - for time in times.iter().flatten() { - *time_occurrence_map.entry(time.unix_timestamp_nanos()).or_insert(0) += 1; - } - - let mut maxima = 0; // Counter for remembering max occurrence of elements. - let mut latest = 0; - - // Find the common cardinality from previously collected - // occurrences of elements. - for (&nano, &count) in &time_occurrence_map { - if count < maxima { - continue; - } - - // We are at or above maxima - if count > maxima || nano > latest { - maxima = count; - latest = nano; - } - } - - if latest == 0 { - return (None, maxima); - } - - if let Ok(time) = OffsetDateTime::from_unix_timestamp_nanos(latest) { - (Some(time), maxima) - } else { - (None, maxima) - } - } - - fn common_etag(etags: &[Option], quorum: usize) -> Option { - let (etag, count) = Self::common_etags(etags); - if count >= quorum { etag } else { None } - } - - fn common_etags(etags: &[Option]) -> (Option, usize) { - let mut etags_map = HashMap::new(); - - for etag in etags.iter().flatten() { - *etags_map.entry(etag).or_insert(0) += 1; - } - - let mut maxima = 0; // Counter for remembering max occurrence of elements. - let mut latest = None; - - for (&etag, &count) in &etags_map { - if count < maxima { - continue; - } - - // We are at or above maxima - if count > maxima { - maxima = count; - latest = Some(etag.clone()); - } - } - - (latest, maxima) - } - - fn list_object_etags(parts_metadata: &[FileInfo], errs: &[Option]) -> Vec> { - let mut etags = vec![None; parts_metadata.len()]; - - for (i, metadata) in parts_metadata.iter().enumerate() { - if errs[i].is_some() { - continue; - } - - if let Some(etag) = metadata.metadata.get("etag") { - etags[i] = Some(etag.clone()) - } - } - - etags - } - - fn list_object_parities(parts_metadata: &[FileInfo], errs: &[Option]) -> Vec { - let total_shards = parts_metadata.len(); - let half = total_shards as i32 / 2; - let mut parities: Vec = vec![-1; total_shards]; - - for (index, metadata) in parts_metadata.iter().enumerate() { - if errs[index].is_some() { - parities[index] = -1; - continue; - } - - if !metadata.is_valid() { - parities[index] = -1; - continue; - } - - if metadata.deleted || metadata.size == 0 { - parities[index] = half; - // } else if metadata.transition_status == "TransitionComplete" { - // TODO: metadata.transition_status - // parities[index] = total_shards - (total_shards / 2 + 1); - } else { - parities[index] = metadata.erasure.parity_blocks as i32; - } - } - parities - } - // Returns per object readQuorum and writeQuorum // readQuorum is the min required disks to read data. // writeQuorum is the min required disks to write data. - #[tracing::instrument(level = "debug", skip(parts_metadata))] - fn object_quorum_from_meta( - parts_metadata: &[FileInfo], - errs: &[Option], - default_parity_count: usize, - ) -> disk::error::Result<(i32, i32)> { - let expected_rquorum = if default_parity_count == 0 { - parts_metadata.len() - } else { - parts_metadata.len() / 2 - }; - - if let Some(err) = reduce_read_quorum_errs(errs, OBJECT_OP_IGNORED_ERRS, expected_rquorum) { - // let object = parts_metadata.first().map(|v| v.name.clone()).unwrap_or_default(); - // error!("object_quorum_from_meta: {:?}, errs={:?}, object={:?}", err, errs, object); - return Err(err); - } - - if default_parity_count == 0 { - return Ok((parts_metadata.len() as i32, parts_metadata.len() as i32)); - } - - let parities = Self::list_object_parities(parts_metadata, errs); - - let parity_blocks = Self::common_parity(&parities, default_parity_count as i32); - - if parity_blocks < 0 { - error!("object_quorum_from_meta: parity_blocks < 0, errs={:?}", errs); - return Err(DiskError::ErasureReadQuorum); - } - - let data_blocks = parts_metadata.len() as i32 - parity_blocks; - let write_quorum = if data_blocks == parity_blocks { - data_blocks + 1 - } else { - data_blocks - }; - - Ok((data_blocks, write_quorum)) - } - - #[tracing::instrument(level = "debug", skip(disks, parts_metadata))] - fn list_online_disks( - disks: &[Option], - parts_metadata: &[FileInfo], - errs: &[Option], - quorum: usize, - ) -> (Vec>, Option, Option) { - let mod_times = Self::list_object_modtimes(parts_metadata, errs); - let etags = Self::list_object_etags(parts_metadata, errs); - - let mod_time = Self::common_time(&mod_times, quorum); - let etag = Self::common_etag(&etags, quorum); - - let mut new_disk = vec![None; disks.len()]; - - for (i, &t) in mod_times.iter().enumerate() { - if parts_metadata[i].is_valid() && mod_time == t { - new_disk[i].clone_from(&disks[i]); - } - } - - (new_disk, mod_time, etag) - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn check_upload_id_exists( - &self, - bucket: &str, - object: &str, - upload_id: &str, - write: bool, - ) -> Result<(FileInfo, Vec)> { - let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); - let disks = self.disks.read().await; - - let disks = disks.clone(); - - let (parts_metadata, errs) = - Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await?; - - let map_err_notfound = |err: DiskError| { - if err == DiskError::FileNotFound { - return StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()); - } - err.into() - }; - - let (read_quorum, write_quorum) = - Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count).map_err(map_err_notfound)?; - - if read_quorum < 0 { - error!("check_upload_id_exists: read_quorum < 0, errs={:?}", errs); - return Err(Error::ErasureReadQuorum); - } - - if write_quorum < 0 { - return Err(Error::ErasureWriteQuorum); - } - - let mut quorum = read_quorum as usize; - if write { - quorum = write_quorum as usize; - - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) { - return Err(map_err_notfound(err)); - } - } else if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) { - return Err(map_err_notfound(err)); - } - - let (_, mod_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, quorum); - - let fi = Self::pick_valid_fileinfo(&parts_metadata, mod_time, etag, quorum)?; - - Ok((fi, parts_metadata)) - } - - fn pick_valid_fileinfo( - metas: &[FileInfo], - mod_time: Option, - etag: Option, - quorum: usize, - ) -> disk::error::Result { - Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum) - } - - fn find_file_info_in_quorum( - metas: &[FileInfo], - mod_time: &Option, - etag: &Option, - quorum: usize, - ) -> disk::error::Result { - if quorum < 1 { - warn!("find_file_info_in_quorum: quorum < 1"); - return Err(DiskError::ErasureReadQuorum); - } - - let mut meta_hashes = vec![None; metas.len()]; - let mut hasher = Sha256::new(); - - for (i, meta) in metas.iter().enumerate() { - if !meta.is_valid() { - debug!( - index = i, - valid = false, - version_id = ?meta.version_id, - mod_time = ?meta.mod_time, - "find_file_info_in_quorum: skipping invalid meta" - ); - continue; - } - - debug!( - index = i, - valid = true, - version_id = ?meta.version_id, - mod_time = ?meta.mod_time, - deleted = meta.deleted, - size = meta.size, - "find_file_info_in_quorum: inspecting meta" - ); - - let etag_only = mod_time.is_none() && etag.is_some() && meta.get_etag().is_some_and(|v| &v == etag.as_ref().unwrap()); - let mod_valid = mod_time == &meta.mod_time; - - if etag_only || mod_valid { - for part in meta.parts.iter() { - hasher.update(format!("part.{}", part.number).as_bytes()); - hasher.update(format!("part.{}", part.size).as_bytes()); - } - - if !meta.deleted && meta.size != 0 { - hasher.update(format!("{}+{}", meta.erasure.data_blocks, meta.erasure.parity_blocks).as_bytes()); - hasher.update(format!("{:?}", meta.erasure.distribution).as_bytes()); - } - - if meta.is_remote() { - // TODO: - } - - // TODO: IsEncrypted - - // TODO: IsCompressed - - meta_hashes[i] = Some(hex(hasher.clone().finalize().as_slice())); - - hasher.reset(); - } else { - debug!( - index = i, - etag_only_match = etag_only, - mod_valid_match = mod_valid, - "find_file_info_in_quorum: meta does not match common etag or mod_time, skipping hash calculation" - ); - } - } - - let mut count_map = HashMap::new(); - - for hash in meta_hashes.iter().flatten() { - *count_map.entry(hash).or_insert(0) += 1; - } - - let mut max_val = None; - let mut max_count = 0; - - for (&val, &count) in &count_map { - if count > max_count { - max_val = Some(val); - max_count = count; - } - } - - if max_count < quorum { - warn!("find_file_info_in_quorum: max_count < quorum, max_val={:?}", max_val); - return Err(DiskError::ErasureReadQuorum); - } - - let mut found_fi = None; - let mut found = false; - - let mut valid_obj_map = HashMap::new(); - - for (i, op_hash) in meta_hashes.iter().enumerate() { - if let Some(hash) = op_hash - && let Some(max_hash) = max_val - && hash == max_hash - { - if metas[i].is_valid() && !found { - found_fi = Some(metas[i].clone()); - found = true; - } - - let props = ObjProps { - mod_time: metas[i].mod_time, - num_versions: metas[i].num_versions, - }; - - *valid_obj_map.entry(props).or_insert(0) += 1; - } - } - - if found { - let mut fi = found_fi.unwrap(); - - for (val, &count) in &valid_obj_map { - if count > quorum { - fi.mod_time = val.mod_time; - fi.num_versions = val.num_versions; - fi.is_latest = val.mod_time.is_none(); - - break; - } - } - - return Ok(fi); - } - - warn!("find_file_info_in_quorum: fileinfo not found"); - - Err(DiskError::ErasureReadQuorum) - } - - #[tracing::instrument(level = "debug", skip(disks))] - async fn read_all_fileinfo( - disks: &[Option], - org_bucket: &str, - bucket: &str, - object: &str, - version_id: &str, - read_data: bool, - healing: bool, - ) -> disk::error::Result<(Vec, Vec>)> { - let mut ress = Vec::with_capacity(disks.len()); - let mut errors = Vec::with_capacity(disks.len()); - let opts = Arc::new(ReadOptions { - read_data, - healing, - ..Default::default() - }); - let org_bucket = Arc::new(org_bucket.to_string()); - let bucket = Arc::new(bucket.to_string()); - let object = Arc::new(object.to_string()); - let version_id = Arc::new(version_id.to_string()); - let futures = disks.iter().map(|disk| { - let disk = disk.clone(); - let opts = opts.clone(); - let org_bucket = org_bucket.clone(); - let bucket = bucket.clone(); - let object = object.clone(); - let version_id = version_id.clone(); - tokio::spawn(async move { - if let Some(disk) = disk { - disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await - } else { - Err(DiskError::DiskNotFound) - } - }) - }); - - // Wait for all tasks to complete - let results = join_all(futures).await; - - for result in results { - match result { - Ok(res) => match res { - Ok(file_info) => { - ress.push(file_info); - errors.push(None); - } - Err(e) => { - ress.push(FileInfo::default()); - errors.push(Some(e)); - } - }, - Err(_) => { - ress.push(FileInfo::default()); - errors.push(Some(DiskError::Unexpected)); - } - } - } - Ok((ress, errors)) - } // Optimized version using batch processor with quorum support - pub async fn read_version_optimized( - &self, - bucket: &str, - object: &str, - version_id: &str, - opts: &ReadOptions, - ) -> Result> { - // Use existing disk selection logic - let disks = self.disks.read().await; - let required_reads = self.format.erasure.sets.len(); - - // Clone parameters outside the closure to avoid lifetime issues - let bucket = bucket.to_string(); - let object = object.to_string(); - let version_id = version_id.to_string(); - let opts = opts.clone(); - - let processor = get_global_processors().read_processor(); - let tasks: Vec<_> = disks - .iter() - .take(required_reads + 2) // Read a few extra for reliability - .filter_map(|disk| { - disk.as_ref().map(|d| { - let disk = d.clone(); - let bucket = bucket.clone(); - let object = object.clone(); - let version_id = version_id.clone(); - let opts = opts.clone(); - - async move { disk.read_version(&bucket, &bucket, &object, &version_id, &opts).await } - }) - }) - .collect(); - - match processor.execute_batch_with_quorum(tasks, required_reads).await { - Ok(results) => Ok(results), - Err(_) => Err(DiskError::FileNotFound.into()), // Use existing error type - } - } - - async fn read_all_xl( - disks: &[Option], - bucket: &str, - object: &str, - read_data: bool, - incl_free_vers: bool, - ) -> (Vec, Vec>) { - let (fileinfos, errs) = Self::read_all_raw_file_info(disks, bucket, object, read_data).await; - - Self::pick_latest_quorum_files_info(fileinfos, errs, bucket, object, read_data, incl_free_vers).await - } - - async fn read_all_raw_file_info( - disks: &[Option], - bucket: &str, - object: &str, - read_data: bool, - ) -> (Vec>, Vec>) { - let mut ress = Vec::with_capacity(disks.len()); - let mut errors = Vec::with_capacity(disks.len()); - - let mut futures = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { - futures.push(async move { - if let Some(disk) = disk { - disk.read_xl(bucket, object, read_data).await - } else { - Err(DiskError::DiskNotFound) - } - }); - } - - let results = join_all(futures).await; - for result in results { - match result { - Ok(res) => { - ress.push(Some(res)); - errors.push(None); - } - Err(e) => { - ress.push(None); - errors.push(Some(e)); - } - } - } - - (ress, errors) - } - - async fn pick_latest_quorum_files_info( - fileinfos: Vec>, - errs: Vec>, - bucket: &str, - object: &str, - read_data: bool, - incl_free_vers: bool, - ) -> (Vec, Vec>) { - let mut metadata_array = vec![None; fileinfos.len()]; - let mut meta_file_infos = vec![FileInfo::default(); fileinfos.len()]; - let mut metadata_shallow_versions = vec![None; fileinfos.len()]; - - let mut v2_bufs = { - if !read_data { - vec![Vec::new(); fileinfos.len()] - } else { - Vec::new() - } - }; - - let mut errs = errs; - - for (idx, info_op) in fileinfos.iter().enumerate() { - if let Some(info) = info_op { - if !read_data { - v2_bufs[idx] = info.buf.clone(); - } - - let xlmeta = match FileMeta::load(&info.buf) { - Ok(res) => res, - Err(err) => { - errs[idx] = Some(err.into()); - continue; - } - }; - - metadata_array[idx] = Some(xlmeta); - meta_file_infos[idx] = FileInfo::default(); - } - } - - for (idx, info_op) in metadata_array.iter().enumerate() { - if let Some(info) = info_op { - metadata_shallow_versions[idx] = Some(info.versions.clone()); - } - } - - let shallow_versions: Vec> = metadata_shallow_versions.iter().flatten().cloned().collect(); - - let read_quorum = fileinfos.len().div_ceil(2); - let versions = merge_file_meta_versions(read_quorum, false, 1, &shallow_versions); - let meta = FileMeta { - versions, - ..Default::default() - }; - - let finfo = match meta.into_fileinfo(bucket, object, "", true, incl_free_vers, true) { - Ok(res) => res, - Err(err) => { - for item in errs.iter_mut() { - if item.is_none() { - *item = Some(err.clone().into()); - } - } - - return (meta_file_infos, errs); - } - }; - - if !finfo.is_valid() { - for item in errs.iter_mut() { - if item.is_none() { - *item = Some(DiskError::FileCorrupt); - } - } - - return (meta_file_infos, errs); - } - - let vid = finfo.version_id.unwrap_or(Uuid::nil()); - - for (idx, meta_op) in metadata_array.iter().enumerate() { - if let Some(meta) = meta_op { - match meta.into_fileinfo(bucket, object, vid.to_string().as_str(), read_data, incl_free_vers, true) { - Ok(res) => meta_file_infos[idx] = res, - Err(err) => errs[idx] = Some(err.into()), - } - } - } - - (meta_file_infos, errs) - } - - async fn read_multiple_files(disks: &[Option], req: ReadMultipleReq, read_quorum: usize) -> Vec { - let mut futures = Vec::with_capacity(disks.len()); - let mut ress = Vec::with_capacity(disks.len()); - let mut errors = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { - let req = req.clone(); - futures.push(async move { - if let Some(disk) = disk { - disk.read_multiple(req).await - } else { - Err(DiskError::DiskNotFound) - } - }); - } - - let results = join_all(futures).await; - for result in results { - match result { - Ok(res) => { - ress.push(Some(res)); - errors.push(None); - } - Err(e) => { - ress.push(None); - errors.push(Some(e)); - } - } - } - - // debug!("ReadMultipleResp ress {:?}", ress); - // debug!("ReadMultipleResp errors {:?}", errors); - - let mut ret = Vec::with_capacity(req.files.len()); - - for want in req.files.iter() { - let mut quorum = 0; - - let mut get_res = ReadMultipleResp::default(); - - for res in ress.iter() { - if res.is_none() { - continue; - } - - let disk_res = res.as_ref().unwrap(); - - for resp in disk_res.iter() { - if !resp.error.is_empty() || !resp.exists { - continue; - } - - if &resp.file != want || resp.bucket != req.bucket || resp.prefix != req.prefix { - continue; - } - quorum += 1; - - if get_res.mod_time > resp.mod_time || get_res.data.len() > resp.data.len() { - continue; - } - - get_res = resp.clone(); - } - } - - if quorum < read_quorum { - // debug!("quorum < read_quorum: {} < {}", quorum, read_quorum); - get_res.exists = false; - get_res.error = Error::ErasureReadQuorum.to_string(); - get_res.data = Vec::new(); - } - - ret.push(get_res); - } - - // log err - - ret - } - - pub async fn connect_disks(&self) { - let rl = self.disks.read().await; - - let disks = rl.clone(); - - // Explicitly release the lock - drop(rl); - - for (i, opdisk) in disks.iter().enumerate() { - if let Some(disk) = opdisk { - if disk.is_online().await && disk.get_disk_location().set_idx.is_some() { - info!("Disk {:?} is online", disk.to_string()); - continue; - } - - let _ = disk.close().await; - } - - if let Some(endpoint) = self.set_endpoints.get(i) { - info!("will renew disk, opdisk: {:?}", opdisk); - self.renew_disk(endpoint).await; - } - } - } - - pub async fn renew_disk(&self, ep: &Endpoint) { - debug!("renew_disk: start {:?}", ep); - - let (new_disk, fm) = match Self::connect_endpoint(ep).await { - Ok(res) => res, - Err(e) => { - warn!("renew_disk: connect_endpoint err {:?}", &e); - if ep.is_local && e == DiskError::UnformattedDisk { - info!("renew_disk unformatteddisk will trigger heal_disk, {:?}", ep); - let set_disk_id = format!("pool_{}_set_{}", ep.pool_idx, ep.set_idx); - let _ = send_heal_disk(set_disk_id, Some(HealChannelPriority::Normal)).await; - } - return; - } - }; - - let (set_idx, disk_idx) = match self.find_disk_index(&fm) { - Ok(res) => res, - Err(e) => { - warn!("renew_disk: find_disk_index err {:?}", e); - return; - } - }; - - // Check that the endpoint matches - - let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await; - - if new_disk.is_local() { - let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await; - let path = new_disk.endpoint().to_string(); - global_local_disk_map.insert(path, Some(new_disk.clone())); - - if is_dist_erasure().await { - let mut local_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.write().await; - local_set_drives[self.pool_index][set_idx][disk_idx] = Some(new_disk.clone()); - } - } - - debug!("renew_disk: update {:?}", fm.erasure.this); - - let mut disk_lock = self.disks.write().await; - disk_lock[disk_idx] = Some(new_disk); - } - - fn find_disk_index(&self, fm: &FormatV3) -> Result<(usize, usize)> { - self.format.check_other(fm)?; - - if fm.erasure.this.is_nil() { - return Err(Error::other("DriveID: offline")); - } - - for i in 0..self.format.erasure.sets.len() { - for j in 0..self.format.erasure.sets[0].len() { - if fm.erasure.this == self.format.erasure.sets[i][j] { - return Ok((i, j)); - } - } - } - - Err(Error::other("DriveID: not found")) - } - - async fn connect_endpoint(ep: &Endpoint) -> disk::error::Result<(DiskStore, FormatV3)> { - let disk = new_disk(ep, &DiskOption::default()).await?; - - let fm = load_format_erasure(&disk, false).await?; - - Ok((disk, fm)) - } // pub async fn walk_dir(&self, opts: &WalkDirOptions) -> (Vec>>, Vec>) { // let disks = self.disks.read().await; @@ -2036,1599 +401,13 @@ impl SetDisks { // Ok(()) // } - #[tracing::instrument(skip(self))] - pub async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> { - let disks = self.disks.read().await; - - let disks = disks.clone(); - - let mut futures = Vec::with_capacity(disks.len()); - let mut errors = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { - futures.push(async move { - if let Some(disk) = disk { - disk.delete( - bucket, - prefix, - DeleteOptions { - recursive: true, - ..Default::default() - }, - ) - .await - } else { - Err(DiskError::DiskNotFound) - } - }); - } - - let results = join_all(futures).await; - for result in results { - match result { - Ok(_) => { - errors.push(None); - } - Err(e) => { - errors.push(Some(e)); - } - } - } - - // debug!("delete_all errs {:?}", &errors); - - Ok(()) - } + // Shuffle the order // Shuffle the order - fn shuffle_disks_and_parts_metadata_by_index( - disks: &[Option], - parts_metadata: &[FileInfo], - fi: &FileInfo, - ) -> (Vec>, Vec) { - let mut shuffled_disks = vec![None; disks.len()]; - let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()]; - let distribution = &fi.erasure.distribution; - - let mut inconsistent = 0; - for (k, v) in parts_metadata.iter().enumerate() { - if disks[k].is_none() { - inconsistent += 1; - continue; - } - - if !v.is_valid() { - inconsistent += 1; - continue; - } - - if distribution[k] != v.erasure.index { - inconsistent += 1; - continue; - } - - let block_idx = distribution[k]; - shuffled_parts_metadata[block_idx - 1] = parts_metadata[k].clone(); - shuffled_disks[block_idx - 1].clone_from(&disks[k]); - } - - if inconsistent < fi.erasure.parity_blocks { - return (shuffled_disks, shuffled_parts_metadata); - } - - Self::shuffle_disks_and_parts_metadata(disks, parts_metadata, fi) - } - - // Shuffle the order - fn shuffle_disks_and_parts_metadata( - disks: &[Option], - parts_metadata: &[FileInfo], - fi: &FileInfo, - ) -> (Vec>, Vec) { - let init = fi.mod_time.is_none(); - - let mut shuffled_disks = vec![None; disks.len()]; - let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()]; - let distribution = &fi.erasure.distribution; - - for (k, v) in disks.iter().enumerate() { - if v.is_none() { - continue; - } - - if !init && !parts_metadata[k].is_valid() { - continue; - } - - // if !init && fi.xlv1 != parts_metadata[k].xlv1 { - // continue; - // } - - let block_idx = distribution[k]; - shuffled_parts_metadata[block_idx - 1] = parts_metadata[k].clone(); - shuffled_disks[block_idx - 1].clone_from(&disks[k]); - } - - (shuffled_disks, shuffled_parts_metadata) - } // Return shuffled partsMetadata depending on distribution. - fn shuffle_parts_metadata(parts_metadata: &[FileInfo], distribution: &[usize]) -> Vec { - if distribution.is_empty() { - return parts_metadata.to_vec(); - } - let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()]; - // Shuffle slice xl metadata for expected distribution. - for index in 0..parts_metadata.len() { - let block_index = distribution[index]; - shuffled_parts_metadata[block_index - 1] = parts_metadata[index].clone(); - } - shuffled_parts_metadata - } // shuffle_disks TODO: use origin value - fn shuffle_disks(disks: &[Option], distribution: &[usize]) -> Vec> { - if distribution.is_empty() { - return disks.to_vec(); - } - - let mut shuffled_disks = vec![None; disks.len()]; - - for (i, v) in disks.iter().enumerate() { - let idx = distribution[i]; - shuffled_disks[idx - 1].clone_from(v); - } - - shuffled_disks - } - - fn shuffle_check_parts(parts_errs: &[usize], distribution: &[usize]) -> Vec { - if distribution.is_empty() { - return parts_errs.to_vec(); - } - let mut shuffled_parts_errs = vec![0; parts_errs.len()]; - for (i, v) in parts_errs.iter().enumerate() { - let idx = distribution[i]; - shuffled_parts_errs[idx - 1] = *v; - } - shuffled_parts_errs - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn get_object_fileinfo( - &self, - bucket: &str, - object: &str, - opts: &ObjectOptions, - read_data: bool, - ) -> Result<(FileInfo, Vec, Vec>)> { - let disks = self.disks.read().await; - - let disks = disks.clone(); - - let vid = opts.version_id.clone().unwrap_or_default(); - - // TODO: optimize concurrency and break once enough slots are available - let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await?; - // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata); - // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs); - - let _min_disks = self.set_drive_count - self.default_parity_count; - - let (read_quorum, _) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) - .map_err(|err| to_object_err(err.into(), vec![bucket, object])) - { - Ok(v) => v, - Err(e) => { - // error!("Self::object_quorum_from_meta: {:?}, bucket: {}, object: {}", &e, bucket, object); - return Err(e); - } - }; - - if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum as usize) { - error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object); - return Err(to_object_err(err.into(), vec![bucket, object])); - } - - let (op_online_disks, mot_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize); - - let fi = Self::pick_valid_fileinfo(&parts_metadata, mot_time, etag, read_quorum as usize)?; - if errs.iter().any(|err| err.is_some()) { - let _ = - rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options( - fi.volume.to_string(), // bucket - Some(fi.name.to_string()), // object_prefix - false, // force_start - Some(HealChannelPriority::Normal), // priority - Some(self.pool_index), // pool_index - Some(self.set_index), // set_index - )) - .await; - } - // debug!("get_object_fileinfo pick fi {:?}", &fi); - - // let online_disks: Vec> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect(); - - Ok((fi, parts_metadata, op_online_disks)) - } - async fn get_object_info_and_quorum( - &self, - bucket: &str, - object: &str, - opts: &ObjectOptions, - ) -> (ObjectInfo, usize, Option) { - let fi = match self.get_object_fileinfo(bucket, object, opts, false).await { - Ok((fi, _, _)) => fi, - Err(e) => return (ObjectInfo::default(), 0, Some(e)), - }; - - let write_quorum = fi.write_quorum(self.default_write_quorum()); - - let oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); - - if !fi.version_purge_status().is_empty() && opts.version_id.is_some() { - return ( - oi, - write_quorum, - Some(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])), - ); - } - - if fi.deleted { - return if opts.version_id.is_none() || opts.delete_marker { - (oi, write_quorum, Some(to_object_err(StorageError::FileNotFound, vec![bucket, object]))) - } else { - ( - oi, - write_quorum, - Some(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])), - ) - }; - } - - (oi, write_quorum, None) - } - - #[allow(clippy::too_many_arguments)] - #[tracing::instrument( - level = "debug", - skip( writer,disks,fi,files), - fields(start_time=?time::OffsetDateTime::now_utc()) - )] - async fn get_object_with_fileinfo( - // &self, - bucket: &str, - object: &str, - offset: usize, - length: i64, - writer: &mut W, - fi: FileInfo, - files: Vec, - disks: &[Option], - set_index: usize, - pool_index: usize, - ) -> Result<()> - where - W: AsyncWrite + Send + Sync + Unpin + 'static, - { - debug!(bucket, object, requested_length = length, offset, "get_object_with_fileinfo start"); - let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, &files, &fi); - - let total_size = fi.size as usize; - - let length = if length < 0 { - fi.size as usize - offset - } else { - length as usize - }; - - if offset > total_size || offset + length > total_size { - error!("get_object_with_fileinfo offset out of range: {}, total_size: {}", offset, total_size); - return Err(Error::other("offset out of range")); - } - - let (part_index, mut part_offset) = fi.to_part_offset(offset)?; - - let mut end_offset = offset; - if length > 0 { - end_offset += length - 1 - } - - let (last_part_index, last_part_relative_offset) = fi.to_part_offset(end_offset)?; - - debug!( - bucket, - object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds" - ); - - let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); - - let part_indices: Vec = (part_index..=last_part_index).collect(); - debug!(bucket, object, ?part_indices, "Multipart part indices to stream"); - - let mut total_read = 0; - for current_part in part_indices { - if total_read == length { - debug!( - bucket, - object, - total_read, - requested_length = length, - part_index = current_part, - "Stopping multipart stream early because accumulated bytes match request" - ); - break; - } - - let part_number = fi.parts[current_part].number; - let part_size = fi.parts[current_part].size; - let mut part_length = part_size - part_offset; - if part_length > (length - total_read) { - part_length = length - total_read - } - - let till_offset = erasure.shard_file_offset(part_offset, part_length, part_size); - - let read_offset = (part_offset / erasure.block_size) * erasure.shard_size(); - - debug!( - bucket, - object, - part_index = current_part, - part_number, - part_offset, - part_size, - part_length, - read_offset, - till_offset, - total_read_before = total_read, - requested_length = length, - "Streaming multipart part" - ); - - let mut readers = Vec::with_capacity(disks.len()); - let mut errors = Vec::with_capacity(disks.len()); - for (idx, disk_op) in disks.iter().enumerate() { - match create_bitrot_reader( - files[idx].data.as_deref(), - disk_op.as_ref(), - bucket, - &format!("{}/{}/part.{}", object, files[idx].data_dir.unwrap_or_default(), part_number), - read_offset, - till_offset, - erasure.shard_size(), - HashAlgorithm::HighwayHash256, - ) - .await - { - Ok(Some(reader)) => { - readers.push(Some(reader)); - errors.push(None); - } - Ok(None) => { - readers.push(None); - errors.push(Some(DiskError::DiskNotFound)); - } - Err(e) => { - readers.push(None); - errors.push(Some(e)); - } - } - } - - let nil_count = errors.iter().filter(|&e| e.is_none()).count(); - if nil_count < erasure.data_shards { - if let Some(read_err) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) { - error!("create_bitrot_reader reduce_read_quorum_errs {:?}", &errors); - return Err(to_object_err(read_err.into(), vec![bucket, object])); - } - error!("create_bitrot_reader not enough disks to read: {:?}", &errors); - return Err(Error::other(format!("not enough disks to read: {errors:?}"))); - } - - // Check if we have missing shards even though we can read successfully - // This happens when a node was offline during write and comes back online - let total_shards = erasure.data_shards + erasure.parity_shards; - let available_shards = nil_count; - let missing_shards = total_shards - available_shards; - - info!( - bucket, - object, - part_number, - total_shards, - available_shards, - missing_shards, - data_shards = erasure.data_shards, - parity_shards = erasure.parity_shards, - "Shard availability check" - ); - - if missing_shards > 0 && available_shards >= erasure.data_shards { - // We have missing shards but enough to read - trigger background heal - info!( - bucket, - object, - part_number, - missing_shards, - available_shards, - pool_index, - set_index, - "Detected missing shards during read, triggering background heal" - ); - if let Err(e) = - rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options( - bucket.to_string(), - Some(object.to_string()), - false, - Some(HealChannelPriority::Normal), - Some(pool_index), - Some(set_index), - )) - .await - { - warn!( - bucket, - object, - part_number, - error = %e, - "Failed to enqueue heal request for missing shards" - ); - } else { - warn!(bucket, object, part_number, "Successfully enqueued heal request for missing shards"); - } - } - - // debug!( - // "read part {} part_offset {},part_length {},part_size {} ", - // part_number, part_offset, part_length, part_size - // ); - let (written, err) = erasure.decode(writer, readers, part_offset, part_length, part_size).await; - debug!( - bucket, - object, - part_index = current_part, - part_number, - part_length, - bytes_written = written, - "Finished decoding multipart part" - ); - if let Some(e) = err { - let de_err: DiskError = e.into(); - let mut has_err = true; - if written == part_length { - match de_err { - DiskError::FileNotFound | DiskError::FileCorrupt => { - error!("erasure.decode err 111 {:?}", &de_err); - if let Err(e) = rustfs_common::heal_channel::send_heal_request( - rustfs_common::heal_channel::create_heal_request_with_options( - bucket.to_string(), - Some(object.to_string()), - false, - Some(HealChannelPriority::Normal), - Some(pool_index), - Some(set_index), - ), - ) - .await - { - warn!( - bucket, - object, - part_number, - error = %e, - "Failed to enqueue heal request after decode error" - ); - } - has_err = false; - } - _ => {} - } - } - - if has_err { - error!("erasure.decode err {} {:?}", written, &de_err); - return Err(de_err.into()); - } - } - - // debug!("ec decode {} written size {}", part_number, n); - - total_read += part_length; - part_offset = 0; - } - - // debug!("read end"); - - debug!(bucket, object, total_read, expected_length = length, "Multipart read finished"); - - Ok(()) - } - - async fn update_object_meta( - &self, - bucket: &str, - object: &str, - fi: FileInfo, - disks: &[Option], - ) -> disk::error::Result<()> { - self.update_object_meta_with_opts(bucket, object, fi, disks, &UpdateMetadataOpts::default()) - .await - } - async fn update_object_meta_with_opts( - &self, - bucket: &str, - object: &str, - fi: FileInfo, - disks: &[Option], - opts: &UpdateMetadataOpts, - ) -> disk::error::Result<()> { - if fi.metadata.is_empty() { - return Ok(()); - } - - let mut futures = Vec::with_capacity(disks.len()); - - let mut errs = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { - let fi = fi.clone(); - futures.push(async move { - if let Some(disk) = disk { - disk.update_metadata(bucket, object, fi, opts).await - } else { - Err(DiskError::DiskNotFound) - } - }) - } - - let results = join_all(futures).await; - for result in results { - match result { - Ok(_) => { - errs.push(None); - } - Err(e) => { - errs.push(Some(e)); - } - } - } - - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, fi.write_quorum(self.default_write_quorum())) { - return Err(err); - } - - Ok(()) - } - - async fn get_online_disk_with_healing(&self, incl_healing: bool) -> Result<(Vec>, bool)> { - let (new_disks, _, healing) = self.get_online_disk_with_healing_and_info(incl_healing).await?; - Ok((new_disks, healing > 0)) - } - - async fn get_online_disk_with_healing_and_info( - &self, - incl_healing: bool, - ) -> Result<(Vec>, Vec, usize)> { - let mut infos = vec![DiskInfo::default(); self.disks.read().await.len()]; - for (idx, disk) in self.disks.write().await.iter().enumerate() { - if let Some(disk) = disk { - match disk.disk_info(&DiskInfoOptions::default()).await { - Ok(disk_info) => infos[idx] = disk_info, - Err(err) => infos[idx].error = err.to_string(), - } - } else { - infos[idx].error = "disk not found".to_string(); - } - } - - let mut new_disks = Vec::new(); - let mut healing_disks = Vec::new(); - let mut scanning_disks = Vec::new(); - let mut new_infos = Vec::new(); - let mut healing_infos = Vec::new(); - let mut scanning_infos = Vec::new(); - let mut healing = 0; - - infos.iter().zip(self.disks.write().await.iter()).for_each(|(info, disk)| { - if info.error.is_empty() { - if info.healing { - healing += 1; - if incl_healing { - healing_disks.push(disk.clone()); - healing_infos.push(info.clone()); - } - } else if !info.scanning { - new_disks.push(disk.clone()); - new_infos.push(info.clone()); - } else { - scanning_disks.push(disk.clone()); - scanning_infos.push(info.clone()); - } - } - }); - - // Prefer non-scanning disks over disks which are currently being scanned. - new_disks.extend(scanning_disks); - new_infos.extend(scanning_infos); - - // Then add healing disks. - new_disks.extend(healing_disks); - new_infos.extend(healing_infos); - - Ok((new_disks, new_infos, healing)) - } - - #[tracing::instrument(skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))] - async fn heal_object( - &self, - bucket: &str, - object: &str, - version_id: &str, - opts: &HealOpts, - ) -> disk::error::Result<(HealResultItem, Option)> { - info!(?opts, "Starting heal_object"); - - let disks = self.get_disks_internal().await; - - let mut result = HealResultItem { - heal_item_type: HealItemType::Object.to_string(), - bucket: bucket.to_string(), - object: object.to_string(), - version_id: version_id.to_string(), - disk_count: disks.len(), - ..Default::default() - }; - - let write_lock_guard = if !opts.no_lock { - let ns_lock = self.new_ns_lock(bucket, object).await?; - Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { - StorageError::other(format!( - "Failed to acquire write lock: {}", - self.format_lock_error_from_error(bucket, object, "write", &e) - )) - })?) - } else { - None - }; - - let version_id_op = { - if version_id.is_empty() { - None - } else { - Some(version_id.to_string()) - } - }; - - let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await?; - - info!( - parts_count = parts_metadata.len(), - bucket = bucket, - object = object, - version_id = version_id, - ?errs, - "File info read complete" - ); - if DiskError::is_all_not_found(&errs) { - warn!( - "heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}", - bucket, object, version_id - ); - let err = if !version_id.is_empty() { - DiskError::FileVersionNotFound - } else { - DiskError::FileNotFound - }; - // Nothing to do, file is already gone. - return Ok(( - self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id) - .await, - Some(err), - )); - } - - info!(parts_count = parts_metadata.len(), "heal_object Initiating quorum check"); - match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) { - Ok((read_quorum, _)) => { - result.parity_blocks = result.disk_count - read_quorum as usize; - result.data_blocks = read_quorum as usize; - - let ((mut online_disks, quorum_mod_time, quorum_etag), disk_len) = { - let disks = self.disks.read().await; - let disk_len = disks.len(); - (Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize), disk_len) - }; - - info!(?parts_metadata, ?errs, ?read_quorum, ?disk_len, "heal_object List disks metadata"); - - info!(?online_disks, ?quorum_mod_time, ?quorum_etag, "heal_object List online disks"); - - let filter_by_etag = quorum_etag.is_some(); - match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) { - Ok(latest_meta) => { - info!("heal_object latest_meta: {:?}", latest_meta); - - let (data_errs_by_disk, data_errs_by_part) = disks_with_all_parts( - &mut online_disks, - &mut parts_metadata, - &errs, - &latest_meta, - filter_by_etag, - bucket, - object, - opts.scan_mode, - ) - .await?; - - info!( - "disks_with_all_parts heal_object results: available_disks count={}, total_disks={}", - online_disks.iter().filter(|d| d.is_some()).count(), - online_disks.len() - ); - - let erasure = if !latest_meta.deleted && !latest_meta.is_remote() { - // Initialize erasure coding - erasure_coding::Erasure::new( - latest_meta.erasure.data_blocks, - latest_meta.erasure.parity_blocks, - latest_meta.erasure.block_size, - ) - } else { - erasure_coding::Erasure::default() - }; - - result.object_size = - ObjectInfo::from_file_info(&latest_meta, bucket, object, true).get_actual_size()? as usize; - // Loop to find number of disks with valid data, per-drive - // data state and a list of outdated disks on which data needs - // to be healed. - let mut out_dated_disks = vec![None; disk_len]; - let mut disks_to_heal_count = 0; - let mut meta_to_heal_count = 0; - - for index in 0..online_disks.len() { - let (yes, is_meta, reason) = should_heal_object_on_disk( - &errs[index], - &data_errs_by_disk[&index], - &parts_metadata[index], - &latest_meta, - ); - - if yes { - out_dated_disks[index] = disks[index].clone(); - disks_to_heal_count += 1; - if is_meta { - meta_to_heal_count += 1; - } - debug!("heal_object Disk {} marked for healing (endpoint={})", index, self.set_endpoints[index]); - } - - let drive_state = match reason { - Some(err) => match err { - DiskError::DiskNotFound => DriveState::Offline.to_string(), - DiskError::FileNotFound - | DiskError::FileVersionNotFound - | DiskError::VolumeNotFound - | DiskError::PartMissingOrCorrupt - | DiskError::OutdatedXLMeta => DriveState::Missing.to_string(), - DiskError::FileCorrupt => DriveState::Corrupt.to_string(), - _ => DriveState::Unknown(err.to_string()).to_string(), - }, - None => DriveState::Ok.to_string(), - }; - result.before.drives.push(HealDriveInfo { - uuid: "".to_string(), - endpoint: self.set_endpoints[index].to_string(), - state: drive_state.to_string(), - }); - - result.after.drives.push(HealDriveInfo { - uuid: "".to_string(), - endpoint: self.set_endpoints[index].to_string(), - state: drive_state.to_string(), - }); - } - - if disks_to_heal_count == 0 { - return Ok((result, None)); - } - - if opts.dry_run { - return Ok((result, None)); - } - - let mut cannot_heal = !latest_meta.deleted && meta_to_heal_count > latest_meta.erasure.parity_blocks; - if cannot_heal && quorum_etag.is_some() { - cannot_heal = false; - } - - if !latest_meta.deleted && !latest_meta.is_remote() { - for (_, part_errs) in data_errs_by_part.iter() { - if count_part_not_success(part_errs) > latest_meta.erasure.parity_blocks { - cannot_heal = true; - break; - } - } - } - - if cannot_heal { - let total_disks = parts_metadata.len(); - let healthy_count = total_disks.saturating_sub(disks_to_heal_count); - let required_data = total_disks.saturating_sub(latest_meta.erasure.parity_blocks); - - error!( - "Data corruption detected for {}/{}: Insufficient healthy shards. Need at least {} data shards, but found only {} healthy disks. (Missing/Corrupt: {}, Parity: {})", - bucket, - object, - required_data, - healthy_count, - disks_to_heal_count, - latest_meta.erasure.parity_blocks - ); - - // Allow for dangling deletes, on versions that have DataDir missing etc. - // this would end up restoring the correct readable versions. - return match self - .delete_if_dang_ling( - bucket, - object, - &parts_metadata, - &errs, - &data_errs_by_part, - ObjectOptions { - version_id: version_id_op.clone(), - ..Default::default() - }, - ) - .await - { - Ok(m) => { - let derr = if !version_id.is_empty() { - DiskError::FileVersionNotFound - } else { - DiskError::FileNotFound - }; - let mut t_errs = Vec::with_capacity(errs.len()); - for _ in 0..errs.len() { - t_errs.push(None); - } - Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, Some(derr))) - } - Err(err) => { - // t_errs = vec![Some(err.clone()]; errs.len()); - let mut t_errs = Vec::with_capacity(errs.len()); - for _ in 0..errs.len() { - t_errs.push(Some(err.clone())); - } - - Ok(( - self.default_heal_result(FileInfo::default(), &t_errs, bucket, object, version_id) - .await, - Some(err), - )) - } - }; - } - - if !latest_meta.deleted && latest_meta.erasure.distribution.len() != online_disks.len() { - let err_str = format!( - "unexpected file distribution ({:?}) from available disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", - latest_meta.erasure.distribution, online_disks, bucket, object, version_id - ); - warn!(err_str); - let err = DiskError::other(err_str); - return Ok(( - self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await, - Some(err), - )); - } - - let latest_disks = Self::shuffle_disks(&online_disks, &latest_meta.erasure.distribution); - if !latest_meta.deleted && latest_meta.erasure.distribution.len() != out_dated_disks.len() { - let err_str = format!( - "unexpected file distribution ({:?}) from outdated disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", - latest_meta.erasure.distribution, out_dated_disks, bucket, object, version_id - ); - warn!(err_str); - let err = DiskError::other(err_str); - return Ok(( - self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await, - Some(err), - )); - } - - if !latest_meta.deleted && latest_meta.erasure.distribution.len() != parts_metadata.len() { - let err_str = format!( - "unexpected file distribution ({:?}) from metadata entries ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", - latest_meta.erasure.distribution, - parts_metadata.len(), - bucket, - object, - version_id - ); - warn!(err_str); - let err = DiskError::other(err_str); - return Ok(( - self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await, - Some(err), - )); - } - - out_dated_disks = Self::shuffle_disks(&out_dated_disks, &latest_meta.erasure.distribution); - let mut parts_metadata = Self::shuffle_parts_metadata(&parts_metadata, &latest_meta.erasure.distribution); - let mut copy_parts_metadata = vec![None; parts_metadata.len()]; - for (index, disk) in latest_disks.iter().enumerate() { - if disk.is_some() { - copy_parts_metadata[index] = Some(parts_metadata[index].clone()); - } - } - - let clean_file_info = |fi: &FileInfo| -> FileInfo { - let mut nfi = fi.clone(); - if !nfi.is_remote() { - nfi.data = None; - nfi.erasure.index = 0; - nfi.erasure.checksums = Vec::new(); - } - nfi - }; - for (index, disk) in out_dated_disks.iter().enumerate() { - if disk.is_some() { - // Make sure to write the FileInfo information - // that is expected to be in quorum. - parts_metadata[index] = clean_file_info(&latest_meta); - } - } - - // We write at temporary location and then rename to final location. - let tmp_id = Uuid::new_v4().to_string(); - let src_data_dir = latest_meta.data_dir.unwrap().to_string(); - let dst_data_dir = latest_meta.data_dir.unwrap(); - - if !latest_meta.deleted && !latest_meta.is_remote() { - let erasure_info = latest_meta.erasure.clone(); - - for (part_index, part) in latest_meta.parts.iter().enumerate() { - let till_offset = erasure.shard_file_offset(0, part.size, part.size); - let checksum_algo = erasure_info.get_checksum_info(part.number).algorithm; - let mut readers = Vec::with_capacity(latest_disks.len()); - let mut writers = Vec::with_capacity(out_dated_disks.len()); - // let mut errors = Vec::with_capacity(out_dated_disks.len()); - - let mut prefer = vec![false; latest_disks.len()]; - for (index, disk) in latest_disks.iter().enumerate() { - let this_part_errs = - Self::shuffle_check_parts(&data_errs_by_part[&part_index], &erasure_info.distribution); - if this_part_errs[index] != CHECK_PART_SUCCESS { - info!( - "reading part {}: index={}, part_errs={:?}, skipping", - part.number, index, this_part_errs[index] - ); - readers.push(None); - continue; - } - - if let (Some(disk), Some(metadata)) = (disk, ©_parts_metadata[index]) { - match create_bitrot_reader( - metadata.data.as_deref(), - Some(disk), - bucket, - &path_join_buf(&[object, &src_data_dir, &format!("part.{}", part.number)]), - 0, - till_offset, - erasure.shard_size(), - checksum_algo.clone(), - ) - .await - { - Ok(Some(reader)) => { - readers.push(Some(reader)); - } - Ok(None) => { - readers.push(None); - continue; - } - Err(e) => { - readers.push(None); - continue; - } - } - - prefer[index] = disk.host_name().is_empty(); - } else { - readers.push(None); - // errors.push(Some(DiskError::DiskNotFound)); - } - } - - let is_inline_buffer = { - if let Some(sc) = GLOBAL_STORAGE_CLASS.get() { - sc.should_inline(erasure.shard_file_size(latest_meta.size), false) - } else { - false - } - }; - // create writers for all disk positions, but only for outdated disks - for (index, disk_op) in out_dated_disks.iter().enumerate() { - if let Some(outdated_disk) = disk_op { - let writer = match create_bitrot_writer( - is_inline_buffer, - Some(outdated_disk), - RUSTFS_META_TMP_BUCKET, - &path_join_buf(&[ - &tmp_id.to_string(), - &dst_data_dir.to_string(), - &format!("part.{}", part.number), - ]), - erasure.shard_file_size(part.size as i64), - erasure.shard_size(), - HashAlgorithm::HighwayHash256, - ) - .await - { - Ok(writer) => writer, - Err(err) => { - info!( - "create_bitrot_writer disk {}, err {:?}, skipping operation", - outdated_disk.to_string(), - err - ); - writers.push(None); - continue; - } - }; - writers.push(Some(writer)); - } else { - writers.push(None); - } - } - - // Heal each part. erasure.Heal() will write the healed - // part to .rustfs/tmp/uuid/ which needs to be renamed - // later to the final location. - erasure.heal(&mut writers, readers, part.size, &prefer).await?; - // close_bitrot_writers(&mut writers).await?; - - for (index, disk_op) in out_dated_disks.iter_mut().enumerate() { - if disk_op.is_none() { - continue; - } - - if writers[index].is_none() { - *disk_op = None; - disks_to_heal_count -= 1; - continue; - } - - parts_metadata[index].data_dir = Some(dst_data_dir); - parts_metadata[index].add_object_part( - part.number, - part.etag.clone(), - part.size, - part.mod_time, - part.actual_size, - part.index.clone(), - part.checksums.clone(), - ); - if is_inline_buffer { - if let Some(writer) = writers[index].take() { - // if let Some(w) = writer.as_any().downcast_ref::() { - // parts_metadata[index].data = Some(w.inline_data().to_vec()); - // } - parts_metadata[index].data = - Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default()); - } - parts_metadata[index].set_inline_data(); - } else { - parts_metadata[index].data = None; - } - } - - if disks_to_heal_count == 0 { - return Ok(( - result, - Some(DiskError::other(format!( - "all drives had write errors, unable to heal {bucket}/{object}" - ))), - )); - } - } - } - // Rename from tmp location to the actual location. - for (index, outdated_disk) in out_dated_disks.iter().enumerate() { - if let Some(disk) = outdated_disk { - // record the index of the updated disks - parts_metadata[index].erasure.index = index + 1; - // Attempt a rename now from healed data to final location. - parts_metadata[index].set_healing(); - - let rename_result = disk - .rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object) - .await; - - if let Err(err) = &rename_result { - self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id) - .await - .map_err(DiskError::other)?; - } else { - self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id) - .await - .map_err(DiskError::other)?; - - if parts_metadata[index].is_remote() { - let rm_data_dir = parts_metadata[index].data_dir.unwrap().to_string(); - - let d_path = Path::new(&encode_dir_object(object)).join(rm_data_dir); - - disk.delete( - bucket, - d_path.to_str().unwrap(), - DeleteOptions { - immediate: true, - - recursive: true, - - ..Default::default() - }, - ) - .await?; - } - - for (i, v) in result.before.drives.iter().enumerate() { - if v.endpoint == disk.endpoint().to_string() { - result.after.drives[i].state = DriveState::Ok.to_string(); - } - } - } - } - } - - Ok((result, None)) - } - Err(err) => Ok((result, Some(err))), - } - } - Err(err) => { - let data_errs_by_part = HashMap::new(); - match self - .delete_if_dang_ling( - bucket, - object, - &parts_metadata, - &errs, - &data_errs_by_part, - ObjectOptions { - version_id: version_id_op.clone(), - ..Default::default() - }, - ) - .await - { - Ok(m) => { - let err = if !version_id.is_empty() { - DiskError::FileVersionNotFound - } else { - DiskError::FileNotFound - }; - Ok((self.default_heal_result(m, &errs, bucket, object, version_id).await, Some(err))) - } - Err(_) => Ok(( - self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id) - .await, - Some(err), - )), - } - } - } - } - - /// Heal directory metadata assuming caller already holds the write lock for `(bucket, object)`. - async fn heal_object_dir_locked( - &self, - bucket: &str, - object: &str, - dry_run: bool, - remove: bool, - ) -> Result<(HealResultItem, Option)> { - let disks = { - let disks = self.disks.read().await; - disks.clone() - }; - let mut result = HealResultItem { - heal_item_type: HealItemType::Object.to_string(), - bucket: bucket.to_string(), - object: object.to_string(), - disk_count: self.disks.read().await.len(), - parity_blocks: self.default_parity_count, - data_blocks: disks.len() - self.default_parity_count, - object_size: 0, - ..Default::default() - }; - - result.before.drives = vec![HealDriveInfo::default(); disks.len()]; - result.after.drives = vec![HealDriveInfo::default(); disks.len()]; - - let errs = stat_all_dirs(&disks, bucket, object).await; - let dang_ling_object = is_object_dir_dang_ling(&errs); - if dang_ling_object && !dry_run && remove { - let mut futures = Vec::with_capacity(disks.len()); - for disk in disks.iter().flatten() { - let disk = disk.clone(); - let bucket = bucket.to_string(); - let object = object.to_string(); - futures.push(tokio::spawn(async move { - let _ = disk - .delete( - &bucket, - &object, - DeleteOptions { - recursive: false, - immediate: false, - ..Default::default() - }, - ) - .await; - })); - } - - // ignore errors - let _ = join_all(futures).await; - } - - for (err, drive) in errs.iter().zip(self.set_endpoints.iter()) { - let endpoint = drive.to_string(); - let drive_state = match err { - Some(err) => match err { - DiskError::DiskNotFound => DriveState::Offline.to_string(), - DiskError::FileNotFound | DiskError::VolumeNotFound => DriveState::Missing.to_string(), - _ => DriveState::Corrupt.to_string(), - }, - None => DriveState::Ok.to_string(), - }; - result.before.drives.push(HealDriveInfo { - uuid: "".to_string(), - endpoint: endpoint.clone(), - state: drive_state.to_string(), - }); - - result.after.drives.push(HealDriveInfo { - uuid: "".to_string(), - endpoint, - state: drive_state.to_string(), - }); - } - - if dang_ling_object || DiskError::is_all_not_found(&errs) { - return Ok((result, Some(DiskError::FileNotFound))); - } - - if dry_run { - // Quit without try to heal the object dir - return Ok((result, None)); - } - for (index, (err, disk)) in errs.iter().zip(disks.iter()).enumerate() { - if let (Some(DiskError::VolumeNotFound | DiskError::FileNotFound), Some(disk)) = (err, disk) { - let vol_path = Path::new(bucket).join(object); - let drive_state = match disk.make_volume(vol_path.to_str().unwrap()).await { - Ok(_) => DriveState::Ok.to_string(), - Err(merr) => match merr { - DiskError::VolumeExists => DriveState::Ok.to_string(), - DiskError::DiskNotFound => DriveState::Offline.to_string(), - _ => DriveState::Corrupt.to_string(), - }, - }; - result.after.drives[index].state = drive_state.to_string(); - } - } - - Ok((result, None)) - } - - #[allow(dead_code)] - /// Heal directory metadata after acquiring the necessary write lock. - async fn heal_object_dir( - &self, - bucket: &str, - object: &str, - dry_run: bool, - remove: bool, - ) -> Result<(HealResultItem, Option)> { - let _write_lock_guard = self - .new_ns_lock(bucket, object) - .await? - .get_write_lock(get_lock_acquire_timeout()) - .await - .map_err(|e| { - let message = format!( - "Failed to acquire write lock: {}", - self.format_lock_error_from_error(bucket, object, "write", &e) - ); - DiskError::other(message) - })?; - - self.heal_object_dir_locked(bucket, object, dry_run, remove).await - } - - async fn default_heal_result( - &self, - lfi: FileInfo, - errs: &[Option], - bucket: &str, - object: &str, - version_id: &str, - ) -> HealResultItem { - let disk_len = { self.disks.read().await.len() }; - let mut result = HealResultItem { - heal_item_type: HealItemType::Object.to_string(), - bucket: bucket.to_string(), - object: object.to_string(), - object_size: lfi.size as usize, - version_id: version_id.to_string(), - disk_count: disk_len, - ..Default::default() - }; - - if lfi.is_valid() { - result.parity_blocks = lfi.erasure.parity_blocks; - } else { - result.parity_blocks = self.default_parity_count; - } - - result.data_blocks = disk_len - result.parity_blocks; - - for (index, disk) in self.disks.read().await.iter().enumerate() { - if disk.is_none() { - result.before.drives.push(HealDriveInfo { - uuid: "".to_string(), - endpoint: self.set_endpoints[index].to_string(), - state: DriveState::Offline.to_string(), - }); - - result.after.drives.push(HealDriveInfo { - uuid: "".to_string(), - endpoint: self.set_endpoints[index].to_string(), - state: DriveState::Offline.to_string(), - }); - } - - let mut drive_state = DriveState::Corrupt; - if let Some(err) = &errs[index] { - if err == &DiskError::FileNotFound || err == &DiskError::VolumeNotFound { - drive_state = DriveState::Missing; - } - } else { - drive_state = DriveState::Ok; - } - - result.before.drives.push(HealDriveInfo { - uuid: "".to_string(), - endpoint: self.set_endpoints[index].to_string(), - state: drive_state.to_string(), - }); - result.after.drives.push(HealDriveInfo { - uuid: "".to_string(), - endpoint: self.set_endpoints[index].to_string(), - state: drive_state.to_string(), - }); - } - result - } - - async fn delete_if_dang_ling( - &self, - bucket: &str, - object: &str, - meta_arr: &[FileInfo], - errs: &[Option], - data_errs_by_part: &HashMap>, - opts: ObjectOptions, - ) -> disk::error::Result { - let (m, can_heal) = is_object_dang_ling(meta_arr, errs, data_errs_by_part); - - if !can_heal { - return Err(DiskError::ErasureReadQuorum); - } - - let mut tags: HashMap = HashMap::new(); - tags.insert("set".to_string(), self.set_index.to_string()); - tags.insert("pool".to_string(), self.pool_index.to_string()); - tags.insert("merrs".to_string(), join_errs(errs)); - tags.insert("derrs".to_string(), format!("{data_errs_by_part:?}")); - if m.is_valid() { - tags.insert("sz".to_string(), m.size.to_string()); - tags.insert( - "mt".to_string(), - m.mod_time - .as_ref() - .map_or(String::new(), |mod_time| mod_time.unix_timestamp().to_string()), - ); - tags.insert("d:p".to_string(), format!("{}:{}", m.erasure.data_blocks, m.erasure.parity_blocks)); - } else { - tags.insert("invalid".to_string(), "1".to_string()); - tags.insert( - "d:p".to_string(), - format!("{}:{}", self.set_drive_count - self.default_parity_count, self.default_parity_count), - ); - } - let mut offline = 0; - for (i, err) in errs.iter().enumerate() { - let mut found = false; - if let Some(err) = err - && err == &DiskError::DiskNotFound - { - found = true; - } - for p in data_errs_by_part { - if let Some(v) = p.1.get(i) - && *v == CHECK_PART_DISK_NOT_FOUND - { - found = true; - break; - } - } - - if found { - offline += 1; - } - } - - if offline > 0 { - tags.insert("offline".to_string(), offline.to_string()); - } - - let mut fi = FileInfo::default(); - if let Some(ref version_id) = opts.version_id { - fi.version_id = Uuid::parse_str(version_id).ok(); - } - - fi.set_tier_free_version_id(&Uuid::new_v4().to_string()); - - let disks = self.get_disks_internal().await; - - let mut futures = Vec::with_capacity(disks.len()); - for disk_op in disks.iter() { - let bucket = bucket.to_string(); - let object = object.to_string(); - let fi = fi.clone(); - futures.push(async move { - if let Some(disk) = disk_op { - disk.delete_version(&bucket, &object, fi, false, DeleteOptions::default()) - .await - } else { - Err(DiskError::DiskNotFound) - } - }); - } - - let results = join_all(futures).await; - for (index, result) in results.into_iter().enumerate() { - let key = format!("ddisk-{index}"); - match result { - Ok(_) => { - tags.insert(key, "".to_string()); - } - Err(e) => { - tags.insert(key, e.to_string()); - } - } - } - - // TODO: audit - - Ok(m) - } - - async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> { - let disks = self.get_disks_internal().await; - let write_quorum = disks.len() / 2 + 1; - - let mut futures = Vec::with_capacity(disks.len()); - - for disk_op in disks.iter() { - let bucket = bucket.to_string(); - let prefix = prefix.to_string(); - futures.push(async move { - if let Some(disk) = disk_op { - disk.delete( - &bucket, - &prefix, - DeleteOptions { - recursive: true, - immediate: true, - ..Default::default() - }, - ) - .await - } else { - Ok(()) - } - }); - } - - let errs = join_all(futures).await.into_iter().map(|v| v.err()).collect::>(); - - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - return Err(err); - } - - Ok(()) - } - - pub async fn update_restore_metadata( - &self, - bucket: &str, - object: &str, - obj_info: &ObjectInfo, - opts: &ObjectOptions, - ) -> Result<()> { - let mut oi = obj_info.clone(); - oi.metadata_only = true; - - oi.user_defined.remove(X_AMZ_RESTORE.as_str()); - - let version_id = oi.version_id.map(|v| v.to_string()); - let _obj = self - .copy_object( - bucket, - object, - bucket, - object, - &mut oi, - &ObjectOptions { - version_id: version_id.clone(), - ..Default::default() - }, - &ObjectOptions { - version_id, - ..Default::default() - }, - ) - .await?; - Ok(()) - } - - async fn check_write_precondition(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Option { - let mut opts = opts.clone(); - - let http_preconditions = opts.http_preconditions?; - opts.http_preconditions = None; - - // Never claim a lock here, to avoid deadlock - // - If no_lock is false, we must have obtained the lock out side of this function - // - If no_lock is true, we should not obtain locks - opts.no_lock = true; - let oi = self.get_object_info(bucket, object, &opts).await; - - match oi { - Ok(oi) => { - // If top level is a delete marker proceed to upload. - if oi.delete_marker { - return None; - } - if should_prevent_write(&oi, http_preconditions.if_none_match, http_preconditions.if_match) { - return Some(StorageError::PreconditionFailed); - } - } - - Err(StorageError::VersionNotFound(_, _, _)) - | Err(StorageError::ObjectNotFound(_, _)) - | Err(StorageError::ErasureReadQuorum) => { - // When the object is not found, - // - if If-Match is set, we should return 404 NotFound - // - if If-None-Match is set, we should be able to proceed with the request - if http_preconditions.if_match.is_some() { - return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string())); - } - } - - Err(e) => { - return Some(e); - } - } - - None - } } #[async_trait::async_trait] diff --git a/crates/ecstore/src/set_disk/heal.rs b/crates/ecstore/src/set_disk/heal.rs new file mode 100644 index 000000000..90038def3 --- /dev/null +++ b/crates/ecstore/src/set_disk/heal.rs @@ -0,0 +1,770 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +impl SetDisks { + #[tracing::instrument(skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))] + pub(super) async fn heal_object( + &self, + bucket: &str, + object: &str, + version_id: &str, + opts: &HealOpts, + ) -> disk::error::Result<(HealResultItem, Option)> { + info!(?opts, "Starting heal_object"); + + let disks = self.get_disks_internal().await; + + let mut result = HealResultItem { + heal_item_type: HealItemType::Object.to_string(), + bucket: bucket.to_string(), + object: object.to_string(), + version_id: version_id.to_string(), + disk_count: disks.len(), + ..Default::default() + }; + + let write_lock_guard = if !opts.no_lock { + let ns_lock = self.new_ns_lock(bucket, object).await?; + Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| { + StorageError::other(format!( + "Failed to acquire write lock: {}", + self.format_lock_error_from_error(bucket, object, "write", &e) + )) + })?) + } else { + None + }; + + let version_id_op = { + if version_id.is_empty() { + None + } else { + Some(version_id.to_string()) + } + }; + + let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await?; + + info!( + parts_count = parts_metadata.len(), + bucket = bucket, + object = object, + version_id = version_id, + ?errs, + "File info read complete" + ); + if DiskError::is_all_not_found(&errs) { + warn!( + "heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}", + bucket, object, version_id + ); + let err = if !version_id.is_empty() { + DiskError::FileVersionNotFound + } else { + DiskError::FileNotFound + }; + // Nothing to do, file is already gone. + return Ok(( + self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id) + .await, + Some(err), + )); + } + + info!(parts_count = parts_metadata.len(), "heal_object Initiating quorum check"); + match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) { + Ok((read_quorum, _)) => { + result.parity_blocks = result.disk_count - read_quorum as usize; + result.data_blocks = read_quorum as usize; + + let ((mut online_disks, quorum_mod_time, quorum_etag), disk_len) = { + let disks = self.disks.read().await; + let disk_len = disks.len(); + (Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize), disk_len) + }; + + info!(?parts_metadata, ?errs, ?read_quorum, ?disk_len, "heal_object List disks metadata"); + + info!(?online_disks, ?quorum_mod_time, ?quorum_etag, "heal_object List online disks"); + + let filter_by_etag = quorum_etag.is_some(); + match Self::pick_valid_fileinfo(&parts_metadata, quorum_mod_time, quorum_etag.clone(), read_quorum as usize) { + Ok(latest_meta) => { + info!("heal_object latest_meta: {:?}", latest_meta); + + let (data_errs_by_disk, data_errs_by_part) = disks_with_all_parts( + &mut online_disks, + &mut parts_metadata, + &errs, + &latest_meta, + filter_by_etag, + bucket, + object, + opts.scan_mode, + ) + .await?; + + info!( + "disks_with_all_parts heal_object results: available_disks count={}, total_disks={}", + online_disks.iter().filter(|d| d.is_some()).count(), + online_disks.len() + ); + + let erasure = if !latest_meta.deleted && !latest_meta.is_remote() { + // Initialize erasure coding + erasure_coding::Erasure::new( + latest_meta.erasure.data_blocks, + latest_meta.erasure.parity_blocks, + latest_meta.erasure.block_size, + ) + } else { + erasure_coding::Erasure::default() + }; + + result.object_size = + ObjectInfo::from_file_info(&latest_meta, bucket, object, true).get_actual_size()? as usize; + // Loop to find number of disks with valid data, per-drive + // data state and a list of outdated disks on which data needs + // to be healed. + let mut out_dated_disks = vec![None; disk_len]; + let mut disks_to_heal_count = 0; + let mut meta_to_heal_count = 0; + + for index in 0..online_disks.len() { + let (yes, is_meta, reason) = should_heal_object_on_disk( + &errs[index], + &data_errs_by_disk[&index], + &parts_metadata[index], + &latest_meta, + ); + + if yes { + out_dated_disks[index] = disks[index].clone(); + disks_to_heal_count += 1; + if is_meta { + meta_to_heal_count += 1; + } + debug!("heal_object Disk {} marked for healing (endpoint={})", index, self.set_endpoints[index]); + } + + let drive_state = match reason { + Some(err) => match err { + DiskError::DiskNotFound => DriveState::Offline.to_string(), + DiskError::FileNotFound + | DiskError::FileVersionNotFound + | DiskError::VolumeNotFound + | DiskError::PartMissingOrCorrupt + | DiskError::OutdatedXLMeta => DriveState::Missing.to_string(), + DiskError::FileCorrupt => DriveState::Corrupt.to_string(), + _ => DriveState::Unknown(err.to_string()).to_string(), + }, + None => DriveState::Ok.to_string(), + }; + result.before.drives.push(HealDriveInfo { + uuid: "".to_string(), + endpoint: self.set_endpoints[index].to_string(), + state: drive_state.to_string(), + }); + + result.after.drives.push(HealDriveInfo { + uuid: "".to_string(), + endpoint: self.set_endpoints[index].to_string(), + state: drive_state.to_string(), + }); + } + + if disks_to_heal_count == 0 { + return Ok((result, None)); + } + + if opts.dry_run { + return Ok((result, None)); + } + + let mut cannot_heal = !latest_meta.deleted && meta_to_heal_count > latest_meta.erasure.parity_blocks; + if cannot_heal && quorum_etag.is_some() { + cannot_heal = false; + } + + if !latest_meta.deleted && !latest_meta.is_remote() { + for (_, part_errs) in data_errs_by_part.iter() { + if count_part_not_success(part_errs) > latest_meta.erasure.parity_blocks { + cannot_heal = true; + break; + } + } + } + + if cannot_heal { + let total_disks = parts_metadata.len(); + let healthy_count = total_disks.saturating_sub(disks_to_heal_count); + let required_data = total_disks.saturating_sub(latest_meta.erasure.parity_blocks); + + error!( + "Data corruption detected for {}/{}: Insufficient healthy shards. Need at least {} data shards, but found only {} healthy disks. (Missing/Corrupt: {}, Parity: {})", + bucket, + object, + required_data, + healthy_count, + disks_to_heal_count, + latest_meta.erasure.parity_blocks + ); + + // Allow for dangling deletes, on versions that have DataDir missing etc. + // this would end up restoring the correct readable versions. + return match self + .delete_if_dang_ling( + bucket, + object, + &parts_metadata, + &errs, + &data_errs_by_part, + ObjectOptions { + version_id: version_id_op.clone(), + ..Default::default() + }, + ) + .await + { + Ok(m) => { + let derr = if !version_id.is_empty() { + DiskError::FileVersionNotFound + } else { + DiskError::FileNotFound + }; + let mut t_errs = Vec::with_capacity(errs.len()); + for _ in 0..errs.len() { + t_errs.push(None); + } + Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, Some(derr))) + } + Err(err) => { + // t_errs = vec![Some(err.clone()]; errs.len()); + let mut t_errs = Vec::with_capacity(errs.len()); + for _ in 0..errs.len() { + t_errs.push(Some(err.clone())); + } + + Ok(( + self.default_heal_result(FileInfo::default(), &t_errs, bucket, object, version_id) + .await, + Some(err), + )) + } + }; + } + + if !latest_meta.deleted && latest_meta.erasure.distribution.len() != online_disks.len() { + let err_str = format!( + "unexpected file distribution ({:?}) from available disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", + latest_meta.erasure.distribution, online_disks, bucket, object, version_id + ); + warn!(err_str); + let err = DiskError::other(err_str); + return Ok(( + self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await, + Some(err), + )); + } + + let latest_disks = Self::shuffle_disks(&online_disks, &latest_meta.erasure.distribution); + if !latest_meta.deleted && latest_meta.erasure.distribution.len() != out_dated_disks.len() { + let err_str = format!( + "unexpected file distribution ({:?}) from outdated disks ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", + latest_meta.erasure.distribution, out_dated_disks, bucket, object, version_id + ); + warn!(err_str); + let err = DiskError::other(err_str); + return Ok(( + self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await, + Some(err), + )); + } + + if !latest_meta.deleted && latest_meta.erasure.distribution.len() != parts_metadata.len() { + let err_str = format!( + "unexpected file distribution ({:?}) from metadata entries ({:?}), looks like backend disks have been manually modified refusing to heal {}/{}({})", + latest_meta.erasure.distribution, + parts_metadata.len(), + bucket, + object, + version_id + ); + warn!(err_str); + let err = DiskError::other(err_str); + return Ok(( + self.default_heal_result(latest_meta, &errs, bucket, object, version_id).await, + Some(err), + )); + } + + out_dated_disks = Self::shuffle_disks(&out_dated_disks, &latest_meta.erasure.distribution); + let mut parts_metadata = Self::shuffle_parts_metadata(&parts_metadata, &latest_meta.erasure.distribution); + let mut copy_parts_metadata = vec![None; parts_metadata.len()]; + for (index, disk) in latest_disks.iter().enumerate() { + if disk.is_some() { + copy_parts_metadata[index] = Some(parts_metadata[index].clone()); + } + } + + let clean_file_info = |fi: &FileInfo| -> FileInfo { + let mut nfi = fi.clone(); + if !nfi.is_remote() { + nfi.data = None; + nfi.erasure.index = 0; + nfi.erasure.checksums = Vec::new(); + } + nfi + }; + for (index, disk) in out_dated_disks.iter().enumerate() { + if disk.is_some() { + // Make sure to write the FileInfo information + // that is expected to be in quorum. + parts_metadata[index] = clean_file_info(&latest_meta); + } + } + + // We write at temporary location and then rename to final location. + let tmp_id = Uuid::new_v4().to_string(); + let src_data_dir = latest_meta.data_dir.unwrap().to_string(); + let dst_data_dir = latest_meta.data_dir.unwrap(); + + if !latest_meta.deleted && !latest_meta.is_remote() { + let erasure_info = latest_meta.erasure.clone(); + + for (part_index, part) in latest_meta.parts.iter().enumerate() { + let till_offset = erasure.shard_file_offset(0, part.size, part.size); + let checksum_algo = erasure_info.get_checksum_info(part.number).algorithm; + let mut readers = Vec::with_capacity(latest_disks.len()); + let mut writers = Vec::with_capacity(out_dated_disks.len()); + // let mut errors = Vec::with_capacity(out_dated_disks.len()); + + let mut prefer = vec![false; latest_disks.len()]; + for (index, disk) in latest_disks.iter().enumerate() { + let this_part_errs = + Self::shuffle_check_parts(&data_errs_by_part[&part_index], &erasure_info.distribution); + if this_part_errs[index] != CHECK_PART_SUCCESS { + info!( + "reading part {}: index={}, part_errs={:?}, skipping", + part.number, index, this_part_errs[index] + ); + readers.push(None); + continue; + } + + if let (Some(disk), Some(metadata)) = (disk, ©_parts_metadata[index]) { + match create_bitrot_reader( + metadata.data.as_deref(), + Some(disk), + bucket, + &path_join_buf(&[object, &src_data_dir, &format!("part.{}", part.number)]), + 0, + till_offset, + erasure.shard_size(), + checksum_algo.clone(), + ) + .await + { + Ok(Some(reader)) => { + readers.push(Some(reader)); + } + Ok(None) => { + readers.push(None); + continue; + } + Err(e) => { + readers.push(None); + continue; + } + } + + prefer[index] = disk.host_name().is_empty(); + } else { + readers.push(None); + // errors.push(Some(DiskError::DiskNotFound)); + } + } + + let is_inline_buffer = { + if let Some(sc) = GLOBAL_STORAGE_CLASS.get() { + sc.should_inline(erasure.shard_file_size(latest_meta.size), false) + } else { + false + } + }; + // create writers for all disk positions, but only for outdated disks + for (index, disk_op) in out_dated_disks.iter().enumerate() { + if let Some(outdated_disk) = disk_op { + let writer = match create_bitrot_writer( + is_inline_buffer, + Some(outdated_disk), + RUSTFS_META_TMP_BUCKET, + &path_join_buf(&[ + &tmp_id.to_string(), + &dst_data_dir.to_string(), + &format!("part.{}", part.number), + ]), + erasure.shard_file_size(part.size as i64), + erasure.shard_size(), + HashAlgorithm::HighwayHash256, + ) + .await + { + Ok(writer) => writer, + Err(err) => { + info!( + "create_bitrot_writer disk {}, err {:?}, skipping operation", + outdated_disk.to_string(), + err + ); + writers.push(None); + continue; + } + }; + writers.push(Some(writer)); + } else { + writers.push(None); + } + } + + // Heal each part. erasure.Heal() will write the healed + // part to .rustfs/tmp/uuid/ which needs to be renamed + // later to the final location. + erasure.heal(&mut writers, readers, part.size, &prefer).await?; + // close_bitrot_writers(&mut writers).await?; + + for (index, disk_op) in out_dated_disks.iter_mut().enumerate() { + if disk_op.is_none() { + continue; + } + + if writers[index].is_none() { + *disk_op = None; + disks_to_heal_count -= 1; + continue; + } + + parts_metadata[index].data_dir = Some(dst_data_dir); + parts_metadata[index].add_object_part( + part.number, + part.etag.clone(), + part.size, + part.mod_time, + part.actual_size, + part.index.clone(), + part.checksums.clone(), + ); + if is_inline_buffer { + if let Some(writer) = writers[index].take() { + // if let Some(w) = writer.as_any().downcast_ref::() { + // parts_metadata[index].data = Some(w.inline_data().to_vec()); + // } + parts_metadata[index].data = + Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default()); + } + parts_metadata[index].set_inline_data(); + } else { + parts_metadata[index].data = None; + } + } + + if disks_to_heal_count == 0 { + return Ok(( + result, + Some(DiskError::other(format!( + "all drives had write errors, unable to heal {bucket}/{object}" + ))), + )); + } + } + } + // Rename from tmp location to the actual location. + for (index, outdated_disk) in out_dated_disks.iter().enumerate() { + if let Some(disk) = outdated_disk { + // record the index of the updated disks + parts_metadata[index].erasure.index = index + 1; + // Attempt a rename now from healed data to final location. + parts_metadata[index].set_healing(); + + let rename_result = disk + .rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object) + .await; + + if let Err(err) = &rename_result { + self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id) + .await + .map_err(DiskError::other)?; + } else { + self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id) + .await + .map_err(DiskError::other)?; + + if parts_metadata[index].is_remote() { + let rm_data_dir = parts_metadata[index].data_dir.unwrap().to_string(); + + let d_path = Path::new(&encode_dir_object(object)).join(rm_data_dir); + + disk.delete( + bucket, + d_path.to_str().unwrap(), + DeleteOptions { + immediate: true, + + recursive: true, + + ..Default::default() + }, + ) + .await?; + } + + for (i, v) in result.before.drives.iter().enumerate() { + if v.endpoint == disk.endpoint().to_string() { + result.after.drives[i].state = DriveState::Ok.to_string(); + } + } + } + } + } + + Ok((result, None)) + } + Err(err) => Ok((result, Some(err))), + } + } + Err(err) => { + let data_errs_by_part = HashMap::new(); + match self + .delete_if_dang_ling( + bucket, + object, + &parts_metadata, + &errs, + &data_errs_by_part, + ObjectOptions { + version_id: version_id_op.clone(), + ..Default::default() + }, + ) + .await + { + Ok(m) => { + let err = if !version_id.is_empty() { + DiskError::FileVersionNotFound + } else { + DiskError::FileNotFound + }; + Ok((self.default_heal_result(m, &errs, bucket, object, version_id).await, Some(err))) + } + Err(_) => Ok(( + self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id) + .await, + Some(err), + )), + } + } + } + } + + pub(super) async fn heal_object_dir_locked( + &self, + bucket: &str, + object: &str, + dry_run: bool, + remove: bool, + ) -> Result<(HealResultItem, Option)> { + let disks = { + let disks = self.disks.read().await; + disks.clone() + }; + let mut result = HealResultItem { + heal_item_type: HealItemType::Object.to_string(), + bucket: bucket.to_string(), + object: object.to_string(), + disk_count: self.disks.read().await.len(), + parity_blocks: self.default_parity_count, + data_blocks: disks.len() - self.default_parity_count, + object_size: 0, + ..Default::default() + }; + + result.before.drives = vec![HealDriveInfo::default(); disks.len()]; + result.after.drives = vec![HealDriveInfo::default(); disks.len()]; + + let errs = stat_all_dirs(&disks, bucket, object).await; + let dang_ling_object = is_object_dir_dang_ling(&errs); + if dang_ling_object && !dry_run && remove { + let mut futures = Vec::with_capacity(disks.len()); + for disk in disks.iter().flatten() { + let disk = disk.clone(); + let bucket = bucket.to_string(); + let object = object.to_string(); + futures.push(tokio::spawn(async move { + let _ = disk + .delete( + &bucket, + &object, + DeleteOptions { + recursive: false, + immediate: false, + ..Default::default() + }, + ) + .await; + })); + } + + // ignore errors + let _ = join_all(futures).await; + } + + for (err, drive) in errs.iter().zip(self.set_endpoints.iter()) { + let endpoint = drive.to_string(); + let drive_state = match err { + Some(err) => match err { + DiskError::DiskNotFound => DriveState::Offline.to_string(), + DiskError::FileNotFound | DiskError::VolumeNotFound => DriveState::Missing.to_string(), + _ => DriveState::Corrupt.to_string(), + }, + None => DriveState::Ok.to_string(), + }; + result.before.drives.push(HealDriveInfo { + uuid: "".to_string(), + endpoint: endpoint.clone(), + state: drive_state.to_string(), + }); + + result.after.drives.push(HealDriveInfo { + uuid: "".to_string(), + endpoint, + state: drive_state.to_string(), + }); + } + + if dang_ling_object || DiskError::is_all_not_found(&errs) { + return Ok((result, Some(DiskError::FileNotFound))); + } + + if dry_run { + // Quit without try to heal the object dir + return Ok((result, None)); + } + for (index, (err, disk)) in errs.iter().zip(disks.iter()).enumerate() { + if let (Some(DiskError::VolumeNotFound | DiskError::FileNotFound), Some(disk)) = (err, disk) { + let vol_path = Path::new(bucket).join(object); + let drive_state = match disk.make_volume(vol_path.to_str().unwrap()).await { + Ok(_) => DriveState::Ok.to_string(), + Err(merr) => match merr { + DiskError::VolumeExists => DriveState::Ok.to_string(), + DiskError::DiskNotFound => DriveState::Offline.to_string(), + _ => DriveState::Corrupt.to_string(), + }, + }; + result.after.drives[index].state = drive_state.to_string(); + } + } + + Ok((result, None)) + } + + pub(super) async fn heal_object_dir( + &self, + bucket: &str, + object: &str, + dry_run: bool, + remove: bool, + ) -> Result<(HealResultItem, Option)> { + let _write_lock_guard = self + .new_ns_lock(bucket, object) + .await? + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|e| { + let message = format!( + "Failed to acquire write lock: {}", + self.format_lock_error_from_error(bucket, object, "write", &e) + ); + DiskError::other(message) + })?; + + self.heal_object_dir_locked(bucket, object, dry_run, remove).await + } + + pub(super) async fn default_heal_result( + &self, + lfi: FileInfo, + errs: &[Option], + bucket: &str, + object: &str, + version_id: &str, + ) -> HealResultItem { + let disk_len = { self.disks.read().await.len() }; + let mut result = HealResultItem { + heal_item_type: HealItemType::Object.to_string(), + bucket: bucket.to_string(), + object: object.to_string(), + object_size: lfi.size as usize, + version_id: version_id.to_string(), + disk_count: disk_len, + ..Default::default() + }; + + if lfi.is_valid() { + result.parity_blocks = lfi.erasure.parity_blocks; + } else { + result.parity_blocks = self.default_parity_count; + } + + result.data_blocks = disk_len - result.parity_blocks; + + for (index, disk) in self.disks.read().await.iter().enumerate() { + if disk.is_none() { + result.before.drives.push(HealDriveInfo { + uuid: "".to_string(), + endpoint: self.set_endpoints[index].to_string(), + state: DriveState::Offline.to_string(), + }); + + result.after.drives.push(HealDriveInfo { + uuid: "".to_string(), + endpoint: self.set_endpoints[index].to_string(), + state: DriveState::Offline.to_string(), + }); + } + + let mut drive_state = DriveState::Corrupt; + if let Some(err) = &errs[index] { + if err == &DiskError::FileNotFound || err == &DiskError::VolumeNotFound { + drive_state = DriveState::Missing; + } + } else { + drive_state = DriveState::Ok; + } + + result.before.drives.push(HealDriveInfo { + uuid: "".to_string(), + endpoint: self.set_endpoints[index].to_string(), + state: drive_state.to_string(), + }); + result.after.drives.push(HealDriveInfo { + uuid: "".to_string(), + endpoint: self.set_endpoints[index].to_string(), + state: drive_state.to_string(), + }); + } + result + } +} diff --git a/crates/ecstore/src/set_disk/list.rs b/crates/ecstore/src/set_disk/list.rs new file mode 100644 index 000000000..d729eafd6 --- /dev/null +++ b/crates/ecstore/src/set_disk/list.rs @@ -0,0 +1,59 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +impl SetDisks { + #[tracing::instrument(skip(self))] + pub async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> { + let disks = self.disks.read().await; + + let disks = disks.clone(); + + let mut futures = Vec::with_capacity(disks.len()); + let mut errors = Vec::with_capacity(disks.len()); + + for disk in disks.iter() { + futures.push(async move { + if let Some(disk) = disk { + disk.delete( + bucket, + prefix, + DeleteOptions { + recursive: true, + ..Default::default() + }, + ) + .await + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + let results = join_all(futures).await; + for result in results { + match result { + Ok(_) => { + errors.push(None); + } + Err(e) => { + errors.push(Some(e)); + } + } + } + + Ok(()) + } +} diff --git a/crates/ecstore/src/set_disk/lock.rs b/crates/ecstore/src/set_disk/lock.rs new file mode 100644 index 000000000..c323ade21 --- /dev/null +++ b/crates/ecstore/src/set_disk/lock.rs @@ -0,0 +1,369 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +impl SetDisks { + pub(super) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String { + match err { + LockResult::Timeout => { + format!("{mode} lock acquisition timed out on {bucket}/{object} (owner={})", self.locker_owner) + } + LockResult::Conflict { + current_owner, + current_mode, + } => format!("{mode} lock conflicted on {bucket}/{object}: held by {current_owner} as {current_mode:?}"), + LockResult::Acquired => format!("unexpected lock state while acquiring {mode} lock on {bucket}/{object}"), + } + } + + pub(super) fn format_lock_error_from_error( + &self, + bucket: &str, + object: &str, + mode: &str, + err: &rustfs_lock::error::LockError, + ) -> String { + match err { + rustfs_lock::error::LockError::Timeout { .. } => { + format!( + "ns_loc: {mode} lock acquisition timed out on {bucket}/{object} (owner={})", + self.locker_owner + ) + } + rustfs_lock::error::LockError::AlreadyLocked { owner, .. } => { + format!("ns_loc: {mode} lock conflicted on {bucket}/{object}: held by {owner}") + } + _ => format!("ns_loc: {mode} lock acquisition failed on {bucket}/{object}: {}", err), + } + } + + pub(super) async fn get_disks_internal(&self) -> Vec> { + let rl = self.disks.read().await; + + rl.clone() + } + + pub async fn get_local_disks(&self) -> Vec> { + let rl = self.disks.read().await; + + let mut disks: Vec> = rl + .clone() + .into_iter() + .filter(|v| v.as_ref().is_some_and(|d| d.is_local())) + .collect(); + + let mut rng = rand::rng(); + + disks.shuffle(&mut rng); + + disks + } + + pub(super) async fn get_online_disks(&self) -> Vec> { + let mut disks = self.get_disks_internal().await; + + // TODO: diskinfo filter online + + let mut new_disk = Vec::with_capacity(disks.len()); + + for disk in disks.iter() { + if let Some(d) = disk + && d.is_online().await + { + new_disk.push(disk.clone()); + } + } + + let mut rng = rand::rng(); + + disks.shuffle(&mut rng); + + new_disk + // let disks = self.get_disks_internal().await; + // let (filtered, _) = self.filter_online_disks(disks).await; + // filtered.into_iter().filter(|disk| disk.is_some()).collect() + } + + pub(super) async fn get_online_local_disks(&self) -> Vec> { + let mut disks = self.get_online_disks().await; + + let mut rng = rand::rng(); + + disks.shuffle(&mut rng); + + disks + .into_iter() + .filter(|v| v.as_ref().is_some_and(|d| d.is_local())) + .collect() + } + + pub async fn get_online_disks_with_healing(&self, incl_healing: bool) -> (Vec, bool) { + let (disks, _, healing) = self.get_online_disks_with_healing_and_info(incl_healing).await; + (disks, healing > 0) + } + + pub async fn get_online_disks_with_healing_and_info(&self, incl_healing: bool) -> (Vec, Vec, usize) { + let mut disks = self.get_disks_internal().await; + + let mut infos = Vec::with_capacity(disks.len()); + + let mut futures = Vec::with_capacity(disks.len()); + let mut numbers: Vec = (0..disks.len()).collect(); + { + let mut rng = rand::rng(); + disks.shuffle(&mut rng); + + numbers.shuffle(&mut rng); + } + + for &i in numbers.iter() { + let disk = disks[i].clone(); + futures.push(async move { + if let Some(disk) = disk { + disk.disk_info(&DiskInfoOptions::default()).await + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + // Use optimized batch processor for disk info retrieval + let processor = get_global_processors().metadata_processor(); + let results = processor.execute_batch(futures).await; + + for result in results { + match result { + Ok(res) => { + infos.push(res); + } + Err(err) => { + infos.push(DiskInfo { + error: err.to_string(), + ..Default::default() + }); + } + } + } + + let mut healing: usize = 0; + + let mut scanning_disks = Vec::new(); + let mut healing_disks = Vec::new(); + let mut scanning_infos = Vec::new(); + let mut healing_infos = Vec::new(); + + let mut new_disks = Vec::new(); + let mut new_infos = Vec::new(); + + for &i in numbers.iter() { + let (info, disk) = (infos[i].clone(), disks[i].clone()); + if !info.error.is_empty() || disk.is_none() { + continue; + } + + if info.healing { + healing += 1; + if incl_healing { + healing_disks.push(disk.unwrap()); + healing_infos.push(info); + } + + continue; + } + + if !info.healing { + new_disks.push(disk.unwrap()); + new_infos.push(info); + } else { + scanning_disks.push(disk.unwrap()); + scanning_infos.push(info); + } + } + + new_disks.extend(scanning_disks); + new_infos.extend(scanning_infos); + new_disks.extend(healing_disks); + new_infos.extend(healing_infos); + + (new_disks, new_infos, healing) + } + + pub(super) async fn _get_local_disks(&self) -> Vec> { + let mut disks = self.get_disks_internal().await; + + let mut rng = rand::rng(); + + disks.shuffle(&mut rng); + + disks + .into_iter() + .filter(|v| v.as_ref().is_some_and(|d| d.is_local())) + .collect() + } + + pub async fn connect_disks(&self) { + let rl = self.disks.read().await; + + let disks = rl.clone(); + + // Explicitly release the lock + drop(rl); + + for (i, opdisk) in disks.iter().enumerate() { + if let Some(disk) = opdisk { + if disk.is_online().await && disk.get_disk_location().set_idx.is_some() { + info!("Disk {:?} is online", disk.to_string()); + continue; + } + + let _ = disk.close().await; + } + + if let Some(endpoint) = self.set_endpoints.get(i) { + info!("will renew disk, opdisk: {:?}", opdisk); + self.renew_disk(endpoint).await; + } + } + } + + pub async fn renew_disk(&self, ep: &Endpoint) { + debug!("renew_disk: start {:?}", ep); + + let (new_disk, fm) = match Self::connect_endpoint(ep).await { + Ok(res) => res, + Err(e) => { + warn!("renew_disk: connect_endpoint err {:?}", &e); + if ep.is_local && e == DiskError::UnformattedDisk { + info!("renew_disk unformatteddisk will trigger heal_disk, {:?}", ep); + let set_disk_id = format!("pool_{}_set_{}", ep.pool_idx, ep.set_idx); + let _ = send_heal_disk(set_disk_id, Some(HealChannelPriority::Normal)).await; + } + return; + } + }; + + let (set_idx, disk_idx) = match self.find_disk_index(&fm) { + Ok(res) => res, + Err(e) => { + warn!("renew_disk: find_disk_index err {:?}", e); + return; + } + }; + + // Check that the endpoint matches + + let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await; + + if new_disk.is_local() { + let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await; + let path = new_disk.endpoint().to_string(); + global_local_disk_map.insert(path, Some(new_disk.clone())); + + if is_dist_erasure().await { + let mut local_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.write().await; + local_set_drives[self.pool_index][set_idx][disk_idx] = Some(new_disk.clone()); + } + } + + debug!("renew_disk: update {:?}", fm.erasure.this); + + let mut disk_lock = self.disks.write().await; + disk_lock[disk_idx] = Some(new_disk); + } + + pub(super) fn find_disk_index(&self, fm: &FormatV3) -> Result<(usize, usize)> { + self.format.check_other(fm)?; + + if fm.erasure.this.is_nil() { + return Err(Error::other("DriveID: offline")); + } + + for i in 0..self.format.erasure.sets.len() { + for j in 0..self.format.erasure.sets[0].len() { + if fm.erasure.this == self.format.erasure.sets[i][j] { + return Ok((i, j)); + } + } + } + + Err(Error::other("DriveID: not found")) + } + + pub(super) async fn connect_endpoint(ep: &Endpoint) -> disk::error::Result<(DiskStore, FormatV3)> { + let disk = new_disk(ep, &DiskOption::default()).await?; + + let fm = load_format_erasure(&disk, false).await?; + + Ok((disk, fm)) + } + + pub(super) async fn get_online_disk_with_healing(&self, incl_healing: bool) -> Result<(Vec>, bool)> { + let (new_disks, _, healing) = self.get_online_disk_with_healing_and_info(incl_healing).await?; + Ok((new_disks, healing > 0)) + } + + pub(super) async fn get_online_disk_with_healing_and_info( + &self, + incl_healing: bool, + ) -> Result<(Vec>, Vec, usize)> { + let mut infos = vec![DiskInfo::default(); self.disks.read().await.len()]; + for (idx, disk) in self.disks.write().await.iter().enumerate() { + if let Some(disk) = disk { + match disk.disk_info(&DiskInfoOptions::default()).await { + Ok(disk_info) => infos[idx] = disk_info, + Err(err) => infos[idx].error = err.to_string(), + } + } else { + infos[idx].error = "disk not found".to_string(); + } + } + + let mut new_disks = Vec::new(); + let mut healing_disks = Vec::new(); + let mut scanning_disks = Vec::new(); + let mut new_infos = Vec::new(); + let mut healing_infos = Vec::new(); + let mut scanning_infos = Vec::new(); + let mut healing = 0; + + infos.iter().zip(self.disks.write().await.iter()).for_each(|(info, disk)| { + if info.error.is_empty() { + if info.healing { + healing += 1; + if incl_healing { + healing_disks.push(disk.clone()); + healing_infos.push(info.clone()); + } + } else if !info.scanning { + new_disks.push(disk.clone()); + new_infos.push(info.clone()); + } else { + scanning_disks.push(disk.clone()); + scanning_infos.push(info.clone()); + } + } + }); + + // Prefer non-scanning disks over disks which are currently being scanned. + new_disks.extend(scanning_disks); + new_infos.extend(scanning_infos); + + // Then add healing disks. + new_disks.extend(healing_disks); + new_infos.extend(healing_infos); + + Ok((new_disks, new_infos, healing)) + } +} diff --git a/crates/ecstore/src/set_disk/metadata.rs b/crates/ecstore/src/set_disk/metadata.rs new file mode 100644 index 000000000..9d729ed2b --- /dev/null +++ b/crates/ecstore/src/set_disk/metadata.rs @@ -0,0 +1,551 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +impl SetDisks { + pub(super) fn reduce_common_data_dir(data_dirs: &Vec>, write_quorum: usize) -> Option { + let mut data_dirs_count = HashMap::new(); + + for ddir in data_dirs { + *data_dirs_count.entry(ddir).or_insert(0) += 1; + } + + let mut max = 0; + let mut data_dir = None; + for (ddir, count) in data_dirs_count { + if count > max { + max = count; + data_dir = *ddir; + } + } + + if max >= write_quorum { data_dir } else { None } + } + + pub(super) fn get_upload_id_dir(bucket: &str, object: &str, upload_id: &str) -> String { + let upload_uuid = base64_simd::URL_SAFE_NO_PAD + .decode_to_vec(upload_id.as_bytes()) + .and_then(|v| { + String::from_utf8(v).map_or(Ok(upload_id.to_owned()), |v| { + let parts: Vec<_> = v.splitn(2, '.').collect(); + if parts.len() == 2 { + Ok(parts[1].to_string()) + } else { + Ok(upload_id.to_string()) + } + }) + }) + .unwrap_or_default(); + + format!("{}/{}", Self::get_multipart_sha_dir(bucket, object), upload_uuid) + } + + pub(super) fn get_multipart_sha_dir(bucket: &str, object: &str) -> String { + let path = format!("{bucket}/{object}"); + let mut hasher = Sha256::new(); + hasher.update(path); + hex(hasher.finalize()) + } + + pub(super) fn common_parity(parities: &[i32], default_parity_count: i32) -> i32 { + let n = parities.len() as i32; + + let mut occ_map: HashMap = HashMap::new(); + for &p in parities { + *occ_map.entry(p).or_insert(0) += 1; + } + + let mut max_occ = 0; + let mut cparity = 0; + for (&parity, &occ) in &occ_map { + if parity == -1 { + // Ignore non defined parity + continue; + } + + let mut read_quorum = n - parity; + if default_parity_count > 0 && parity == 0 { + // In this case, parity == 0 implies that this object version is a + // delete marker + read_quorum = n / 2 + 1; + } + if occ < read_quorum { + // Ignore this parity since we don't have enough shards for read quorum + continue; + } + + if occ > max_occ { + max_occ = occ; + cparity = parity; + } + } + + if max_occ == 0 { + // Did not find anything useful + return -1; + } + cparity + } + + pub(super) fn list_object_modtimes(parts_metadata: &[FileInfo], errs: &[Option]) -> Vec> { + let mut times = vec![None; parts_metadata.len()]; + + for (i, metadata) in parts_metadata.iter().enumerate() { + if errs[i].is_some() { + continue; + } + + times[i] = metadata.mod_time + } + + times + } + + pub(super) fn common_time(times: &[Option], quorum: usize) -> Option { + let (time, count) = Self::common_time_and_occurrence(times); + if count >= quorum { time } else { None } + } + + pub(super) fn common_time_and_occurrence(times: &[Option]) -> (Option, usize) { + let mut time_occurrence_map = HashMap::new(); + + // Ignore the uuid sentinel and count the rest. + for time in times.iter().flatten() { + *time_occurrence_map.entry(time.unix_timestamp_nanos()).or_insert(0) += 1; + } + + let mut maxima = 0; // Counter for remembering max occurrence of elements. + let mut latest = 0; + + // Find the common cardinality from previously collected + // occurrences of elements. + for (&nano, &count) in &time_occurrence_map { + if count < maxima { + continue; + } + + // We are at or above maxima + if count > maxima || nano > latest { + maxima = count; + latest = nano; + } + } + + if latest == 0 { + return (None, maxima); + } + + if let Ok(time) = OffsetDateTime::from_unix_timestamp_nanos(latest) { + (Some(time), maxima) + } else { + (None, maxima) + } + } + + pub(super) fn common_etag(etags: &[Option], quorum: usize) -> Option { + let (etag, count) = Self::common_etags(etags); + if count >= quorum { etag } else { None } + } + + pub(super) fn common_etags(etags: &[Option]) -> (Option, usize) { + let mut etags_map = HashMap::new(); + + for etag in etags.iter().flatten() { + *etags_map.entry(etag).or_insert(0) += 1; + } + + let mut maxima = 0; // Counter for remembering max occurrence of elements. + let mut latest = None; + + for (&etag, &count) in &etags_map { + if count < maxima { + continue; + } + + // We are at or above maxima + if count > maxima { + maxima = count; + latest = Some(etag.clone()); + } + } + + (latest, maxima) + } + + pub(super) fn list_object_etags(parts_metadata: &[FileInfo], errs: &[Option]) -> Vec> { + let mut etags = vec![None; parts_metadata.len()]; + + for (i, metadata) in parts_metadata.iter().enumerate() { + if errs[i].is_some() { + continue; + } + + if let Some(etag) = metadata.metadata.get("etag") { + etags[i] = Some(etag.clone()) + } + } + + etags + } + + pub(super) fn list_object_parities(parts_metadata: &[FileInfo], errs: &[Option]) -> Vec { + let total_shards = parts_metadata.len(); + let half = total_shards as i32 / 2; + let mut parities: Vec = vec![-1; total_shards]; + + for (index, metadata) in parts_metadata.iter().enumerate() { + if errs[index].is_some() { + parities[index] = -1; + continue; + } + + if !metadata.is_valid() { + parities[index] = -1; + continue; + } + + if metadata.deleted || metadata.size == 0 { + parities[index] = half; + // } else if metadata.transition_status == "TransitionComplete" { + // TODO: metadata.transition_status + // parities[index] = total_shards - (total_shards / 2 + 1); + } else { + parities[index] = metadata.erasure.parity_blocks as i32; + } + } + parities + } + + #[tracing::instrument(level = "debug", skip(parts_metadata))] + pub(super) fn object_quorum_from_meta( + parts_metadata: &[FileInfo], + errs: &[Option], + default_parity_count: usize, + ) -> disk::error::Result<(i32, i32)> { + let expected_rquorum = if default_parity_count == 0 { + parts_metadata.len() + } else { + parts_metadata.len() / 2 + }; + + if let Some(err) = reduce_read_quorum_errs(errs, OBJECT_OP_IGNORED_ERRS, expected_rquorum) { + // let object = parts_metadata.first().map(|v| v.name.clone()).unwrap_or_default(); + // error!("object_quorum_from_meta: {:?}, errs={:?}, object={:?}", err, errs, object); + return Err(err); + } + + if default_parity_count == 0 { + return Ok((parts_metadata.len() as i32, parts_metadata.len() as i32)); + } + + let parities = Self::list_object_parities(parts_metadata, errs); + + let parity_blocks = Self::common_parity(&parities, default_parity_count as i32); + + if parity_blocks < 0 { + error!("object_quorum_from_meta: parity_blocks < 0, errs={:?}", errs); + return Err(DiskError::ErasureReadQuorum); + } + + let data_blocks = parts_metadata.len() as i32 - parity_blocks; + let write_quorum = if data_blocks == parity_blocks { + data_blocks + 1 + } else { + data_blocks + }; + + Ok((data_blocks, write_quorum)) + } + + #[tracing::instrument(level = "debug", skip(disks, parts_metadata))] + pub(super) fn list_online_disks( + disks: &[Option], + parts_metadata: &[FileInfo], + errs: &[Option], + quorum: usize, + ) -> (Vec>, Option, Option) { + let mod_times = Self::list_object_modtimes(parts_metadata, errs); + let etags = Self::list_object_etags(parts_metadata, errs); + + let mod_time = Self::common_time(&mod_times, quorum); + let etag = Self::common_etag(&etags, quorum); + + let mut new_disk = vec![None; disks.len()]; + + for (i, &t) in mod_times.iter().enumerate() { + if parts_metadata[i].is_valid() && mod_time == t { + new_disk[i].clone_from(&disks[i]); + } + } + + (new_disk, mod_time, etag) + } + + pub(super) fn pick_valid_fileinfo( + metas: &[FileInfo], + mod_time: Option, + etag: Option, + quorum: usize, + ) -> disk::error::Result { + Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum) + } + + pub(super) fn find_file_info_in_quorum( + metas: &[FileInfo], + mod_time: &Option, + etag: &Option, + quorum: usize, + ) -> disk::error::Result { + if quorum < 1 { + warn!("find_file_info_in_quorum: quorum < 1"); + return Err(DiskError::ErasureReadQuorum); + } + + let mut meta_hashes = vec![None; metas.len()]; + let mut hasher = Sha256::new(); + + for (i, meta) in metas.iter().enumerate() { + if !meta.is_valid() { + debug!( + index = i, + valid = false, + version_id = ?meta.version_id, + mod_time = ?meta.mod_time, + "find_file_info_in_quorum: skipping invalid meta" + ); + continue; + } + + debug!( + index = i, + valid = true, + version_id = ?meta.version_id, + mod_time = ?meta.mod_time, + deleted = meta.deleted, + size = meta.size, + "find_file_info_in_quorum: inspecting meta" + ); + + let etag_only = mod_time.is_none() && etag.is_some() && meta.get_etag().is_some_and(|v| &v == etag.as_ref().unwrap()); + let mod_valid = mod_time == &meta.mod_time; + + if etag_only || mod_valid { + for part in meta.parts.iter() { + hasher.update(format!("part.{}", part.number).as_bytes()); + hasher.update(format!("part.{}", part.size).as_bytes()); + } + + if !meta.deleted && meta.size != 0 { + hasher.update(format!("{}+{}", meta.erasure.data_blocks, meta.erasure.parity_blocks).as_bytes()); + hasher.update(format!("{:?}", meta.erasure.distribution).as_bytes()); + } + + if meta.is_remote() { + // TODO: + } + + // TODO: IsEncrypted + + // TODO: IsCompressed + + meta_hashes[i] = Some(hex(hasher.clone().finalize().as_slice())); + + hasher.reset(); + } else { + debug!( + index = i, + etag_only_match = etag_only, + mod_valid_match = mod_valid, + "find_file_info_in_quorum: meta does not match common etag or mod_time, skipping hash calculation" + ); + } + } + + let mut count_map = HashMap::new(); + + for hash in meta_hashes.iter().flatten() { + *count_map.entry(hash).or_insert(0) += 1; + } + + let mut max_val = None; + let mut max_count = 0; + + for (&val, &count) in &count_map { + if count > max_count { + max_val = Some(val); + max_count = count; + } + } + + if max_count < quorum { + warn!("find_file_info_in_quorum: max_count < quorum, max_val={:?}", max_val); + return Err(DiskError::ErasureReadQuorum); + } + + let mut found_fi = None; + let mut found = false; + + let mut valid_obj_map = HashMap::new(); + + for (i, op_hash) in meta_hashes.iter().enumerate() { + if let Some(hash) = op_hash + && let Some(max_hash) = max_val + && hash == max_hash + { + if metas[i].is_valid() && !found { + found_fi = Some(metas[i].clone()); + found = true; + } + + let props = ObjProps { + mod_time: metas[i].mod_time, + num_versions: metas[i].num_versions, + }; + + *valid_obj_map.entry(props).or_insert(0) += 1; + } + } + + if found { + let mut fi = found_fi.unwrap(); + + for (val, &count) in &valid_obj_map { + if count > quorum { + fi.mod_time = val.mod_time; + fi.num_versions = val.num_versions; + fi.is_latest = val.mod_time.is_none(); + + break; + } + } + + return Ok(fi); + } + + warn!("find_file_info_in_quorum: fileinfo not found"); + + Err(DiskError::ErasureReadQuorum) + } + + pub(super) fn shuffle_disks_and_parts_metadata_by_index( + disks: &[Option], + parts_metadata: &[FileInfo], + fi: &FileInfo, + ) -> (Vec>, Vec) { + let mut shuffled_disks = vec![None; disks.len()]; + let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()]; + let distribution = &fi.erasure.distribution; + + let mut inconsistent = 0; + for (k, v) in parts_metadata.iter().enumerate() { + if disks[k].is_none() { + inconsistent += 1; + continue; + } + + if !v.is_valid() { + inconsistent += 1; + continue; + } + + if distribution[k] != v.erasure.index { + inconsistent += 1; + continue; + } + + let block_idx = distribution[k]; + shuffled_parts_metadata[block_idx - 1] = parts_metadata[k].clone(); + shuffled_disks[block_idx - 1].clone_from(&disks[k]); + } + + if inconsistent < fi.erasure.parity_blocks { + return (shuffled_disks, shuffled_parts_metadata); + } + + Self::shuffle_disks_and_parts_metadata(disks, parts_metadata, fi) + } + + pub(super) fn shuffle_disks_and_parts_metadata( + disks: &[Option], + parts_metadata: &[FileInfo], + fi: &FileInfo, + ) -> (Vec>, Vec) { + let init = fi.mod_time.is_none(); + + let mut shuffled_disks = vec![None; disks.len()]; + let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()]; + let distribution = &fi.erasure.distribution; + + for (k, v) in disks.iter().enumerate() { + if v.is_none() { + continue; + } + + if !init && !parts_metadata[k].is_valid() { + continue; + } + + // if !init && fi.xlv1 != parts_metadata[k].xlv1 { + // continue; + // } + + let block_idx = distribution[k]; + shuffled_parts_metadata[block_idx - 1] = parts_metadata[k].clone(); + shuffled_disks[block_idx - 1].clone_from(&disks[k]); + } + + (shuffled_disks, shuffled_parts_metadata) + } + + pub(super) fn shuffle_parts_metadata(parts_metadata: &[FileInfo], distribution: &[usize]) -> Vec { + if distribution.is_empty() { + return parts_metadata.to_vec(); + } + let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()]; + // Shuffle slice xl metadata for expected distribution. + for index in 0..parts_metadata.len() { + let block_index = distribution[index]; + shuffled_parts_metadata[block_index - 1] = parts_metadata[index].clone(); + } + shuffled_parts_metadata + } + + pub(super) fn shuffle_disks(disks: &[Option], distribution: &[usize]) -> Vec> { + if distribution.is_empty() { + return disks.to_vec(); + } + + let mut shuffled_disks = vec![None; disks.len()]; + + for (i, v) in disks.iter().enumerate() { + let idx = distribution[i]; + shuffled_disks[idx - 1].clone_from(v); + } + + shuffled_disks + } + + pub(super) fn shuffle_check_parts(parts_errs: &[usize], distribution: &[usize]) -> Vec { + if distribution.is_empty() { + return parts_errs.to_vec(); + } + let mut shuffled_parts_errs = vec![0; parts_errs.len()]; + for (i, v) in parts_errs.iter().enumerate() { + let idx = distribution[i]; + shuffled_parts_errs[idx - 1] = *v; + } + shuffled_parts_errs + } +} diff --git a/crates/ecstore/src/set_disk/multipart.rs b/crates/ecstore/src/set_disk/multipart.rs new file mode 100644 index 000000000..0efb96f26 --- /dev/null +++ b/crates/ecstore/src/set_disk/multipart.rs @@ -0,0 +1,146 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +impl SetDisks { + pub(super) async fn list_parts( + disks: &[Option], + part_path: &str, + read_quorum: usize, + ) -> disk::error::Result> { + let mut futures = Vec::with_capacity(disks.len()); + for (i, disk) in disks.iter().enumerate() { + futures.push(async move { + if let Some(disk) = disk { + disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1) + .await + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + let mut errs = Vec::with_capacity(disks.len()); + let mut object_parts = Vec::with_capacity(disks.len()); + + let results = join_all(futures).await; + for result in results { + match result { + Ok(res) => { + errs.push(None); + object_parts.push(res); + } + Err(e) => { + errs.push(Some(e)); + object_parts.push(vec![]); + } + } + } + + if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { + return Err(err); + } + + let mut part_quorum_map: HashMap = HashMap::new(); + + for drive_parts in object_parts { + let mut parts_with_meta_count: HashMap = HashMap::new(); + + // part files can be either part.N or part.N.meta + for part_path in drive_parts { + if let Some(num_str) = part_path.strip_prefix("part.") { + if let Some(meta_idx) = num_str.find(".meta") { + if let Ok(part_num) = num_str[..meta_idx].parse::() { + *parts_with_meta_count.entry(part_num).or_insert(0) += 1; + } + } else if let Ok(part_num) = num_str.parse::() { + *parts_with_meta_count.entry(part_num).or_insert(0) += 1; + } + } + } + + // Include only part.N.meta files with corresponding part.N + for (&part_num, &cnt) in &parts_with_meta_count { + if cnt >= 2 { + *part_quorum_map.entry(part_num).or_insert(0) += 1; + } + } + } + + let mut part_numbers = Vec::with_capacity(part_quorum_map.len()); + for (part_num, count) in part_quorum_map { + if count >= read_quorum { + part_numbers.push(part_num); + } + } + + part_numbers.sort(); + + Ok(part_numbers) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub(super) async fn check_upload_id_exists( + &self, + bucket: &str, + object: &str, + upload_id: &str, + write: bool, + ) -> Result<(FileInfo, Vec)> { + let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); + let disks = self.disks.read().await; + + let disks = disks.clone(); + + let (parts_metadata, errs) = + Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await?; + + let map_err_notfound = |err: DiskError| { + if err == DiskError::FileNotFound { + return StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()); + } + err.into() + }; + + let (read_quorum, write_quorum) = + Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count).map_err(map_err_notfound)?; + + if read_quorum < 0 { + error!("check_upload_id_exists: read_quorum < 0, errs={:?}", errs); + return Err(Error::ErasureReadQuorum); + } + + if write_quorum < 0 { + return Err(Error::ErasureWriteQuorum); + } + + let mut quorum = read_quorum as usize; + if write { + quorum = write_quorum as usize; + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) { + return Err(map_err_notfound(err)); + } + } else if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) { + return Err(map_err_notfound(err)); + } + + let (_, mod_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, quorum); + + let fi = Self::pick_valid_fileinfo(&parts_metadata, mod_time, etag, quorum)?; + + Ok((fi, parts_metadata)) + } +} diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs new file mode 100644 index 000000000..c2cfb74f6 --- /dev/null +++ b/crates/ecstore/src/set_disk/read.rs @@ -0,0 +1,808 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +impl SetDisks { + pub(super) async fn read_parts( + disks: &[Option], + bucket: &str, + part_meta_paths: &[String], + part_numbers: &[usize], + read_quorum: usize, + ) -> disk::error::Result> { + let mut errs = Vec::with_capacity(disks.len()); + let mut object_parts = Vec::with_capacity(disks.len()); + + // Use batch processor for better performance + let processor = get_global_processors().read_processor(); + let bucket = bucket.to_string(); + let part_meta_paths = part_meta_paths.to_vec(); + + let tasks: Vec<_> = disks + .iter() + .map(|disk| { + let disk = disk.clone(); + let bucket = bucket.clone(); + let part_meta_paths = part_meta_paths.clone(); + + async move { + if let Some(disk) = disk { + disk.read_parts(&bucket, &part_meta_paths).await + } else { + Err(DiskError::DiskNotFound) + } + } + }) + .collect(); + + let results = processor.execute_batch(tasks).await; + for result in results { + match result { + Ok(res) => { + errs.push(None); + object_parts.push(res); + } + Err(e) => { + errs.push(Some(e)); + object_parts.push(vec![]); + } + } + } + + if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { + return Err(err); + } + + let mut ret = vec![ObjectPartInfo::default(); part_meta_paths.len()]; + + for (part_idx, part_info) in part_meta_paths.iter().enumerate() { + let mut part_meta_quorum = HashMap::new(); + let mut part_infos = Vec::new(); + for (j, parts) in object_parts.iter().enumerate() { + if parts.len() != part_meta_paths.len() { + *part_meta_quorum.entry(part_info.clone()).or_insert(0) += 1; + continue; + } + + if !parts[part_idx].etag.is_empty() { + *part_meta_quorum.entry(parts[part_idx].etag.clone()).or_insert(0) += 1; + part_infos.push(parts[part_idx].clone()); + continue; + } + + *part_meta_quorum.entry(part_info.clone()).or_insert(0) += 1; + } + + let mut max_quorum = 0; + let mut max_etag = None; + let mut max_part_meta = None; + for (etag, quorum) in part_meta_quorum.iter() { + if quorum > &max_quorum { + max_quorum = *quorum; + max_etag = Some(etag); + max_part_meta = Some(etag); + } + } + + let mut found = None; + for info in part_infos.iter() { + if let Some(etag) = max_etag + && info.etag == *etag + { + found = Some(info.clone()); + break; + } + + if let Some(part_meta) = max_part_meta + && info.etag.is_empty() + && part_meta.ends_with(format!("part.{0}.meta", info.number).as_str()) + { + found = Some(info.clone()); + break; + } + } + + if let (Some(found), Some(max_etag)) = (found, max_etag) + && !found.etag.is_empty() + && part_meta_quorum.get(max_etag).unwrap_or(&0) >= &read_quorum + { + ret[part_idx] = found.clone(); + } else { + ret[part_idx] = ObjectPartInfo { + number: part_numbers[part_idx], + error: Some(format!("part.{} not found", part_numbers[part_idx])), + ..Default::default() + }; + } + } + + Ok(ret) + } + + #[tracing::instrument(level = "debug", skip(disks))] + pub(super) async fn read_all_fileinfo( + disks: &[Option], + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + read_data: bool, + healing: bool, + ) -> disk::error::Result<(Vec, Vec>)> { + let mut ress = Vec::with_capacity(disks.len()); + let mut errors = Vec::with_capacity(disks.len()); + let opts = Arc::new(ReadOptions { + read_data, + healing, + ..Default::default() + }); + let org_bucket = Arc::new(org_bucket.to_string()); + let bucket = Arc::new(bucket.to_string()); + let object = Arc::new(object.to_string()); + let version_id = Arc::new(version_id.to_string()); + let futures = disks.iter().map(|disk| { + let disk = disk.clone(); + let opts = opts.clone(); + let org_bucket = org_bucket.clone(); + let bucket = bucket.clone(); + let object = object.clone(); + let version_id = version_id.clone(); + tokio::spawn(async move { + if let Some(disk) = disk { + disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await + } else { + Err(DiskError::DiskNotFound) + } + }) + }); + + // Wait for all tasks to complete + let results = join_all(futures).await; + + for result in results { + match result { + Ok(res) => match res { + Ok(file_info) => { + ress.push(file_info); + errors.push(None); + } + Err(e) => { + ress.push(FileInfo::default()); + errors.push(Some(e)); + } + }, + Err(_) => { + ress.push(FileInfo::default()); + errors.push(Some(DiskError::Unexpected)); + } + } + } + Ok((ress, errors)) + } + + pub async fn read_version_optimized( + &self, + bucket: &str, + object: &str, + version_id: &str, + opts: &ReadOptions, + ) -> Result> { + // Use existing disk selection logic + let disks = self.disks.read().await; + let required_reads = self.format.erasure.sets.len(); + + // Clone parameters outside the closure to avoid lifetime issues + let bucket = bucket.to_string(); + let object = object.to_string(); + let version_id = version_id.to_string(); + let opts = opts.clone(); + + let processor = get_global_processors().read_processor(); + let tasks: Vec<_> = disks + .iter() + .take(required_reads + 2) // Read a few extra for reliability + .filter_map(|disk| { + disk.as_ref().map(|d| { + let disk = d.clone(); + let bucket = bucket.clone(); + let object = object.clone(); + let version_id = version_id.clone(); + let opts = opts.clone(); + + async move { disk.read_version(&bucket, &bucket, &object, &version_id, &opts).await } + }) + }) + .collect(); + + match processor.execute_batch_with_quorum(tasks, required_reads).await { + Ok(results) => Ok(results), + Err(_) => Err(DiskError::FileNotFound.into()), // Use existing error type + } + } + + pub(super) async fn read_all_xl( + disks: &[Option], + bucket: &str, + object: &str, + read_data: bool, + incl_free_vers: bool, + ) -> (Vec, Vec>) { + let (fileinfos, errs) = Self::read_all_raw_file_info(disks, bucket, object, read_data).await; + + Self::pick_latest_quorum_files_info(fileinfos, errs, bucket, object, read_data, incl_free_vers).await + } + + pub(super) async fn read_all_raw_file_info( + disks: &[Option], + bucket: &str, + object: &str, + read_data: bool, + ) -> (Vec>, Vec>) { + let mut ress = Vec::with_capacity(disks.len()); + let mut errors = Vec::with_capacity(disks.len()); + + let mut futures = Vec::with_capacity(disks.len()); + + for disk in disks.iter() { + futures.push(async move { + if let Some(disk) = disk { + disk.read_xl(bucket, object, read_data).await + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + let results = join_all(futures).await; + for result in results { + match result { + Ok(res) => { + ress.push(Some(res)); + errors.push(None); + } + Err(e) => { + ress.push(None); + errors.push(Some(e)); + } + } + } + + (ress, errors) + } + + pub(super) async fn pick_latest_quorum_files_info( + fileinfos: Vec>, + errs: Vec>, + bucket: &str, + object: &str, + read_data: bool, + incl_free_vers: bool, + ) -> (Vec, Vec>) { + let mut metadata_array = vec![None; fileinfos.len()]; + let mut meta_file_infos = vec![FileInfo::default(); fileinfos.len()]; + let mut metadata_shallow_versions = vec![None; fileinfos.len()]; + + let mut v2_bufs = { + if !read_data { + vec![Vec::new(); fileinfos.len()] + } else { + Vec::new() + } + }; + + let mut errs = errs; + + for (idx, info_op) in fileinfos.iter().enumerate() { + if let Some(info) = info_op { + if !read_data { + v2_bufs[idx] = info.buf.clone(); + } + + let xlmeta = match FileMeta::load(&info.buf) { + Ok(res) => res, + Err(err) => { + errs[idx] = Some(err.into()); + continue; + } + }; + + metadata_array[idx] = Some(xlmeta); + meta_file_infos[idx] = FileInfo::default(); + } + } + + for (idx, info_op) in metadata_array.iter().enumerate() { + if let Some(info) = info_op { + metadata_shallow_versions[idx] = Some(info.versions.clone()); + } + } + + let shallow_versions: Vec> = metadata_shallow_versions.iter().flatten().cloned().collect(); + + let read_quorum = fileinfos.len().div_ceil(2); + let versions = merge_file_meta_versions(read_quorum, false, 1, &shallow_versions); + let meta = FileMeta { + versions, + ..Default::default() + }; + + let finfo = match meta.into_fileinfo(bucket, object, "", true, incl_free_vers, true) { + Ok(res) => res, + Err(err) => { + for item in errs.iter_mut() { + if item.is_none() { + *item = Some(err.clone().into()); + } + } + + return (meta_file_infos, errs); + } + }; + + if !finfo.is_valid() { + for item in errs.iter_mut() { + if item.is_none() { + *item = Some(DiskError::FileCorrupt); + } + } + + return (meta_file_infos, errs); + } + + let vid = finfo.version_id.unwrap_or(Uuid::nil()); + + for (idx, meta_op) in metadata_array.iter().enumerate() { + if let Some(meta) = meta_op { + match meta.into_fileinfo(bucket, object, vid.to_string().as_str(), read_data, incl_free_vers, true) { + Ok(res) => meta_file_infos[idx] = res, + Err(err) => errs[idx] = Some(err.into()), + } + } + } + + (meta_file_infos, errs) + } + + pub(super) async fn read_multiple_files( + disks: &[Option], + req: ReadMultipleReq, + read_quorum: usize, + ) -> Vec { + let mut futures = Vec::with_capacity(disks.len()); + let mut ress = Vec::with_capacity(disks.len()); + let mut errors = Vec::with_capacity(disks.len()); + + for disk in disks.iter() { + let req = req.clone(); + futures.push(async move { + if let Some(disk) = disk { + disk.read_multiple(req).await + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + let results = join_all(futures).await; + for result in results { + match result { + Ok(res) => { + ress.push(Some(res)); + errors.push(None); + } + Err(e) => { + ress.push(None); + errors.push(Some(e)); + } + } + } + + // debug!("ReadMultipleResp ress {:?}", ress); + // debug!("ReadMultipleResp errors {:?}", errors); + + let mut ret = Vec::with_capacity(req.files.len()); + + for want in req.files.iter() { + let mut quorum = 0; + + let mut get_res = ReadMultipleResp::default(); + + for res in ress.iter() { + if res.is_none() { + continue; + } + + let disk_res = res.as_ref().unwrap(); + + for resp in disk_res.iter() { + if !resp.error.is_empty() || !resp.exists { + continue; + } + + if &resp.file != want || resp.bucket != req.bucket || resp.prefix != req.prefix { + continue; + } + quorum += 1; + + if get_res.mod_time > resp.mod_time || get_res.data.len() > resp.data.len() { + continue; + } + + get_res = resp.clone(); + } + } + + if quorum < read_quorum { + // debug!("quorum < read_quorum: {} < {}", quorum, read_quorum); + get_res.exists = false; + get_res.error = Error::ErasureReadQuorum.to_string(); + get_res.data = Vec::new(); + } + + ret.push(get_res); + } + + // log err + + ret + } + + #[tracing::instrument(level = "debug", skip(self))] + pub(super) async fn get_object_fileinfo( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + read_data: bool, + ) -> Result<(FileInfo, Vec, Vec>)> { + let disks = self.disks.read().await; + + let disks = disks.clone(); + + let vid = opts.version_id.clone().unwrap_or_default(); + + // TODO: optimize concurrency and break once enough slots are available + let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await?; + // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata); + // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs); + + let _min_disks = self.set_drive_count - self.default_parity_count; + + let (read_quorum, _) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count) + .map_err(|err| to_object_err(err.into(), vec![bucket, object])) + { + Ok(v) => v, + Err(e) => { + // error!("Self::object_quorum_from_meta: {:?}, bucket: {}, object: {}", &e, bucket, object); + return Err(e); + } + }; + + if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum as usize) { + error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object); + return Err(to_object_err(err.into(), vec![bucket, object])); + } + + let (op_online_disks, mot_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize); + + let fi = Self::pick_valid_fileinfo(&parts_metadata, mot_time, etag, read_quorum as usize)?; + if errs.iter().any(|err| err.is_some()) { + let _ = + rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options( + fi.volume.to_string(), // bucket + Some(fi.name.to_string()), // object_prefix + false, // force_start + Some(HealChannelPriority::Normal), // priority + Some(self.pool_index), // pool_index + Some(self.set_index), // set_index + )) + .await; + } + // debug!("get_object_fileinfo pick fi {:?}", &fi); + + // let online_disks: Vec> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect(); + + Ok((fi, parts_metadata, op_online_disks)) + } + + pub(super) async fn get_object_info_and_quorum( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + ) -> (ObjectInfo, usize, Option) { + let fi = match self.get_object_fileinfo(bucket, object, opts, false).await { + Ok((fi, _, _)) => fi, + Err(e) => return (ObjectInfo::default(), 0, Some(e)), + }; + + let write_quorum = fi.write_quorum(self.default_write_quorum()); + + let oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended); + + if !fi.version_purge_status().is_empty() && opts.version_id.is_some() { + return ( + oi, + write_quorum, + Some(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])), + ); + } + + if fi.deleted { + return if opts.version_id.is_none() || opts.delete_marker { + (oi, write_quorum, Some(to_object_err(StorageError::FileNotFound, vec![bucket, object]))) + } else { + ( + oi, + write_quorum, + Some(to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])), + ) + }; + } + + (oi, write_quorum, None) + } + + #[allow(clippy::too_many_arguments)] + pub(super) async fn get_object_with_fileinfo( + // &self, + bucket: &str, + object: &str, + offset: usize, + length: i64, + writer: &mut W, + fi: FileInfo, + files: Vec, + disks: &[Option], + set_index: usize, + pool_index: usize, + ) -> Result<()> + where + W: AsyncWrite + Send + Sync + Unpin + 'static, + { + debug!(bucket, object, requested_length = length, offset, "get_object_with_fileinfo start"); + let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, &files, &fi); + + let total_size = fi.size as usize; + + let length = if length < 0 { + fi.size as usize - offset + } else { + length as usize + }; + + if offset > total_size || offset + length > total_size { + error!("get_object_with_fileinfo offset out of range: {}, total_size: {}", offset, total_size); + return Err(Error::other("offset out of range")); + } + + let (part_index, mut part_offset) = fi.to_part_offset(offset)?; + + let mut end_offset = offset; + if length > 0 { + end_offset += length - 1 + } + + let (last_part_index, last_part_relative_offset) = fi.to_part_offset(end_offset)?; + + debug!( + bucket, + object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds" + ); + + let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); + + let part_indices: Vec = (part_index..=last_part_index).collect(); + debug!(bucket, object, ?part_indices, "Multipart part indices to stream"); + + let mut total_read = 0; + for current_part in part_indices { + if total_read == length { + debug!( + bucket, + object, + total_read, + requested_length = length, + part_index = current_part, + "Stopping multipart stream early because accumulated bytes match request" + ); + break; + } + + let part_number = fi.parts[current_part].number; + let part_size = fi.parts[current_part].size; + let mut part_length = part_size - part_offset; + if part_length > (length - total_read) { + part_length = length - total_read + } + + let till_offset = erasure.shard_file_offset(part_offset, part_length, part_size); + + let read_offset = (part_offset / erasure.block_size) * erasure.shard_size(); + + debug!( + bucket, + object, + part_index = current_part, + part_number, + part_offset, + part_size, + part_length, + read_offset, + till_offset, + total_read_before = total_read, + requested_length = length, + "Streaming multipart part" + ); + + let mut readers = Vec::with_capacity(disks.len()); + let mut errors = Vec::with_capacity(disks.len()); + for (idx, disk_op) in disks.iter().enumerate() { + match create_bitrot_reader( + files[idx].data.as_deref(), + disk_op.as_ref(), + bucket, + &format!("{}/{}/part.{}", object, files[idx].data_dir.unwrap_or_default(), part_number), + read_offset, + till_offset, + erasure.shard_size(), + HashAlgorithm::HighwayHash256, + ) + .await + { + Ok(Some(reader)) => { + readers.push(Some(reader)); + errors.push(None); + } + Ok(None) => { + readers.push(None); + errors.push(Some(DiskError::DiskNotFound)); + } + Err(e) => { + readers.push(None); + errors.push(Some(e)); + } + } + } + + let nil_count = errors.iter().filter(|&e| e.is_none()).count(); + if nil_count < erasure.data_shards { + if let Some(read_err) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) { + error!("create_bitrot_reader reduce_read_quorum_errs {:?}", &errors); + return Err(to_object_err(read_err.into(), vec![bucket, object])); + } + error!("create_bitrot_reader not enough disks to read: {:?}", &errors); + return Err(Error::other(format!("not enough disks to read: {errors:?}"))); + } + + // Check if we have missing shards even though we can read successfully + // This happens when a node was offline during write and comes back online + let total_shards = erasure.data_shards + erasure.parity_shards; + let available_shards = nil_count; + let missing_shards = total_shards - available_shards; + + info!( + bucket, + object, + part_number, + total_shards, + available_shards, + missing_shards, + data_shards = erasure.data_shards, + parity_shards = erasure.parity_shards, + "Shard availability check" + ); + + if missing_shards > 0 && available_shards >= erasure.data_shards { + // We have missing shards but enough to read - trigger background heal + info!( + bucket, + object, + part_number, + missing_shards, + available_shards, + pool_index, + set_index, + "Detected missing shards during read, triggering background heal" + ); + if let Err(e) = + rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options( + bucket.to_string(), + Some(object.to_string()), + false, + Some(HealChannelPriority::Normal), + Some(pool_index), + Some(set_index), + )) + .await + { + warn!( + bucket, + object, + part_number, + error = %e, + "Failed to enqueue heal request for missing shards" + ); + } else { + warn!(bucket, object, part_number, "Successfully enqueued heal request for missing shards"); + } + } + + // debug!( + // "read part {} part_offset {},part_length {},part_size {} ", + // part_number, part_offset, part_length, part_size + // ); + let (written, err) = erasure.decode(writer, readers, part_offset, part_length, part_size).await; + debug!( + bucket, + object, + part_index = current_part, + part_number, + part_length, + bytes_written = written, + "Finished decoding multipart part" + ); + if let Some(e) = err { + let de_err: DiskError = e.into(); + let mut has_err = true; + if written == part_length { + match de_err { + DiskError::FileNotFound | DiskError::FileCorrupt => { + error!("erasure.decode err 111 {:?}", &de_err); + if let Err(e) = rustfs_common::heal_channel::send_heal_request( + rustfs_common::heal_channel::create_heal_request_with_options( + bucket.to_string(), + Some(object.to_string()), + false, + Some(HealChannelPriority::Normal), + Some(pool_index), + Some(set_index), + ), + ) + .await + { + warn!( + bucket, + object, + part_number, + error = %e, + "Failed to enqueue heal request after decode error" + ); + } + has_err = false; + } + _ => {} + } + } + + if has_err { + error!("erasure.decode err {} {:?}", written, &de_err); + return Err(de_err.into()); + } + } + + // debug!("ec decode {} written size {}", part_number, n); + + total_read += part_length; + part_offset = 0; + } + + // debug!("read end"); + + debug!(bucket, object, total_read, expected_length = length, "Multipart read finished"); + + Ok(()) + } +} diff --git a/crates/ecstore/src/set_disk/replication.rs b/crates/ecstore/src/set_disk/replication.rs new file mode 100644 index 000000000..f71c99885 --- /dev/null +++ b/crates/ecstore/src/set_disk/replication.rs @@ -0,0 +1,50 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +impl SetDisks { + pub async fn update_restore_metadata( + &self, + bucket: &str, + object: &str, + obj_info: &ObjectInfo, + _opts: &ObjectOptions, + ) -> Result<()> { + let mut oi = obj_info.clone(); + oi.metadata_only = true; + + oi.user_defined.remove(X_AMZ_RESTORE.as_str()); + + let version_id = oi.version_id.map(|v| v.to_string()); + let _obj = self + .copy_object( + bucket, + object, + bucket, + object, + &mut oi, + &ObjectOptions { + version_id: version_id.clone(), + ..Default::default() + }, + &ObjectOptions { + version_id, + ..Default::default() + }, + ) + .await?; + Ok(()) + } +} diff --git a/crates/ecstore/src/set_disk/write.rs b/crates/ecstore/src/set_disk/write.rs new file mode 100644 index 000000000..f0a939bc4 --- /dev/null +++ b/crates/ecstore/src/set_disk/write.rs @@ -0,0 +1,629 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; + +impl SetDisks { + pub(super) fn default_read_quorum(&self) -> usize { + self.set_drive_count - self.default_parity_count + } + + pub(super) fn default_write_quorum(&self) -> usize { + let mut data_count = self.set_drive_count - self.default_parity_count; + if data_count == self.default_parity_count { + data_count += 1 + } + + data_count + } + + #[tracing::instrument(level = "debug", skip(disks, file_infos))] + #[allow(clippy::type_complexity)] + pub(super) async fn rename_data( + disks: &[Option], + src_bucket: &str, + src_object: &str, + file_infos: &[FileInfo], + dst_bucket: &str, + dst_object: &str, + write_quorum: usize, + ) -> disk::error::Result<(Vec>, Option>, Option)> { + let mut futures = Vec::with_capacity(disks.len()); + + let mut errs = Vec::with_capacity(disks.len()); + + let src_bucket = Arc::new(src_bucket.to_string()); + let src_object = Arc::new(src_object.to_string()); + let dst_bucket = Arc::new(dst_bucket.to_string()); + let dst_object = Arc::new(dst_object.to_string()); + + for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() { + let mut file_info = file_info.clone(); + let disk = disk.clone(); + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + let dst_object = dst_object.clone(); + let dst_bucket = dst_bucket.clone(); + + futures.push(tokio::spawn(async move { + if file_info.erasure.index == 0 { + file_info.erasure.index = i + 1; + } + + if !file_info.is_valid() { + return Err(DiskError::FileCorrupt); + } + + if let Some(disk) = disk { + disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object) + .await + } else { + Err(DiskError::DiskNotFound) + } + })); + } + + let mut disk_versions = vec![None; disks.len()]; + let mut data_dirs = vec![None; disks.len()]; + + let results = join_all(futures).await; + + for (idx, result) in results.iter().enumerate() { + match result.as_ref().map_err(|_| DiskError::Unexpected)? { + Ok(res) => { + data_dirs[idx] = res.old_data_dir; + disk_versions[idx].clone_from(&res.sign); + errs.push(None); + } + Err(e) => { + errs.push(Some(e.clone())); + } + } + } + + let mut futures = Vec::with_capacity(disks.len()); + if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + // TODO: add concurrency + for (i, err) in errs.iter().enumerate() { + if err.is_some() { + continue; + } + + if let Some(disk) = disks[i].as_ref() { + let fi = file_infos[i].clone(); + let old_data_dir = data_dirs[i]; + let disk = disk.clone(); + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + futures.push(tokio::spawn(async move { + let _ = disk + .delete_version( + &src_bucket, + &src_object, + fi, + false, + DeleteOptions { + undo_write: true, + old_data_dir, + ..Default::default() + }, + ) + .await + .map_err(|e| { + debug!("rename_data delete_version err {:?}", e); + e + }); + })); + } + } + + let _ = join_all(futures).await; + return Err(ret_err); + } + + let versions = None; + // TODO: reduceCommonVersions + + let data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum); + + // // TODO: reduce_common_data_dir + // if let Some(old_dir) = rename_ress + // .iter() + // .filter_map(|v| if v.is_some() { v.as_ref().unwrap().old_data_dir } else { None }) + // .map(|v| v.to_string()) + // .next() + // { + // let cm_errs = self.commit_rename_data_dir(&shuffle_disks, &bucket, &object, &old_dir).await; + // warn!("put_object commit_rename_data_dir:{:?}", &cm_errs); + // } + + // self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?; + + Ok((Self::eval_disks(disks, &errs), versions, data_dir)) + } + + #[allow(dead_code)] + #[tracing::instrument(level = "debug", skip(self, disks))] + pub(super) async fn commit_rename_data_dir( + &self, + disks: &[Option], + bucket: &str, + object: &str, + data_dir: &str, + write_quorum: usize, + ) -> disk::error::Result<()> { + let file_path = Arc::new(format!("{object}/{data_dir}")); + let bucket = Arc::new(bucket.to_string()); + let futures = disks.iter().map(|disk| { + let file_path = file_path.clone(); + let bucket = bucket.clone(); + let disk = disk.clone(); + tokio::spawn(async move { + if let Some(disk) = disk { + (disk + .delete( + &bucket, + &file_path, + DeleteOptions { + recursive: true, + ..Default::default() + }, + ) + .await) + .err() + } else { + Some(DiskError::DiskNotFound) + } + }) + }); + let errs: Vec> = join_all(futures) + .await + .into_iter() + .map(|e| e.unwrap_or(Some(DiskError::Unexpected))) + .collect(); + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + return Err(err); + } + + Ok(()) + } + + #[tracing::instrument(skip(self))] + pub(super) async fn cleanup_multipart_path(&self, paths: &[String]) { + let disks = self.get_disks_internal().await; + + let mut errs = Vec::with_capacity(disks.len()); + + // Use improved simple batch processor instead of join_all for better performance + let processor = get_global_processors().write_processor(); + + let tasks: Vec<_> = disks + .iter() + .map(|disk| { + let disk = disk.clone(); + let paths = paths.to_vec(); + + async move { + if let Some(disk) = disk { + disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &paths).await + } else { + Err(DiskError::DiskNotFound) + } + } + }) + .collect(); + + let results = processor.execute_batch(tasks).await; + for result in results { + match result { + Ok(_) => { + errs.push(None); + } + Err(e) => { + errs.push(Some(e)); + } + } + } + + if errs.iter().any(|e| e.is_some()) { + warn!("cleanup_multipart_path errs {:?}", &errs); + } + } + + #[tracing::instrument(skip(disks, meta))] + #[allow(clippy::too_many_arguments)] + pub(super) async fn rename_part( + &self, + disks: &[Option], + src_bucket: &str, + src_object: &str, + dst_bucket: &str, + dst_object: &str, + meta: Bytes, + write_quorum: usize, + ) -> disk::error::Result>> { + let src_bucket = Arc::new(src_bucket.to_string()); + let src_object = Arc::new(src_object.to_string()); + let dst_bucket = Arc::new(dst_bucket.to_string()); + let dst_object = Arc::new(dst_object.to_string()); + + let mut errs = Vec::with_capacity(disks.len()); + + let futures = disks.iter().map(|disk| { + let disk = disk.clone(); + let meta = meta.clone(); + let src_bucket = src_bucket.clone(); + let src_object = src_object.clone(); + let dst_bucket = dst_bucket.clone(); + let dst_object = dst_object.clone(); + tokio::spawn(async move { + if let Some(disk) = disk { + disk.rename_part(&src_bucket, &src_object, &dst_bucket, &dst_object, meta) + .await + } else { + Err(DiskError::DiskNotFound) + } + }) + }); + + let results = join_all(futures).await; + for result in results { + match result? { + Ok(_) => { + errs.push(None); + } + Err(e) => { + errs.push(Some(e)); + } + } + } + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + warn!("rename_part errs {:?}", &errs); + self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")]) + .await; + return Err(err); + } + + let disks = Self::eval_disks(disks, &errs); + Ok(disks) + } + + pub(super) fn eval_disks(disks: &[Option], errs: &[Option]) -> Vec> { + if disks.len() != errs.len() { + return Vec::new(); + } + + let mut online_disks = vec![None; disks.len()]; + + for (i, err_op) in errs.iter().enumerate() { + if err_op.is_none() { + online_disks[i].clone_from(&disks[i]); + } + } + + online_disks + } + + #[tracing::instrument(skip(disks, files))] + pub(super) async fn write_unique_file_info( + disks: &[Option], + org_bucket: &str, + bucket: &str, + prefix: &str, + files: &[FileInfo], + write_quorum: usize, + ) -> disk::error::Result<()> { + let mut futures = Vec::with_capacity(disks.len()); + let mut errs = Vec::with_capacity(disks.len()); + + for (i, disk) in disks.iter().enumerate() { + let mut file_info = files[i].clone(); + file_info.erasure.index = i + 1; + futures.push(async move { + if let Some(disk) = disk { + disk.write_metadata(org_bucket, bucket, prefix, file_info).await + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + let results = join_all(futures).await; + for result in results { + match result { + Ok(_) => { + errs.push(None); + } + Err(e) => { + errs.push(Some(e)); + } + } + } + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + // TODO: add concurrency + for (i, err) in errs.iter().enumerate() { + if err.is_some() { + continue; + } + + if let Some(disk) = disks[i].as_ref() { + let _ = disk + .delete( + bucket, + &path_join_buf(&[prefix, STORAGE_FORMAT_FILE]), + DeleteOptions { + recursive: true, + ..Default::default() + }, + ) + .await + .map_err(|e| { + warn!("write meta revert err {:?}", e); + e + }); + } + } + + return Err(err); + } + Ok(()) + } + + pub(super) async fn update_object_meta( + &self, + bucket: &str, + object: &str, + fi: FileInfo, + disks: &[Option], + ) -> disk::error::Result<()> { + self.update_object_meta_with_opts(bucket, object, fi, disks, &UpdateMetadataOpts::default()) + .await + } + + pub(super) async fn update_object_meta_with_opts( + &self, + bucket: &str, + object: &str, + fi: FileInfo, + disks: &[Option], + opts: &UpdateMetadataOpts, + ) -> disk::error::Result<()> { + if fi.metadata.is_empty() { + return Ok(()); + } + + let mut futures = Vec::with_capacity(disks.len()); + + let mut errs = Vec::with_capacity(disks.len()); + + for disk in disks.iter() { + let fi = fi.clone(); + futures.push(async move { + if let Some(disk) = disk { + disk.update_metadata(bucket, object, fi, opts).await + } else { + Err(DiskError::DiskNotFound) + } + }) + } + + let results = join_all(futures).await; + for result in results { + match result { + Ok(_) => { + errs.push(None); + } + Err(e) => { + errs.push(Some(e)); + } + } + } + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, fi.write_quorum(self.default_write_quorum())) { + return Err(err); + } + + Ok(()) + } + + pub(super) async fn delete_if_dang_ling( + &self, + bucket: &str, + object: &str, + meta_arr: &[FileInfo], + errs: &[Option], + data_errs_by_part: &HashMap>, + opts: ObjectOptions, + ) -> disk::error::Result { + let (m, can_heal) = is_object_dang_ling(meta_arr, errs, data_errs_by_part); + + if !can_heal { + return Err(DiskError::ErasureReadQuorum); + } + + let mut tags: HashMap = HashMap::new(); + tags.insert("set".to_string(), self.set_index.to_string()); + tags.insert("pool".to_string(), self.pool_index.to_string()); + tags.insert("merrs".to_string(), join_errs(errs)); + tags.insert("derrs".to_string(), format!("{data_errs_by_part:?}")); + if m.is_valid() { + tags.insert("sz".to_string(), m.size.to_string()); + tags.insert( + "mt".to_string(), + m.mod_time + .as_ref() + .map_or(String::new(), |mod_time| mod_time.unix_timestamp().to_string()), + ); + tags.insert("d:p".to_string(), format!("{}:{}", m.erasure.data_blocks, m.erasure.parity_blocks)); + } else { + tags.insert("invalid".to_string(), "1".to_string()); + tags.insert( + "d:p".to_string(), + format!("{}:{}", self.set_drive_count - self.default_parity_count, self.default_parity_count), + ); + } + let mut offline = 0; + for (i, err) in errs.iter().enumerate() { + let mut found = false; + if let Some(err) = err + && err == &DiskError::DiskNotFound + { + found = true; + } + for p in data_errs_by_part { + if let Some(v) = p.1.get(i) + && *v == CHECK_PART_DISK_NOT_FOUND + { + found = true; + break; + } + } + + if found { + offline += 1; + } + } + + if offline > 0 { + tags.insert("offline".to_string(), offline.to_string()); + } + + let mut fi = FileInfo::default(); + if let Some(ref version_id) = opts.version_id { + fi.version_id = Uuid::parse_str(version_id).ok(); + } + + fi.set_tier_free_version_id(&Uuid::new_v4().to_string()); + + let disks = self.get_disks_internal().await; + + let mut futures = Vec::with_capacity(disks.len()); + for disk_op in disks.iter() { + let bucket = bucket.to_string(); + let object = object.to_string(); + let fi = fi.clone(); + futures.push(async move { + if let Some(disk) = disk_op { + disk.delete_version(&bucket, &object, fi, false, DeleteOptions::default()) + .await + } else { + Err(DiskError::DiskNotFound) + } + }); + } + + let results = join_all(futures).await; + for (index, result) in results.into_iter().enumerate() { + let key = format!("ddisk-{index}"); + match result { + Ok(_) => { + tags.insert(key, "".to_string()); + } + Err(e) => { + tags.insert(key, e.to_string()); + } + } + } + + // TODO: audit + + Ok(m) + } + + pub(super) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> { + let disks = self.get_disks_internal().await; + let write_quorum = disks.len() / 2 + 1; + + let mut futures = Vec::with_capacity(disks.len()); + + for disk_op in disks.iter() { + let bucket = bucket.to_string(); + let prefix = prefix.to_string(); + futures.push(async move { + if let Some(disk) = disk_op { + disk.delete( + &bucket, + &prefix, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + } else { + Ok(()) + } + }); + } + + let errs = join_all(futures).await.into_iter().map(|v| v.err()).collect::>(); + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + return Err(err); + } + + Ok(()) + } + + pub(super) async fn check_write_precondition( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + ) -> Option { + let mut opts = opts.clone(); + + let http_preconditions = opts.http_preconditions?; + opts.http_preconditions = None; + + // Never claim a lock here, to avoid deadlock + // - If no_lock is false, we must have obtained the lock out side of this function + // - If no_lock is true, we should not obtain locks + opts.no_lock = true; + let oi = self.get_object_info(bucket, object, &opts).await; + + match oi { + Ok(oi) => { + // If top level is a delete marker proceed to upload. + if oi.delete_marker { + return None; + } + if should_prevent_write(&oi, http_preconditions.if_none_match, http_preconditions.if_match) { + return Some(StorageError::PreconditionFailed); + } + } + + Err(StorageError::VersionNotFound(_, _, _)) + | Err(StorageError::ObjectNotFound(_, _)) + | Err(StorageError::ErasureReadQuorum) => { + // When the object is not found, + // - if If-Match is set, we should return 404 NotFound + // - if If-None-Match is set, we should be able to proceed with the request + if http_preconditions.if_match.is_some() { + return Some(StorageError::ObjectNotFound(bucket.to_string(), object.to_string())); + } + } + + Err(e) => { + return Some(e); + } + } + + None + } +}