// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use crate::config::storageclass; use crate::disk::error_reduce::{count_errs, reduce_write_quorum_errs}; use crate::disk::{self, DiskAPI}; use crate::error::{Error, Result}; use crate::{ disk::{ DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET, error::DiskError, format::{FormatErasureVersion, FormatMetaVersion, FormatV3}, new_disk, }, layout::endpoints::Endpoints, }; use futures::future::join_all; use std::collections::{HashMap, hash_map::Entry}; use tracing::{debug, error, info, warn}; use uuid::Uuid; pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec>, Vec>) { let mut futures = Vec::with_capacity(eps.as_ref().len()); for ep in eps.as_ref().iter() { futures.push(new_disk(ep, opt)); } let mut res = Vec::with_capacity(eps.as_ref().len()); let mut errors = Vec::with_capacity(eps.as_ref().len()); let results = join_all(futures).await; for result in results { match result { Ok(s) => { res.push(Some(s)); errors.push(None); } Err(e) => { res.push(None); errors.push(Some(e)); } } } (res, errors) } pub async fn connect_load_init_formats( first_disk: bool, disks: &[Option], set_count: usize, set_drive_count: usize, deployment_id: Option, ) -> Result { let (formats, errs) = load_format_erasure_all(disks, false).await; check_disk_fatal_errs(&errs)?; check_format_erasure_values(&formats, set_drive_count)?; if first_disk && should_init_erasure_disks(&errs) { // UnformattedDisk, try migrate from MinIO format first, else create new format info!("first_disk && should_init_erasure_disks"); match try_migrate_format(disks, set_count, set_drive_count).await { Ok(LegacyFormatOutcome::Migrated(fm)) => { info!("Migrated format from MinIO config"); return Ok(*fm); } Ok(LegacyFormatOutcome::Incompatible) => { // A MinIO format.json was found on disk but could not be migrated // (topology/version mismatch or parse failure). Falling through to // create a FRESH RustFS format changes the object placement layout, // so the pre-existing MinIO objects will not be readable. Surface // this loudly instead of silently discarding the legacy data. error!( "Detected MinIO format.json on disk but could NOT migrate it; initializing a fresh RustFS format instead. \ Existing MinIO objects will not be readable under the new format. Ensure the RustFS pool / erasure-set \ topology exactly matches the original MinIO deployment, and that no stale .rustfs.sys/format.json remains." ); } Ok(LegacyFormatOutcome::None) => {} Err(e) => { warn!("MinIO format migration attempt failed, will initialize a fresh format: {e}"); } } let fm = init_format_erasure(disks, set_count, set_drive_count, deployment_id).await?; return Ok(fm); } info!( "first_disk: {}, should_init_erasure_disks: {}", first_disk, should_init_erasure_disks(&errs) ); let unformatted = quorum_unformatted_disks(&errs); if unformatted && !first_disk { return Err(Error::NotFirstDisk); } if unformatted && first_disk { return Err(Error::FirstDiskWait); } let fm = get_format_erasure_in_quorum(&formats)?; Ok(fm) } pub fn quorum_unformatted_disks(errs: &[Option]) -> bool { count_errs(errs, &DiskError::UnformattedDisk) > (errs.len() / 2) } pub fn should_init_erasure_disks(errs: &[Option]) -> bool { count_errs(errs, &DiskError::UnformattedDisk) == errs.len() } pub fn check_disk_fatal_errs(errs: &[Option]) -> disk::error::Result<()> { if count_errs(errs, &DiskError::UnsupportedDisk) == errs.len() { return Err(DiskError::UnsupportedDisk); } if count_errs(errs, &DiskError::FileAccessDenied) == errs.len() { return Err(DiskError::FileAccessDenied); } if count_errs(errs, &DiskError::DiskNotDir) == errs.len() { return Err(DiskError::DiskNotDir); } Ok(()) } async fn init_format_erasure( disks: &[Option], set_count: usize, set_drive_count: usize, deployment_id: Option, ) -> Result { let fm = FormatV3::new(set_count, set_drive_count); let mut fms = vec![None; disks.len()]; for i in 0..set_count { for j in 0..set_drive_count { let idx = i * set_drive_count + j; let mut newfm = fm.clone(); newfm.erasure.this = fm.erasure.sets[i][j]; if let Some(id) = deployment_id { newfm.id = id; } fms[idx] = Some(newfm); } } save_format_file_all(disks, &fms).await?; get_format_erasure_in_quorum(&fms) } /// Outcome of attempting to migrate an on-disk MinIO `format.json`. enum LegacyFormatOutcome { /// A compatible MinIO format was found and migrated into RustFS format files. /// Boxed to keep the enum small (`FormatV3` is large; the others are unit variants). Migrated(Box), /// A MinIO `format.json` was present but could not be migrated (topology / /// version mismatch, or a parse failure). The caller must decide how to /// proceed; creating a fresh format would leave the legacy objects unreadable. Incompatible, /// No MinIO `format.json` was present on any disk (a normal fresh install). None, } /// Tries to migrate an on-disk MinIO `format.json` into RustFS format files. /// /// Returns [`LegacyFormatOutcome`] describing whether a legacy format was present /// and, if so, whether it was compatible. `Err` is only returned for genuine IO /// failures while persisting the migrated format. async fn try_migrate_format( disks: &[Option], set_count: usize, set_drive_count: usize, ) -> Result { let mut legacy_seen = false; for disk in disks.iter().flatten() { let data = match disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await { Ok(d) if !d.is_empty() => d, _ => continue, }; // A non-empty MinIO format.json exists on at least one disk. legacy_seen = true; let fm = match FormatV3::try_from(data.as_ref()) { Ok(fm) => fm, Err(e) => { warn!("failed to parse MinIO format.json, skipping this disk: {e}"); continue; } }; let Some(first_set) = fm.erasure.sets.first() else { warn!("MinIO format.json has empty erasure.sets, skipping this disk"); continue; }; if fm.erasure.sets.len() != set_count || first_set.len() != set_drive_count { warn!( "MinIO format topology mismatch: got {}x{}, expected {}x{}; skipping migration for this disk", fm.erasure.sets.len(), first_set.len(), set_count, set_drive_count ); continue; } if fm.erasure.version != FormatErasureVersion::V3 { warn!( "MinIO format erasure version is not V3 ({:?}); skipping migration for this disk", fm.erasure.version ); continue; } let mut fms = vec![None; disks.len()]; for (idx, disk_opt) in disks.iter().enumerate() { if disk_opt.is_none() { continue; } let set_idx = idx / set_drive_count; let disk_idx = idx % set_drive_count; if set_idx >= fm.erasure.sets.len() || disk_idx >= fm.erasure.sets[set_idx].len() { continue; } let mut newfm = fm.clone(); newfm.erasure.this = fm.erasure.sets[set_idx][disk_idx]; fms[idx] = Some(newfm); } save_format_file_all(disks, &fms).await?; return Ok(LegacyFormatOutcome::Migrated(Box::new(get_format_erasure_in_quorum(&fms)?))); } Ok(if legacy_seen { LegacyFormatOutcome::Incompatible } else { LegacyFormatOutcome::None }) } pub fn get_format_erasure_in_quorum(formats: &[Option]) -> Result { let mut countmap = HashMap::new(); for f in formats.iter() { if f.is_some() { let ds = f.as_ref().unwrap().drives(); let v = countmap.entry(ds); match v { Entry::Occupied(mut entry) => *entry.get_mut() += 1, Entry::Vacant(vacant) => { vacant.insert(1); } }; } } let (max_drives, max_count) = countmap.iter().max_by_key(|&(_, c)| c).unwrap_or((&0, &0)); if *max_drives == 0 || *max_count <= formats.len() / 2 { warn!("get_format_erasure_in_quorum fi: {:?}", &formats); return Err(Error::ErasureReadQuorum); } let format = formats .iter() .find(|f| f.as_ref().is_some_and(|v| v.drives().eq(max_drives))) .ok_or(Error::ErasureReadQuorum)?; let mut format = format.as_ref().unwrap().clone(); format.erasure.this = Uuid::nil(); Ok(format) } pub fn check_format_erasure_values( formats: &[Option], // disks: &Vec>, set_drive_count: usize, ) -> Result<()> { for f in formats.iter() { if f.is_none() { continue; } let f = f.as_ref().unwrap(); check_format_erasure_value(f)?; let first_set = f.erasure.sets.first().ok_or_else(|| Error::other("erasure.sets is empty"))?; if formats.len() != f.erasure.sets.len() * first_set.len() { return Err(Error::other(format!( "formats length for erasure.sets does not match: got {}, expected {}", formats.len(), f.erasure.sets.len() * first_set.len() ))); } if first_set.len() != set_drive_count { return Err(Error::other(format!( "erasure set length for set_drive_count does not match: got {}, expected {}", first_set.len(), set_drive_count ))); } } Ok(()) } fn check_format_erasure_value(format: &FormatV3) -> Result<()> { if format.version != FormatMetaVersion::V1 { return Err(Error::other("invalid FormatMetaVersion")); } if format.erasure.version != FormatErasureVersion::V3 { return Err(Error::other("invalid FormatErasureVersion")); } Ok(()) } // load_format_erasure_all reads all format.json files pub async fn load_format_erasure_all(disks: &[Option], heal: bool) -> (Vec>, Vec>) { let mut futures = Vec::with_capacity(disks.len()); let mut datas = Vec::with_capacity(disks.len()); let mut errors = Vec::with_capacity(disks.len()); for disk in disks.iter() { futures.push(async move { if let Some(disk) = disk { load_format_erasure(disk, heal).await } else { Err(DiskError::DiskNotFound) } }); } let results = join_all(futures).await; for (i, result) in results.into_iter().enumerate() { match result { Ok(s) => { if !heal { let _ = disks[i].as_ref().unwrap().set_disk_id(Some(s.erasure.this)).await; } datas.push(Some(s)); errors.push(None); } Err(e) => { datas.push(None); errors.push(Some(e)); } } } // Log aggregation summary of format load results let ok_count = errors.iter().filter(|e| e.is_none()).count(); let err_count = errors.iter().filter(|e| e.is_some()).count(); // Count occurrences of each unique error let mut err_counts: HashMap = HashMap::new(); for err in errors.iter().flatten() { *err_counts.entry(format!("{err}")).or_default() += 1; } if !err_counts.is_empty() { debug!( disks_ok = ok_count, disks_err = err_count, disks_total = disks.len(), "load format erasure all errors: {:?}", err_counts ); } (datas, errors) } pub async fn load_format_erasure(disk: &DiskStore, heal: bool) -> disk::error::Result { let data = disk .read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE) .await .map_err(|e| match e { DiskError::FileNotFound => DiskError::UnformattedDisk, DiskError::DiskNotFound => DiskError::UnformattedDisk, _ => { warn!("load_format_erasure err: {:?} {:?}", disk.to_string(), e); e } })?; let mut fm = FormatV3::try_from(data.as_ref())?; if heal { let info = disk .disk_info(&DiskInfoOptions { noop: heal, ..Default::default() }) .await?; fm.disk_info = Some(info); } Ok(fm) } async fn save_format_file_all(disks: &[Option], formats: &[Option]) -> disk::error::Result<()> { let mut futures = Vec::with_capacity(disks.len()); for (i, disk) in disks.iter().enumerate() { futures.push(save_format_file(disk, &formats[i])); } let mut errors = Vec::with_capacity(disks.len()); let results = join_all(futures).await; for result in results { match result { Ok(_) => { errors.push(None); } Err(e) => { errors.push(Some(e)); } } } if let Some(e) = reduce_write_quorum_errs(&errors, &[], disks.len()) { return Err(e); } Ok(()) } pub async fn save_format_file(disk: &Option, format: &Option) -> disk::error::Result<()> { let Some(disk) = disk else { return Err(DiskError::DiskNotFound); }; let Some(format) = format else { return Err(DiskError::other("format is none")); }; let json_data = format.to_json()?; let tmpfile = Uuid::new_v4().to_string(); disk.write_all(RUSTFS_META_BUCKET, tmpfile.as_str(), json_data.into_bytes().into()) .await?; disk.rename_file(RUSTFS_META_BUCKET, tmpfile.as_str(), RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE) .await?; disk.set_disk_id(Some(format.erasure.this)).await?; Ok(()) } pub fn ec_drives_no_config(set_drive_count: usize) -> Result { let parity = storageclass::default_parity_count(set_drive_count); storageclass::validate_parity(parity, set_drive_count)?; Ok(parity) } #[cfg(test)] mod tests { use super::*; #[test] fn ec_drives_no_config_uses_topology_defaults() { assert_eq!(ec_drives_no_config(1).expect("single-drive topology should resolve"), 0); assert_eq!(ec_drives_no_config(2).expect("two-drive topology should resolve"), 1); assert_eq!(ec_drives_no_config(6).expect("six-drive topology should resolve"), 3); } } // #[derive(Debug, PartialEq, thiserror::Error)] // pub enum ErasureError { // #[error("erasure read quorum")] // ErasureReadQuorum, // #[error("erasure write quorum")] // _ErasureWriteQuorum, // #[error("not first disk")] // NotFirstDisk, // #[error("first disk wait")] // FirstDiskWait, // #[error("invalid part id {0}")] // InvalidPart(usize), // } // impl ErasureError { // pub fn is(&self, err: &Error) -> bool { // if let Some(e) = err.downcast_ref::() { // return self == e; // } // false // } // } // impl ErasureError { // pub fn to_u32(&self) -> u32 { // match self { // ErasureError::ErasureReadQuorum => 0x01, // ErasureError::_ErasureWriteQuorum => 0x02, // ErasureError::NotFirstDisk => 0x03, // ErasureError::FirstDiskWait => 0x04, // ErasureError::InvalidPart(_) => 0x05, // } // } // pub fn from_u32(error: u32) -> Option { // match error { // 0x01 => Some(ErasureError::ErasureReadQuorum), // 0x02 => Some(ErasureError::_ErasureWriteQuorum), // 0x03 => Some(ErasureError::NotFirstDisk), // 0x04 => Some(ErasureError::FirstDiskWait), // 0x05 => Some(ErasureError::InvalidPart(Default::default())), // _ => None, // } // } // }