use super::BitrotWriterWrapper; use super::Erasure; use crate::disk::error::Error; use crate::disk::error_reduce::count_errs; use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs}; use bytes::Bytes; use futures::StreamExt; use futures::stream::FuturesUnordered; use std::sync::Arc; use std::vec; use tokio::io::AsyncRead; use tokio::sync::mpsc; use tracing::error; pub(crate) struct MultiWriter<'a> { writers: &'a mut [Option], write_quorum: usize, errs: Vec>, } impl<'a> MultiWriter<'a> { pub fn new(writers: &'a mut [Option], write_quorum: usize) -> Self { let length = writers.len(); MultiWriter { writers, write_quorum, errs: vec![None; length], } } async fn write_shard(writer_opt: &mut Option, err: &mut Option, shard: &Bytes) { match writer_opt { Some(writer) => { match writer.write(shard).await { Ok(n) => { if n < shard.len() { *err = Some(Error::ShortWrite); *writer_opt = None; // Mark as failed } else { *err = None; } } Err(e) => { *err = Some(Error::from(e)); } } } None => { *err = Some(Error::DiskNotFound); } } } pub async fn write(&mut self, data: Vec) -> std::io::Result<()> { assert_eq!(data.len(), self.writers.len()); { let mut futures = FuturesUnordered::new(); for ((writer_opt, err), shard) in self.writers.iter_mut().zip(self.errs.iter_mut()).zip(data.iter()) { if err.is_some() { continue; // Skip if we already have an error for this writer } futures.push(Self::write_shard(writer_opt, err, shard)); } while let Some(()) = futures.next().await {} } let nil_count = self.errs.iter().filter(|&e| e.is_none()).count(); if nil_count >= self.write_quorum { return Ok(()); } if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) { error!( "reduce_write_quorum_errs: {:?}, offline-disks={}/{}, errs={:?}", write_err, count_errs(&self.errs, &Error::DiskNotFound), self.writers.len(), self.errs ); return Err(std::io::Error::other(format!( "Failed to write data: {} (offline-disks={}/{})", write_err, count_errs(&self.errs, &Error::DiskNotFound), self.writers.len() ))); } Err(std::io::Error::other(format!( "Failed to write data: (offline-disks={}/{}): {}", count_errs(&self.errs, &Error::DiskNotFound), self.writers.len(), self.errs .iter() .map(|e| e.as_ref().map_or("".to_string(), |e| e.to_string())) .collect::>() .join(", ") ))) } pub async fn _shutdown(&mut self) -> std::io::Result<()> { for writer in self.writers.iter_mut().flatten() { writer.shutdown().await?; } Ok(()) } } impl Erasure { pub async fn encode( self: Arc, mut reader: R, writers: &mut [Option], quorum: usize, ) -> std::io::Result<(R, usize)> where R: AsyncRead + Send + Sync + Unpin + 'static, { let (tx, mut rx) = mpsc::channel::>(8); let task = tokio::spawn(async move { let block_size = self.block_size; let mut total = 0; let mut buf = vec![0u8; block_size]; loop { match rustfs_utils::read_full(&mut reader, &mut buf).await { Ok(n) if n > 0 => { total += n; let res = self.encode_data(&buf[..n])?; if let Err(err) = tx.send(res).await { return Err(std::io::Error::other(format!("Failed to send encoded data : {}", err))); } } Ok(_) => break, Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { break; } Err(e) => { return Err(e); } } } Ok((reader, total)) }); let mut writers = MultiWriter::new(writers, quorum); while let Some(block) = rx.recv().await { if block.is_empty() { break; } writers.write(block).await?; } let (reader, total) = task.await??; // writers.shutdown().await?; Ok((reader, total)) } }