gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-31 09:18:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix bitrot

Browse Source
This commit is contained in:
weisd
2025-06-18 18:22:07 +08:00
parent 85421657ad
commit 5b1b527851
12 changed files with 177 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/rio/src/http_reader.rs
+57 -45
View File
@@ -46,7 +46,7 @@ pin_project! {
impl HttpReader { impl HttpReader {
pub async fn new(url: String, method: Method, headers: HeaderMap, body: Option<Vec<u8>>) -> io::Result<Self> { pub async fn new(url: String, method: Method, headers: HeaderMap, body: Option<Vec<u8>>) -> io::Result<Self> {
http_log!("[HttpReader::new] url: {url}, method: {method:?}, headers: {headers:?}"); // http_log!("[HttpReader::new] url: {url}, method: {method:?}, headers: {headers:?}");
Self::with_capacity(url, method, headers, body, 0).await Self::with_capacity(url, method, headers, body, 0).await
} }
/// Create a new HttpReader from a URL. The request is performed immediately. /// Create a new HttpReader from a URL. The request is performed immediately.
@@ -57,10 +57,10 @@ impl HttpReader {
body: Option<Vec<u8>>, body: Option<Vec<u8>>,
_read_buf_size: usize, _read_buf_size: usize,
) -> io::Result<Self> { ) -> io::Result<Self> {
http_log!( // http_log!(
"[HttpReader::with_capacity] url: {url}, method: {method:?}, headers: {headers:?}, buf_size: {}", // "[HttpReader::with_capacity] url: {url}, method: {method:?}, headers: {headers:?}, buf_size: {}",
_read_buf_size // _read_buf_size
); // );
// First, check if the connection is available (HEAD) // First, check if the connection is available (HEAD)
let client = get_http_client(); let client = get_http_client();
let head_resp = client.head(&url).headers(headers.clone()).send().await; let head_resp = client.head(&url).headers(headers.clone()).send().await;
@@ -119,12 +119,12 @@ impl HttpReader {
impl AsyncRead for HttpReader { impl AsyncRead for HttpReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> { fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
http_log!( // http_log!(
"[HttpReader::poll_read] url: {}, method: {:?}, buf.remaining: {}", // "[HttpReader::poll_read] url: {}, method: {:?}, buf.remaining: {}",
self.url, // self.url,
self.method, // self.method,
buf.remaining() // buf.remaining()
); // );
// Read from the inner stream // Read from the inner stream
Pin::new(&mut self.inner).poll_read(cx, buf) Pin::new(&mut self.inner).poll_read(cx, buf)
} }
@@ -157,20 +157,20 @@ impl Stream for ReceiverStream {
type Item = Result<Bytes, std::io::Error>; type Item = Result<Bytes, std::io::Error>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let poll = Pin::new(&mut self.receiver).poll_recv(cx); let poll = Pin::new(&mut self.receiver).poll_recv(cx);
match &poll { // match &poll {
Poll::Ready(Some(Some(bytes))) => { // Poll::Ready(Some(Some(bytes))) => {
http_log!("[ReceiverStream] poll_next: got {} bytes", bytes.len()); // // http_log!("[ReceiverStream] poll_next: got {} bytes", bytes.len());
} // }
Poll::Ready(Some(None)) => { // Poll::Ready(Some(None)) => {
http_log!("[ReceiverStream] poll_next: sender shutdown"); // // http_log!("[ReceiverStream] poll_next: sender shutdown");
} // }
Poll::Ready(None) => { // Poll::Ready(None) => {
http_log!("[ReceiverStream] poll_next: channel closed"); // // http_log!("[ReceiverStream] poll_next: channel closed");
} // }
Poll::Pending => { // Poll::Pending => {
// http_log!("[ReceiverStream] poll_next: pending"); // // http_log!("[ReceiverStream] poll_next: pending");
} // }
} // }
match poll { match poll {
Poll::Ready(Some(Some(bytes))) => Poll::Ready(Some(Ok(bytes))), Poll::Ready(Some(Some(bytes))) => Poll::Ready(Some(Ok(bytes))),
Poll::Ready(Some(None)) => Poll::Ready(None), // Sender shutdown Poll::Ready(Some(None)) => Poll::Ready(None), // Sender shutdown
@@ -196,7 +196,7 @@ pin_project! {
impl HttpWriter { impl HttpWriter {
/// Create a new HttpWriter for the given URL. The HTTP request is performed in the background. /// Create a new HttpWriter for the given URL. The HTTP request is performed in the background.
pub async fn new(url: String, method: Method, headers: HeaderMap) -> io::Result<Self> { pub async fn new(url: String, method: Method, headers: HeaderMap) -> io::Result<Self> {
http_log!("[HttpWriter::new] url: {url}, method: {method:?}, headers: {headers:?}"); // http_log!("[HttpWriter::new] url: {url}, method: {method:?}, headers: {headers:?}");
let url_clone = url.clone(); let url_clone = url.clone();
let method_clone = method.clone(); let method_clone = method.clone();
let headers_clone = headers.clone(); let headers_clone = headers.clone();
@@ -206,13 +206,13 @@ impl HttpWriter {
let resp = client.put(&url).headers(headers.clone()).body(Vec::new()).send().await; let resp = client.put(&url).headers(headers.clone()).body(Vec::new()).send().await;
match resp { match resp {
Ok(resp) => { Ok(resp) => {
http_log!("[HttpWriter::new] empty PUT status: {}", resp.status()); // http_log!("[HttpWriter::new] empty PUT status: {}", resp.status());
if !resp.status().is_success() { if !resp.status().is_success() {
return Err(Error::other(format!("Empty PUT failed: status {}", resp.status()))); return Err(Error::other(format!("Empty PUT failed: status {}", resp.status())));
} }
} }
Err(e) => { Err(e) => {
http_log!("[HttpWriter::new] empty PUT error: {e}"); // http_log!("[HttpWriter::new] empty PUT error: {e}");
return Err(Error::other(format!("Empty PUT failed: {e}"))); return Err(Error::other(format!("Empty PUT failed: {e}")));
} }
} }
@@ -223,9 +223,9 @@ impl HttpWriter {
let handle = tokio::spawn(async move { let handle = tokio::spawn(async move {
let stream = ReceiverStream { receiver }; let stream = ReceiverStream { receiver };
let body = reqwest::Body::wrap_stream(stream); let body = reqwest::Body::wrap_stream(stream);
http_log!( // http_log!(
"[HttpWriter::spawn] sending HTTP request: url={url_clone}, method={method_clone:?}, headers={headers_clone:?}" // "[HttpWriter::spawn] sending HTTP request: url={url_clone}, method={method_clone:?}, headers={headers_clone:?}"
); // );
let client = get_http_client(); let client = get_http_client();
let request = client let request = client
@@ -238,7 +238,7 @@ impl HttpWriter {
match response { match response {
Ok(resp) => { Ok(resp) => {
http_log!("[HttpWriter::spawn] got response: status={}", resp.status()); // http_log!("[HttpWriter::spawn] got response: status={}", resp.status());
if !resp.status().is_success() { if !resp.status().is_success() {
let _ = err_tx.send(Error::other(format!( let _ = err_tx.send(Error::other(format!(
"HttpWriter HTTP request failed with non-200 status {}", "HttpWriter HTTP request failed with non-200 status {}",
@@ -248,17 +248,17 @@ impl HttpWriter {
} }
} }
Err(e) => { Err(e) => {
http_log!("[HttpWriter::spawn] HTTP request error: {e}"); // http_log!("[HttpWriter::spawn] HTTP request error: {e}");
let _ = err_tx.send(Error::other(format!("HTTP request failed: {}", e))); let _ = err_tx.send(Error::other(format!("HTTP request failed: {}", e)));
return Err(Error::other(format!("HTTP request failed: {}", e))); return Err(Error::other(format!("HTTP request failed: {}", e)));
} }
} }
http_log!("[HttpWriter::spawn] HTTP request completed, exiting"); // http_log!("[HttpWriter::spawn] HTTP request completed, exiting");
Ok(()) Ok(())
}); });
http_log!("[HttpWriter::new] connection established successfully"); // http_log!("[HttpWriter::new] connection established successfully");
Ok(Self { Ok(Self {
url, url,
method, method,
@@ -285,12 +285,12 @@ impl HttpWriter {
impl AsyncWrite for HttpWriter { impl AsyncWrite for HttpWriter {
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> { fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
http_log!( // http_log!(
"[HttpWriter::poll_write] url: {}, method: {:?}, buf.len: {}", // "[HttpWriter::poll_write] url: {}, method: {:?}, buf.len: {}",
self.url, // self.url,
self.method, // self.method,
buf.len() // buf.len()
); // );
if let Ok(e) = Pin::new(&mut self.err_rx).try_recv() { if let Ok(e) = Pin::new(&mut self.err_rx).try_recv() {
return Poll::Ready(Err(e)); return Poll::Ready(Err(e));
} }
@@ -307,12 +307,19 @@ impl AsyncWrite for HttpWriter {
} }
fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> { fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
// let url = self.url.clone();
// let method = self.method.clone();
if !self.finish { if !self.finish {
http_log!("[HttpWriter::poll_shutdown] url: {}, method: {:?}", self.url, self.method); // http_log!("[HttpWriter::poll_shutdown] url: {}, method: {:?}", url, method);
self.sender self.sender
.try_send(None) .try_send(None)
.map_err(|e| Error::other(format!("HttpWriter shutdown error: {}", e)))?; .map_err(|e| Error::other(format!("HttpWriter shutdown error: {}", e)))?;
http_log!("[HttpWriter::poll_shutdown] sent shutdown signal to HTTP request"); // http_log!(
// "[HttpWriter::poll_shutdown] sent shutdown signal to HTTP request, url: {}, method: {:?}",
// url,
// method
// );
self.finish = true; self.finish = true;
} }
@@ -320,13 +327,18 @@ impl AsyncWrite for HttpWriter {
use futures::FutureExt; use futures::FutureExt;
match Pin::new(&mut self.get_mut().handle).poll_unpin(_cx) { match Pin::new(&mut self.get_mut().handle).poll_unpin(_cx) {
Poll::Ready(Ok(_)) => { Poll::Ready(Ok(_)) => {
http_log!("[HttpWriter::poll_shutdown] HTTP request finished successfully"); // http_log!(
// "[HttpWriter::poll_shutdown] HTTP request finished successfully, url: {}, method: {:?}",
// url,
// method
// );
} }
Poll::Ready(Err(e)) => { Poll::Ready(Err(e)) => {
http_log!("[HttpWriter::poll_shutdown] HTTP request failed: {e}"); // http_log!("[HttpWriter::poll_shutdown] HTTP request failed: {e}, url: {}, method: {:?}", url, method);
return Poll::Ready(Err(Error::other(format!("HTTP request failed: {}", e)))); return Poll::Ready(Err(Error::other(format!("HTTP request failed: {}", e))));
} }
Poll::Pending => { Poll::Pending => {
// http_log!("[HttpWriter::poll_shutdown] HTTP request pending, url: {}, method: {:?}", url, method);
return Poll::Pending; return Poll::Pending;
} }
} }
ecstore/src/bitrot.rs
+1 -1
View File
@@ -28,7 +28,7 @@ pub async fn create_bitrot_reader(
checksum_algo: HashAlgorithm, checksum_algo: HashAlgorithm,
) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> { ) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
// Calculate the total length to read, including the checksum overhead // Calculate the total length to read, including the checksum overhead
let length = offset.div_ceil(shard_size) * checksum_algo.size() + length; let length = length.div_ceil(shard_size) * checksum_algo.size() + length;
if let Some(data) = inline_data { if let Some(data) = inline_data {
// Use inline data // Use inline data
ecstore/src/disk/local.rs
+3 -5
View File
@@ -1399,8 +1399,6 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "debug", skip(self))] #[tracing::instrument(level = "debug", skip(self))]
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, _file_size: i64) -> Result<FileWriter> { async fn create_file(&self, origvolume: &str, volume: &str, path: &str, _file_size: i64) -> Result<FileWriter> {
// warn!("disk create_file: origvolume: {}, volume: {}, path: {}", origvolume, volume, path);
if !origvolume.is_empty() { if !origvolume.is_empty() {
let origvolume_dir = self.get_bucket_path(origvolume)?; let origvolume_dir = self.get_bucket_path(origvolume)?;
if !skip_access_checks(origvolume) { if !skip_access_checks(origvolume) {
@@ -1431,8 +1429,6 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "debug", skip(self))] #[tracing::instrument(level = "debug", skip(self))]
// async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result<File> { // async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result<File> {
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> { async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
warn!("disk append_file: volume: {}, path: {}", volume, path);
let volume_dir = self.get_bucket_path(volume)?; let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) { if !skip_access_checks(volume) {
access(&volume_dir) access(&volume_dir)
@@ -1497,7 +1493,9 @@ impl DiskAPI for LocalDisk {
return Err(DiskError::FileCorrupt); return Err(DiskError::FileCorrupt);
} }
f.seek(SeekFrom::Start(offset as u64)).await?; if offset > 0 {
f.seek(SeekFrom::Start(offset as u64)).await?;
}
Ok(Box::new(f)) Ok(Box::new(f))
} }
ecstore/src/disk/remote.rs
+15 -2
View File
@@ -608,7 +608,14 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))] #[tracing::instrument(level = "debug", skip(self))]
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> { async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
info!("read_file_stream {}/{}/{}", self.endpoint.to_string(), volume, path); // warn!(
// "disk remote read_file_stream {}/{}/{} offset={} length={}",
// self.endpoint.to_string(),
// volume,
// path,
// offset,
// length
// );
let url = format!( let url = format!(
"{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}", "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}",
self.endpoint.grid_host(), self.endpoint.grid_host(),
@@ -641,7 +648,13 @@ impl DiskAPI for RemoteDisk {
#[tracing::instrument(level = "debug", skip(self))] #[tracing::instrument(level = "debug", skip(self))]
async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<FileWriter> { async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, file_size: i64) -> Result<FileWriter> {
info!("create_file {}/{}/{}", self.endpoint.to_string(), volume, path); // warn!(
// "disk remote create_file {}/{}/{} file_size={}",
// self.endpoint.to_string(),
// volume,
// path,
// file_size
// );
let url = format!( let url = format!(
"{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}", "{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}",
ecstore/src/erasure_coding/bitrot.rs
+44 -37
View File
@@ -1,7 +1,9 @@
use bytes::Bytes; use bytes::Bytes;
use pin_project_lite::pin_project; use pin_project_lite::pin_project;
use rustfs_utils::{HashAlgorithm, read_full, write_all}; use rustfs_utils::HashAlgorithm;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite}; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tracing::error;
use uuid::Uuid;
pin_project! { pin_project! {
/// BitrotReader reads (hash+data) blocks from an async reader and verifies hash integrity. /// BitrotReader reads (hash+data) blocks from an async reader and verifies hash integrity.
@@ -12,10 +14,11 @@ pin_project! {
shard_size: usize, shard_size: usize,
buf: Vec<u8>, buf: Vec<u8>,
hash_buf: Vec<u8>, hash_buf: Vec<u8>,
hash_read: usize, // hash_read: usize,
data_buf: Vec<u8>, // data_buf: Vec<u8>,
data_read: usize, // data_read: usize,
hash_checked: bool, // hash_checked: bool,
id: Uuid,
} }
} }
@@ -32,10 +35,11 @@ where
shard_size, shard_size,
buf: Vec::new(), buf: Vec::new(),
hash_buf: vec![0u8; hash_size], hash_buf: vec![0u8; hash_size],
hash_read: 0, // hash_read: 0,
data_buf: Vec::new(), // data_buf: Vec::new(),
data_read: 0, // data_read: 0,
hash_checked: false, // hash_checked: false,
id: Uuid::new_v4(),
} }
} }
@@ -51,30 +55,31 @@ where
let hash_size = self.hash_algo.size(); let hash_size = self.hash_algo.size();
// Read hash // Read hash
let mut hash_buf = vec![0u8; hash_size];
if hash_size > 0 { if hash_size > 0 {
self.inner.read_exact(&mut hash_buf).await?; self.inner.read_exact(&mut self.hash_buf).await.map_err(|e| {
error!("bitrot reader read hash error: {}", e);
e
})?;
} }
let data_len = read_full(&mut self.inner, out).await?; // Read data
let mut data_len = 0;
// // Read data while data_len < out.len() {
// let mut data_len = 0; let n = self.inner.read(&mut out[data_len..]).await.map_err(|e| {
// while data_len < out.len() { error!("bitrot reader read data error: {}", e);
// let n = self.inner.read(&mut out[data_len..]).await?; e
// if n == 0 { })?;
// break; if n == 0 {
// } break;
// data_len += n; }
// // Only read up to one shard_size block data_len += n;
// if data_len >= self.shard_size { }
// break;
// }
// }
if hash_size > 0 { if hash_size > 0 {
let actual_hash = self.hash_algo.hash_encode(&out[..data_len]); let actual_hash = self.hash_algo.hash_encode(&out[..data_len]);
if actual_hash.as_ref() != hash_buf.as_slice() { if actual_hash.as_ref() != self.hash_buf.as_slice() {
error!("bitrot reader hash mismatch, id={} data_len={}, out_len={}", self.id, data_len, out.len());
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch")); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
} }
} }
@@ -145,22 +150,20 @@ where
self.buf.extend_from_slice(buf); self.buf.extend_from_slice(buf);
// Write hash+data in one call self.inner.write_all(&self.buf).await?;
let mut n = write_all(&mut self.inner, &self.buf).await?;
if n < hash_algo.size() { self.inner.flush().await?;
return Err(std::io::Error::new(
std::io::ErrorKind::WriteZero,
"short write: not enough bytes written",
));
}
n -= hash_algo.size(); let n = self.buf.len();
self.buf.clear(); self.buf.clear();
Ok(n) Ok(n)
} }
pub async fn shutdown(&mut self) -> std::io::Result<()> {
self.inner.shutdown().await
}
} }
pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize { pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize {
@@ -330,6 +333,10 @@ impl BitrotWriterWrapper {
self.bitrot_writer.write(buf).await self.bitrot_writer.write(buf).await
} }
pub async fn shutdown(&mut self) -> std::io::Result<()> {
self.bitrot_writer.shutdown().await
}
/// Extract the inline buffer data, consuming the wrapper /// Extract the inline buffer data, consuming the wrapper
pub fn into_inline_data(self) -> Option<Vec<u8>> { pub fn into_inline_data(self) -> Option<Vec<u8>> {
match self.writer_type { match self.writer_type {
ecstore/src/erasure_coding/decode.rs
+21 -23
View File
@@ -67,36 +67,34 @@ where
} }
// 使用并发读取所有分片 // 使用并发读取所有分片
let mut read_futs = Vec::with_capacity(self.readers.len());
let read_futs: Vec<_> = self for (i, opt_reader) in self.readers.iter_mut().enumerate() {
.readers let future = if let Some(reader) = opt_reader.as_mut() {
.iter_mut() Box::pin(async move {
.enumerate()
.map(|(i, opt_reader)| {
if let Some(reader) = opt_reader.as_mut() {
let mut buf = vec![0u8; shard_size]; let mut buf = vec![0u8; shard_size];
// 需要move i, buf match reader.read(&mut buf).await {
Some(async move { Ok(n) => {
match reader.read(&mut buf).await { buf.truncate(n);
Ok(n) => { (i, Ok(buf))
buf.truncate(n);
(i, Ok(buf))
}
Err(e) => (i, Err(Error::from(e))),
} }
}) Err(e) => (i, Err(Error::from(e))),
} else { }
None }) as std::pin::Pin<Box<dyn std::future::Future<Output = (usize, Result<Vec<u8>, Error>)> + Send>>
} } else {
}) // reader是None时返回FileNotFound错误
.collect(); Box::pin(async move { (i, Err(Error::FileNotFound)) })
as std::pin::Pin<Box<dyn std::future::Future<Output = (usize, Result<Vec<u8>, Error>)> + Send>>
};
read_futs.push(future);
}
// 过滤掉None，join_all let results = join_all(read_futs).await;
let mut results = join_all(read_futs.into_iter().flatten()).await;
let mut shards: Vec<Option<Vec<u8>>> = vec![None; self.readers.len()]; let mut shards: Vec<Option<Vec<u8>>> = vec![None; self.readers.len()];
let mut errs = vec![None; self.readers.len()]; let mut errs = vec![None; self.readers.len()];
for (i, shard) in results.drain(..) {
for (i, shard) in results.into_iter() {
match shard { match shard {
Ok(data) => { Ok(data) => {
if !data.is_empty() { if !data.is_empty() {
ecstore/src/erasure_coding/encode.rs
+8 -1
View File
@@ -97,6 +97,13 @@ impl<'a> MultiWriter<'a> {
.join(", ") .join(", ")
))) )))
} }
pub async fn _shutdown(&mut self) -> std::io::Result<()> {
for writer in self.writers.iter_mut().flatten() {
writer.shutdown().await?;
}
Ok(())
}
} }
impl Erasure { impl Erasure {
@@ -147,7 +154,7 @@ impl Erasure {
} }
let (reader, total) = task.await??; let (reader, total) = task.await??;
// writers.shutdown().await?;
Ok((reader, total)) Ok((reader, total))
} }
} }
ecstore/src/erasure_coding/erasure.rs
+16 -2
View File
@@ -555,6 +555,13 @@ mod tests {
use super::*; use super::*;
#[test]
fn test_shard_file_size_cases2() {
let erasure = Erasure::new(12, 4, 1024 * 1024);
assert_eq!(erasure.shard_file_size(1572864), 131074);
}
#[test] #[test]
fn test_shard_file_size_cases() { fn test_shard_file_size_cases() {
let erasure = Erasure::new(4, 2, 8); let erasure = Erasure::new(4, 2, 8);
@@ -577,6 +584,8 @@ mod tests {
assert_eq!(erasure.shard_file_size(1248739), 312186); // 1248739/8=156092, last=3, 3 div_ceil 4=1, 156092*2+1=312185 assert_eq!(erasure.shard_file_size(1248739), 312186); // 1248739/8=156092, last=3, 3 div_ceil 4=1, 156092*2+1=312185
assert_eq!(erasure.shard_file_size(43), 12); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11 assert_eq!(erasure.shard_file_size(43), 12); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11
assert_eq!(erasure.shard_file_size(1572864), 393216); // 43/8=5, last=3, 3 div_ceil 4=1, 5*2+1=11
} }
#[test] #[test]
@@ -677,9 +686,14 @@ mod tests {
#[test] #[test]
fn test_shard_file_offset() { fn test_shard_file_offset() {
let erasure = Erasure::new(4, 2, 8); let erasure = Erasure::new(8, 8, 1024 * 1024);
let offset = erasure.shard_file_offset(0, 16, 32); let offset = erasure.shard_file_offset(0, 86, 86);
println!("offset={}", offset);
assert!(offset > 0); assert!(offset > 0);
let total_length = erasure.shard_file_size(86);
println!("total_length={}", total_length);
assert!(total_length > 0);
} }
#[tokio::test] #[tokio::test]
ecstore/src/rebalance.rs
+2 -2
View File
@@ -243,7 +243,7 @@ impl ECStore {
return Err(err); return Err(err);
} }
error!("rebalanceMeta: not found, rebalance not started"); warn!("rebalanceMeta: not found, rebalance not started");
} }
} }
@@ -501,7 +501,7 @@ impl ECStore {
if let Some(meta) = rebalance_meta.as_mut() { if let Some(meta) = rebalance_meta.as_mut() {
meta.cancel = Some(tx) meta.cancel = Some(tx)
} else { } else {
error!("start_rebalance: rebalance_meta is None exit"); warn!("start_rebalance: rebalance_meta is None exit");
return; return;
} }
ecstore/src/set_disk.rs
-2
View File
@@ -1859,8 +1859,6 @@ impl SetDisks {
let (last_part_index, _) = fi.to_part_offset(end_offset)?; let (last_part_index, _) = fi.to_part_offset(end_offset)?;
// let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size); let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let mut total_readed = 0; let mut total_readed = 0;
rustfs/src/storage/ecfs.rs
+2 -1
View File
@@ -91,6 +91,7 @@ use tokio_stream::wrappers::ReceiverStream;
use tokio_tar::Archive; use tokio_tar::Archive;
use tokio_util::io::ReaderStream; use tokio_util::io::ReaderStream;
use tokio_util::io::StreamReader; use tokio_util::io::StreamReader;
use tracing::debug;
use tracing::error; use tracing::error;
use tracing::info; use tracing::info;
use tracing::warn; use tracing::warn;
@@ -1787,7 +1788,7 @@ impl S3 for FS {
let object_lock_configuration = match metadata_sys::get_object_lock_config(&bucket).await { let object_lock_configuration = match metadata_sys::get_object_lock_config(&bucket).await {
Ok((cfg, _created)) => Some(cfg), Ok((cfg, _created)) => Some(cfg),
Err(err) => { Err(err) => {
warn!("get_object_lock_config err {:?}", err); debug!("get_object_lock_config err {:?}", err);
None None
} }
}; };
scripts/dev_rustfs.sh
+8 -8
View File
@@ -9,14 +9,14 @@ UNZIP_TARGET="./"
SERVER_LIST=( SERVER_LIST=(
"root@172.23.215.2" # node1 "root@node1" # node1
"root@172.23.215.4" # node2 "root@node2" # node2
"root@172.23.215.7" # node3 "root@node3" # node3
"root@172.23.215.3" # node4 "root@node4" # node4
"root@172.23.215.8" # node5 # "root@node5" # node5
"root@172.23.215.5" # node6 # "root@node6" # node6
"root@172.23.215.9" # node7 # "root@node7" # node7
"root@172.23.215.6" # node8 # "root@node8" # node8
) )
REMOTE_TMP="~/rustfs" REMOTE_TMP="~/rustfs"