Merge pull request #122 from rustfs/fix/content-length

Fix/content length

Cargo.lock

@@ -2135,6 +2135,7 @@ version = "0.1.0"
 dependencies = [
  "admin",
  "async-trait",
+ "atoi",
  "axum",
  "bytes",
  "clap",

ecstore/src/erasure.rs

@@ -67,34 +67,36 @@ impl Erasure {
         );
 
         let mut total: usize = 0;
+
         loop {
-            let new_len = {
-                let remain = total_size - total;
-                if remain > self.block_size {
-                    self.block_size
-                } else {
-                    remain
-                }
-            };
-
-            if new_len == 0 {
-                break;
-            }
-
-            self.buf.resize(new_len, 0u8);
-
-            match reader.read_exact(&mut self.buf).await {
-                Ok(res) => res,
-                Err(e) => {
-                    if let ErrorKind::UnexpectedEof = e.kind() {
-                        break;
+            if total_size > 0 {
+                let new_len = {
+                    let remain = total_size - total;
+                    if remain > self.block_size {
+                        self.block_size
                     } else {
-                        return Err(Error::new(e));
+                        remain
                     }
-                }
-            };
+                };
 
-            total += self.buf.len();
+                if new_len == 0 && total > 0 {
+                    break;
+                }
+
+                self.buf.resize(new_len, 0u8);
+
+                match reader.read_exact(&mut self.buf).await {
+                    Ok(res) => res,
+                    Err(e) => {
+                        if let ErrorKind::UnexpectedEof = e.kind() {
+                            break;
+                        } else {
+                            return Err(Error::new(e));
+                        }
+                    }
+                };
+                total += self.buf.len();
+            }
 
             let blocks = self.encode_data(&self.buf)?;
             let mut errs = Vec::new();
@@ -112,6 +114,9 @@ impl Erasure {
 
             let none_count = errs.iter().filter(|&x| x.is_none()).count();
             if none_count >= write_quorum {
+                if total_size == 0 {
+                    break;
+                }
                 continue;
             }
 
@@ -119,6 +124,10 @@ impl Erasure {
                 warn!("Erasure encode errs {:?}", &errs);
                 return Err(err);
             }
+
+            if total_size == 0 {
+                break;
+            }
         }
 
         Ok(total)

ecstore/src/utils/path.rs

@@ -1,4 +1,4 @@
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 
 const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__";
 

ecstore/src/xhttp.rs

@@ -1,2 +1,3 @@
 pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
 pub const AMZ_STORAGE_CLASS: &str = "x-amz-storage-class";
+pub const AMZ_DECODED_CONTENT_LENGTH: &str = "X-Amz-Decoded-Content-Length";

rustfs/Cargo.toml

@@ -57,6 +57,7 @@ router = { version = "0.0.1", path = "../router" }
 matchit = "0.8.4"
 shadow-rs = "0.35.2"
 const-str = { version = "0.5.7", features = ["std", "proc"] }
+atoi = "2.0.0"
 
 [build-dependencies]
 prost-build.workspace = true

rustfs/src/storage/ecfs.rs

@@ -588,7 +588,19 @@ impl S3 for FS {
 
         let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
 
-        let Some(content_length) = content_length else { return Err(s3_error!(IncompleteBody)) };
+        let content_length = match content_length {
+            Some(c) => c,
+            None => {
+                if let Some(val) = req.headers.get(xhttp::AMZ_DECODED_CONTENT_LENGTH) {
+                    match atoi::atoi::<i64>(val.as_bytes()) {
+                        Some(x) => x,
+                        None => return Err(s3_error!(UnexpectedContent)),
+                    }
+                } else {
+                    return Err(s3_error!(UnexpectedContent));
+                }
+            }
+        };
 
         let mut reader = PutObjReader::new(body, content_length as usize);
 
@@ -677,7 +689,19 @@ impl S3 for FS {
         // let upload_id =
 
         let body = body.ok_or_else(|| s3_error!(IncompleteBody))?;
-        let content_length = content_length.ok_or_else(|| s3_error!(IncompleteBody))?;
+        let content_length = match content_length {
+            Some(c) => c,
+            None => {
+                if let Some(val) = req.headers.get(xhttp::AMZ_DECODED_CONTENT_LENGTH) {
+                    match atoi::atoi::<i64>(val.as_bytes()) {
+                        Some(x) => x,
+                        None => return Err(s3_error!(UnexpectedContent)),
+                    }
+                } else {
+                    return Err(s3_error!(UnexpectedContent));
+                }
+            }
+        };
 
         // mc cp step 4
         let mut data = PutObjReader::new(body, content_length as usize);