From 1a6fa86fc2466aa498160f332989cd388fd5e55f Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 18 Jul 2024 16:25:46 +0800 Subject: [PATCH] todo: get_object_reader --- Cargo.lock | 2 ++ ecstore/Cargo.toml | 1 + ecstore/src/sets.rs | 17 +++++++++++++++-- ecstore/src/store.rs | 21 +++++++++++++++++++-- ecstore/src/store_api.rs | 16 ++++++++++++++++ rustfs/Cargo.toml | 1 + rustfs/src/storage/ecfs.rs | 8 ++------ 7 files changed, 56 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c55451eb2..02491fda1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -318,6 +318,7 @@ dependencies = [ "crc32fast", "futures", "hex-simd", + "http", "lazy_static", "netif", "path-absolutize", @@ -1093,6 +1094,7 @@ dependencies = [ "ecstore", "http", "hyper-util", + "mime", "s3s", "time", "tokio", diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 224eb2d89..082fea7e6 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -37,6 +37,7 @@ base64-simd = "0.8.0" sha2 = "0.10.8" hex-simd = "0.8.0" path-clean = "1.0.1" +http.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index bdf3d1f82..fc3cd12f8 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use http::HeaderMap; use uuid::Uuid; use crate::{ @@ -7,8 +8,8 @@ use crate::{ format::{DistributionAlgoVersion, FormatV3}, set_disk::SetDisks, store_api::{ - BucketInfo, BucketOptions, CompletePart, FileInfo, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, - PartInfo, PutObjReader, StorageAPI, + BucketInfo, BucketOptions, CompletePart, FileInfo, GetObjectReader, HTTPRangeSpec, MakeBucketOptions, + MultipartUploadResult, ObjectInfo, ObjectOptions, PartInfo, PutObjReader, StorageAPI, }, utils::hash, }; @@ -132,6 +133,18 @@ impl StorageAPI for Sets { async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { self.get_disks_by_key(object).get_object_info(bucket, object, opts).await } + async fn get_Object_reader( + &self, + bucket: &str, + object: &str, + range: HTTPRangeSpec, + h: HeaderMap, + opts: &ObjectOptions, + ) -> Result { + self.get_disks_by_key(object) + .get_Object_reader(bucket, object, range, h, opts) + .await + } async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<()> { self.get_disks_by_key(object).put_object(bucket, object, data, opts).await } diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 9148f9556..9549246e3 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use anyhow::{Error, Result}; +use http::HeaderMap; use s3s::{dto::StreamingBlob, Body}; use uuid::Uuid; @@ -14,8 +15,8 @@ use crate::{ peer::{PeerS3Client, S3PeerSys}, sets::Sets, store_api::{ - BucketInfo, BucketOptions, CompletePart, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, PartInfo, - PutObjReader, StorageAPI, + BucketInfo, BucketOptions, CompletePart, GetObjectReader, HTTPRangeSpec, MakeBucketOptions, MultipartUploadResult, + ObjectInfo, ObjectOptions, PartInfo, PutObjReader, StorageAPI, }, store_init, utils, }; @@ -160,6 +161,22 @@ impl StorageAPI for ECStore { unimplemented!() } + async fn get_Object_reader( + &self, + bucket: &str, + object: &str, + range: HTTPRangeSpec, + h: HeaderMap, + opts: &ObjectOptions, + ) -> Result { + let object = utils::path::encode_dir_object(object); + + if self.single_pool() { + return self.pools[0].get_Object_reader(bucket, object.as_str(), range, h, opts).await; + } + + unimplemented!() + } async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<()> { // checkPutObjectArgs diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 477c9e0fc..deb7af4eb 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use http::HeaderMap; use rmp_serde::Serializer; use s3s::dto::StreamingBlob; use serde::{Deserialize, Serialize}; @@ -236,6 +237,13 @@ impl PutObjReader { } } +pub struct GetObjectReader { + pub stream: StreamingBlob, + pub object_info: ObjectInfo, +} + +pub struct HTTPRangeSpec {} + #[derive(Debug)] pub struct ObjectOptions { // Use the maximum parity (N/2), used when saving server configuration files @@ -302,6 +310,14 @@ pub trait StorageAPI { async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>; async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result; async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; + async fn get_Object_reader( + &self, + bucket: &str, + object: &str, + range: HTTPRangeSpec, + h: HeaderMap, + opts: &ObjectOptions, + ) -> Result; async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<()>; async fn put_object_part( &self, diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 6d283c7ef..2cac5e658 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -22,3 +22,4 @@ clap = { version = "4.5.7", features = ["derive"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time"] } hyper-util = { version = "0.1.5", features = ["tokio", "server-auto", "server-graceful"] } http.workspace = true +mime = "0.3.17" diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 20249144a..79f5421ed 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -6,7 +6,6 @@ use ecstore::store_api::MultipartUploadResult; use ecstore::store_api::ObjectOptions; use ecstore::store_api::PutObjReader; use ecstore::store_api::StorageAPI; -use http::header::HeaderValue; use s3s::dto::*; use s3s::s3_error; use s3s::S3Error; @@ -15,6 +14,7 @@ use s3s::S3Result; use s3s::S3; use s3s::{S3Request, S3Response}; use std::fmt::Debug; +use std::str::FromStr; use anyhow::Result; use ecstore::store::ECStore; @@ -172,11 +172,7 @@ impl S3 for FS { let info = try_!(self.store.get_object_info(&bucket, &key, &ObjectOptions::default()).await); debug!("info {:?}", info); - let content_type = { - let m = HeaderValue::from_static("hello"); - - ContentType::try_from_header_value(m)? - }; + let content_type = try_!(ContentType::from_str("application/x-msdownload")); let output = HeadObjectOutput { content_length: Some(try_!(i64::try_from(info.size))),