gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-18 02:33:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: narrow storage compatibility surfaces (#3585)

Browse Source 
* refactor: narrow storage compatibility surfaces

* refactor: narrow observability compatibility surfaces
This commit is contained in:
安正超
2026-06-19 00:56:14 +08:00
committed by GitHub
parent f11b07bf83
commit a860f2b40c
13 changed files with 587 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/s3select-api/src/object_store.rs
+18 -18
View File
@@ -12,12 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_compat::ecstore::error::{
StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
use crate::storage_compat::{
SELECT_DEFAULT_READ_BUFFER_SIZE, SelectGetObjectReader, SelectObjectInfo, SelectObjectOptions, SelectStorageError,
SelectStore, resolve_select_object_store_handle, select_default_read_buffer_size_u64, select_is_err_bucket_not_found,
select_is_err_object_not_found, select_is_err_version_not_found,
};
use crate::storage_compat::ecstore::resolve_object_store_handle;
use crate::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use crate::storage_compat::ecstore::store::ECStore;
use async_trait::async_trait;
use bytes::Bytes;
use chrono::Utc;
@@ -50,8 +49,6 @@ use tokio::io::{AsyncRead, ReadBuf};
use tokio_util::io::ReaderStream;
use transform_stream::AsyncTryStream;
use crate::storage_compat::{SelectGetObjectReader, SelectObjectInfo, SelectObjectOptions};
/// Maximum allowed object size for JSON DOCUMENT mode.
///
/// JSON DOCUMENT format requires loading the entire file into memory for DOM
@@ -83,7 +80,7 @@ pub struct EcObjectStore {
/// this key in the root JSON object before flattening.
json_sub_path: Option<String>,
store: Arc<ECStore>,
store: Arc<SelectStore>,
}
#[derive(Clone, Copy, Debug)]
@@ -111,7 +108,7 @@ pub struct InvalidScanRange;
impl EcObjectStore {
pub fn new(input: Arc<SelectObjectContentInput>) -> S3Result<Self> {
let Some(store) = resolve_object_store_handle() else {
let Some(store) = resolve_select_object_store_handle() else {
return Err(s3_error!(InternalError, "ec store not inited"));
};
@@ -236,7 +233,7 @@ impl EcObjectStore {
return Ok(Bytes::new());
}
let mut end = (DEFAULT_READ_BUFFER_SIZE as u64).min(object_size);
let mut end = select_default_read_buffer_size_u64().min(object_size);
loop {
let bytes = self.read_raw_range_with_opts(0..end, opts).await?;
if let Some(pos) = find_delimiter(&bytes, delimiter) {
@@ -333,8 +330,8 @@ fn find_delimiter(bytes: &[u8], delimiter: &[u8]) -> Option<usize> {
bytes.windows(delimiter.len()).position(|window| window == delimiter)
}
fn map_storage_error(bucket: &str, object: &str, err: StorageError) -> o_Error {
if is_err_bucket_not_found(&err) || is_err_object_not_found(&err) || is_err_version_not_found(&err) {
fn map_storage_error(bucket: &str, object: &str, err: SelectStorageError) -> o_Error {
if select_is_err_bucket_not_found(&err) || select_is_err_object_not_found(&err) || select_is_err_version_not_found(&err) {
return o_Error::NotFound {
path: format!("{bucket}/{object}"),
source: err.to_string().into(),
@@ -461,7 +458,7 @@ impl ObjectStore for EcObjectStore {
GetResultPayload::Stream(stream::empty().boxed())
} else if options.range.is_some() {
let size = (result_range.end - result_range.start) as usize;
let stream = bytes_stream(ReaderStream::with_capacity(reader.stream, DEFAULT_READ_BUFFER_SIZE), size).boxed();
let stream = bytes_stream(ReaderStream::with_capacity(reader.stream, SELECT_DEFAULT_READ_BUFFER_SIZE), size).boxed();
GetResultPayload::Stream(stream)
} else if self.is_json_document {
// JSON DOCUMENT mode: gate on object size before doing any I/O.
@@ -501,7 +498,7 @@ impl ObjectStore for EcObjectStore {
None
};
let stream = scan_range_stream(
ReaderStream::with_capacity(reader.stream, DEFAULT_READ_BUFFER_SIZE),
ReaderStream::with_capacity(reader.stream, SELECT_DEFAULT_READ_BUFFER_SIZE),
delimiter,
scan_range,
include_header && header.is_none(),
@@ -516,14 +513,17 @@ impl ObjectStore for EcObjectStore {
GetResultPayload::Stream(convert_field_delimiter_stream(stream, self.need_convert.then(|| self.delimiter.clone())))
} else if self.need_convert {
let stream = bytes_stream(
ReaderStream::with_capacity(ConvertStream::new(reader.stream, self.delimiter.clone()), DEFAULT_READ_BUFFER_SIZE),
ReaderStream::with_capacity(
ConvertStream::new(reader.stream, self.delimiter.clone()),
SELECT_DEFAULT_READ_BUFFER_SIZE,
),
original_size as usize,
)
.boxed();
GetResultPayload::Stream(stream)
} else {
let stream = bytes_stream(
ReaderStream::with_capacity(reader.stream, DEFAULT_READ_BUFFER_SIZE),
ReaderStream::with_capacity(reader.stream, SELECT_DEFAULT_READ_BUFFER_SIZE),
original_size as usize,
)
.boxed();
@@ -587,7 +587,7 @@ impl<R> ConvertStream<R> {
ConvertStream {
inner,
converter: DelimiterConverter::new(delimiter.into_bytes()),
read_buf: vec![0; DEFAULT_READ_BUFFER_SIZE],
read_buf: vec![0; SELECT_DEFAULT_READ_BUFFER_SIZE],
pending: Vec::new(),
pending_pos: 0,
eof: false,
@@ -607,7 +607,7 @@ impl<R: AsyncRead + Unpin> AsyncRead for ConvertStream<R> {
return Poll::Ready(Ok(()));
}
let read_len = DEFAULT_READ_BUFFER_SIZE.min(buf.remaining().max(1));
let read_len = SELECT_DEFAULT_READ_BUFFER_SIZE.min(buf.remaining().max(1));
let bytes_read = {
let mut read_buf = ReadBuf::new(&mut this.read_buf[..read_len]);
ready!(Pin::new(&mut *this.inner).poll_read(cx, &mut read_buf))?;
crates/s3select-api/src/storage_compat.rs
+24 -4
View File
@@ -12,10 +12,30 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) use rustfs_ecstore::{error, resolve_object_store_handle, set_disk, store};
}
pub(crate) type SelectGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub(crate) type SelectObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) type SelectObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type SelectStorageError = rustfs_ecstore::error::StorageError;
pub(crate) type SelectStore = rustfs_ecstore::store::ECStore;
pub(crate) const SELECT_DEFAULT_READ_BUFFER_SIZE: usize = rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
pub(crate) fn select_default_read_buffer_size_u64() -> u64 {
u64::try_from(SELECT_DEFAULT_READ_BUFFER_SIZE).unwrap_or(u64::MAX)
}
pub(crate) fn resolve_select_object_store_handle() -> Option<std::sync::Arc<SelectStore>> {
rustfs_ecstore::resolve_object_store_handle()
}
pub(crate) fn select_is_err_bucket_not_found(err: &SelectStorageError) -> bool {
rustfs_ecstore::error::is_err_bucket_not_found(err)
}
pub(crate) fn select_is_err_object_not_found(err: &SelectStorageError) -> bool {
rustfs_ecstore::error::is_err_object_not_found(err)
}
pub(crate) fn select_is_err_version_not_found(err: &SelectStorageError) -> bool {
rustfs_ecstore::error::is_err_version_not_found(err)
}