gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 21:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/observability-metrics

Browse Source 
# Conflicts:
#	.github/workflows/build.yml
#	.github/workflows/ci.yml
#	Cargo.lock
#	Cargo.toml
#	appauth/src/token.rs
#	crates/config/src/config.rs
#	crates/event-notifier/examples/simple.rs
#	crates/event-notifier/src/global.rs
#	crates/event-notifier/src/lib.rs
#	crates/event-notifier/src/notifier.rs
#	crates/event-notifier/src/store.rs
#	crates/filemeta/src/filemeta.rs
#	crates/notify/examples/webhook.rs
#	crates/utils/Cargo.toml
#	ecstore/Cargo.toml
#	ecstore/src/cmd/bucket_replication.rs
#	ecstore/src/config/com.rs
#	ecstore/src/disk/error.rs
#	ecstore/src/disk/mod.rs
#	ecstore/src/set_disk.rs
#	ecstore/src/store_api.rs
#	ecstore/src/store_list_objects.rs
#	iam/Cargo.toml
#	iam/src/manager.rs
#	policy/Cargo.toml
#	rustfs/src/admin/rpc.rs
#	rustfs/src/main.rs
#	rustfs/src/storage/mod.rs
This commit is contained in:
houseme
2025-06-19 13:16:48 +08:00
parent 09e8bc8f02 450db14305
commit e6b019c29d
249 changed files with 25137 additions and 11731 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/store_list_objects.rs
+69 -58
View File
@@ -1,31 +1,31 @@
use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::versioning::VersioningApi;
use crate::cache_value::metacache_set::{list_path_raw, ListPathRawOptions};
use crate::disk::error::{is_all_not_found, is_all_volume_not_found, is_err_eof, DiskError};
use crate::disk::{
DiskInfo, DiskStore, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry,
MetadataResolutionParams,
use crate::disk::error::DiskError;
use crate::disk::{DiskInfo, DiskStore};
use crate::error::{
is_all_not_found, is_all_volume_not_found, is_err_bucket_not_found, to_object_err, Error, Result, StorageError,
};
use crate::error::clone_err;
use crate::file_meta::merge_file_meta_versions;
use crate::peer::is_reserved_or_invalid_bucket;
use crate::set_disk::SetDisks;
use crate::store::check_list_objs_args;
use crate::store_api::{FileInfo, ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectOptions};
use crate::store_err::{is_err_bucket_not_found, to_object_err, StorageError};
use crate::utils::path::{self, base_dir_from_prefix, SLASH_SEPARATOR};
use crate::store_api::{ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectOptions};
use crate::StorageAPI;
use crate::{store::ECStore, store_api::ListObjectsV2Info};
use common::error::{Error, Result};
use futures::future::join_all;
use rand::seq::SliceRandom;
use rustfs_filemeta::{
merge_file_meta_versions, FileInfo, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry,
MetadataResolutionParams,
};
use rustfs_utils::path::{self, base_dir_from_prefix, SLASH_SEPARATOR};
use std::collections::HashMap;
use std::io::ErrorKind;
use std::sync::Arc;
use tokio::sync::broadcast::{self, Receiver as B_Receiver};
use tokio::sync::mpsc::{self, Receiver, Sender};
use tracing::error;
use tracing::{error, warn};
use uuid::Uuid;
use crate::disk::fs::SLASH_SEPARATOR;
const MAX_OBJECT_LIST: i32 = 1000;
// const MAX_DELETE_LIST: i32 = 1000;
@@ -280,13 +280,13 @@ impl ECStore {
.list_path(&opts)
.await
.unwrap_or_else(|err| MetaCacheEntriesSortedResult {
err: Some(err),
err: Some(err.into()),
..Default::default()
});
if let Some(err) = &list_result.err {
if !is_err_eof(err) {
return Err(to_object_err(list_result.err.unwrap(), vec![bucket, prefix]));
if let Some(err) = list_result.err.clone() {
if err != rustfs_filemeta::Error::Unexpected {
return Err(to_object_err(err.into(), vec![bucket, prefix]));
}
}
@@ -296,11 +296,13 @@ impl ECStore {
// contextCanceled
let mut get_objects = list_result
.entries
.unwrap_or_default()
.file_infos(bucket, prefix, delimiter.clone())
.await;
let mut get_objects = ObjectInfo::from_meta_cache_entries_sorted(
&list_result.entries.unwrap_or_default(),
bucket,
prefix,
delimiter.clone(),
)
.await;
let is_truncated = {
if max_keys > 0 && get_objects.len() > max_keys as usize {
@@ -363,7 +365,8 @@ impl ECStore {
max_keys: i32,
) -> Result<ListObjectVersionsInfo> {
if marker.is_none() && version_marker.is_some() {
return Err(Error::new(StorageError::NotImplemented));
warn!("inner_list_object_versions: marker is none and version_marker is some");
return Err(StorageError::NotImplemented);
}
// if marker set, limit +1
@@ -382,14 +385,14 @@ impl ECStore {
let mut list_result = match self.list_path(&opts).await {
Ok(res) => res,
Err(err) => MetaCacheEntriesSortedResult {
err: Some(err),
err: Some(err.into()),
..Default::default()
},
};
if let Some(err) = &list_result.err {
if !is_err_eof(err) {
return Err(to_object_err(list_result.err.unwrap(), vec![bucket, prefix]));
if let Some(err) = list_result.err.clone() {
if err != rustfs_filemeta::Error::Unexpected {
return Err(to_object_err(err.into(), vec![bucket, prefix]));
}
}
@@ -397,11 +400,13 @@ impl ECStore {
result.forward_past(opts.marker);
}
let mut get_objects = list_result
.entries
.unwrap_or_default()
.file_info_versions(bucket, prefix, delimiter.clone(), version_marker)
.await;
let mut get_objects = ObjectInfo::from_meta_cache_entries_sorted(
&list_result.entries.unwrap_or_default(),
bucket,
prefix,
delimiter.clone(),
)
.await;
let is_truncated = {
if max_keys > 0 && get_objects.len() > max_keys as usize {
@@ -471,16 +476,16 @@ impl ECStore {
if let Some(marker) = &o.marker {
if !o.prefix.is_empty() && !marker.starts_with(&o.prefix) {
return Err(Error::new(std::io::Error::from(ErrorKind::UnexpectedEof)));
return Err(Error::Unexpected);
}
}
if o.limit == 0 {
return Err(Error::new(std::io::Error::from(ErrorKind::UnexpectedEof)));
return Err(Error::Unexpected);
}
if o.prefix.starts_with(SLASH_SEPARATOR) {
return Err(Error::new(std::io::Error::from(ErrorKind::UnexpectedEof)));
return Err(Error::Unexpected);
}
let slash_separator = Some(SLASH_SEPARATOR.to_owned());
@@ -535,6 +540,9 @@ impl ECStore {
error!("gather_results err {:?}", err);
let _ = err_tx2.send(Arc::new(err));
}
// cancel call exit spawns
let _ = cancel_tx.send(true);
});
let mut result = {
@@ -545,12 +553,12 @@ impl ECStore {
match res{
Ok(o) => {
error!("list_path err_rx.recv() ok {:?}", &o);
MetaCacheEntriesSortedResult{ entries: None, err: Some(clone_err(o.as_ref())) }
MetaCacheEntriesSortedResult{ entries: None, err: Some(o.as_ref().clone().into()) }
},
Err(err) => {
error!("list_path err_rx.recv() err {:?}", &err);
MetaCacheEntriesSortedResult{ entries: None, err: Some(Error::new(err)) }
MetaCacheEntriesSortedResult{ entries: None, err: Some(rustfs_filemeta::Error::other(err)) }
},
}
},
@@ -560,9 +568,6 @@ impl ECStore {
}
};
// cancel call exit spawns
cancel_tx.send(true)?;
// wait spawns exit
join_all(vec![job1, job2]).await;
@@ -583,7 +588,7 @@ impl ECStore {
}
if !truncated {
result.err = Some(Error::new(std::io::Error::from(ErrorKind::UnexpectedEof)));
result.err = Some(Error::Unexpected.into());
}
}
@@ -617,7 +622,7 @@ impl ECStore {
tokio::spawn(async move {
if let Err(err) = merge_entry_channels(rx, inputs, sender.clone(), 1).await {
println!("merge_entry_channels err {:?}", err)
error!("merge_entry_channels err {:?}", err)
}
});
@@ -643,7 +648,7 @@ impl ECStore {
if is_all_not_found(&errs) {
if is_all_volume_not_found(&errs) {
return Err(Error::new(DiskError::VolumeNotFound));
return Err(StorageError::VolumeNotFound);
}
return Ok(Vec::new());
@@ -655,11 +660,11 @@ impl ECStore {
for err in errs.iter() {
if let Some(err) = err {
if is_err_eof(err) {
if err == &Error::Unexpected {
continue;
}
return Err(clone_err(err));
return Err(err.clone());
} else {
all_at_eof = false;
continue;
@@ -772,7 +777,7 @@ impl ECStore {
}
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<Error>]| {
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
Box::pin({
let value = tx2.clone();
let resolver = resolver.clone();
@@ -813,7 +818,7 @@ impl ECStore {
if !sent_err {
let item = ObjectInfoOrErr {
item: None,
err: Some(err),
err: Some(err.into()),
};
if let Err(err) = result.send(item).await {
@@ -832,7 +837,7 @@ impl ECStore {
if let Some(fiter) = opts.filter {
if fiter(&fi) {
let item = ObjectInfoOrErr {
item: Some(fi.to_object_info(&bucket, &fi.name, {
item: Some(ObjectInfo::from_file_info(&fi, &bucket, &fi.name, {
if let Some(v) = &vcf {
v.versioned(&fi.name)
} else {
@@ -848,7 +853,7 @@ impl ECStore {
}
} else {
let item = ObjectInfoOrErr {
item: Some(fi.to_object_info(&bucket, &fi.name, {
item: Some(ObjectInfo::from_file_info(&fi, &bucket, &fi.name, {
if let Some(v) = &vcf {
v.versioned(&fi.name)
} else {
@@ -870,7 +875,7 @@ impl ECStore {
Err(err) => {
let item = ObjectInfoOrErr {
item: None,
err: Some(err),
err: Some(err.into()),
};
if let Err(err) = result.send(item).await {
@@ -888,7 +893,7 @@ impl ECStore {
if let Some(fiter) = opts.filter {
if fiter(fi) {
let item = ObjectInfoOrErr {
item: Some(fi.to_object_info(&bucket, &fi.name, {
item: Some(ObjectInfo::from_file_info(fi, &bucket, &fi.name, {
if let Some(v) = &vcf {
v.versioned(&fi.name)
} else {
@@ -904,7 +909,7 @@ impl ECStore {
}
} else {
let item = ObjectInfoOrErr {
item: Some(fi.to_object_info(&bucket, &fi.name, {
item: Some(ObjectInfo::from_file_info(fi, &bucket, &fi.name, {
if let Some(v) = &vcf {
v.versioned(&fi.name)
} else {
@@ -1012,7 +1017,8 @@ async fn gather_results(
}),
err: None,
})
.await?;
.await
.map_err(Error::other)?;
returned = true;
sender = None;
@@ -1031,9 +1037,10 @@ async fn gather_results(
o: MetaCacheEntries(entrys.clone()),
..Default::default()
}),
err: Some(Error::new(std::io::Error::new(ErrorKind::UnexpectedEof, "Unexpected EOF"))),
err: Some(Error::Unexpected.into()),
})
.await?;
.await
.map_err(Error::other)?;
}
Ok(())
@@ -1072,12 +1079,15 @@ async fn merge_entry_channels(
has_entry = in_channels[0].recv()=>{
if let Some(entry) = has_entry{
// warn!("merge_entry_channels entry {}", &entry.name);
out_channel.send(entry).await?;
out_channel.send(entry).await.map_err(Error::other)?;
} else {
return Ok(())
}
},
_ = rx.recv()=>return Err(Error::msg("cancel")),
_ = rx.recv()=>{
warn!("merge_entry_channels rx.recv() cancel");
return Ok(())
},
}
}
}
@@ -1207,7 +1217,7 @@ async fn merge_entry_channels(
if let Some(best_entry) = &best {
if best_entry.name > last {
out_channel.send(best_entry.clone()).await?;
out_channel.send(best_entry.clone()).await.map_err(Error::other)?;
last = best_entry.name.clone();
}
top[best_idx] = None; // Replace entry we just sent
@@ -1290,7 +1300,7 @@ impl SetDisks {
}
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<Error>]| {
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
Box::pin({
let value = tx2.clone();
let resolver = resolver.clone();
@@ -1308,6 +1318,7 @@ impl SetDisks {
},
)
.await
.map_err(Error::other)
}
}