refactor: clean runtime and test storage boundaries (#3573)

* refactor: clean runtime observability select boundaries

* refactor: clean test harness fuzz storage boundaries
This commit is contained in:
安正超
2026-06-18 18:09:25 +08:00
committed by GitHub
parent 8313767f75
commit c098184c49
49 changed files with 414 additions and 170 deletions
+1
View File
@@ -13,6 +13,7 @@
// limitations under the License.
mod reliant;
mod storage_compat;
// Common utilities for all E2E tests
#[cfg(test)]
@@ -15,8 +15,8 @@
// Used by test_distributed_lock_4_nodes_grpc in lock.rs
#![allow(dead_code)]
use crate::storage_compat::ecstore::rpc::node_service_time_out_client_no_auth;
use async_trait::async_trait;
use rustfs_ecstore::rpc::node_service_time_out_client_no_auth;
use rustfs_lock::{
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
types::{LockMetadata, LockPriority},
@@ -41,7 +41,10 @@ impl GrpcLockClient {
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, rustfs_ecstore::rpc::TonicInterceptor>,
tonic::service::interceptor::InterceptedService<
tonic::transport::Channel,
crate::storage_compat::ecstore::rpc::TonicInterceptor,
>,
>,
> {
node_service_time_out_client_no_auth(&self.addr)
@@ -14,10 +14,10 @@
// limitations under the License.
use crate::common::workspace_root;
use crate::storage_compat::ecstore::disk::{VolumeInfo, WalkDirOptions};
use crate::storage_compat::ecstore::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use futures::future::join_all;
use rmp_serde::{Deserializer, Serializer};
use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions};
use rustfs_ecstore::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
use rustfs_protos::proto_gen::node_service::WalkDirRequest;
use rustfs_protos::{
@@ -15,13 +15,13 @@
use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
};
use crate::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys;
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
use aws_sdk_s3::{Client, Config};
use http::header::{CONTENT_TYPE, HOST};
use reqwest::StatusCode;
use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
use rustfs_madmin::{
AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore as ecstore;
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod storage_compat;
@@ -0,0 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore as ecstore;
+8 -4
View File
@@ -14,8 +14,10 @@
//! test endpoint index settings
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
mod common;
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
use crate::common::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use std::net::SocketAddr;
use tempfile::TempDir;
use tokio_util::sync::CancellationToken;
@@ -71,10 +73,12 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
}
// test ECStore initialization
rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await?;
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()).await?;
let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = rustfs_ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?;
let ecstore =
crate::common::storage_compat::ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await?;
println!("ECStore initialized successfully with {} pools", ecstore.pools.len());
+25 -8
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod common;
use rustfs_heal::heal::{
event::{HealEvent, Severity},
task::{HealPriority, HealType},
@@ -20,7 +22,7 @@ use rustfs_heal::heal::{
#[test]
fn test_heal_event_to_heal_request_no_panic() {
use rustfs_ecstore::disk::endpoint::Endpoint;
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
// Test that invalid pool/set indices don't cause panic
// Create endpoint using try_from or similar method
@@ -45,7 +47,7 @@ fn test_heal_event_to_heal_request_no_panic() {
#[test]
fn test_heal_event_to_heal_request_valid_indices() {
use rustfs_ecstore::disk::endpoint::Endpoint;
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
// Test that valid indices work correctly
let endpoint_result = Endpoint::try_from("http://localhost:9000");
@@ -190,11 +192,14 @@ fn test_heal_task_status_atomic_update() {
}
async fn get_disk_status(
&self,
_endpoint: &rustfs_ecstore::disk::endpoint::Endpoint,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
}
async fn format_disk(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_heal::Result<()> {
async fn format_disk(
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<()> {
Ok(())
}
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
@@ -248,7 +253,10 @@ fn test_heal_task_status_atomic_update() {
) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
Ok((vec![], None, false))
}
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<rustfs_ecstore::disk::DiskStore> {
async fn get_disk_for_resume(
&self,
_set_disk_id: &str,
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
Err(rustfs_heal::Error::other("Not implemented in mock"))
}
}
@@ -318,11 +326,17 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok(Vec::new())
}
async fn get_disk_status(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_heal::Result<DiskStatus> {
async fn get_disk_status(
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<DiskStatus> {
Ok(DiskStatus::Ok)
}
async fn format_disk(&self, _endpoint: &rustfs_ecstore::disk::endpoint::Endpoint) -> rustfs_heal::Result<()> {
async fn format_disk(
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<()> {
Ok(())
}
@@ -392,7 +406,10 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok((Vec::new(), None, false))
}
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<rustfs_ecstore::disk::DiskStore> {
async fn get_disk_for_resume(
&self,
_set_disk_id: &str,
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
Err(rustfs_heal::Error::other("not implemented"))
}
}
+9 -5
View File
@@ -12,13 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use http::HeaderMap;
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_ecstore::{
mod common;
use crate::common::storage_compat::ecstore::{
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
store::ECStore,
};
use http::HeaderMap;
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_heal::heal::{
manager::{HealConfig, HealManager},
storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
@@ -122,7 +124,9 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
// format disks (only first time)
rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
// create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
let port = 9001; // for simplicity
@@ -140,7 +144,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Create heal storage layer
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
+9 -9
View File
@@ -179,20 +179,20 @@ impl Error {
}
}
impl From<rustfs_ecstore::error::StorageError> for Error {
fn from(e: rustfs_ecstore::error::StorageError) -> Self {
impl From<crate::storage_compat::ecstore::error::StorageError> for Error {
fn from(e: crate::storage_compat::ecstore::error::StorageError) -> Self {
match e {
rustfs_ecstore::error::StorageError::ConfigNotFound => Error::ConfigNotFound,
crate::storage_compat::ecstore::error::StorageError::ConfigNotFound => Error::ConfigNotFound,
_ => Error::other(e),
}
}
}
impl From<Error> for rustfs_ecstore::error::StorageError {
impl From<Error> for crate::storage_compat::ecstore::error::StorageError {
fn from(e: Error) -> Self {
match e {
Error::ConfigNotFound => rustfs_ecstore::error::StorageError::ConfigNotFound,
_ => rustfs_ecstore::error::StorageError::other(e),
Error::ConfigNotFound => crate::storage_compat::ecstore::error::StorageError::ConfigNotFound,
_ => crate::storage_compat::ecstore::error::StorageError::other(e),
}
}
}
@@ -330,13 +330,13 @@ mod tests {
#[test]
fn test_iam_error_from_storage_error() {
// Test conversion from StorageError
let storage_error = rustfs_ecstore::error::StorageError::ConfigNotFound;
let storage_error = crate::storage_compat::ecstore::error::StorageError::ConfigNotFound;
let iam_error: Error = storage_error.into();
assert_eq!(iam_error, Error::ConfigNotFound);
// Test reverse conversion
let back_to_storage: rustfs_ecstore::error::StorageError = iam_error.into();
assert_eq!(back_to_storage, rustfs_ecstore::error::StorageError::ConfigNotFound);
let back_to_storage: crate::storage_compat::ecstore::error::StorageError = iam_error.into();
assert_eq!(back_to_storage, crate::storage_compat::ecstore::error::StorageError::ConfigNotFound);
}
#[test]
+2 -1
View File
@@ -13,9 +13,9 @@
// limitations under the License.
use crate::error::{Error, Result};
use crate::storage_compat::ecstore::store::ECStore;
use manager::IamCache;
use oidc::OidcSys;
use rustfs_ecstore::store::ECStore;
use std::sync::{Arc, OnceLock};
use store::object::ObjectStore;
use sys::IamSys;
@@ -33,6 +33,7 @@ pub mod keyring;
pub mod manager;
pub mod oidc;
pub mod oidc_state;
mod storage_compat;
pub mod store;
pub mod sys;
pub mod utils;
+1 -1
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use crate::error::{Error, Result, is_err_config_not_found};
use crate::storage_compat::ecstore::global::is_first_cluster_node_local;
use crate::sys::{get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp};
use crate::{
cache::{Cache, CacheEntity, LockedCache},
@@ -25,7 +26,6 @@ use crate::{
};
use futures::future::join_all;
use rustfs_credentials::{Credentials, EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, get_global_action_cred};
use rustfs_ecstore::global::is_first_cluster_node_local;
use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq, GroupDesc};
use rustfs_policy::{
arn::ARN,
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore as ecstore;
+9 -9
View File
@@ -14,6 +14,14 @@
use super::{GroupInfo, MappedPolicy, Store, UserType};
use crate::error::{Error, Result, is_err_config_not_found, is_err_no_such_group};
use crate::storage_compat::ecstore::error::{Error as EcstoreError, StorageError, classify_system_path_failure_reason};
use crate::storage_compat::ecstore::{
config::{
RUSTFS_CONFIG_PREFIX,
com::{delete_config, read_config_no_lock, read_config_with_metadata, save_config, save_config_with_opts},
},
store::ECStore,
};
use crate::{
cache::{Cache, CacheEntity},
error::{is_err_no_such_policy, is_err_no_such_user},
@@ -22,14 +30,6 @@ use crate::{
};
use futures::future::join_all;
use rustfs_credentials::get_global_action_cred;
use rustfs_ecstore::error::{Error as EcstoreError, StorageError, classify_system_path_failure_reason};
use rustfs_ecstore::{
config::{
RUSTFS_CONFIG_PREFIX,
com::{delete_config, read_config_no_lock, read_config_with_metadata, save_config, save_config_with_opts},
},
store::ECStore,
};
use rustfs_io_metrics::record_system_path_failure;
use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
use rustfs_storage_api::{HTTPPreconditions, ListOperations as _, ObjectInfoOrErr as StorageObjectInfoOrErr};
@@ -589,7 +589,7 @@ impl ObjectStore {
match read_config_no_lock(self.object_api.clone(), &probe_path).await {
Ok(_) => Ok(()),
Err(rustfs_ecstore::error::StorageError::ConfigNotFound) => Err(Error::other(format!(
Err(crate::storage_compat::ecstore::error::StorageError::ConfigNotFound) => Err(Error::other(format!(
"Storage metadata not ready: probe object '{}' not found (expected IAM config to be initialized)",
probe_path
))),
+1 -1
View File
@@ -19,13 +19,13 @@ use crate::error::{Error, Result};
use crate::manager::extract_jwt_claims;
use crate::manager::get_default_policyes;
use crate::manager::{IamCache, IamSyncMetricsSnapshot};
use crate::storage_compat::ecstore::notification_sys::get_global_notification_sys;
use crate::store::GroupInfo;
use crate::store::MappedPolicy;
use crate::store::Store;
use crate::store::UserType;
use crate::utils::{extract_claims, extract_claims_allow_missing_exp};
use rustfs_credentials::{Credentials, EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, get_global_action_cred};
use rustfs_ecstore::notification_sys::get_global_notification_sys;
use rustfs_madmin::AddOrUpdateUserReq;
use rustfs_madmin::GroupDesc;
use rustfs_policy::arn::ARN;
+3 -3
View File
@@ -316,13 +316,13 @@ impl NotifyConfigManager {
where
F: FnMut(&mut Config) -> bool,
{
let Some(store) = rustfs_ecstore::global::resolve_object_store_handle() else {
let Some(store) = crate::storage_compat::ecstore::global::resolve_object_store_handle() else {
return Err(NotificationError::StorageNotAvailable(
"Failed to save target configuration: server storage not initialized".to_string(),
));
};
let mut new_config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
let mut new_config = crate::storage_compat::ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| NotificationError::ReadConfig(e.to_string()))?;
@@ -338,7 +338,7 @@ impl NotifyConfigManager {
return Ok(());
}
rustfs_ecstore::config::com::save_server_config(store, &new_config)
crate::storage_compat::ecstore::config::com::save_server_config(store, &new_config)
.await
.map_err(|e| NotificationError::SaveConfig(e.to_string()))?;
+3 -1
View File
@@ -12,4 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub type NotifyObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) use rustfs_ecstore as ecstore;
pub type NotifyObjectInfo = ecstore::store_api::ObjectInfo;
+1
View File
@@ -64,6 +64,7 @@ mod error;
mod global;
mod logging;
pub mod metrics;
mod storage_compat;
mod telemetry;
pub use cleaner::*;
+1 -1
View File
@@ -81,8 +81,8 @@ use crate::metrics::stats_collector::{
collect_ilm_metric_stats, collect_internode_network_stats, collect_process_metric_bundle, collect_replication_stats,
collect_scanner_metric_stats, collect_system_cpu_and_memory_stats_with,
};
use crate::storage_compat::ecstore::global::get_global_bucket_monitor;
use rustfs_audit::audit_target_metrics;
use rustfs_ecstore::global::get_global_bucket_monitor;
use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics};
use rustfs_utils::get_env_opt_u64;
use serde::Serialize;
+7 -7
View File
@@ -26,16 +26,16 @@ use crate::metrics::collectors::{
DriveDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats,
ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
};
use crate::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState};
use crate::storage_compat::ecstore::bucket::metadata_sys::get_quota_config;
use crate::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
use crate::storage_compat::ecstore::data_usage::load_data_usage_from_backend;
use crate::storage_compat::ecstore::global::get_global_bucket_monitor;
use crate::storage_compat::ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
use crate::storage_compat::ecstore::resolve_object_store_handle;
use chrono::Utc;
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::global_metrics;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState};
use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
use rustfs_ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
use rustfs_ecstore::data_usage::load_data_usage_from_backend;
use rustfs_ecstore::global::get_global_bucket_monitor;
use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_iam::{get_global_iam_sys, oidc::oidc_plugin_authn_metrics_snapshot};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore as ecstore;
+6 -4
View File
@@ -12,6 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_compat::ecstore::error::{
StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
use crate::storage_compat::ecstore::resolve_object_store_handle;
use crate::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use crate::storage_compat::ecstore::store::ECStore;
use async_trait::async_trait;
use bytes::Bytes;
use chrono::Utc;
@@ -25,10 +31,6 @@ use object_store::{
};
use pin_project_lite::pin_project;
use rustfs_common::DEFAULT_DELIMITER;
use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use rustfs_ecstore::store::ECStore;
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
use s3s::S3Result;
use s3s::dto::SelectObjectContentInput;
+3 -1
View File
@@ -12,7 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ecstore::store_api::{
pub(crate) use rustfs_ecstore as ecstore;
use ecstore::store_api::{
GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions,
};
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod storage_compat;
@@ -0,0 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore as ecstore;
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
use rustfs_ecstore::{
mod common;
use crate::common::storage_compat::ecstore::{
bucket::lifecycle::lifecycle::TransitionOptions,
bucket::metadata::BUCKET_LIFECYCLE_CONFIG,
bucket::{
@@ -33,6 +33,8 @@ use rustfs_ecstore::{
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
},
};
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
use rustfs_filemeta::FileMeta;
use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk;
@@ -123,7 +125,9 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
// format disks (only first time)
rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
let port = 9002; // for simplicity
@@ -141,10 +145,11 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Initialize background expiry workers
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
// Store in global once lock
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
@@ -192,7 +197,9 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
rustfs_ecstore::store::init_local_disks(endpoint_pools.clone()).await.unwrap();
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
@@ -207,10 +214,11 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
if init_expiry {
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
}
(disk_paths, ecstore)
@@ -870,7 +878,7 @@ mod serial_tests {
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
// Verify lifecycle configuration was set
match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
match crate::common::storage_compat::ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
Ok(bucket_meta) => {
assert!(bucket_meta.lifecycle_config.is_some());
println!("✅ Bucket metadata retrieved successfully");
@@ -1276,7 +1284,8 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner fallback runs"
);
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1337,7 +1346,8 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner cleanup runs"
);
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1704,7 +1714,8 @@ mod serial_tests {
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1808,7 +1819,8 @@ mod serial_tests {
.await
.expect("Failed to set noncurrent lifecycle configuration");
rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;