mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
refactor: centralize external ECStore facade aliases (#3746)
This commit is contained in:
@@ -16,7 +16,7 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rustfs_ecstore::api::rpc::{self as ecstore_rpc, TonicInterceptor};
|
||||
use rustfs_ecstore::api::rpc as ecstore_rpc;
|
||||
use rustfs_lock::{
|
||||
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockMetadata, LockPriority},
|
||||
@@ -25,6 +25,8 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
|
||||
use tonic::Request;
|
||||
use tracing::{info, warn};
|
||||
|
||||
type TonicInterceptor = ecstore_rpc::TonicInterceptor;
|
||||
|
||||
/// gRPC lock client without authentication for testing
|
||||
/// Similar to RemoteClient but uses no_auth client
|
||||
#[derive(Debug, Clone)]
|
||||
|
||||
@@ -16,10 +16,8 @@
|
||||
use crate::common::workspace_root;
|
||||
use futures::future::join_all;
|
||||
use rmp_serde::{Deserializer, Serializer};
|
||||
use rustfs_ecstore::api::{
|
||||
disk::{VolumeInfo, WalkDirOptions},
|
||||
rpc::{self as ecstore_rpc, TonicInterceptor},
|
||||
};
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_ecstore::api::rpc as ecstore_rpc;
|
||||
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
|
||||
use rustfs_protos::proto_gen::node_service::WalkDirRequest;
|
||||
use rustfs_protos::{
|
||||
@@ -38,6 +36,10 @@ use tonic::codegen::tokio_stream::StreamExt;
|
||||
|
||||
const CLUSTER_ADDR: &str = "http://localhost:9000";
|
||||
|
||||
type TonicInterceptor = ecstore_rpc::TonicInterceptor;
|
||||
type VolumeInfo = ecstore_disk::VolumeInfo;
|
||||
type WalkDirOptions = ecstore_disk::WalkDirOptions;
|
||||
|
||||
fn signature_interceptor() -> TonicInterceptor {
|
||||
TonicInterceptor::Signature(ecstore_rpc::gen_tonic_signature_interceptor())
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_madmin::{
|
||||
AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
|
||||
ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
|
||||
@@ -37,6 +37,7 @@ use time::Duration as TimeDuration;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
type BucketTargetSys = ecstore_bucket::bucket_target_sys::BucketTargetSys;
|
||||
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
struct ReplicationResetStatusResponse {
|
||||
|
||||
@@ -14,15 +14,19 @@
|
||||
|
||||
//! test endpoint index settings
|
||||
|
||||
use rustfs_ecstore::api::{
|
||||
disk::endpoint::Endpoint,
|
||||
layout::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
storage::{ECStore, init_local_disks},
|
||||
};
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_ecstore::api::layout as ecstore_layout;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use std::net::SocketAddr;
|
||||
use tempfile::TempDir;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
type ECStore = ecstore_storage::ECStore;
|
||||
type Endpoint = ecstore_disk::endpoint::Endpoint;
|
||||
type EndpointServerPools = ecstore_layout::EndpointServerPools;
|
||||
type Endpoints = ecstore_layout::Endpoints;
|
||||
type PoolEndpoints = ecstore_layout::PoolEndpoints;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_endpoint_index_settings() -> anyhow::Result<()> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
@@ -74,7 +78,7 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
|
||||
}
|
||||
|
||||
// test ECStore initialization
|
||||
init_local_disks(endpoint_pools.clone()).await?;
|
||||
ecstore_storage::init_local_disks(endpoint_pools.clone()).await?;
|
||||
|
||||
let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?;
|
||||
|
||||
@@ -12,13 +12,16 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint};
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_heal::heal::{
|
||||
event::{HealEvent, Severity},
|
||||
task::{HealPriority, HealType},
|
||||
utils,
|
||||
};
|
||||
|
||||
type DiskStore = ecstore_disk::DiskStore;
|
||||
type Endpoint = ecstore_disk::endpoint::Endpoint;
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_to_heal_request_no_panic() {
|
||||
// Test that invalid pool/set indices don't cause panic
|
||||
|
||||
@@ -14,12 +14,10 @@
|
||||
|
||||
use http::HeaderMap;
|
||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_ecstore::api::{
|
||||
bucket::metadata_sys::init_bucket_metadata_sys,
|
||||
disk::endpoint::Endpoint,
|
||||
layout::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
storage::{ECStore, init_local_disks},
|
||||
};
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_ecstore::api::layout as ecstore_layout;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
|
||||
@@ -41,6 +39,12 @@ const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
|
||||
const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
|
||||
const NON_INLINE_TEST_DATA_SIZE: usize = 256 * 1024 + 137;
|
||||
|
||||
type ECStore = ecstore_storage::ECStore;
|
||||
type Endpoint = ecstore_disk::endpoint::Endpoint;
|
||||
type EndpointServerPools = ecstore_layout::EndpointServerPools;
|
||||
type Endpoints = ecstore_layout::Endpoints;
|
||||
type PoolEndpoints = ecstore_layout::PoolEndpoints;
|
||||
|
||||
fn non_inline_test_data() -> Vec<u8> {
|
||||
(0..NON_INLINE_TEST_DATA_SIZE).map(|idx| (idx % 251) as u8).collect()
|
||||
}
|
||||
@@ -123,7 +127,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
|
||||
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
|
||||
|
||||
// format disks (only first time)
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
ecstore_storage::init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
// create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
|
||||
let port = 9001; // for simplicity
|
||||
@@ -141,7 +145,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
// Create heal storage layer
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
|
||||
|
||||
@@ -15,10 +15,11 @@
|
||||
use crate::error::{Error, Result};
|
||||
use manager::IamCache;
|
||||
use oidc::OidcSys;
|
||||
use rustfs_ecstore::api::{
|
||||
config as ecstore_config, error as ecstore_error, global as ecstore_global, notification as ecstore_notification,
|
||||
storage as ecstore_storage,
|
||||
};
|
||||
use rustfs_ecstore::api::config as ecstore_config;
|
||||
use rustfs_ecstore::api::error as ecstore_error;
|
||||
use rustfs_ecstore::api::global as ecstore_global;
|
||||
use rustfs_ecstore::api::notification as ecstore_notification;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use store::object::ObjectStore;
|
||||
use sys::IamSys;
|
||||
|
||||
@@ -20,7 +20,9 @@ use rustfs_config::notify::{
|
||||
NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
|
||||
};
|
||||
use rustfs_config::server_config::{Config, KVS};
|
||||
use rustfs_ecstore::api::{config as ecstore_config, global as ecstore_global, storage as ecstore_storage};
|
||||
use rustfs_ecstore::api::config as ecstore_config;
|
||||
use rustfs_ecstore::api::global as ecstore_global;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use rustfs_targets::{Target, arn::TargetID};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
@@ -29,10 +29,12 @@ use crate::metrics::collectors::{
|
||||
use chrono::Utc;
|
||||
use rustfs_common::heal_channel::HealScanMode;
|
||||
use rustfs_common::metrics::global_metrics;
|
||||
use rustfs_ecstore::api::{
|
||||
bucket as ecstore_bucket, capacity as ecstore_capacity, data_usage as ecstore_data_usage, error as ecstore_error,
|
||||
global as ecstore_global, storage as ecstore_storage,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_ecstore::api::capacity as ecstore_capacity;
|
||||
use rustfs_ecstore::api::data_usage as ecstore_data_usage;
|
||||
use rustfs_ecstore::api::error as ecstore_error;
|
||||
use rustfs_ecstore::api::global as ecstore_global;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use rustfs_iam::{get_global_iam_sys, oidc::oidc_plugin_authn_metrics_snapshot};
|
||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||
use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};
|
||||
|
||||
@@ -35,9 +35,10 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_ecstore::api::{
|
||||
bucket as ecstore_bucket, error as ecstore_error, global as ecstore_global, storage as ecstore_storage,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_ecstore::api::error as ecstore_error;
|
||||
use rustfs_ecstore::api::global as ecstore_global;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
|
||||
pub mod account;
|
||||
pub mod acl;
|
||||
|
||||
@@ -25,9 +25,10 @@ use object_store::{
|
||||
};
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_common::DEFAULT_DELIMITER;
|
||||
use rustfs_ecstore::api::{
|
||||
error as ecstore_error, global as ecstore_global, set_disk as ecstore_set_disk, storage as ecstore_storage,
|
||||
};
|
||||
use rustfs_ecstore::api::error as ecstore_error;
|
||||
use rustfs_ecstore::api::global as ecstore_global;
|
||||
use rustfs_ecstore::api::set_disk as ecstore_set_disk;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
||||
use s3s::S3Result;
|
||||
use s3s::dto::SelectObjectContentInput;
|
||||
|
||||
@@ -12,29 +12,20 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use ecstore_bucket::metadata_sys;
|
||||
use ecstore_disk::DiskAPI as _;
|
||||
use ecstore_global::GLOBAL_TierConfigMgr;
|
||||
use ecstore_tier::warm_backend::{WarmBackend as ScannerWarmBackend, build_transition_put_options};
|
||||
use futures::FutureExt;
|
||||
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
|
||||
use rustfs_ecstore::api::{
|
||||
bucket::{
|
||||
lifecycle::{
|
||||
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry},
|
||||
lifecycle::TransitionOptions,
|
||||
},
|
||||
metadata::BUCKET_LIFECYCLE_CONFIG,
|
||||
metadata_sys::{self, init_bucket_metadata_sys},
|
||||
versioning_sys::BucketVersioningSys,
|
||||
},
|
||||
capacity::path2_bucket_object_with_base_path,
|
||||
client::transition_api::{ReadCloser, ReaderImpl},
|
||||
disk::{DiskAPI as _, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk},
|
||||
global::GLOBAL_TierConfigMgr,
|
||||
layout::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
storage::{ECStore, init_local_disks},
|
||||
tier::{
|
||||
tier_config::{TierConfig, TierMinIO, TierType},
|
||||
warm_backend::{WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
},
|
||||
};
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_ecstore::api::capacity as ecstore_capacity;
|
||||
use rustfs_ecstore::api::client as ecstore_client;
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_ecstore::api::global as ecstore_global;
|
||||
use rustfs_ecstore::api::layout as ecstore_layout;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use rustfs_ecstore::api::tier as ecstore_tier;
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_scanner::scanner_folder::ScannerItem;
|
||||
use rustfs_scanner::scanner_io::ScannerIODisk;
|
||||
@@ -66,6 +57,23 @@ use uuid::Uuid;
|
||||
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
|
||||
static INIT: Once = Once::new();
|
||||
const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
const BUCKET_LIFECYCLE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
const STORAGE_FORMAT_FILE: &str = ecstore_disk::STORAGE_FORMAT_FILE;
|
||||
|
||||
type BucketVersioningSys = ecstore_bucket::versioning_sys::BucketVersioningSys;
|
||||
type ECStore = ecstore_storage::ECStore;
|
||||
type Endpoint = ecstore_disk::endpoint::Endpoint;
|
||||
type EndpointServerPools = ecstore_layout::EndpointServerPools;
|
||||
type Endpoints = ecstore_layout::Endpoints;
|
||||
type PoolEndpoints = ecstore_layout::PoolEndpoints;
|
||||
type DiskOption = ecstore_disk::DiskOption;
|
||||
type ReadCloser = ecstore_client::transition_api::ReadCloser;
|
||||
type ReaderImpl = ecstore_client::transition_api::ReaderImpl;
|
||||
type TierConfig = ecstore_tier::tier_config::TierConfig;
|
||||
type TierMinIO = ecstore_tier::tier_config::TierMinIO;
|
||||
type TierType = ecstore_tier::tier_config::TierType;
|
||||
type TransitionOptions = ecstore_bucket::lifecycle::lifecycle::TransitionOptions;
|
||||
type WarmBackendGetOpts = ecstore_tier::warm_backend::WarmBackendGetOpts;
|
||||
|
||||
fn init_tracing() {
|
||||
INIT.call_once(|| {
|
||||
@@ -125,7 +133,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
|
||||
|
||||
// format disks (only first time)
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
ecstore_storage::init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
|
||||
let port = 9002; // for simplicity
|
||||
@@ -143,10 +151,10 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
// Initialize background expiry workers
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
|
||||
// Store in global once lock
|
||||
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
|
||||
@@ -194,7 +202,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
|
||||
};
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
ecstore_storage::init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||
@@ -209,10 +217,10 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
if init_expiry {
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
}
|
||||
|
||||
(disk_paths, ecstore)
|
||||
@@ -507,7 +515,7 @@ async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usi
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
let disk = ecstore_disk::new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
@@ -591,7 +599,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
let disk = ecstore_disk::new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
@@ -602,7 +610,8 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
|
||||
.expect("failed to open local disk");
|
||||
let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE);
|
||||
let relative_path = metadata_path.to_string_lossy().to_string();
|
||||
let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
|
||||
let (_, scanner_path) =
|
||||
ecstore_capacity::path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
|
||||
let file_type = fs::metadata(&metadata_path)
|
||||
.await
|
||||
.expect("failed to stat object metadata")
|
||||
@@ -633,7 +642,7 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
let disk = ecstore_disk::new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
@@ -644,7 +653,8 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
|
||||
.expect("failed to open local disk");
|
||||
let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE);
|
||||
let relative_path = metadata_path.to_string_lossy().to_string();
|
||||
let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
|
||||
let (_, scanner_path) =
|
||||
ecstore_capacity::path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
|
||||
let file_type = fs::metadata(&metadata_path)
|
||||
.await
|
||||
.expect("failed to stat object metadata")
|
||||
@@ -961,9 +971,12 @@ mod serial_tests {
|
||||
.await
|
||||
.expect("Failed to upload transition metadata test object");
|
||||
|
||||
enqueue_transition_for_existing_objects(ecstore.clone(), put_bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned put object");
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
put_bucket.as_str(),
|
||||
)
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned put object");
|
||||
|
||||
let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
@@ -1031,9 +1044,12 @@ mod serial_tests {
|
||||
.await
|
||||
.expect("Failed to complete multipart upload");
|
||||
|
||||
enqueue_transition_for_existing_objects(ecstore.clone(), multipart_bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned multipart object");
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
multipart_bucket.as_str(),
|
||||
)
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned multipart object");
|
||||
|
||||
let multipart_info = wait_for_transition(&ecstore, multipart_bucket.as_str(), multipart_object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
@@ -1082,9 +1098,12 @@ mod serial_tests {
|
||||
.await
|
||||
.expect("Failed to copy object");
|
||||
|
||||
enqueue_transition_for_existing_objects(ecstore.clone(), dst_bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned copied object");
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
dst_bucket.as_str(),
|
||||
)
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned copied object");
|
||||
|
||||
let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
@@ -1105,9 +1124,12 @@ mod serial_tests {
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transition for existing objects");
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
bucket_name.as_str(),
|
||||
)
|
||||
.await
|
||||
.expect("Failed to enqueue transition for existing objects");
|
||||
|
||||
let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
@@ -1182,9 +1204,12 @@ mod serial_tests {
|
||||
.await
|
||||
.expect("Failed to complete multipart upload");
|
||||
|
||||
enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned restore object");
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
bucket_name.as_str(),
|
||||
)
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned restore object");
|
||||
|
||||
let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
@@ -1254,9 +1279,12 @@ mod serial_tests {
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, initial_payload).await;
|
||||
enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned object");
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
bucket_name.as_str(),
|
||||
)
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned object");
|
||||
|
||||
let transitioned = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
@@ -1278,7 +1306,7 @@ mod serial_tests {
|
||||
"stale transitioned remote object should still exist before scanner fallback runs"
|
||||
);
|
||||
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
assert!(
|
||||
@@ -1339,7 +1367,7 @@ mod serial_tests {
|
||||
"stale transitioned remote object should still exist before scanner cleanup runs"
|
||||
);
|
||||
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
assert!(
|
||||
@@ -1382,9 +1410,12 @@ mod serial_tests {
|
||||
let remote_object = transitioned.transitioned_object.name.clone();
|
||||
assert!(backend.objects.lock().await.contains_key(&remote_object));
|
||||
|
||||
enqueue_transition_for_existing_objects(ecstore.clone(), bucket_name.as_str())
|
||||
.await
|
||||
.expect("existing-object backfill should succeed after compensation transition");
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
bucket_name.as_str(),
|
||||
)
|
||||
.await
|
||||
.expect("existing-object backfill should succeed after compensation transition");
|
||||
|
||||
let info = wait_for_transition(&ecstore, bucket_name.as_str(), object_name, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
@@ -1706,7 +1737,7 @@ mod serial_tests {
|
||||
|
||||
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
|
||||
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
assert!(
|
||||
@@ -1810,7 +1841,7 @@ mod serial_tests {
|
||||
.await
|
||||
.expect("Failed to set noncurrent lifecycle configuration");
|
||||
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
|
||||
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
|
||||
@@ -5,17 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-rustfs-ecstore-boundary-cleanup`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128`.
|
||||
- Based on: API-128 slice.
|
||||
- Branch: `overtrue/arch-external-ecstore-facade-aliases`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129`.
|
||||
- Based on: API-129 slice.
|
||||
- PR type for this branch: `pure-move`
|
||||
- Runtime behavior changes: none.
|
||||
- Rust code changes: route RustFS crate startup, server, capacity, config,
|
||||
table-catalog, workload, and S3 API consumers through the storage owner
|
||||
ECStore boundary instead of direct ECStore facade imports.
|
||||
- CI/script changes: narrow RustFS direct ECStore import allowance to the
|
||||
app/admin/storage owner modules.
|
||||
- Docs changes: record the API-129 RustFS internal ECStore boundary cleanup.
|
||||
- Rust code changes: centralize non-ECStore grouped/raw ECStore facade imports
|
||||
behind per-module `ecstore_*` aliases across external owner modules, tests,
|
||||
and fuzz targets.
|
||||
- CI/script changes: reject grouped `rustfs_ecstore::api::{...}` imports and
|
||||
raw `rustfs_ecstore::api::<module>::...` subpaths outside ECStore.
|
||||
- Docs changes: record the API-130 external ECStore facade alias cleanup.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -3845,6 +3845,23 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
migration/layer guards, formatting, diff hygiene, Rust risk scan, branch
|
||||
freshness check, pre-commit quality gate, and three-expert review.
|
||||
|
||||
- [x] `API-130` Centralize external ECStore facade alias imports.
|
||||
- Do: replace grouped and raw-subpath `rustfs_ecstore::api` imports in IAM,
|
||||
notify, observability, Swift, S3 Select, e2e helpers, heal/scanner tests,
|
||||
and fuzz targets with per-module `ecstore_*` aliases plus local type
|
||||
aliases or module-qualified calls.
|
||||
- Acceptance: non-ECStore source no longer uses grouped
|
||||
`rustfs_ecstore::api::{...}` imports or raw
|
||||
`rustfs_ecstore::api::<module>::...` subpaths, while owner alias imports
|
||||
remain explicit.
|
||||
- Must preserve: IAM config IO, notify config persistence, observability
|
||||
metrics collection, Swift metadata access, S3 Select object-store access,
|
||||
e2e RPC helpers, heal/scanner ECStore test setup, and fuzz validation
|
||||
semantics.
|
||||
- Verification: focused external crate compile, grouped/raw facade residual
|
||||
scans, migration/layer guards, formatting, diff hygiene, Rust risk scan,
|
||||
branch freshness check, pre-commit quality gate, and three-expert review.
|
||||
|
||||
## Next PRs
|
||||
|
||||
1. `pure-move`: continue pruning remaining facade compatibility and owner boundaries.
|
||||
@@ -3853,14 +3870,27 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | pass | API-129 keeps ECStore facade exposure in the storage owner boundary and migrates RustFS internal consumers without new wrappers. |
|
||||
| Migration preservation | pass | Startup, readiness, server, capacity, table-catalog, workload, and S3 helper call paths keep the same ECStore symbols through `crate::storage`. |
|
||||
| Testing/verification | pass | Full pre-commit, focused RustFS compile, residual scan, migration/layer guards, formatting, diff hygiene, and diff-only Rust risk scan passed. |
|
||||
| Quality/architecture | pass | API-130 keeps external ECStore facade access explicit through per-module `ecstore_*` aliases without adding behavior wrappers. |
|
||||
| Migration preservation | pass | IAM, notify, obs, Swift, S3 Select, e2e, heal/scanner test, and fuzz call paths keep the same ECStore symbols through local aliases. |
|
||||
| Testing/verification | pass | Focused external crate compile, residual scans, migration/layer guards, formatting, diff hygiene, and diff-only Rust risk scan passed. |
|
||||
|
||||
## Verification Notes
|
||||
|
||||
Passed before push:
|
||||
|
||||
- Issue #660 API-130 current slice:
|
||||
- `cargo check --tests -p rustfs-notify -p rustfs-obs -p rustfs-protocols -p rustfs-s3select-api -p e2e_test -p rustfs-heal -p rustfs-scanner -p rustfs-iam`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `make pre-commit`: passed, including clippy, script tests, nextest
|
||||
`6518 passed, 111 skipped`, and doc-tests.
|
||||
- Grouped/raw ECStore facade residual scan outside ECStore: passed.
|
||||
- Rust risk scan: diff-only scan found path-rewritten existing test
|
||||
unwraps/expects only; no new risky behavior added.
|
||||
|
||||
- Issue #660 API-129 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use rustfs_ecstore::api::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
|
||||
fn parse_case(data: &[u8]) -> (String, String) {
|
||||
let text = String::from_utf8_lossy(data);
|
||||
@@ -24,10 +22,10 @@ fn looks_like_ipv4(text: &str) -> bool {
|
||||
fuzz_target!(|data: &[u8]| {
|
||||
let (bucket, object) = parse_case(data);
|
||||
|
||||
let strict_bucket_ok = check_valid_bucket_name_strict(&bucket).is_ok();
|
||||
let valid_bucket_for_object_ops = strict_bucket_ok || is_meta_bucketname(&bucket);
|
||||
let pair_ok = check_bucket_and_object_names(&bucket, &object).is_ok();
|
||||
let list_ok = check_list_objs_args(&bucket, &object, &None).is_ok();
|
||||
let strict_bucket_ok = ecstore_bucket::utils::check_valid_bucket_name_strict(&bucket).is_ok();
|
||||
let valid_bucket_for_object_ops = strict_bucket_ok || ecstore_bucket::utils::is_meta_bucketname(&bucket);
|
||||
let pair_ok = ecstore_bucket::utils::check_bucket_and_object_names(&bucket, &object).is_ok();
|
||||
let list_ok = ecstore_bucket::utils::check_list_objs_args(&bucket, &object, &None).is_ok();
|
||||
|
||||
if strict_bucket_ok {
|
||||
let trimmed = bucket.trim();
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use rustfs_ecstore::api::bucket::utils::{
|
||||
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_utils::path::{clean, path_join};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
@@ -94,9 +92,9 @@ fuzz_target!(|data: &[u8]| {
|
||||
let (bucket, base_object, extra, flags) = parse_case(data);
|
||||
let object = materialize_object(base_object, &extra, &flags);
|
||||
|
||||
let has_bad_component = has_bad_path_component(&object);
|
||||
let is_valid_prefix = is_valid_object_prefix(&object);
|
||||
let length_and_slash_ok = check_object_name_for_length_and_slash(&bucket, &object).is_ok();
|
||||
let has_bad_component = ecstore_bucket::utils::has_bad_path_component(&object);
|
||||
let is_valid_prefix = ecstore_bucket::utils::is_valid_object_prefix(&object);
|
||||
let length_and_slash_ok = ecstore_bucket::utils::check_object_name_for_length_and_slash(&bucket, &object).is_ok();
|
||||
|
||||
if object.is_empty() || !(is_valid_prefix && length_and_slash_ok) {
|
||||
return;
|
||||
|
||||
@@ -97,6 +97,8 @@ OUTER_CONSUMER_COMPAT_RAW_FACADE_PATH_HITS_FILE="${TMP_DIR}/outer_consumer_compa
|
||||
RUSTFS_ROOT_E2E_COMPAT_RAW_FACADE_PATH_HITS_FILE="${TMP_DIR}/rustfs_root_e2e_compat_raw_facade_path_hits.txt"
|
||||
ALL_STORAGE_COMPAT_RAW_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_raw_facade_path_hits.txt"
|
||||
ALL_STORAGE_COMPAT_GROUPED_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/all_storage_compat_grouped_facade_import_hits.txt"
|
||||
ALL_ECSTORE_API_GROUPED_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/all_ecstore_api_grouped_facade_import_hits.txt"
|
||||
ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE="${TMP_DIR}/all_ecstore_api_raw_subpath_hits.txt"
|
||||
ALL_STORAGE_COMPAT_SELF_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_self_facade_path_hits.txt"
|
||||
RUSTFS_LOCAL_COMPAT_OWNER_SELF_PATH_HITS_FILE="${TMP_DIR}/rustfs_local_compat_owner_self_path_hits.txt"
|
||||
RUSTFS_ROOT_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_root_compat_relative_consumer_hits.txt"
|
||||
@@ -1042,6 +1044,28 @@ if [[ -s "$ALL_STORAGE_COMPAT_GROUPED_FACADE_IMPORT_HITS_FILE" ]]; then
|
||||
report_failure "storage compatibility boundaries must import ECStore facade modules through explicit per-module ecstore_* aliases: $(paste -sd '; ' "$ALL_STORAGE_COMPAT_GROUPED_FACADE_IMPORT_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename '^use rustfs_ecstore::api::\{' rustfs/src crates fuzz \
|
||||
--glob '*.rs' \
|
||||
--glob '!crates/ecstore/**' || true
|
||||
) >"$ALL_ECSTORE_API_GROUPED_FACADE_IMPORT_HITS_FILE"
|
||||
|
||||
if [[ -s "$ALL_ECSTORE_API_GROUPED_FACADE_IMPORT_HITS_FILE" ]]; then
|
||||
report_failure "non-ECStore sources must import ECStore facade modules through explicit per-module ecstore_* aliases: $(paste -sd '; ' "$ALL_ECSTORE_API_GROUPED_FACADE_IMPORT_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' rustfs/src crates fuzz \
|
||||
--glob '*.rs' \
|
||||
--glob '!crates/ecstore/**' || true
|
||||
) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE"
|
||||
|
||||
if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then
|
||||
report_failure "non-ECStore sources must keep raw ECStore facade subpaths behind local ecstore_* module aliases: $(paste -sd '; ' "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::(?:admin|app|storage)::storage_compat::ecstore_|crate::storage_compat::ecstore_' rustfs/src crates fuzz \
|
||||
|
||||
Reference in New Issue
Block a user