gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 11:56:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(runtime): route RustFS runtime consumers through storage owner (#3756)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-23 05:17:56 +08:00
committed by GitHub
parent e0b79aa00c
commit 198fd4f150
50 changed files with 1622 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/ecstore_compat.rs
+74
View File
@@ -0,0 +1,74 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
GLOBAL_ExpiryState as ECSTORE_GLOBAL_EXPIRY_STATE, apply_expiry_rule as ecstore_apply_expiry_rule,
apply_transition_rule as ecstore_apply_transition_rule,
};
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE,
};
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
};
pub(crate) use rustfs_ecstore::api::bucket::replication::{
ReplicationConfig as EcstoreReplicationConfig, ReplicationConfigurationExt as EcstoreReplicationConfigurationExt,
ReplicationHealQueueResult as EcstoreReplicationHealQueueResult,
ReplicationQueueAdmission as EcstoreReplicationQueueAdmission,
queue_replication_heal_internal as ecstore_queue_replication_heal_internal,
};
pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi as EcstoreVersioningApi;
pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys as EcstoreBucketVersioningSys;
pub(crate) use rustfs_ecstore::api::cache::{
ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw,
};
pub(crate) use rustfs_ecstore::api::capacity::{
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
};
pub(crate) use rustfs_ecstore::api::config::com::{read_config as ecstore_read_config, save_config as ecstore_save_config};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::config::init as ecstore_config_init;
pub(crate) use rustfs_ecstore::api::config::storageclass::{
RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
};
pub(crate) use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
pub(crate) use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
ScanGuard as EcstoreScanGuard,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{
DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk,
};
pub(crate) use rustfs_ecstore::api::error::{
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
};
pub(crate) use rustfs_ecstore::api::global::{
GLOBAL_TierConfigMgr as ECSTORE_GLOBAL_TIER_CONFIG_MGR, is_erasure as ecstore_is_erasure,
is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle,
};
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
pub(crate) use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
crates/scanner/src/lib.rs
+18 -52
View File
@@ -20,59 +20,25 @@
rust_2018_idioms
)]
mod ecstore_compat;
use ecstore_compat::{
ECSTORE_BUCKET_META_PREFIX, ECSTORE_GLOBAL_EXPIRY_STATE, ECSTORE_GLOBAL_TIER_CONFIG_MGR, ECSTORE_RUSTFS_META_BUCKET,
ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS, ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE,
EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError,
EcstoreDiskInfo, EcstoreDiskInfoOptions, EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator,
EcstoreEvent, EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfig,
EcstoreReplicationConfigurationExt, EcstoreReplicationHealQueueResult, EcstoreReplicationQueueAdmission, EcstoreResultType,
EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi,
ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket,
ecstore_list_path_raw, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_queue_replication_heal_internal, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
ecstore_resolve_object_store_handle, ecstore_save_config,
};
#[cfg(test)]
use ecstore_compat::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
use http::HeaderMap;
use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
GLOBAL_ExpiryState as ECSTORE_GLOBAL_EXPIRY_STATE, apply_expiry_rule as ecstore_apply_expiry_rule,
apply_transition_rule as ecstore_apply_transition_rule,
};
use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator;
use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE,
};
use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
};
use rustfs_ecstore::api::bucket::replication::{
ReplicationConfig as EcstoreReplicationConfig, ReplicationConfigurationExt as EcstoreReplicationConfigurationExt,
ReplicationHealQueueResult as EcstoreReplicationHealQueueResult,
ReplicationQueueAdmission as EcstoreReplicationQueueAdmission,
queue_replication_heal_internal as ecstore_queue_replication_heal_internal,
};
use rustfs_ecstore::api::bucket::versioning::VersioningApi as EcstoreVersioningApi;
use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys as EcstoreBucketVersioningSys;
use rustfs_ecstore::api::cache::{ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw};
use rustfs_ecstore::api::capacity::{
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
};
use rustfs_ecstore::api::config::com::{read_config as ecstore_read_config, save_config as ecstore_save_config};
#[cfg(test)]
use rustfs_ecstore::api::config::init as ecstore_config_init;
use rustfs_ecstore::api::config::storageclass::{RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD};
use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
#[cfg(test)]
use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
ScanGuard as EcstoreScanGuard,
};
#[cfg(test)]
use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk};
use rustfs_ecstore::api::error::{Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError};
use rustfs_ecstore::api::global::{
GLOBAL_TierConfigMgr as ECSTORE_GLOBAL_TIER_CONFIG_MGR, is_erasure as ecstore_is_erasure,
is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle,
};
use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectToDelete};
use std::path::PathBuf;
use std::sync::Arc;
crates/scanner/tests/ecstore_test_compat/mod.rs
+20
View File
@@ -0,0 +1,20 @@
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
enqueue_transition_for_existing_objects, init_background_expiry,
};
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::TransitionOptions;
pub(crate) use rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
};
pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys;
pub(crate) use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path;
pub(crate) use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl};
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk};
pub(crate) use rustfs_ecstore::api::global::GLOBAL_TierConfigMgr;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
pub(crate) use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType};
pub(crate) use rustfs_ecstore::api::tier::warm_backend::{
WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options,
};
crates/scanner/tests/lifecycle_integration_test.rs
+9 -20
View File
@@ -12,28 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod ecstore_test_compat;
use ecstore_test_compat::{
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskAPI as _, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerWarmBackend, TierConfig, TierMinIO,
TierType, TransitionOptions, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys, init_local_disks, new_disk,
path2_bucket_object_with_base_path, update_bucket_metadata,
};
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
use rustfs_ecstore::api::bucket::{
lifecycle::{
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry},
lifecycle::TransitionOptions,
},
metadata::BUCKET_LIFECYCLE_CONFIG,
metadata_sys::{get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata},
versioning_sys::BucketVersioningSys,
};
use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path;
use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl};
use rustfs_ecstore::api::disk::endpoint::Endpoint;
use rustfs_ecstore::api::disk::{DiskAPI as _, DiskOption, STORAGE_FORMAT_FILE, new_disk};
use rustfs_ecstore::api::global::GLOBAL_TierConfigMgr;
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
use rustfs_ecstore::api::tier::{
tier_config::{TierConfig, TierMinIO, TierType},
warm_backend::{WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options},
};
use rustfs_filemeta::FileMeta;
use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk;