gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 03:22:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: wrap disk rpc compat trait methods (#3707)

Browse Source
This commit is contained in:
安正超
2026-06-22 06:31:24 +08:00
committed by GitHub
parent 7eb9a4759e
commit c13f42e9a0
18 changed files with 460 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_folder.rs
+4 -5
View File
@@ -39,7 +39,6 @@ use rustfs_common::metrics::{
IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn,
current_path_updater, global_metrics,
};
use rustfs_ecstore::api::disk::DiskAPI as _;
use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ReplicateObjectInfo, ReplicationStatusType, ReplicationType,
};
@@ -53,10 +52,10 @@ use tracing::{debug, error, warn};
use crate::storage_compat::{
BucketVersioningSys, Disk, DiskError, DiskInfoOptions, Evaluator, Event, LcEventSrc, ListPathRawOptions, ObjectOpts,
ReplicationConfig, ReplicationQueueAdmission, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
ScannerVersioningConfigExt as _, StorageError, apply_expiry_rule, apply_transition_rule, enqueue_global_newer_noncurrent,
is_erasure, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object, path2_bucket_object_with_base_path,
queue_replication_heal_internal,
ReplicationConfig, ReplicationQueueAdmission, ScannerDiskExt as _, ScannerLifecycleConfigExt as _,
ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, StorageError, apply_expiry_rule, apply_transition_rule,
enqueue_global_newer_noncurrent, is_erasure, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object,
path2_bucket_object_with_base_path, queue_replication_heal_internal,
};
use crate::{ScannerObjectInfo as ObjectInfo, ScannerObjectToDelete as ObjectToDelete};
crates/scanner/src/scanner_io.rs
+4 -4
View File
@@ -26,7 +26,6 @@ use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics};
#[cfg(test)]
use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
use rustfs_ecstore::api::disk::DiskAPI as _;
use rustfs_filemeta::FileMeta;
use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
use rustfs_utils::path::path_join_buf;
@@ -46,9 +45,10 @@ use tracing::{debug, error, warn};
use crate::ScannerObjectInfo as ObjectInfo;
use crate::storage_compat::{
BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result,
ReplicationConfig, STORAGE_FORMAT_FILE, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_global_free_version, get_lifecycle_config,
get_object_lock_config, get_replication_config, list_global_tiers, resolve_scanner_object_store_handle, storageclass,
ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _,
ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_global_free_version,
get_lifecycle_config, get_object_lock_config, get_replication_config, list_global_tiers, resolve_scanner_object_store_handle,
storageclass,
};
pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file";
crates/scanner/src/storage_compat.rs
+37
View File
@@ -19,6 +19,7 @@ use rustfs_ecstore::api::{
set_disk as ecstore_set_disk, storage as ecstore_storage, tier as ecstore_tier,
};
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectToDelete};
use std::path::PathBuf;
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
@@ -30,7 +31,9 @@ pub(crate) const TRANSITION_COMPLETE: &str = ecstore_bucket::lifecycle::lifecycl
pub(crate) type Disk = ecstore_disk::Disk;
#[cfg(test)]
pub(crate) type DiskStore = ecstore_disk::DiskStore;
pub(crate) type DiskLocation = ecstore_disk::DiskLocation;
pub(crate) type DiskError = ecstore_disk::error::DiskError;
pub(crate) type DiskResult<T> = ecstore_disk::error::Result<T>;
pub(crate) type ECStore = ecstore_storage::ECStore;
pub(crate) type EcstoreError = ecstore_error::Error;
pub(crate) type EcstoreResult<T> = ecstore_error::Result<T>;
@@ -45,6 +48,7 @@ pub(crate) type ObjectOpts = ecstore_bucket::lifecycle::lifecycle::ObjectOpts;
pub(crate) type ReplicationConfig = ecstore_bucket::replication::ReplicationConfig;
pub(crate) type ReplicationHealQueueResult = ecstore_bucket::replication::ReplicationHealQueueResult;
pub(crate) type ReplicationQueueAdmission = ecstore_bucket::replication::ReplicationQueueAdmission;
pub(crate) type ScanGuard = ecstore_disk::ScanGuard;
pub(crate) type SetDisks = ecstore_set_disk::SetDisks;
pub(crate) type StorageError = ecstore_error::StorageError;
@@ -133,6 +137,39 @@ impl ScannerVersioningConfigExt for s3s::dto::VersioningConfiguration {
}
}
pub(crate) trait ScannerDiskExt {
async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult<ecstore_disk::DiskInfo>;
async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<ecstore_disk::Bytes>;
fn path(&self) -> PathBuf;
fn get_disk_location(&self) -> DiskLocation;
fn start_scan(&self) -> ScanGuard;
}
impl<T> ScannerDiskExt for T
where
T: ecstore_disk::DiskAPI,
{
async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult<ecstore_disk::DiskInfo> {
ecstore_disk::DiskAPI::disk_info(self, opts).await
}
async fn read_metadata(&self, volume: &str, path: &str) -> DiskResult<ecstore_disk::Bytes> {
ecstore_disk::DiskAPI::read_metadata(self, volume, path).await
}
fn path(&self) -> PathBuf {
ecstore_disk::DiskAPI::path(self)
}
fn get_disk_location(&self) -> DiskLocation {
ecstore_disk::DiskAPI::get_disk_location(self)
}
fn start_scan(&self) -> ScanGuard {
ecstore_disk::DiskAPI::start_scan(self)
}
}
pub(crate) async fn apply_transition_rule(event: &Event, src: &LcEventSrc, oi: &ScannerObjectInfo) -> bool {
ecstore_bucket::lifecycle::bucket_lifecycle_ops::apply_transition_rule(event, src, oi).await
}
crates/scanner/tests/common/storage_compat.rs
+14
View File
@@ -41,8 +41,22 @@ pub(crate) type TierConfigMgr = ecstore_tier::tier::TierConfigMgr;
pub(crate) type TierMinIO = ecstore_tier::tier_config::TierMinIO;
pub(crate) type TierType = ecstore_tier::tier_config::TierType;
pub(crate) type TransitionOptions = ecstore_bucket::lifecycle::lifecycle::TransitionOptions;
pub(crate) use ecstore_tier::warm_backend::WarmBackend as ScannerWarmBackend;
pub(crate) type WarmBackendGetOpts = ecstore_tier::warm_backend::WarmBackendGetOpts;
pub(crate) trait ScannerTestDiskExt {
async fn read_metadata(&self, volume: &str, path: &str) -> ecstore_disk::error::Result<ecstore_disk::Bytes>;
}
impl<T> ScannerTestDiskExt for T
where
T: ecstore_disk::DiskAPI,
{
async fn read_metadata(&self, volume: &str, path: &str) -> ecstore_disk::error::Result<ecstore_disk::Bytes> {
ecstore_disk::DiskAPI::read_metadata(self, volume, path).await
}
}
#[allow(non_snake_case)]
pub(crate) fn EndpointServerPools(pools: Vec<PoolEndpoints>) -> EndpointServerPools {
ecstore_layout::EndpointServerPools::from(pools)
crates/scanner/tests/lifecycle_integration_test.rs
+5 -7
View File
@@ -16,15 +16,13 @@ mod common;
use crate::common::storage_compat::{
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, TierConfig, TierMinIO, TierType,
TransitionOptions, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys, init_local_disks, new_disk,
path2_bucket_object_with_base_path, update_bucket_metadata,
GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerTestDiskExt as _,
ScannerWarmBackend, TierConfig, TierMinIO, TierType, TransitionOptions, WarmBackendGetOpts, build_transition_put_options,
enqueue_transition_for_existing_objects, get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys,
init_local_disks, new_disk, path2_bucket_object_with_base_path, update_bucket_metadata,
};
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
use rustfs_ecstore::api::disk::DiskAPI as _;
use rustfs_ecstore::api::tier::warm_backend::WarmBackend;
use rustfs_filemeta::FileMeta;
use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk;
@@ -687,7 +685,7 @@ impl MockWarmBackend {
}
#[async_trait::async_trait]
impl WarmBackend for MockWarmBackend {
impl ScannerWarmBackend for MockWarmBackend {
async fn put(&self, object: &str, r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
let bytes = self.read_bytes(r).await?;
Ok(self.put_bytes(object, bytes, HashMap::new()).await)