refactor: route scanner heal storage boundaries (#3888)

This commit is contained in:
Zhengchao An
2026-06-26 05:24:41 +08:00
committed by GitHub
parent f7a9a7142b
commit 4d6ea453a7
20 changed files with 380 additions and 154 deletions
+2 -2
View File
@@ -558,13 +558,13 @@ mod tests {
async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> { async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> {
Ok(()) Ok(())
} }
async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<rustfs_storage_api::BucketInfo>> { async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<crate::heal::storage_api::BucketInfo>> {
Ok(None) Ok(None)
} }
async fn heal_bucket_metadata(&self, _bucket: &str) -> crate::Result<()> { async fn heal_bucket_metadata(&self, _bucket: &str) -> crate::Result<()> {
Ok(()) Ok(())
} }
async fn list_buckets(&self) -> crate::Result<Vec<rustfs_storage_api::BucketInfo>> { async fn list_buckets(&self) -> crate::Result<Vec<crate::heal::storage_api::BucketInfo>> {
Ok(vec![]) Ok(vec![])
} }
async fn object_exists(&self, _bucket: &str, _object: &str) -> crate::Result<bool> { async fn object_exists(&self, _bucket: &str, _object: &str) -> crate::Result<bool> {
+1 -2
View File
@@ -2393,9 +2393,8 @@ mod tests {
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealType}; use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
use rustfs_common::heal_channel::{HealOpts, HealRequestSource}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
use rustfs_madmin::heal_commands::HealResultItem; use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::BucketInfo;
use super::super::{DiskStore, Endpoint}; use super::super::{DiskStore, Endpoint, storage_api::BucketInfo};
struct MockStorage; struct MockStorage;
+9 -13
View File
@@ -19,21 +19,17 @@ pub mod manager;
pub mod progress; pub mod progress;
pub mod resume; pub mod resume;
pub mod storage; pub mod storage;
pub(crate) mod storage_api;
pub mod task; pub mod task;
pub mod utils; pub mod utils;
use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME; use storage_api::{
use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint; ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_GLOBAL_LOCAL_DISK_MAP, ECSTORE_RUSTFS_META_BUCKET,
use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult}; EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore,
use rustfs_ecstore::api::disk::{ EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations,
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, DeleteOptions as EcstoreDeleteOptions,
DiskAPI as EcstoreDiskAPI, DiskStore as EcstoreDiskStore, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
}; };
#[cfg(test)] #[cfg(test)]
use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk}; use storage_api::{EcstoreDiskOption, ecstore_new_disk};
use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP;
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
pub use erasure_healer::ErasureSetHealer; pub use erasure_healer::ErasureSetHealer;
pub use manager::{HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts}; pub use manager::{HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts};
@@ -116,6 +112,6 @@ where
} }
} }
pub type HealObjectInfo = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectInfo; pub type HealObjectInfo = <ECStore as ObjectOperations>::ObjectInfo;
pub type HealObjectOptions = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectOptions; pub type HealObjectOptions = <ECStore as ObjectOperations>::ObjectOptions;
pub type HealPutObjReader = <ECStore as rustfs_storage_api::ObjectIO>::PutObjectReader; pub type HealPutObjReader = <ECStore as ObjectIO>::PutObjectReader;
+4 -4
View File
@@ -16,13 +16,13 @@ use crate::{Error, Result};
use async_trait::async_trait; use async_trait::async_trait;
use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_madmin::heal_commands::HealResultItem; use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
ObjectOperations as _, StorageAdminApi,
};
use std::sync::Arc; use std::sync::Arc;
use tracing::{debug, error, warn}; use tracing::{debug, error, warn};
use super::storage_api::{
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
ObjectOperations as _, StorageAdminApi,
};
use super::{DiskStore, ECStore, Endpoint, StorageError}; use super::{DiskStore, ECStore, Endpoint, StorageError};
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME;
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
pub(crate) use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, DeleteOptions as EcstoreDeleteOptions,
DiskAPI as EcstoreDiskAPI, DiskStore as EcstoreDiskStore, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
pub(crate) use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
pub(crate) use rustfs_storage_api::{
BucketInfo, BucketOperations, DiskSetSelector, HealOperations, ListOperations, ObjectIO, ObjectOperations, StorageAdminApi,
};
+1 -1
View File
@@ -2227,10 +2227,10 @@ mod tests {
use super::*; use super::*;
use crate::heal::storage::{DiskStatus, HealObjectInfo}; use crate::heal::storage::{DiskStatus, HealObjectInfo};
use rustfs_madmin::heal_commands::HealResultItem; use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::BucketInfo;
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Mutex; use std::sync::Mutex;
use super::super::storage_api::BucketInfo;
#[derive(Default)] #[derive(Default)]
struct MockStorage { struct MockStorage {
listed: Mutex<bool>, listed: Mutex<bool>,
+5 -3
View File
@@ -14,13 +14,15 @@
//! test endpoint index settings //! test endpoint index settings
use rustfs_ecstore::api::disk::endpoint::Endpoint;
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
use std::net::SocketAddr; use std::net::SocketAddr;
use tempfile::TempDir; use tempfile::TempDir;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
#[path = "endpoint_index_test/storage_api.rs"]
mod storage_api;
use storage_api::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_endpoint_index_settings() -> anyhow::Result<()> { async fn test_endpoint_index_settings() -> anyhow::Result<()> {
let temp_dir = TempDir::new()?; let temp_dir = TempDir::new()?;
@@ -0,0 +1,17 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
+9 -5
View File
@@ -12,13 +12,17 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint};
use rustfs_heal::heal::{ use rustfs_heal::heal::{
event::{HealEvent, Severity}, event::{HealEvent, Severity},
task::{HealPriority, HealType}, task::{HealPriority, HealType},
utils, utils,
}; };
#[path = "heal_bug_fixes_test/storage_api.rs"]
mod storage_api;
use storage_api::{BucketInfo, DiskStore, Endpoint};
#[test] #[test]
fn test_heal_event_to_heal_request_no_panic() { fn test_heal_event_to_heal_request_no_panic() {
// Test that invalid pool/set indices don't cause panic // Test that invalid pool/set indices don't cause panic
@@ -191,13 +195,13 @@ fn test_heal_task_status_atomic_update() {
async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> { async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> {
Ok(()) Ok(())
} }
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> { async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<BucketInfo>> {
Ok(None) Ok(None)
} }
async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> { async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> {
Ok(()) Ok(())
} }
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<rustfs_storage_api::BucketInfo>> { async fn list_buckets(&self) -> rustfs_heal::Result<Vec<BucketInfo>> {
Ok(vec![]) Ok(vec![])
} }
async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> { async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> {
@@ -320,7 +324,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok(()) Ok(())
} }
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> { async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<BucketInfo>> {
Ok(None) Ok(None)
} }
@@ -328,7 +332,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok(()) Ok(())
} }
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<rustfs_storage_api::BucketInfo>> { async fn list_buckets(&self) -> rustfs_heal::Result<Vec<BucketInfo>> {
Ok(Vec::new()) Ok(Vec::new())
} }
@@ -0,0 +1,16 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint};
pub(crate) use rustfs_storage_api::BucketInfo;
+9 -6
View File
@@ -14,16 +14,11 @@
use http::HeaderMap; use http::HeaderMap;
use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
use rustfs_ecstore::api::disk::endpoint::Endpoint;
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
use rustfs_heal::heal::{ use rustfs_heal::heal::{
manager::{HealConfig, HealManager}, manager::{HealConfig, HealManager},
storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI}, storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType}, task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
}; };
use rustfs_storage_api::{BucketOperations, ObjectIO as _, ObjectOperations as _};
use serial_test::serial; use serial_test::serial;
use std::{ use std::{
path::{Path, PathBuf}, path::{Path, PathBuf},
@@ -35,6 +30,14 @@ use tokio_util::sync::CancellationToken;
use tracing::info; use tracing::info;
use walkdir::WalkDir; use walkdir::WalkDir;
#[path = "heal_integration_test/storage_api.rs"]
mod storage_api;
use storage_api::{
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO as _, ObjectOperations as _,
PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
};
const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25); const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250); const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
const NON_INLINE_TEST_DATA_SIZE: usize = 256 * 1024 + 137; const NON_INLINE_TEST_DATA_SIZE: usize = 256 * 1024 + 137;
@@ -132,7 +135,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
// init bucket metadata system // init bucket metadata system
let buckets_list = ecstore let buckets_list = ecstore
.list_bucket(&rustfs_storage_api::BucketOptions { .list_bucket(&BucketOptions {
no_metadata: true, no_metadata: true,
..Default::default() ..Default::default()
}) })
@@ -0,0 +1,19 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectIO, ObjectOperations};
+20 -55
View File
@@ -22,61 +22,25 @@
use http::HeaderMap; use http::HeaderMap;
use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config as config_get_global_server_config}; use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config as config_get_global_server_config};
use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule,
get_global_expiry_state as ecstore_get_global_expiry_state,
};
use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator;
use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE,
};
use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
};
use rustfs_ecstore::api::bucket::replication::{
ReplicationConfig as EcstoreReplicationConfig, ReplicationConfigurationExt as EcstoreReplicationConfigurationExt,
ReplicationHealQueueResult as EcstoreReplicationHealQueueResult,
ReplicationQueueAdmission as EcstoreReplicationQueueAdmission,
queue_replication_heal_internal as ecstore_queue_replication_heal_internal,
};
use rustfs_ecstore::api::bucket::versioning::VersioningApi as EcstoreVersioningApi;
use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys as EcstoreBucketVersioningSys;
use rustfs_ecstore::api::cache::{ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw};
use rustfs_ecstore::api::capacity::{
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
};
use rustfs_ecstore::api::config::com::{read_config as ecstore_read_config, save_config as ecstore_save_config};
#[cfg(test)]
use rustfs_ecstore::api::config::init as ecstore_config_init;
use rustfs_ecstore::api::config::storageclass::{RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD};
use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
#[cfg(test)]
use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
ScanGuard as EcstoreScanGuard,
};
#[cfg(test)]
use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk};
use rustfs_ecstore::api::error::{Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError};
use rustfs_ecstore::api::global::{
get_global_tier_config_mgr as ecstore_get_global_tier_config_mgr, is_erasure as ecstore_is_erasure,
is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle,
};
use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectToDelete};
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use storage_api::{
ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS,
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk,
EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, EcstoreDiskLocation,
EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, EcstoreLifecycle,
EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfig, EcstoreReplicationConfigurationExt,
EcstoreReplicationHealQueueResult, EcstoreReplicationQueueAdmission, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks,
EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi, HTTPRangeSpec, ObjectIO, ObjectOperations,
ObjectToDelete, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_get_global_expiry_state,
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket,
ecstore_list_path_raw, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_queue_replication_heal_internal, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
ecstore_resolve_object_store_handle, ecstore_save_config,
};
#[cfg(test)]
use storage_api::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
pub mod data_usage_define; pub mod data_usage_define;
@@ -87,6 +51,7 @@ pub mod scanner_budget;
pub mod scanner_folder; pub mod scanner_folder;
pub mod scanner_io; pub mod scanner_io;
pub mod sleeper; pub mod sleeper;
pub(crate) mod storage_api;
pub use data_usage_define::*; pub use data_usage_define::*;
pub use error::ScannerError; pub use error::ScannerError;
@@ -151,8 +116,8 @@ pub(crate) type SetDisks = EcstoreSetDisks;
pub(crate) type StorageError = EcstoreStorageError; pub(crate) type StorageError = EcstoreStorageError;
pub type ScannerGetObjectReader = <ECStore as ObjectIO>::GetObjectReader; pub type ScannerGetObjectReader = <ECStore as ObjectIO>::GetObjectReader;
pub type ScannerObjectInfo = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectInfo; pub type ScannerObjectInfo = <ECStore as ObjectOperations>::ObjectInfo;
pub type ScannerObjectOptions = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectOptions; pub type ScannerObjectOptions = <ECStore as ObjectOperations>::ObjectOptions;
pub type ScannerObjectToDelete = ObjectToDelete; pub type ScannerObjectToDelete = ObjectToDelete;
pub type ScannerPutObjReader = <ECStore as ObjectIO>::PutObjectReader; pub type ScannerPutObjReader = <ECStore as ObjectIO>::PutObjectReader;
+4 -4
View File
@@ -38,13 +38,13 @@ use rustfs_config::{
ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_CYCLE_MAX_OBJECTS,
}; };
use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS}; use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
use rustfs_storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::sync::mpsc; use tokio::sync::mpsc;
use tokio::time::{Duration, Instant}; use tokio::time::{Duration, Instant};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn}; use tracing::{debug, error, info, instrument, warn};
use crate::storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _};
use crate::{ use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config, get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config,
@@ -1148,9 +1148,9 @@ mod tests {
} }
#[async_trait::async_trait] #[async_trait::async_trait]
impl rustfs_storage_api::ObjectIO for MemoryConfigStore { impl crate::storage_api::ObjectIO for MemoryConfigStore {
type Error = EcstoreError; type Error = EcstoreError;
type RangeSpec = rustfs_storage_api::HTTPRangeSpec; type RangeSpec = crate::storage_api::HTTPRangeSpec;
type HeaderMap = http::HeaderMap; type HeaderMap = http::HeaderMap;
type ObjectOptions = ObjectOptions; type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo; type ObjectInfo = ObjectInfo;
@@ -1161,7 +1161,7 @@ mod tests {
&self, &self,
bucket: &str, bucket: &str,
object: &str, object: &str,
_range: Option<rustfs_storage_api::HTTPRangeSpec>, _range: Option<crate::storage_api::HTTPRangeSpec>,
_h: http::HeaderMap, _h: http::HeaderMap,
_opts: &ObjectOptions, _opts: &ObjectOptions,
) -> EcstoreResult<GetObjectReader> { ) -> EcstoreResult<GetObjectReader> {
+1 -1
View File
@@ -27,7 +27,6 @@ use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, e
#[cfg(test)] #[cfg(test)]
use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS}; use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
use rustfs_filemeta::FileMeta; use rustfs_filemeta::FileMeta;
use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
use rustfs_utils::path::path_join_buf; use rustfs_utils::path::path_join_buf;
use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration}; use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration};
use std::collections::HashMap; use std::collections::HashMap;
@@ -43,6 +42,7 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn}; use tracing::{debug, error, warn};
use crate::ScannerObjectInfo as ObjectInfo; use crate::ScannerObjectInfo as ObjectInfo;
use crate::storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
use crate::{ use crate::{
BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result,
ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _,
+78
View File
@@ -0,0 +1,78 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule,
get_global_expiry_state as ecstore_get_global_expiry_state,
};
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE,
};
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
};
pub(crate) use rustfs_ecstore::api::bucket::replication::{
ReplicationConfig as EcstoreReplicationConfig, ReplicationConfigurationExt as EcstoreReplicationConfigurationExt,
ReplicationHealQueueResult as EcstoreReplicationHealQueueResult,
ReplicationQueueAdmission as EcstoreReplicationQueueAdmission,
queue_replication_heal_internal as ecstore_queue_replication_heal_internal,
};
pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi as EcstoreVersioningApi;
pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys as EcstoreBucketVersioningSys;
pub(crate) use rustfs_ecstore::api::cache::{
ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw,
};
pub(crate) use rustfs_ecstore::api::capacity::{
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
};
pub(crate) use rustfs_ecstore::api::config::com::{read_config as ecstore_read_config, save_config as ecstore_save_config};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::config::init as ecstore_config_init;
pub(crate) use rustfs_ecstore::api::config::storageclass::{
RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
};
pub(crate) use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
pub(crate) use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
ScanGuard as EcstoreScanGuard,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{
DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk,
};
pub(crate) use rustfs_ecstore::api::error::{
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
};
pub(crate) use rustfs_ecstore::api::global::{
get_global_tier_config_mgr as ecstore_get_global_tier_config_mgr, is_erasure as ecstore_is_erasure,
is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle,
};
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
pub(crate) use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
pub(crate) use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectOperations,
ObjectToDelete, StorageAdminApi,
};
@@ -14,25 +14,6 @@
use futures::FutureExt; use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT; use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
use rustfs_ecstore::api::bucket::lifecycle::{
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry},
lifecycle::TransitionOptions,
};
use rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
use rustfs_ecstore::api::bucket::metadata_sys::{
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
};
use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys;
use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path;
use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl};
use rustfs_ecstore::api::disk::{DiskAPI as _, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk};
use rustfs_ecstore::api::global::get_global_tier_config_mgr;
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType};
use rustfs_ecstore::api::tier::warm_backend::{
WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options,
};
use rustfs_filemeta::FileMeta; use rustfs_filemeta::FileMeta;
use rustfs_scanner::scanner_folder::ScannerItem; use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk; use rustfs_scanner::scanner_io::ScannerIODisk;
@@ -40,9 +21,6 @@ use rustfs_scanner::{
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
scanner::init_data_scanner, scanner::init_data_scanner,
}; };
use rustfs_storage_api::{
BucketOperations, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, ObjectOperations as _,
};
use rustfs_utils::path::path_join_buf; use rustfs_utils::path::path_join_buf;
use s3s::dto::RestoreRequest; use s3s::dto::RestoreRequest;
use serial_test::serial; use serial_test::serial;
@@ -61,6 +39,17 @@ use tokio_util::sync::CancellationToken;
use tracing::info; use tracing::info;
use uuid::Uuid; use uuid::Uuid;
mod storage_api;
use storage_api::{
BUCKET_LIFECYCLE_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart, DiskAPI as _, DiskOption,
ECStore, Endpoint, EndpointServerPools, Endpoints, ListOperations as _, MakeBucketOptions, MultipartOperations as _,
ObjectIO as _, ObjectOperations as _, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerWarmBackend,
TierConfig, TierMinIO, TierType, TransitionOptions, WarmBackendGetOpts, build_transition_put_options,
enqueue_transition_for_existing_objects, get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry,
init_bucket_metadata_sys, init_local_disks, new_disk, path2_bucket_object_with_base_path, update_bucket_metadata,
};
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new(); static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
static INIT: Once = Once::new(); static INIT: Once = Once::new();
const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15); const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
@@ -134,7 +123,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
// init bucket metadata system // init bucket metadata system
let buckets_list = ecstore let buckets_list = ecstore
.list_bucket(&rustfs_storage_api::BucketOptions { .list_bucket(&BucketOptions {
no_metadata: true, no_metadata: true,
..Default::default() ..Default::default()
}) })
@@ -200,7 +189,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
.unwrap(); .unwrap();
let buckets_list = ecstore let buckets_list = ecstore
.list_bucket(&rustfs_storage_api::BucketOptions { .list_bucket(&BucketOptions {
no_metadata: true, no_metadata: true,
..Default::default() ..Default::default()
}) })
@@ -1021,7 +1010,7 @@ mod serial_tests {
multipart_bucket.as_str(), multipart_bucket.as_str(),
multipart_object, multipart_object,
&upload.upload_id, &upload.upload_id,
vec![rustfs_storage_api::CompletePart { vec![CompletePart {
part_num: 1, part_num: 1,
etag: part.etag.clone(), etag: part.etag.clone(),
..Default::default() ..Default::default()
@@ -1166,12 +1155,12 @@ mod serial_tests {
object_name, object_name,
&upload.upload_id, &upload.upload_id,
vec![ vec![
rustfs_storage_api::CompletePart { CompletePart {
part_num: 1, part_num: 1,
etag: uploaded_part1.etag.clone(), etag: uploaded_part1.etag.clone(),
..Default::default() ..Default::default()
}, },
rustfs_storage_api::CompletePart { CompletePart {
part_num: 2, part_num: 2,
etag: uploaded_part2.etag.clone(), etag: uploaded_part2.etag.clone(),
..Default::default() ..Default::default()
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::{
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry},
lifecycle::TransitionOptions,
};
pub(crate) use rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
};
pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys;
pub(crate) use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path;
pub(crate) use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl};
pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk};
pub(crate) use rustfs_ecstore::api::global::get_global_tier_config_mgr;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
pub(crate) use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType};
pub(crate) use rustfs_ecstore::api::tier::warm_backend::{
WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options,
};
pub(crate) use rustfs_storage_api::{
BucketOperations, BucketOptions, CompletePart, ListOperations, MakeBucketOptions, MultipartOperations, ObjectIO,
ObjectOperations,
};
+28 -8
View File
@@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context ## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-storage-api-boundary-batch` - Branch: `overtrue/arch-external-crate-storage-boundary-sweep`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218`. - Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221`.
- Based on: API-218 branch; branch routes root/server/startup, app, and admin storage contract consumers through local `storage_api` boundaries and folds app S3 helper forwarding into the app boundary. - Based on: API-221 branch; branch routes scanner and heal source plus integration test storage symbols through local `storage_api` boundaries.
- PR type for this branch: `consumer-migration` - PR type for this branch: `consumer-migration`
- Runtime behavior changes: none. - Runtime behavior changes: none.
- Rust code changes: route replication pool, outbound TLS generation, runtime - Rust code changes: route replication pool, outbound TLS generation, runtime
@@ -55,8 +55,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
capacity, workload admission, table catalog, init, protocol clients, and capacity, workload admission, table catalog, init, protocol clients, and
config tests through a root-local storage_api boundary, plus root/server/startup config tests through a root-local storage_api boundary, plus root/server/startup
`rustfs_storage_api` contract imports through the same boundary, app/admin `rustfs_storage_api` contract imports through the same boundary, app/admin
`rustfs_storage_api` contract imports through their local boundaries, and app `rustfs_storage_api` contract imports through their local boundaries, app
S3 helper forwarding through `app::storage_api`. S3 helper forwarding through `app::storage_api`, and scanner/heal source and
test ECStore plus storage contract imports through crate-local `storage_api`
boundaries.
- CI/script changes: lock completed owner and test/fuzz boundaries against - CI/script changes: lock completed owner and test/fuzz boundaries against
bare/glob imports, scattered raw ECStore facade subpaths, and startup bare/glob imports, scattered raw ECStore facade subpaths, and startup
runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade
@@ -66,8 +68,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
event-bridge thin module regressions, plus IAM runtime-source bypasses; event-bridge thin module regressions, plus IAM runtime-source bypasses;
accept the reviewed AppContext resolver reverse dependencies in the layer accept the reviewed AppContext resolver reverse dependencies in the layer
baseline, and block direct admin AppContext resolver consumers outside the baseline, and block direct admin AppContext resolver consumers outside the
admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, and keep app S3 helper imports routed through `app::storage_api`. admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, and reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries.
- Docs changes: record the API-136 through API-221 owner facade and lifecycle - Docs changes: record the API-136 through API-222 owner facade and lifecycle
runtime-source cleanup. runtime-source cleanup.
## Phase 0 Tasks ## Phase 0 Tasks
@@ -5211,14 +5213,32 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
layer guards, diff hygiene, residual app S3 helper scan, Rust risk scan, layer guards, diff hygiene, residual app S3 helper scan, Rust risk scan,
fast PR gate, and full PR gate before PR. fast PR gate, and full PR gate before PR.
- [x] `API-222` Route scanner and heal storage imports through local storage_api boundaries.
- Do: move scanner and heal source/test ECStore and `rustfs_storage_api`
imports into crate-local `storage_api` boundary modules, then route
scanner runtime, scanner lifecycle integration tests, heal runtime, and
heal integration tests through those local boundaries.
- Acceptance: migrated scanner/heal consumers no longer import ECStore or
storage contract symbols directly, and migration rules reject direct
scanner/heal bypasses outside the reviewed boundary modules.
- Must preserve: scanner lifecycle, tier transition, replication heal queue,
bucket scanner IO, heal local disk/indexing, bucket/object heal storage,
endpoint setup, and heal transient-object test behavior.
- Verification: focused scanner/heal compile coverage, formatting,
migration and layer guards, diff hygiene, residual scanner/heal boundary
scan, Rust risk scan, fast PR gate, and full PR gate before PR.
## Next PRs ## Next PRs
1. `consumer-migration`: continue larger outer/owner facade batches after API-221. 1. `consumer-migration`: continue larger owner/external crate storage-boundary batches after API-222.
## Pre-Push Review Log ## Pre-Push Review Log
| Expert | Status | Notes | | Expert | Status | Notes |
|---|---|---| |---|---|---|
| Quality/architecture | pass | API-222 routes scanner and heal ECStore/storage contract imports through crate-local storage_api boundaries. |
| Migration preservation | pass | Scanner lifecycle/tier/replication IO and heal disk/object/bucket test contracts keep the same ECStore and storage-api implementations. |
| Testing/verification | pass | Focused scanner/heal compile coverage, formatting, migration/layer guards, residual boundary scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. |
| Quality/architecture | pass | API-221 folds app S3 helper forwarding into app storage_api and removes the standalone app s3_api forwarding module. | | Quality/architecture | pass | API-221 folds app S3 helper forwarding into app storage_api and removes the standalone app s3_api forwarding module. |
| Migration preservation | pass | Bucket/object/multipart list and multipart helper parsers/builders keep the same storage S3 API implementations and owner metadata helper. | | Migration preservation | pass | Bucket/object/multipart list and multipart helper parsers/builders keep the same storage S3 API implementations and owner metadata helper. |
| Testing/verification | pass | Focused RustFS compile/tests, formatting, migration/layer guards, residual app S3 helper scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. | | Testing/verification | pass | Focused RustFS compile/tests, formatting, migration/layer guards, residual app S3 helper scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. |
+75 -23
View File
@@ -131,6 +131,10 @@ RUSTFS_APP_SERVER_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_ap
RUSTFS_HEAL_TEST_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_heal_test_local_compat_relative_consumer_hits.txt" RUSTFS_HEAL_TEST_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_heal_test_local_compat_relative_consumer_hits.txt"
STANDALONE_CRATE_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/standalone_crate_local_compat_relative_consumer_hits.txt" STANDALONE_CRATE_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/standalone_crate_local_compat_relative_consumer_hits.txt"
SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/scanner_bucket_storage_compat_module_hits.txt" SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/scanner_bucket_storage_compat_module_hits.txt"
SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE="${TMP_DIR}/scanner_storage_api_source_bypass_hits.txt"
SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE="${TMP_DIR}/scanner_storage_api_test_bypass_hits.txt"
HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE="${TMP_DIR}/heal_storage_api_source_bypass_hits.txt"
HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE="${TMP_DIR}/heal_storage_api_test_bypass_hits.txt"
NOTIFY_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/notify_storage_compat_module_hits.txt" NOTIFY_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/notify_storage_compat_module_hits.txt"
OBS_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE="${TMP_DIR}/obs_storage_compat_passthrough_hits.txt" OBS_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE="${TMP_DIR}/obs_storage_compat_passthrough_hits.txt"
E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE="${TMP_DIR}/e2e_storage_compat_rpc_passthrough_hits.txt" E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE="${TMP_DIR}/e2e_storage_compat_rpc_passthrough_hits.txt"
@@ -757,7 +761,7 @@ fi
--glob '!**/ecstore_test_compat/**' \ --glob '!**/ecstore_test_compat/**' \
--glob '!**/ecstore_fuzz_compat.rs' \ --glob '!**/ecstore_fuzz_compat.rs' \
--glob '!target/**' \ --glob '!target/**' \
| rg -v '^(rustfs/src/(admin/mod|app/mod|storage/mod)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true | rg -v '^(rustfs/src/(admin/mod|app/mod|storage/mod)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true
) | ) |
cat >"$DIRECT_ECSTORE_IMPORT_HITS_FILE" cat >"$DIRECT_ECSTORE_IMPORT_HITS_FILE"
@@ -1106,7 +1110,7 @@ fi
--glob '!**/ecstore_compat.rs' \ --glob '!**/ecstore_compat.rs' \
--glob '!**/ecstore_test_compat.rs' \ --glob '!**/ecstore_test_compat.rs' \
--glob '!**/ecstore_test_compat/**' | --glob '!**/ecstore_test_compat/**' |
rg -v '^(fuzz/fuzz_targets/bucket_validation\.rs|fuzz/fuzz_targets/path_containment\.rs|crates/e2e_test/src/reliant/grpc_lock_client\.rs|crates/e2e_test/src/reliant/node_interact_test\.rs|crates/e2e_test/src/replication_extension_test\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/endpoint_index_test\.rs|crates/heal/tests/heal_bug_fixes_test\.rs|crates/heal/tests/heal_integration_test\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/mod\.rs):' || true rg -v '^(fuzz/fuzz_targets/bucket_validation\.rs|fuzz/fuzz_targets/path_containment\.rs|crates/e2e_test/src/reliant/grpc_lock_client\.rs|crates/e2e_test/src/reliant/node_interact_test\.rs|crates/e2e_test/src/replication_extension_test\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/mod\.rs):' || true
) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE"
if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then
@@ -1116,17 +1120,17 @@ fi
( (
cd "$ROOT_DIR" cd "$ROOT_DIR"
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \ rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \
crates/heal/src/heal/mod.rs \ crates/heal/src/heal/storage_api.rs \
crates/heal/tests/endpoint_index_test.rs \ crates/heal/tests/endpoint_index_test/storage_api.rs \
crates/heal/tests/heal_bug_fixes_test.rs \ crates/heal/tests/heal_bug_fixes_test/storage_api.rs \
crates/heal/tests/heal_integration_test.rs \ crates/heal/tests/heal_integration_test/storage_api.rs \
crates/iam/src/lib.rs \ crates/iam/src/lib.rs \
crates/notify/src/lib.rs \ crates/notify/src/lib.rs \
crates/obs/src/metrics/mod.rs \ crates/obs/src/metrics/mod.rs \
crates/protocols/src/swift/mod.rs \ crates/protocols/src/swift/mod.rs \
crates/s3select-api/src/lib.rs \ crates/s3select-api/src/lib.rs \
crates/scanner/src/lib.rs \ crates/scanner/src/storage_api.rs \
crates/scanner/tests/lifecycle_integration_test.rs \ crates/scanner/tests/storage_api/mod.rs \
crates/e2e_test/src/reliant/grpc_lock_client.rs \ crates/e2e_test/src/reliant/grpc_lock_client.rs \
crates/e2e_test/src/reliant/node_interact_test.rs \ crates/e2e_test/src/reliant/node_interact_test.rs \
crates/e2e_test/src/replication_extension_test.rs \ crates/e2e_test/src/replication_extension_test.rs \
@@ -1144,17 +1148,17 @@ fi
( (
cd "$ROOT_DIR" cd "$ROOT_DIR"
rg -n --with-filename '^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+\s*;|^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+::\*\s*;' \ rg -n --with-filename '^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+\s*;|^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+::\*\s*;' \
crates/heal/src/heal/mod.rs \ crates/heal/src/heal/storage_api.rs \
crates/heal/tests/endpoint_index_test.rs \ crates/heal/tests/endpoint_index_test/storage_api.rs \
crates/heal/tests/heal_bug_fixes_test.rs \ crates/heal/tests/heal_bug_fixes_test/storage_api.rs \
crates/heal/tests/heal_integration_test.rs \ crates/heal/tests/heal_integration_test/storage_api.rs \
crates/iam/src/lib.rs \ crates/iam/src/lib.rs \
crates/notify/src/lib.rs \ crates/notify/src/lib.rs \
crates/obs/src/metrics/mod.rs \ crates/obs/src/metrics/mod.rs \
crates/protocols/src/swift/mod.rs \ crates/protocols/src/swift/mod.rs \
crates/s3select-api/src/lib.rs \ crates/s3select-api/src/lib.rs \
crates/scanner/src/lib.rs \ crates/scanner/src/storage_api.rs \
crates/scanner/tests/lifecycle_integration_test.rs \ crates/scanner/tests/storage_api/mod.rs \
crates/e2e_test/src/reliant/grpc_lock_client.rs \ crates/e2e_test/src/reliant/grpc_lock_client.rs \
crates/e2e_test/src/reliant/node_interact_test.rs \ crates/e2e_test/src/reliant/node_interact_test.rs \
crates/e2e_test/src/replication_extension_test.rs \ crates/e2e_test/src/replication_extension_test.rs \
@@ -1172,17 +1176,17 @@ fi
( (
cd "$ROOT_DIR" cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' \ rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' \
crates/heal/src/heal/mod.rs \ crates/heal/src/heal/storage_api.rs \
crates/heal/tests/endpoint_index_test.rs \ crates/heal/tests/endpoint_index_test/storage_api.rs \
crates/heal/tests/heal_bug_fixes_test.rs \ crates/heal/tests/heal_bug_fixes_test/storage_api.rs \
crates/heal/tests/heal_integration_test.rs \ crates/heal/tests/heal_integration_test/storage_api.rs \
crates/iam/src/lib.rs \ crates/iam/src/lib.rs \
crates/notify/src/lib.rs \ crates/notify/src/lib.rs \
crates/obs/src/metrics/mod.rs \ crates/obs/src/metrics/mod.rs \
crates/protocols/src/swift/mod.rs \ crates/protocols/src/swift/mod.rs \
crates/s3select-api/src/lib.rs \ crates/s3select-api/src/lib.rs \
crates/scanner/src/lib.rs \ crates/scanner/src/storage_api.rs \
crates/scanner/tests/lifecycle_integration_test.rs \ crates/scanner/tests/storage_api/mod.rs \
crates/e2e_test/src/reliant/grpc_lock_client.rs \ crates/e2e_test/src/reliant/grpc_lock_client.rs \
crates/e2e_test/src/reliant/node_interact_test.rs \ crates/e2e_test/src/reliant/node_interact_test.rs \
crates/e2e_test/src/replication_extension_test.rs \ crates/e2e_test/src/replication_extension_test.rs \
@@ -1306,7 +1310,7 @@ fi
crates/scanner/src \ crates/scanner/src \
--glob '*.rs' \ --glob '*.rs' \
--glob '!**/ecstore_compat.rs' | --glob '!**/ecstore_compat.rs' |
rg -v '^(crates/heal/src/heal/mod.rs|crates/iam/src/(lib|runtime_sources).rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/lib.rs):' || true rg -v '^(crates/heal/src/heal/storage_api.rs|crates/iam/src/(lib|runtime_sources).rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/storage_api.rs):' || true
) >"$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE" ) >"$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE"
if [[ -s "$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then if [[ -s "$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then
@@ -1594,7 +1598,7 @@ fi
crates/scanner/tests \ crates/scanner/tests \
crates/e2e_test/src \ crates/e2e_test/src \
--glob '*.rs' \ --glob '*.rs' \
| rg -v '^(crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/scanner/tests/lifecycle_integration_test\.rs):' || true | rg -v '^(crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs):' || true
) >"$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE" ) >"$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE"
if [[ -s "$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then if [[ -s "$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then
@@ -1618,7 +1622,7 @@ fi
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \ rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \
crates/heal/src crates/iam/src crates/notify/src crates/obs/src crates/protocols/src crates/s3select-api/src crates/scanner/src \ crates/heal/src crates/iam/src crates/notify/src crates/obs/src crates/protocols/src crates/s3select-api/src crates/scanner/src \
--glob '*.rs' | --glob '*.rs' |
rg -v '^(crates/heal/src/heal/mod.rs|crates/iam/src/lib.rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/lib.rs):' || true rg -v '^(crates/heal/src/heal/storage_api.rs|crates/iam/src/lib.rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/storage_api.rs):' || true
) >"$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE" ) >"$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE"
if [[ -s "$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE" ]]; then if [[ -s "$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE" ]]; then
@@ -2020,6 +2024,54 @@ if [[ -s "$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then
report_failure "scanner storage compatibility must expose bucket contracts as explicit aliases: $(paste -sd '; ' "$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE")" report_failure "scanner storage compatibility must expose bucket contracts as explicit aliases: $(paste -sd '; ' "$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE")"
fi fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \
crates/scanner/src \
--glob '*.rs' |
rg -v '^crates/scanner/src/storage_api\.rs:' || true
) >"$SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE"
if [[ -s "$SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE" ]]; then
report_failure "scanner source files must route ECStore and storage-api symbols through scanner::storage_api: $(paste -sd '; ' "$SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \
crates/scanner/tests \
--glob '*.rs' |
rg -v '^crates/scanner/tests/storage_api/mod\.rs:' || true
) >"$SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE"
if [[ -s "$SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE" ]]; then
report_failure "scanner tests must route ECStore and storage-api symbols through scanner test storage_api: $(paste -sd '; ' "$SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \
crates/heal/src/heal \
--glob '*.rs' |
rg -v '^crates/heal/src/heal/storage_api\.rs:' || true
) >"$HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE"
if [[ -s "$HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE" ]]; then
report_failure "heal source files must route ECStore and storage-api symbols through heal::storage_api: $(paste -sd '; ' "$HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \
crates/heal/tests \
--glob '*.rs' |
rg -v '^crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs:' || true
) >"$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE"
if [[ -s "$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE" ]]; then
report_failure "heal tests must route ECStore and storage-api symbols through heal test storage_api: $(paste -sd '; ' "$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE")"
fi
( (
cd "$ROOT_DIR" cd "$ROOT_DIR"
if [[ -f crates/notify/src/storage_compat.rs ]]; then if [[ -f crates/notify/src/storage_compat.rs ]]; then