refactor: route scanner heal storage boundaries (#3888)

This commit is contained in:
Zhengchao An
2026-06-26 05:24:41 +08:00
committed by GitHub
parent f7a9a7142b
commit 4d6ea453a7
20 changed files with 380 additions and 154 deletions
+2 -2
View File
@@ -558,13 +558,13 @@ mod tests {
async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> {
Ok(())
}
async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<rustfs_storage_api::BucketInfo>> {
async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<crate::heal::storage_api::BucketInfo>> {
Ok(None)
}
async fn heal_bucket_metadata(&self, _bucket: &str) -> crate::Result<()> {
Ok(())
}
async fn list_buckets(&self) -> crate::Result<Vec<rustfs_storage_api::BucketInfo>> {
async fn list_buckets(&self) -> crate::Result<Vec<crate::heal::storage_api::BucketInfo>> {
Ok(vec![])
}
async fn object_exists(&self, _bucket: &str, _object: &str) -> crate::Result<bool> {
+1 -2
View File
@@ -2393,9 +2393,8 @@ mod tests {
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::BucketInfo;
use super::super::{DiskStore, Endpoint};
use super::super::{DiskStore, Endpoint, storage_api::BucketInfo};
struct MockStorage;
+9 -13
View File
@@ -19,21 +19,17 @@ pub mod manager;
pub mod progress;
pub mod resume;
pub mod storage;
pub(crate) mod storage_api;
pub mod task;
pub mod utils;
use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME;
use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, DeleteOptions as EcstoreDeleteOptions,
DiskAPI as EcstoreDiskAPI, DiskStore as EcstoreDiskStore, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
use storage_api::{
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_GLOBAL_LOCAL_DISK_MAP, ECSTORE_RUSTFS_META_BUCKET,
EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore,
EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations,
};
#[cfg(test)]
use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP;
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
use storage_api::{EcstoreDiskOption, ecstore_new_disk};
pub use erasure_healer::ErasureSetHealer;
pub use manager::{HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts};
@@ -116,6 +112,6 @@ where
}
}
pub type HealObjectInfo = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
pub type HealObjectOptions = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
pub type HealPutObjReader = <ECStore as rustfs_storage_api::ObjectIO>::PutObjectReader;
pub type HealObjectInfo = <ECStore as ObjectOperations>::ObjectInfo;
pub type HealObjectOptions = <ECStore as ObjectOperations>::ObjectOptions;
pub type HealPutObjReader = <ECStore as ObjectIO>::PutObjectReader;
+4 -4
View File
@@ -16,13 +16,13 @@ use crate::{Error, Result};
use async_trait::async_trait;
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
ObjectOperations as _, StorageAdminApi,
};
use std::sync::Arc;
use tracing::{debug, error, warn};
use super::storage_api::{
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
ObjectOperations as _, StorageAdminApi,
};
use super::{DiskStore, ECStore, Endpoint, StorageError};
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME;
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
pub(crate) use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, DeleteOptions as EcstoreDeleteOptions,
DiskAPI as EcstoreDiskAPI, DiskStore as EcstoreDiskStore, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
pub(crate) use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
pub(crate) use rustfs_storage_api::{
BucketInfo, BucketOperations, DiskSetSelector, HealOperations, ListOperations, ObjectIO, ObjectOperations, StorageAdminApi,
};
+1 -1
View File
@@ -2227,10 +2227,10 @@ mod tests {
use super::*;
use crate::heal::storage::{DiskStatus, HealObjectInfo};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::BucketInfo;
use std::collections::HashMap;
use std::sync::Mutex;
use super::super::storage_api::BucketInfo;
#[derive(Default)]
struct MockStorage {
listed: Mutex<bool>,
+5 -3
View File
@@ -14,13 +14,15 @@
//! test endpoint index settings
use rustfs_ecstore::api::disk::endpoint::Endpoint;
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
use std::net::SocketAddr;
use tempfile::TempDir;
use tokio_util::sync::CancellationToken;
#[path = "endpoint_index_test/storage_api.rs"]
mod storage_api;
use storage_api::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn test_endpoint_index_settings() -> anyhow::Result<()> {
let temp_dir = TempDir::new()?;
@@ -0,0 +1,17 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
+9 -5
View File
@@ -12,13 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint};
use rustfs_heal::heal::{
event::{HealEvent, Severity},
task::{HealPriority, HealType},
utils,
};
#[path = "heal_bug_fixes_test/storage_api.rs"]
mod storage_api;
use storage_api::{BucketInfo, DiskStore, Endpoint};
#[test]
fn test_heal_event_to_heal_request_no_panic() {
// Test that invalid pool/set indices don't cause panic
@@ -191,13 +195,13 @@ fn test_heal_task_status_atomic_update() {
async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> {
Ok(())
}
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<BucketInfo>> {
Ok(None)
}
async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> {
Ok(())
}
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<rustfs_storage_api::BucketInfo>> {
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<BucketInfo>> {
Ok(vec![])
}
async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> {
@@ -320,7 +324,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok(())
}
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<BucketInfo>> {
Ok(None)
}
@@ -328,7 +332,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok(())
}
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<rustfs_storage_api::BucketInfo>> {
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<BucketInfo>> {
Ok(Vec::new())
}
@@ -0,0 +1,16 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint};
pub(crate) use rustfs_storage_api::BucketInfo;
+9 -6
View File
@@ -14,16 +14,11 @@
use http::HeaderMap;
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
use rustfs_ecstore::api::disk::endpoint::Endpoint;
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
use rustfs_heal::heal::{
manager::{HealConfig, HealManager},
storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
};
use rustfs_storage_api::{BucketOperations, ObjectIO as _, ObjectOperations as _};
use serial_test::serial;
use std::{
path::{Path, PathBuf},
@@ -35,6 +30,14 @@ use tokio_util::sync::CancellationToken;
use tracing::info;
use walkdir::WalkDir;
#[path = "heal_integration_test/storage_api.rs"]
mod storage_api;
use storage_api::{
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO as _, ObjectOperations as _,
PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
};
const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
const NON_INLINE_TEST_DATA_SIZE: usize = 256 * 1024 + 137;
@@ -132,7 +135,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
// init bucket metadata system
let buckets_list = ecstore
.list_bucket(&rustfs_storage_api::BucketOptions {
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
@@ -0,0 +1,19 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectIO, ObjectOperations};
+20 -55
View File
@@ -22,61 +22,25 @@
use http::HeaderMap;
use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config as config_get_global_server_config};
use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule,
get_global_expiry_state as ecstore_get_global_expiry_state,
};
use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator;
use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE,
};
use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
};
use rustfs_ecstore::api::bucket::replication::{
ReplicationConfig as EcstoreReplicationConfig, ReplicationConfigurationExt as EcstoreReplicationConfigurationExt,
ReplicationHealQueueResult as EcstoreReplicationHealQueueResult,
ReplicationQueueAdmission as EcstoreReplicationQueueAdmission,
queue_replication_heal_internal as ecstore_queue_replication_heal_internal,
};
use rustfs_ecstore::api::bucket::versioning::VersioningApi as EcstoreVersioningApi;
use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys as EcstoreBucketVersioningSys;
use rustfs_ecstore::api::cache::{ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw};
use rustfs_ecstore::api::capacity::{
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
};
use rustfs_ecstore::api::config::com::{read_config as ecstore_read_config, save_config as ecstore_save_config};
#[cfg(test)]
use rustfs_ecstore::api::config::init as ecstore_config_init;
use rustfs_ecstore::api::config::storageclass::{RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD};
use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
#[cfg(test)]
use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
ScanGuard as EcstoreScanGuard,
};
#[cfg(test)]
use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk};
use rustfs_ecstore::api::error::{Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError};
use rustfs_ecstore::api::global::{
get_global_tier_config_mgr as ecstore_get_global_tier_config_mgr, is_erasure as ecstore_is_erasure,
is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle,
};
use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectToDelete};
use std::path::PathBuf;
use std::sync::Arc;
use storage_api::{
ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS,
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk,
EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, EcstoreDiskLocation,
EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc, EcstoreLifecycle,
EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfig, EcstoreReplicationConfigurationExt,
EcstoreReplicationHealQueueResult, EcstoreReplicationQueueAdmission, EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks,
EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi, HTTPRangeSpec, ObjectIO, ObjectOperations,
ObjectToDelete, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_get_global_expiry_state,
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket,
ecstore_list_path_raw, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
ecstore_queue_replication_heal_internal, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
ecstore_resolve_object_store_handle, ecstore_save_config,
};
#[cfg(test)]
use storage_api::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
use tokio_util::sync::CancellationToken;
pub mod data_usage_define;
@@ -87,6 +51,7 @@ pub mod scanner_budget;
pub mod scanner_folder;
pub mod scanner_io;
pub mod sleeper;
pub(crate) mod storage_api;
pub use data_usage_define::*;
pub use error::ScannerError;
@@ -151,8 +116,8 @@ pub(crate) type SetDisks = EcstoreSetDisks;
pub(crate) type StorageError = EcstoreStorageError;
pub type ScannerGetObjectReader = <ECStore as ObjectIO>::GetObjectReader;
pub type ScannerObjectInfo = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
pub type ScannerObjectOptions = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
pub type ScannerObjectInfo = <ECStore as ObjectOperations>::ObjectInfo;
pub type ScannerObjectOptions = <ECStore as ObjectOperations>::ObjectOptions;
pub type ScannerObjectToDelete = ObjectToDelete;
pub type ScannerPutObjReader = <ECStore as ObjectIO>::PutObjectReader;
+4 -4
View File
@@ -38,13 +38,13 @@ use rustfs_config::{
ENV_SCANNER_CYCLE_MAX_OBJECTS,
};
use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
use rustfs_storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _};
use serde::{Deserialize, Serialize};
use tokio::sync::mpsc;
use tokio::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn};
use crate::storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _};
use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config,
@@ -1148,9 +1148,9 @@ mod tests {
}
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectIO for MemoryConfigStore {
impl crate::storage_api::ObjectIO for MemoryConfigStore {
type Error = EcstoreError;
type RangeSpec = rustfs_storage_api::HTTPRangeSpec;
type RangeSpec = crate::storage_api::HTTPRangeSpec;
type HeaderMap = http::HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
@@ -1161,7 +1161,7 @@ mod tests {
&self,
bucket: &str,
object: &str,
_range: Option<rustfs_storage_api::HTTPRangeSpec>,
_range: Option<crate::storage_api::HTTPRangeSpec>,
_h: http::HeaderMap,
_opts: &ObjectOptions,
) -> EcstoreResult<GetObjectReader> {
+1 -1
View File
@@ -27,7 +27,6 @@ use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, e
#[cfg(test)]
use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
use rustfs_filemeta::FileMeta;
use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
use rustfs_utils::path::path_join_buf;
use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration};
use std::collections::HashMap;
@@ -43,6 +42,7 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, warn};
use crate::ScannerObjectInfo as ObjectInfo;
use crate::storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
use crate::{
BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result,
ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _,
+78
View File
@@ -0,0 +1,78 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule,
get_global_expiry_state as ecstore_get_global_expiry_state,
};
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator;
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE,
};
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
};
pub(crate) use rustfs_ecstore::api::bucket::replication::{
ReplicationConfig as EcstoreReplicationConfig, ReplicationConfigurationExt as EcstoreReplicationConfigurationExt,
ReplicationHealQueueResult as EcstoreReplicationHealQueueResult,
ReplicationQueueAdmission as EcstoreReplicationQueueAdmission,
queue_replication_heal_internal as ecstore_queue_replication_heal_internal,
};
pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi as EcstoreVersioningApi;
pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys as EcstoreBucketVersioningSys;
pub(crate) use rustfs_ecstore::api::cache::{
ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw,
};
pub(crate) use rustfs_ecstore::api::capacity::{
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
};
pub(crate) use rustfs_ecstore::api::config::com::{read_config as ecstore_read_config, save_config as ecstore_save_config};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::config::init as ecstore_config_init;
pub(crate) use rustfs_ecstore::api::config::storageclass::{
RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
};
pub(crate) use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
pub(crate) use rustfs_ecstore::api::disk::{
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
ScanGuard as EcstoreScanGuard,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{
DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk,
};
pub(crate) use rustfs_ecstore::api::error::{
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
};
pub(crate) use rustfs_ecstore::api::global::{
get_global_tier_config_mgr as ecstore_get_global_tier_config_mgr, is_erasure as ecstore_is_erasure,
is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle,
};
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
pub(crate) use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
pub(crate) use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectOperations,
ObjectToDelete, StorageAdminApi,
};
@@ -14,25 +14,6 @@
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
use rustfs_ecstore::api::bucket::lifecycle::{
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry},
lifecycle::TransitionOptions,
};
use rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
use rustfs_ecstore::api::bucket::metadata_sys::{
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
};
use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys;
use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path;
use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl};
use rustfs_ecstore::api::disk::{DiskAPI as _, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk};
use rustfs_ecstore::api::global::get_global_tier_config_mgr;
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType};
use rustfs_ecstore::api::tier::warm_backend::{
WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options,
};
use rustfs_filemeta::FileMeta;
use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk;
@@ -40,9 +21,6 @@ use rustfs_scanner::{
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
scanner::init_data_scanner,
};
use rustfs_storage_api::{
BucketOperations, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, ObjectOperations as _,
};
use rustfs_utils::path::path_join_buf;
use s3s::dto::RestoreRequest;
use serial_test::serial;
@@ -61,6 +39,17 @@ use tokio_util::sync::CancellationToken;
use tracing::info;
use uuid::Uuid;
mod storage_api;
use storage_api::{
BUCKET_LIFECYCLE_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart, DiskAPI as _, DiskOption,
ECStore, Endpoint, EndpointServerPools, Endpoints, ListOperations as _, MakeBucketOptions, MultipartOperations as _,
ObjectIO as _, ObjectOperations as _, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerWarmBackend,
TierConfig, TierMinIO, TierType, TransitionOptions, WarmBackendGetOpts, build_transition_put_options,
enqueue_transition_for_existing_objects, get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry,
init_bucket_metadata_sys, init_local_disks, new_disk, path2_bucket_object_with_base_path, update_bucket_metadata,
};
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
static INIT: Once = Once::new();
const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
@@ -134,7 +123,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
// init bucket metadata system
let buckets_list = ecstore
.list_bucket(&rustfs_storage_api::BucketOptions {
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
@@ -200,7 +189,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
.unwrap();
let buckets_list = ecstore
.list_bucket(&rustfs_storage_api::BucketOptions {
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
@@ -1021,7 +1010,7 @@ mod serial_tests {
multipart_bucket.as_str(),
multipart_object,
&upload.upload_id,
vec![rustfs_storage_api::CompletePart {
vec![CompletePart {
part_num: 1,
etag: part.etag.clone(),
..Default::default()
@@ -1166,12 +1155,12 @@ mod serial_tests {
object_name,
&upload.upload_id,
vec![
rustfs_storage_api::CompletePart {
CompletePart {
part_num: 1,
etag: uploaded_part1.etag.clone(),
..Default::default()
},
rustfs_storage_api::CompletePart {
CompletePart {
part_num: 2,
etag: uploaded_part2.etag.clone(),
..Default::default()
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::{
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry},
lifecycle::TransitionOptions,
};
pub(crate) use rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
};
pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys;
pub(crate) use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path;
pub(crate) use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl};
pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk};
pub(crate) use rustfs_ecstore::api::global::get_global_tier_config_mgr;
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
pub(crate) use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType};
pub(crate) use rustfs_ecstore::api::tier::warm_backend::{
WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options,
};
pub(crate) use rustfs_storage_api::{
BucketOperations, BucketOptions, CompletePart, ListOperations, MakeBucketOptions, MultipartOperations, ObjectIO,
ObjectOperations,
};
+28 -8
View File
@@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-storage-api-boundary-batch`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218`.
- Based on: API-218 branch; branch routes root/server/startup, app, and admin storage contract consumers through local `storage_api` boundaries and folds app S3 helper forwarding into the app boundary.
- Branch: `overtrue/arch-external-crate-storage-boundary-sweep`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221`.
- Based on: API-221 branch; branch routes scanner and heal source plus integration test storage symbols through local `storage_api` boundaries.
- PR type for this branch: `consumer-migration`
- Runtime behavior changes: none.
- Rust code changes: route replication pool, outbound TLS generation, runtime
@@ -55,8 +55,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
capacity, workload admission, table catalog, init, protocol clients, and
config tests through a root-local storage_api boundary, plus root/server/startup
`rustfs_storage_api` contract imports through the same boundary, app/admin
`rustfs_storage_api` contract imports through their local boundaries, and app
S3 helper forwarding through `app::storage_api`.
`rustfs_storage_api` contract imports through their local boundaries, app
S3 helper forwarding through `app::storage_api`, and scanner/heal source and
test ECStore plus storage contract imports through crate-local `storage_api`
boundaries.
- CI/script changes: lock completed owner and test/fuzz boundaries against
bare/glob imports, scattered raw ECStore facade subpaths, and startup
runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade
@@ -66,8 +68,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
event-bridge thin module regressions, plus IAM runtime-source bypasses;
accept the reviewed AppContext resolver reverse dependencies in the layer
baseline, and block direct admin AppContext resolver consumers outside the
admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, and keep app S3 helper imports routed through `app::storage_api`.
- Docs changes: record the API-136 through API-221 owner facade and lifecycle
admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, and reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries.
- Docs changes: record the API-136 through API-222 owner facade and lifecycle
runtime-source cleanup.
## Phase 0 Tasks
@@ -5211,14 +5213,32 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
layer guards, diff hygiene, residual app S3 helper scan, Rust risk scan,
fast PR gate, and full PR gate before PR.
- [x] `API-222` Route scanner and heal storage imports through local storage_api boundaries.
- Do: move scanner and heal source/test ECStore and `rustfs_storage_api`
imports into crate-local `storage_api` boundary modules, then route
scanner runtime, scanner lifecycle integration tests, heal runtime, and
heal integration tests through those local boundaries.
- Acceptance: migrated scanner/heal consumers no longer import ECStore or
storage contract symbols directly, and migration rules reject direct
scanner/heal bypasses outside the reviewed boundary modules.
- Must preserve: scanner lifecycle, tier transition, replication heal queue,
bucket scanner IO, heal local disk/indexing, bucket/object heal storage,
endpoint setup, and heal transient-object test behavior.
- Verification: focused scanner/heal compile coverage, formatting,
migration and layer guards, diff hygiene, residual scanner/heal boundary
scan, Rust risk scan, fast PR gate, and full PR gate before PR.
## Next PRs
1. `consumer-migration`: continue larger outer/owner facade batches after API-221.
1. `consumer-migration`: continue larger owner/external crate storage-boundary batches after API-222.
## Pre-Push Review Log
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | pass | API-222 routes scanner and heal ECStore/storage contract imports through crate-local storage_api boundaries. |
| Migration preservation | pass | Scanner lifecycle/tier/replication IO and heal disk/object/bucket test contracts keep the same ECStore and storage-api implementations. |
| Testing/verification | pass | Focused scanner/heal compile coverage, formatting, migration/layer guards, residual boundary scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. |
| Quality/architecture | pass | API-221 folds app S3 helper forwarding into app storage_api and removes the standalone app s3_api forwarding module. |
| Migration preservation | pass | Bucket/object/multipart list and multipart helper parsers/builders keep the same storage S3 API implementations and owner metadata helper. |
| Testing/verification | pass | Focused RustFS compile/tests, formatting, migration/layer guards, residual app S3 helper scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. |
+75 -23
View File
@@ -131,6 +131,10 @@ RUSTFS_APP_SERVER_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_ap
RUSTFS_HEAL_TEST_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_heal_test_local_compat_relative_consumer_hits.txt"
STANDALONE_CRATE_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/standalone_crate_local_compat_relative_consumer_hits.txt"
SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/scanner_bucket_storage_compat_module_hits.txt"
SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE="${TMP_DIR}/scanner_storage_api_source_bypass_hits.txt"
SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE="${TMP_DIR}/scanner_storage_api_test_bypass_hits.txt"
HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE="${TMP_DIR}/heal_storage_api_source_bypass_hits.txt"
HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE="${TMP_DIR}/heal_storage_api_test_bypass_hits.txt"
NOTIFY_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/notify_storage_compat_module_hits.txt"
OBS_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE="${TMP_DIR}/obs_storage_compat_passthrough_hits.txt"
E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE="${TMP_DIR}/e2e_storage_compat_rpc_passthrough_hits.txt"
@@ -757,7 +761,7 @@ fi
--glob '!**/ecstore_test_compat/**' \
--glob '!**/ecstore_fuzz_compat.rs' \
--glob '!target/**' \
| rg -v '^(rustfs/src/(admin/mod|app/mod|storage/mod)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true
| rg -v '^(rustfs/src/(admin/mod|app/mod|storage/mod)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true
) |
cat >"$DIRECT_ECSTORE_IMPORT_HITS_FILE"
@@ -1106,7 +1110,7 @@ fi
--glob '!**/ecstore_compat.rs' \
--glob '!**/ecstore_test_compat.rs' \
--glob '!**/ecstore_test_compat/**' |
rg -v '^(fuzz/fuzz_targets/bucket_validation\.rs|fuzz/fuzz_targets/path_containment\.rs|crates/e2e_test/src/reliant/grpc_lock_client\.rs|crates/e2e_test/src/reliant/node_interact_test\.rs|crates/e2e_test/src/replication_extension_test\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/endpoint_index_test\.rs|crates/heal/tests/heal_bug_fixes_test\.rs|crates/heal/tests/heal_integration_test\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/mod\.rs):' || true
rg -v '^(fuzz/fuzz_targets/bucket_validation\.rs|fuzz/fuzz_targets/path_containment\.rs|crates/e2e_test/src/reliant/grpc_lock_client\.rs|crates/e2e_test/src/reliant/node_interact_test\.rs|crates/e2e_test/src/replication_extension_test\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/mod\.rs):' || true
) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE"
if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then
@@ -1116,17 +1120,17 @@ fi
(
cd "$ROOT_DIR"
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \
crates/heal/src/heal/mod.rs \
crates/heal/tests/endpoint_index_test.rs \
crates/heal/tests/heal_bug_fixes_test.rs \
crates/heal/tests/heal_integration_test.rs \
crates/heal/src/heal/storage_api.rs \
crates/heal/tests/endpoint_index_test/storage_api.rs \
crates/heal/tests/heal_bug_fixes_test/storage_api.rs \
crates/heal/tests/heal_integration_test/storage_api.rs \
crates/iam/src/lib.rs \
crates/notify/src/lib.rs \
crates/obs/src/metrics/mod.rs \
crates/protocols/src/swift/mod.rs \
crates/s3select-api/src/lib.rs \
crates/scanner/src/lib.rs \
crates/scanner/tests/lifecycle_integration_test.rs \
crates/scanner/src/storage_api.rs \
crates/scanner/tests/storage_api/mod.rs \
crates/e2e_test/src/reliant/grpc_lock_client.rs \
crates/e2e_test/src/reliant/node_interact_test.rs \
crates/e2e_test/src/replication_extension_test.rs \
@@ -1144,17 +1148,17 @@ fi
(
cd "$ROOT_DIR"
rg -n --with-filename '^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+\s*;|^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+::\*\s*;' \
crates/heal/src/heal/mod.rs \
crates/heal/tests/endpoint_index_test.rs \
crates/heal/tests/heal_bug_fixes_test.rs \
crates/heal/tests/heal_integration_test.rs \
crates/heal/src/heal/storage_api.rs \
crates/heal/tests/endpoint_index_test/storage_api.rs \
crates/heal/tests/heal_bug_fixes_test/storage_api.rs \
crates/heal/tests/heal_integration_test/storage_api.rs \
crates/iam/src/lib.rs \
crates/notify/src/lib.rs \
crates/obs/src/metrics/mod.rs \
crates/protocols/src/swift/mod.rs \
crates/s3select-api/src/lib.rs \
crates/scanner/src/lib.rs \
crates/scanner/tests/lifecycle_integration_test.rs \
crates/scanner/src/storage_api.rs \
crates/scanner/tests/storage_api/mod.rs \
crates/e2e_test/src/reliant/grpc_lock_client.rs \
crates/e2e_test/src/reliant/node_interact_test.rs \
crates/e2e_test/src/replication_extension_test.rs \
@@ -1172,17 +1176,17 @@ fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' \
crates/heal/src/heal/mod.rs \
crates/heal/tests/endpoint_index_test.rs \
crates/heal/tests/heal_bug_fixes_test.rs \
crates/heal/tests/heal_integration_test.rs \
crates/heal/src/heal/storage_api.rs \
crates/heal/tests/endpoint_index_test/storage_api.rs \
crates/heal/tests/heal_bug_fixes_test/storage_api.rs \
crates/heal/tests/heal_integration_test/storage_api.rs \
crates/iam/src/lib.rs \
crates/notify/src/lib.rs \
crates/obs/src/metrics/mod.rs \
crates/protocols/src/swift/mod.rs \
crates/s3select-api/src/lib.rs \
crates/scanner/src/lib.rs \
crates/scanner/tests/lifecycle_integration_test.rs \
crates/scanner/src/storage_api.rs \
crates/scanner/tests/storage_api/mod.rs \
crates/e2e_test/src/reliant/grpc_lock_client.rs \
crates/e2e_test/src/reliant/node_interact_test.rs \
crates/e2e_test/src/replication_extension_test.rs \
@@ -1306,7 +1310,7 @@ fi
crates/scanner/src \
--glob '*.rs' \
--glob '!**/ecstore_compat.rs' |
rg -v '^(crates/heal/src/heal/mod.rs|crates/iam/src/(lib|runtime_sources).rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/lib.rs):' || true
rg -v '^(crates/heal/src/heal/storage_api.rs|crates/iam/src/(lib|runtime_sources).rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/storage_api.rs):' || true
) >"$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE"
if [[ -s "$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then
@@ -1594,7 +1598,7 @@ fi
crates/scanner/tests \
crates/e2e_test/src \
--glob '*.rs' \
| rg -v '^(crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/scanner/tests/lifecycle_integration_test\.rs):' || true
| rg -v '^(crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs):' || true
) >"$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE"
if [[ -s "$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then
@@ -1618,7 +1622,7 @@ fi
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \
crates/heal/src crates/iam/src crates/notify/src crates/obs/src crates/protocols/src crates/s3select-api/src crates/scanner/src \
--glob '*.rs' |
rg -v '^(crates/heal/src/heal/mod.rs|crates/iam/src/lib.rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/lib.rs):' || true
rg -v '^(crates/heal/src/heal/storage_api.rs|crates/iam/src/lib.rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/storage_api.rs):' || true
) >"$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE"
if [[ -s "$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE" ]]; then
@@ -2020,6 +2024,54 @@ if [[ -s "$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then
report_failure "scanner storage compatibility must expose bucket contracts as explicit aliases: $(paste -sd '; ' "$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \
crates/scanner/src \
--glob '*.rs' |
rg -v '^crates/scanner/src/storage_api\.rs:' || true
) >"$SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE"
if [[ -s "$SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE" ]]; then
report_failure "scanner source files must route ECStore and storage-api symbols through scanner::storage_api: $(paste -sd '; ' "$SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \
crates/scanner/tests \
--glob '*.rs' |
rg -v '^crates/scanner/tests/storage_api/mod\.rs:' || true
) >"$SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE"
if [[ -s "$SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE" ]]; then
report_failure "scanner tests must route ECStore and storage-api symbols through scanner test storage_api: $(paste -sd '; ' "$SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \
crates/heal/src/heal \
--glob '*.rs' |
rg -v '^crates/heal/src/heal/storage_api\.rs:' || true
) >"$HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE"
if [[ -s "$HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE" ]]; then
report_failure "heal source files must route ECStore and storage-api symbols through heal::storage_api: $(paste -sd '; ' "$HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \
crates/heal/tests \
--glob '*.rs' |
rg -v '^crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs:' || true
) >"$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE"
if [[ -s "$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE" ]]; then
report_failure "heal tests must route ECStore and storage-api symbols through heal test storage_api: $(paste -sd '; ' "$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
if [[ -f crates/notify/src/storage_compat.rs ]]; then