gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: remove remaining compatibility bridges (#3738)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-22 16:53:14 +08:00
committed by GitHub
parent a66350b645
commit b63b076275
37 changed files with 324 additions and 691 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/heal/tests/common/mod.rs
-15
View File
@@ -1,15 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod storage_compat;
crates/heal/tests/common/storage_compat.rs
-43
View File
@@ -1,43 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code, unused_imports)]
use std::sync::Arc;
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::disk as ecstore_disk;
use rustfs_ecstore::api::error as ecstore_error;
use rustfs_ecstore::api::layout as ecstore_layout;
use rustfs_ecstore::api::storage as ecstore_storage;
pub(crate) type DiskStore = ecstore_disk::DiskStore;
pub(crate) type ECStore = ecstore_storage::ECStore;
pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint;
pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools;
pub(crate) type Endpoints = ecstore_layout::Endpoints;
pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
#[allow(non_snake_case)]
pub(crate) fn EndpointServerPools(pools: Vec<PoolEndpoints>) -> EndpointServerPools {
ecstore_layout::EndpointServerPools::from(pools)
}
pub(crate) async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await;
}
pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> ecstore_error::Result<()> {
ecstore_storage::init_local_disks(endpoint_pools).await
}
crates/heal/tests/endpoint_index_test.rs
+6 -4
View File
@@ -14,9 +14,11 @@
//! test endpoint index settings
mod common;
use common::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
use rustfs_ecstore::api::{
disk::endpoint::Endpoint,
layout::{EndpointServerPools, Endpoints, PoolEndpoints},
storage::{ECStore, init_local_disks},
};
use std::net::SocketAddr;
use tempfile::TempDir;
use tokio_util::sync::CancellationToken;
@@ -58,7 +60,7 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
// validate all endpoint indexes are in valid range
for (i, ep) in endpoints.iter().enumerate() {
crates/heal/tests/heal_bug_fixes_test.rs
+7 -15
View File
@@ -12,8 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod common;
use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint};
use rustfs_heal::heal::{
event::{HealEvent, Severity},
task::{HealPriority, HealType},
@@ -22,8 +21,6 @@ use rustfs_heal::heal::{
#[test]
fn test_heal_event_to_heal_request_no_panic() {
use common::storage_compat::Endpoint;
// Test that invalid pool/set indices don't cause panic
// Create endpoint using try_from or similar method
let endpoint_result = Endpoint::try_from("http://localhost:9000");
@@ -47,8 +44,6 @@ fn test_heal_event_to_heal_request_no_panic() {
#[test]
fn test_heal_event_to_heal_request_valid_indices() {
use common::storage_compat::Endpoint;
// Test that valid indices work correctly
let endpoint_result = Endpoint::try_from("http://localhost:9000");
if let Ok(mut endpoint) = endpoint_result {
@@ -190,13 +185,10 @@ fn test_heal_task_status_atomic_update() {
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Vec<u8>> {
Ok(vec![])
}
async fn get_disk_status(
&self,
_endpoint: &common::storage_compat::Endpoint,
) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
async fn get_disk_status(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
}
async fn format_disk(&self, _endpoint: &common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> {
Ok(())
}
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
@@ -250,7 +242,7 @@ fn test_heal_task_status_atomic_update() {
) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
Ok((vec![], None, false))
}
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<common::storage_compat::DiskStore> {
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<DiskStore> {
Err(rustfs_heal::Error::other("Not implemented in mock"))
}
}
@@ -320,11 +312,11 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok(Vec::new())
}
async fn get_disk_status(&self, _endpoint: &common::storage_compat::Endpoint) -> rustfs_heal::Result<DiskStatus> {
async fn get_disk_status(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<DiskStatus> {
Ok(DiskStatus::Ok)
}
async fn format_disk(&self, _endpoint: &common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> {
Ok(())
}
@@ -394,7 +386,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok((Vec::new(), None, false))
}
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<common::storage_compat::DiskStore> {
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<DiskStore> {
Err(rustfs_heal::Error::other("not implemented"))
}
}
crates/heal/tests/heal_integration_test.rs
+7 -6
View File
@@ -12,13 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod common;
use common::storage_compat::{
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
};
use http::HeaderMap;
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_ecstore::api::{
bucket::metadata_sys::init_bucket_metadata_sys,
disk::endpoint::Endpoint,
layout::{EndpointServerPools, Endpoints, PoolEndpoints},
storage::{ECStore, init_local_disks},
};
use rustfs_heal::heal::{
manager::{HealConfig, HealManager},
storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
@@ -119,7 +120,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
// format disks (only first time)
init_local_disks(endpoint_pools.clone()).await.unwrap();
crates/scanner/tests/common/mod.rs
-15
View File
@@ -1,15 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod storage_compat;
crates/scanner/tests/common/storage_compat.rs
-124
View File
@@ -1,124 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(dead_code, unused_imports)]
use std::collections::HashMap;
use std::sync::Arc;
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::capacity as ecstore_capacity;
use rustfs_ecstore::api::client as ecstore_client;
use rustfs_ecstore::api::disk as ecstore_disk;
use rustfs_ecstore::api::error as ecstore_error;
use rustfs_ecstore::api::global as ecstore_global;
use rustfs_ecstore::api::layout as ecstore_layout;
use rustfs_ecstore::api::storage as ecstore_storage;
use rustfs_ecstore::api::tier as ecstore_tier;
use time::OffsetDateTime;
pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
pub(crate) const STORAGE_FORMAT_FILE: &str = ecstore_disk::STORAGE_FORMAT_FILE;
pub(crate) type BucketMetadata = ecstore_bucket::metadata::BucketMetadata;
pub(crate) type BucketVersioningSys = ecstore_bucket::versioning_sys::BucketVersioningSys;
pub(crate) type DiskOption = ecstore_disk::DiskOption;
pub(crate) type ECStore = ecstore_storage::ECStore;
pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint;
pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools;
pub(crate) type Endpoints = ecstore_layout::Endpoints;
pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
pub(crate) type ReadCloser = ecstore_client::transition_api::ReadCloser;
pub(crate) type ReaderImpl = ecstore_client::transition_api::ReaderImpl;
pub(crate) type TierConfig = ecstore_tier::tier_config::TierConfig;
pub(crate) type TierConfigMgr = ecstore_tier::tier::TierConfigMgr;
pub(crate) type TierMinIO = ecstore_tier::tier_config::TierMinIO;
pub(crate) type TierType = ecstore_tier::tier_config::TierType;
pub(crate) type TransitionOptions = ecstore_bucket::lifecycle::lifecycle::TransitionOptions;
pub(crate) use ecstore_tier::warm_backend::WarmBackend as ScannerWarmBackend;
pub(crate) type WarmBackendGetOpts = ecstore_tier::warm_backend::WarmBackendGetOpts;
pub(crate) trait ScannerTestDiskExt {
async fn read_metadata(&self, volume: &str, path: &str) -> ecstore_disk::error::Result<ecstore_disk::Bytes>;
}
impl<T> ScannerTestDiskExt for T
where
T: ecstore_disk::DiskAPI,
{
async fn read_metadata(&self, volume: &str, path: &str) -> ecstore_disk::error::Result<ecstore_disk::Bytes> {
ecstore_disk::DiskAPI::read_metadata(self, volume, path).await
}
}
#[allow(non_snake_case)]
pub(crate) fn EndpointServerPools(pools: Vec<PoolEndpoints>) -> EndpointServerPools {
ecstore_layout::EndpointServerPools::from(pools)
}
pub(crate) struct GlobalTierConfigMgrCompat;
#[allow(non_upper_case_globals)]
pub(crate) static GLOBAL_TierConfigMgr: GlobalTierConfigMgrCompat = GlobalTierConfigMgrCompat;
impl std::ops::Deref for GlobalTierConfigMgrCompat {
type Target = Arc<tokio::sync::RwLock<TierConfigMgr>>;
fn deref(&self) -> &Self::Target {
&ecstore_global::GLOBAL_TierConfigMgr
}
}
pub(crate) fn build_transition_put_options(
storage_class: String,
metadata: HashMap<String, String>,
) -> ecstore_client::api_put_object::PutObjectOptions {
ecstore_tier::warm_backend::build_transition_put_options(storage_class, metadata)
}
pub(crate) async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> ecstore_error::Result<()> {
ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(api, bucket).await
}
pub(crate) async fn init_background_expiry(api: Arc<ECStore>) {
ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(api).await;
}
pub(crate) async fn get_bucket_metadata(bucket: &str) -> ecstore_error::Result<Arc<BucketMetadata>> {
ecstore_bucket::metadata_sys::get(bucket).await
}
pub(crate) async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await;
}
pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> ecstore_error::Result<()> {
ecstore_storage::init_local_disks(endpoint_pools).await
}
pub(crate) async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> ecstore_disk::error::Result<ecstore_disk::DiskStore> {
ecstore_disk::new_disk(ep, opt).await
}
pub(crate) fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) {
ecstore_capacity::path2_bucket_object_with_base_path(base_path, path)
}
pub(crate) async fn update_bucket_metadata(
bucket: &str,
config_file: &str,
data: Vec<u8>,
) -> ecstore_error::Result<OffsetDateTime> {
ecstore_bucket::metadata_sys::update(bucket, config_file, data).await
}
crates/scanner/tests/lifecycle_integration_test.rs
+35 -23
View File
@@ -12,17 +12,29 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod common;
use common::storage_compat::{
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerTestDiskExt as _,
ScannerWarmBackend, TierConfig, TierMinIO, TierType, TransitionOptions, WarmBackendGetOpts, build_transition_put_options,
enqueue_transition_for_existing_objects, get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys,
init_local_disks, new_disk, path2_bucket_object_with_base_path, update_bucket_metadata,
};
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
use rustfs_ecstore::api::{
bucket::{
lifecycle::{
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry},
lifecycle::TransitionOptions,
},
metadata::BUCKET_LIFECYCLE_CONFIG,
metadata_sys::{self, init_bucket_metadata_sys},
versioning_sys::BucketVersioningSys,
},
capacity::path2_bucket_object_with_base_path,
client::transition_api::{ReadCloser, ReaderImpl},
disk::{DiskAPI as _, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk},
global::GLOBAL_TierConfigMgr,
layout::{EndpointServerPools, Endpoints, PoolEndpoints},
storage::{ECStore, init_local_disks},
tier::{
tier_config::{TierConfig, TierMinIO, TierType},
warm_backend::{WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options},
},
};
use rustfs_filemeta::FileMeta;
use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk;
@@ -110,7 +122,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
// format disks (only first time)
init_local_disks(endpoint_pools.clone()).await.unwrap();
@@ -181,7 +193,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]);
init_local_disks(endpoint_pools.clone()).await.unwrap();
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
@@ -269,7 +281,7 @@ async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box<dyn std::erro
</Rule>
</LifecycleConfiguration>"#;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
Ok(())
}
@@ -293,7 +305,7 @@ async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box<
</Rule>
</LifecycleConfiguration>"#;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
Ok(())
}
@@ -316,7 +328,7 @@ async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64)
</LifecycleConfiguration>"#
);
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
Ok(())
}
@@ -360,7 +372,7 @@ async fn set_bucket_lifecycle_transition_with_tier(
</LifecycleConfiguration>"#
);
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
Ok(())
}
@@ -595,7 +607,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
.await
.expect("failed to stat object metadata")
.file_type();
let lifecycle = get_bucket_metadata(bucket)
let lifecycle = metadata_sys::get(bucket)
.await
.expect("failed to load bucket metadata")
.lifecycle_config
@@ -860,7 +872,7 @@ mod serial_tests {
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
// Verify lifecycle configuration was set
match get_bucket_metadata(bucket_name.as_str()).await {
match metadata_sys::get(bucket_name.as_str()).await {
Ok(bucket_meta) => {
assert!(bucket_meta.lifecycle_config.is_some());
println!("✅ Bucket metadata retrieved successfully");
@@ -1417,7 +1429,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
@@ -1502,7 +1514,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
@@ -1729,7 +1741,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
@@ -1794,7 +1806,7 @@ mod serial_tests {
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>"#;
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await
.expect("Failed to set noncurrent lifecycle configuration");
@@ -1832,7 +1844,7 @@ mod serial_tests {
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>"#;
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await
.expect("Failed to set noncurrent lifecycle configuration");
@@ -1919,7 +1931,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
docs/architecture/migration-progress.md
+87 -11
View File
@@ -5,16 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-runtime-local-compat-bridges`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116`.
- Based on: API-116 slice.
- Branch: `overtrue/arch-test-fuzz-compat-bridge-cleanup`
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123`.
- Based on: API-123 slice.
- PR type for this branch: `pure-move`
- Runtime behavior changes: none.
- Rust code changes: remove app/admin, storage core, nested, handler, capacity,
server, and S3 API secondary compatibility bridges.
- CI/script changes: guard against reintroducing the removed secondary bridge
modules or their consumer paths.
- Docs changes: record the API-117 through API-121 compatibility bridge cleanup.
- Rust code changes: remove heal/scanner test and fuzz storage compatibility
bridges, then route their consumers directly to ECStore API owner modules.
- CI/script changes: allow the migrated test/fuzz targets to import owner APIs
directly and reject reintroduced test/fuzz bridge modules.
- Docs changes: record the API-124 test/fuzz bridge cleanup.
## Phase 0 Tasks
@@ -709,6 +709,48 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
- Verification: RustFS compile coverage, runtime local bridge residual scan,
migration and layer guards, formatting, diff hygiene, path-only risk
review, and three-expert review.
- [x] `API-122` Remove root one-off compatibility bridges.
- Completed slice: replace config test, error mapping, runtime capability,
table catalog, and workload admission consumers with direct ECStore API
imports, then delete the root one-off bridge modules.
- Acceptance: the deleted bridge files and module declarations are gone;
migration rules reject reintroduced files, declarations, or bridge
references.
- Must preserve: config disk-layout tests, API error mapping, runtime
topology snapshots, table-catalog paths and lock behavior, and workload
admission snapshots.
- Verification: RustFS compile coverage, root one-off bridge residual scan,
migration and layer guards, formatting, diff hygiene, Rust risk scan, and
three-expert review.
- [x] `API-123` Remove startup storage compatibility bridge.
- Completed slice: replace startup storage, notification, bucket metadata,
service, shutdown, server, lifecycle, IAM, background, fs guard, and init
consumers with direct ECStore API owner imports, then delete
`startup_storage_compat.rs`.
- Acceptance: startup/init consumers no longer route through the startup
compatibility bridge; migration rules reject the deleted file, module
declaration, or bridge references.
- Must preserve: endpoint parsing, unsupported filesystem policy, ECStore
initialization, global endpoint/erasure registration, local disk and lock
client initialization, config migration/retry behavior, metadata/IAM
migration, notification startup, background replication, scanner/heal
startup and shutdown, and readiness marking.
- Verification: RustFS compile coverage, startup bridge residual scan,
migration and layer guards, formatting, diff hygiene, Rust risk scan, and
three-expert review.
- [x] `API-124` Remove test and fuzz storage compatibility bridges.
- Completed slice: replace heal tests, scanner lifecycle tests, and bucket/path
fuzz targets with direct ECStore API owner imports, then delete their local
`storage_compat.rs` modules.
- Acceptance: migrated test/fuzz targets no longer route through local
storage compatibility bridges; migration rules reject deleted files, module
declarations, or bridge references.
- Must preserve: heal endpoint indexing, heal mock storage signatures,
lifecycle metadata updates, scanner warm-tier mocks, fuzz target validation
invariants, and direct compile coverage for affected crates/targets.
- Verification: heal/scanner test compile coverage, fuzz target compile
coverage, test/fuzz bridge residual scan, migration and layer guards,
formatting, diff hygiene, Rust risk scan, and three-expert review.
- [x] `G-012` Inventory placement and repair invariants.
- Acceptance:
[`placement-repair-invariants.md`](placement-repair-invariants.md) records
@@ -3742,14 +3784,48 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | pass | API-121 removes capacity, server, and S3 API local bridge modules while keeping direct owner API calls explicit. |
| Migration preservation | pass | The extended guard rejects deleted runtime bridge files, module declarations, and local bridge consumers. |
| Testing/verification | pass | RustFS compile, runtime local bridge residual scan, migration guard, layer guard, formatting, diff hygiene, pre-commit, and path-only risk review passed. |
| Quality/architecture | pass | API-124 removes only test/fuzz bridge modules and keeps call sites on explicit ECStore owner APIs. |
| Migration preservation | pass | The migration guard now rejects the deleted test/fuzz bridge files, module declarations, and bridge consumers. |
| Testing/verification | pass | Heal/scanner test compile, fuzz target compile, residual scan, migration guard, layer guard, formatting, and diff hygiene passed. |
## Verification Notes
Passed before push:
- Issue #660 API-124 current slice:
- `cargo check --tests -p rustfs-heal -p rustfs-scanner`: passed.
- `cargo check --manifest-path fuzz/Cargo.toml --bins`: passed; transient `fuzz/Cargo.lock` refresh was restored to avoid dependency churn.
- `cargo fmt --all`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- Test/fuzz compatibility bridge residual scan: passed.
- Rust risk scan: reviewed pre-existing test-only unwrap/expect/panic/unsafe usage; no new production risk.
- Issue #660 API-123 current slice:
- `cargo check -p rustfs --tests`: passed.
- `cargo fmt --all`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- Startup compatibility bridge residual scan: passed.
- Rust risk scan: passed.
- Issue #660 API-122 current slice:
- `cargo check -p rustfs --tests`: passed.
- `cargo fmt --all`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- Root one-off compatibility bridge residual scan: passed.
- Rust risk scan: passed.
- Issue #660 API-121 current slice:
- `cargo check -p rustfs --tests`: passed.
- `cargo fmt --all`: passed.
fuzz/fuzz_targets/bucket_validation.rs
+1 -4
View File
@@ -1,10 +1,7 @@
#![no_main]
#[path = "bucket_validation/storage_compat.rs"]
mod storage_compat;
use libfuzzer_sys::fuzz_target;
use self::storage_compat::{
use rustfs_ecstore::api::bucket::utils::{
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
};
fuzz/fuzz_targets/bucket_validation/storage_compat.rs
-36
View File
@@ -1,36 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::error as ecstore_error;
pub(crate) fn check_bucket_and_object_names(bucket: &str, object: &str) -> ecstore_error::Result<()> {
ecstore_bucket::utils::check_bucket_and_object_names(bucket, object)
}
pub(crate) fn check_list_objs_args(
bucket: &str,
prefix: &str,
marker: &Option<String>,
) -> ecstore_error::Result<()> {
ecstore_bucket::utils::check_list_objs_args(bucket, prefix, marker)
}
pub(crate) fn check_valid_bucket_name_strict(bucket_name: &str) -> ecstore_error::Result<()> {
ecstore_bucket::utils::check_valid_bucket_name_strict(bucket_name)
}
pub(crate) fn is_meta_bucketname(name: &str) -> bool {
ecstore_bucket::utils::is_meta_bucketname(name)
}
fuzz/fuzz_targets/path_containment.rs
+3 -4
View File
@@ -1,10 +1,9 @@
#![no_main]
#[path = "path_containment/storage_compat.rs"]
mod storage_compat;
use libfuzzer_sys::fuzz_target;
use self::storage_compat::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
use rustfs_ecstore::api::bucket::utils::{
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
};
use rustfs_utils::path::{clean, path_join};
use std::path::{Path, PathBuf};
fuzz/fuzz_targets/path_containment/storage_compat.rs
-31
View File
@@ -1,31 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::error as ecstore_error;
pub(crate) fn check_object_name_for_length_and_slash(
bucket: &str,
object: &str,
) -> ecstore_error::Result<()> {
ecstore_bucket::utils::check_object_name_for_length_and_slash(bucket, object)
}
pub(crate) fn has_bad_path_component(path: &str) -> bool {
ecstore_bucket::utils::has_bad_path_component(path)
}
pub(crate) fn is_valid_object_prefix(object: &str) -> bool {
ecstore_bucket::utils::is_valid_object_prefix(object)
}
rustfs/src/config/config_test.rs
+2 -2
View File
@@ -15,10 +15,10 @@
#[cfg(test)]
#[allow(unsafe_op_in_unsafe_fn)]
mod tests {
use super::super::super::config_storage_compat::DisksLayout;
use crate::config::{CommandResult, Config, Opt, TlsCommands};
use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
use rustfs_ecstore::api::layout::DisksLayout;
use serial_test::serial;
use std::env;
@@ -262,7 +262,7 @@ mod tests {
#[test]
#[serial]
fn test_volumes_and_disk_layout_parsing() {
use super::super::super::config_storage_compat::DisksLayout;
use rustfs_ecstore::api::layout::DisksLayout;
// Test case 1: Single volume path
let args = vec!["rustfs", "/data/vol1"];
rustfs/src/config_storage_compat.rs
-19
View File
@@ -1,19 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
use rustfs_ecstore::api::layout as ecstore_layout;
#[cfg(test)]
pub(crate) type DisksLayout = ecstore_layout::DisksLayout;
rustfs/src/error.rs
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::error_storage_compat::{QuotaError, StorageError};
use rustfs_ecstore::api::{bucket::quota::QuotaError, error::StorageError};
use rustfs_storage_api::HTTPRangeError;
use s3s::{S3Error, S3ErrorCode};
rustfs/src/error_storage_compat.rs
-19
View File
@@ -1,19 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::error as ecstore_error;
pub(crate) type QuotaError = ecstore_bucket::quota::QuotaError;
pub(crate) type StorageError = ecstore_error::StorageError;
rustfs/src/init.rs
+16 -14
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::startup_storage_compat::{get_global_region, get_notification_config};
use crate::server::ShutdownHandle;
use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations};
use crate::{admin, config, version};
@@ -20,6 +19,7 @@ use rustfs_config::{
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
ENV_RUSTFS_BUFFER_DEFAULT_SIZE, ENV_RUSTFS_BUFFER_MAX_SIZE, ENV_RUSTFS_BUFFER_MIN_SIZE, ENV_UPDATE_CHECK, RUSTFS_REGION,
};
use rustfs_ecstore::api::{bucket::metadata_sys as ecstore_metadata_sys, global as ecstore_global};
use rustfs_notify::notifier_global;
use rustfs_targets::arn::{ARN, TargetIDError};
use rustfs_utils::get_env_usize;
@@ -157,7 +157,7 @@ fn arn_to_target_id(arn_str: &str) -> Result<rustfs_targets::arn::TargetID, Targ
/// * `buckets` - A vector of bucket names to process
#[instrument(skip_all)]
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
let global_region = get_global_region();
let global_region = ecstore_global::get_global_region();
let region = global_region
.as_ref()
.filter(|r| !r.as_str().is_empty())
@@ -174,18 +174,20 @@ pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
RUSTFS_REGION
});
for bucket in buckets.iter() {
let has_notification_config = get_notification_config(bucket).await.unwrap_or_else(|err| {
warn!(
target: "rustfs::init",
event = "notification_config_load_failed",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
error = ?err,
"Failed to load bucket notification configuration"
);
None
});
let has_notification_config = ecstore_metadata_sys::get_notification_config(bucket)
.await
.unwrap_or_else(|err| {
warn!(
target: "rustfs::init",
event = "notification_config_load_failed",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
error = ?err,
"Failed to load bucket notification configuration"
);
None
});
match has_notification_config {
Some(cfg) => {
rustfs/src/lib.rs
-6
View File
@@ -57,11 +57,9 @@ pub mod auth;
pub mod auth_keystone;
pub mod capacity;
pub mod config;
pub(crate) mod config_storage_compat;
pub mod delete_tail_activity;
pub mod embedded;
pub mod error;
pub(crate) mod error_storage_compat;
pub mod init;
pub mod license;
pub mod memory_observability;
@@ -69,7 +67,6 @@ pub mod profiling;
#[cfg(any(feature = "ftps", feature = "webdav", feature = "sftp"))]
pub mod protocols;
pub mod runtime_capabilities;
pub(crate) mod runtime_capabilities_storage_compat;
pub mod server;
pub(crate) mod startup_audit;
pub(crate) mod startup_auth;
@@ -93,16 +90,13 @@ pub(crate) mod startup_server;
pub(crate) mod startup_services;
pub(crate) mod startup_shutdown;
pub(crate) mod startup_storage;
pub(crate) mod startup_storage_compat;
pub(crate) mod startup_tls_material;
pub mod storage;
pub(crate) mod table_catalog;
pub(crate) mod table_catalog_storage_compat;
pub mod tls;
pub mod update;
pub mod version;
pub mod workload_admission;
pub(crate) mod workload_admission_storage_compat;
// Re-export from rustfs_utils so that config sub-modules can use
// `crate::apply_external_env_compat` without breaking.
rustfs/src/runtime_capabilities.rs
+6 -4
View File
@@ -12,14 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ecstore::api::{cluster as ecstore_cluster, layout::EndpointServerPools};
use rustfs_storage_api::{
CapabilitySnapshotError, CapabilityStatus, DiskCapabilities, MemorySamplingState, ObservabilitySnapshot,
ObservabilitySnapshotProvider, PlatformSupport, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider,
UserspaceProfilingCapability,
};
use super::runtime_capabilities_storage_compat::{EndpointServerPools, topology_snapshot_from_endpoint_pools_with_capabilities};
const NOT_WIRED_INTO_RUNTIME: &str = "not wired into runtime";
const STORAGE_MEDIA_NOT_REPORTED: &str = "storage media not reported by endpoints";
const FAILURE_DOMAIN_NOT_REPORTED: &str = "failure domain labels not reported by endpoints";
@@ -80,7 +79,7 @@ impl TopologySnapshotProvider for EndpointTopologySnapshotProvider {
}
pub fn topology_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> TopologySnapshot {
topology_snapshot_from_endpoint_pools_with_capabilities(
ecstore_cluster::topology_snapshot_from_endpoint_pools_with_capabilities(
endpoint_pools,
TopologyCapabilities {
profiling: cpu_profiling_status(),
@@ -131,8 +130,11 @@ fn cgroup_memory_status() -> CapabilityStatus {
#[cfg(test)]
mod tests {
use super::super::runtime_capabilities_storage_compat::{Endpoint, Endpoints, PoolEndpoints};
use super::*;
use rustfs_ecstore::api::{
disk::endpoint::Endpoint,
layout::{Endpoints, PoolEndpoints},
};
use rustfs_storage_api::{CapabilityState, ObservabilitySnapshotProvider, TopologySnapshotProvider};
#[tokio::test]
rustfs/src/runtime_capabilities_storage_compat.rs
-34
View File
@@ -1,34 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ecstore::api::cluster as ecstore_cluster;
#[cfg(test)]
use rustfs_ecstore::api::disk as ecstore_disk;
use rustfs_ecstore::api::layout as ecstore_layout;
#[cfg(test)]
pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint;
pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools;
#[cfg(test)]
pub(crate) type Endpoints = ecstore_layout::Endpoints;
#[cfg(test)]
pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
pub(crate) fn topology_snapshot_from_endpoint_pools_with_capabilities(
endpoint_pools: &EndpointServerPools,
capabilities: rustfs_storage_api::TopologyCapabilities,
disk_capabilities: rustfs_storage_api::DiskCapabilities,
) -> rustfs_storage_api::TopologySnapshot {
ecstore_cluster::topology_snapshot_from_endpoint_pools_with_capabilities(endpoint_pools, capabilities, disk_capabilities)
}
rustfs/src/startup_background.rs
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::startup_storage_compat::ECStore;
use rustfs_ecstore::api::storage::ECStore;
use rustfs_heal::{create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager};
use rustfs_utils::get_env_bool_with_aliases;
use std::{io::Result, sync::Arc};
rustfs/src/startup_bucket_metadata.rs
+10 -9
View File
@@ -12,8 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::startup_storage_compat::{
ECStore, get_global_replication_pool, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config,
use rustfs_ecstore::api::{
bucket::{metadata_sys as ecstore_metadata_sys, migration as ecstore_migration, replication as ecstore_replication},
storage::ECStore,
};
use rustfs_storage_api::{BucketOperations, BucketOptions};
use std::{
@@ -33,9 +34,9 @@ pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>) -
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
try_migrate_bucket_metadata(store.clone()).await;
init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
try_migrate_iam_config(store).await;
ecstore_migration::try_migrate_bucket_metadata(store.clone()).await;
ecstore_metadata_sys::init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
ecstore_migration::try_migrate_iam_config(store).await;
Ok(buckets)
}
@@ -51,14 +52,14 @@ pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: Cance
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
try_migrate_bucket_metadata(store.clone()).await;
ecstore_migration::try_migrate_bucket_metadata(store.clone()).await;
if let Some(pool) = get_global_replication_pool() {
if let Some(pool) = ecstore_replication::get_global_replication_pool() {
pool.init_resync(ctx, buckets.clone()).await?;
}
try_migrate_iam_config(store.clone()).await;
init_bucket_metadata_sys(store, buckets.clone()).await;
ecstore_migration::try_migrate_iam_config(store.clone()).await;
ecstore_metadata_sys::init_bucket_metadata_sys(store, buckets.clone()).await;
Ok(buckets)
}
rustfs/src/startup_fs_guard.rs
+1 -1
View File
@@ -12,11 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::startup_storage_compat::EndpointServerPools;
use rustfs_config::{
DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
};
use rustfs_ecstore::api::layout::EndpointServerPools;
use std::collections::BTreeSet;
use std::io::{Error, Result};
use tracing::warn;
rustfs/src/startup_iam.rs
+1 -1
View File
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::startup_storage_compat::ECStore;
use crate::app::context::AppContext;
use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready};
use rustfs_common::{GlobalReadiness, SystemStage};
use rustfs_ecstore::api::storage::ECStore;
use rustfs_iam::init_iam_sys;
use rustfs_kms::KmsServiceManager;
use std::future::Future;
rustfs/src/startup_lifecycle.rs
+1 -1
View File
@@ -17,9 +17,9 @@ use crate::{
startup_iam::{IamBootstrapDisposition, publish_ready_for_iam_bootstrap},
startup_services::StartupServiceRuntime,
startup_shutdown::run_startup_shutdown_sequence,
startup_storage_compat::ECStore,
};
use rustfs_common::GlobalReadiness;
use rustfs_ecstore::api::storage::ECStore;
use rustfs_scanner::init_data_scanner;
use std::{
io::Result,
rustfs/src/startup_notification.rs
+7 -7
View File
@@ -12,8 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::startup_storage_compat::{EcstoreResult, EndpointServerPools, new_global_notification_sys};
use crate::init::add_bucket_notification_configuration;
use rustfs_ecstore::api::{error as ecstore_error, layout::EndpointServerPools, notification as ecstore_notification};
use std::{
future::Future,
io::{Error, Result},
@@ -50,14 +50,14 @@ pub(crate) async fn init_notification_runtime(endpoint_pools: EndpointServerPool
})
}
pub(crate) async fn init_notification_system(endpoint_pools: EndpointServerPools) -> EcstoreResult<()> {
init_notification_system_with(|| new_global_notification_sys(endpoint_pools)).await
pub(crate) async fn init_notification_system(endpoint_pools: EndpointServerPools) -> ecstore_error::Result<()> {
init_notification_system_with(|| ecstore_notification::new_global_notification_sys(endpoint_pools)).await
}
async fn init_notification_system_with<InitFn, InitFuture>(init_notification: InitFn) -> EcstoreResult<()>
async fn init_notification_system_with<InitFn, InitFuture>(init_notification: InitFn) -> ecstore_error::Result<()>
where
InitFn: FnOnce() -> InitFuture,
InitFuture: Future<Output = EcstoreResult<()>>,
InitFuture: Future<Output = ecstore_error::Result<()>>,
{
init_notification().await
}
@@ -76,11 +76,11 @@ fn log_embedded_optional_service_skipped(service: &str, err: impl std::fmt::Disp
#[cfg(test)]
mod tests {
use super::init_notification_system_with;
use rustfs_ecstore::api::error::Error as EcstoreError;
#[tokio::test]
async fn notification_system_returns_source_error() {
let result =
init_notification_system_with(|| async { Err(super::super::startup_storage_compat::EcstoreError::FaultyDisk) }).await;
let result = init_notification_system_with(|| async { Err(EcstoreError::FaultyDisk) }).await;
assert!(result.is_err());
}
rustfs/src/startup_server.rs
+1 -1
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::startup_storage_compat::{set_global_region, set_global_rustfs_port};
use crate::{
capacity::capacity_integration::init_capacity_management,
config::Config,
@@ -20,6 +19,7 @@ use crate::{
};
use rustfs_common::{GlobalReadiness, set_global_addr};
use rustfs_credentials::init_global_action_credentials;
use rustfs_ecstore::api::global::{set_global_region, set_global_rustfs_port};
use rustfs_utils::net::parse_and_resolve_address;
use std::{
io::{Error, Result},
rustfs/src/startup_services.rs
+1 -1
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::startup_storage_compat::{ECStore, EndpointServerPools};
use crate::{
config::Config,
init::{init_buffer_profile_system, init_kms_system},
@@ -29,6 +28,7 @@ use crate::{
startup_optional_runtime_sidecars::{OptionalRuntimeServices, init_optional_runtime_services},
};
use rustfs_common::GlobalReadiness;
use rustfs_ecstore::api::{layout::EndpointServerPools, storage::ECStore};
use std::{io::Result, sync::Arc};
use tokio_util::sync::CancellationToken;
rustfs/src/startup_shutdown.rs
+1 -1
View File
@@ -17,8 +17,8 @@ use crate::{
startup_optional_runtime_sidecars::{
OptionalRuntimeServices, prepare_optional_runtime_shutdowns, shutdown_optional_runtime_services,
},
startup_storage_compat::shutdown_background_services,
};
use rustfs_ecstore::api::global::shutdown_background_services;
use rustfs_heal::shutdown_ahm_services;
use rustfs_utils::get_env_bool_with_aliases;
use std::path::Path;
rustfs/src/startup_storage.rs
+22 -21
View File
@@ -12,12 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::startup_storage_compat::{
ECStore, EndpointServerPools, init_background_replication, init_ecstore_config, init_global_config_sys, init_local_disks,
init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config, update_erasure_type,
};
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
use rustfs_common::{GlobalReadiness, SystemStage};
use rustfs_ecstore::api::storage::ECStore;
use rustfs_ecstore::api::{
bucket::replication as ecstore_replication, config as ecstore_config, global as ecstore_global, layout::EndpointServerPools,
storage as ecstore_storage,
};
use std::{
io::{Error, Result},
net::SocketAddr,
@@ -70,8 +71,8 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
.map_err(Error::other)?;
enforce_unsupported_fs_policy(&endpoint_pools)?;
set_global_endpoints(endpoint_pools.as_ref().clone());
update_erasure_type(setup_type).await;
ecstore_global::set_global_endpoints(endpoint_pools.as_ref().clone());
ecstore_global::update_erasure_type(setup_type).await;
debug!(
target: "rustfs::main::run",
@@ -82,7 +83,7 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
state = "starting",
"starting local disk initialization"
);
init_local_disks(endpoint_pools.clone())
ecstore_storage::init_local_disks(endpoint_pools.clone())
.await
.inspect_err(|err| {
error!(
@@ -97,8 +98,8 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
);
})
.map_err(Error::other)?;
prewarm_local_disk_id_map().await;
init_lock_clients(endpoint_pools.clone());
ecstore_storage::prewarm_local_disk_id_map().await;
ecstore_storage::init_lock_clients(endpoint_pools.clone());
log_storage_pool_layout(&endpoint_pools);
@@ -114,13 +115,13 @@ pub(crate) async fn init_embedded_startup_storage_foundation(
.map_err(|err| Error::other(format!("endpoints: {err}")))?;
enforce_unsupported_fs_policy(&endpoint_pools).map_err(|err| Error::other(format!("unsupported fs guard: {err}")))?;
set_global_endpoints(endpoint_pools.as_ref().clone());
update_erasure_type(setup_type).await;
ecstore_global::set_global_endpoints(endpoint_pools.as_ref().clone());
ecstore_global::update_erasure_type(setup_type).await;
init_local_disks(endpoint_pools.clone())
ecstore_storage::init_local_disks(endpoint_pools.clone())
.await
.map_err(|err| Error::other(format!("local disks: {err}")))?;
init_lock_clients(endpoint_pools.clone());
ecstore_storage::init_lock_clients(endpoint_pools.clone());
Ok(endpoint_pools)
}
@@ -158,7 +159,7 @@ pub(crate) async fn init_startup_storage_runtime(
init_startup_storage_global_config(store.clone()).await?;
readiness.mark_stage(SystemStage::StorageReady);
init_background_replication(store.clone()).await;
ecstore_replication::init_background_replication(store.clone()).await;
Ok(StartupStorageRuntime {
store,
@@ -189,17 +190,17 @@ pub(crate) async fn init_embedded_startup_storage_runtime(
init_embedded_startup_storage_global_config(store.clone()).await?;
readiness.mark_stage(SystemStage::StorageReady);
init_background_replication(store.clone()).await;
ecstore_replication::init_background_replication(store.clone()).await;
Ok(StartupStorageRuntime { store, shutdown_token })
}
async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
init_ecstore_config();
try_migrate_server_config(store.clone()).await;
ecstore_config::init();
ecstore_config::try_migrate_server_config(store.clone()).await;
let mut retry_count = 0;
while let Err(e) = init_global_config_sys(store.clone()).await {
while let Err(e) = ecstore_config::init_global_config_sys(store.clone()).await {
let next_retry_count = retry_count + 1;
error!(
target: "rustfs::main::run",
@@ -224,11 +225,11 @@ async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
}
async fn init_embedded_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
init_ecstore_config();
try_migrate_server_config(store.clone()).await;
ecstore_config::init();
ecstore_config::try_migrate_server_config(store.clone()).await;
let mut retry = 0;
while let Err(err) = init_global_config_sys(store.clone()).await {
while let Err(err) = ecstore_config::init_global_config_sys(store.clone()).await {
retry += 1;
if retry > GLOBAL_CONFIG_INIT_MAX_RETRIES {
return Err(Error::other(format!(
rustfs/src/startup_storage_compat.rs
-105
View File
@@ -1,105 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::config as ecstore_config;
use rustfs_ecstore::api::error as ecstore_error;
use rustfs_ecstore::api::global as ecstore_global;
use rustfs_ecstore::api::layout as ecstore_layout;
use rustfs_ecstore::api::notification as ecstore_notification;
use rustfs_ecstore::api::storage as ecstore_storage;
pub(crate) type ECStore = ecstore_storage::ECStore;
#[cfg(test)]
pub(crate) type EcstoreError = ecstore_error::Error;
pub(crate) type EcstoreResult<T> = ecstore_error::Result<T>;
pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools;
pub(crate) async fn get_notification_config(bucket: &str) -> ecstore_error::Result<Option<s3s::dto::NotificationConfiguration>> {
ecstore_bucket::metadata_sys::get_notification_config(bucket).await
}
pub(crate) async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await;
}
pub(crate) async fn try_migrate_bucket_metadata(store: Arc<ECStore>) {
ecstore_bucket::migration::try_migrate_bucket_metadata(store).await;
}
pub(crate) async fn try_migrate_iam_config(store: Arc<ECStore>) {
ecstore_bucket::migration::try_migrate_iam_config(store).await;
}
pub(crate) fn get_global_replication_pool() -> Option<Arc<ecstore_bucket::replication::DynReplicationPool>> {
ecstore_bucket::replication::get_global_replication_pool()
}
pub(crate) async fn init_background_replication(storage: Arc<ECStore>) {
ecstore_bucket::replication::init_background_replication(storage).await;
}
pub(crate) fn init_ecstore_config() {
ecstore_config::init();
}
pub(crate) async fn init_global_config_sys(api: Arc<ECStore>) -> EcstoreResult<()> {
ecstore_config::init_global_config_sys(api).await
}
pub(crate) async fn try_migrate_server_config(api: Arc<ECStore>) {
ecstore_config::try_migrate_server_config(api).await;
}
pub(crate) fn get_global_region() -> Option<s3s::region::Region> {
ecstore_global::get_global_region()
}
pub(crate) fn set_global_endpoints(eps: Vec<ecstore_layout::PoolEndpoints>) {
ecstore_global::set_global_endpoints(eps);
}
pub(crate) fn set_global_region(region: s3s::region::Region) {
ecstore_global::set_global_region(region);
}
pub(crate) fn set_global_rustfs_port(value: u16) {
ecstore_global::set_global_rustfs_port(value);
}
pub(crate) fn shutdown_background_services() {
ecstore_global::shutdown_background_services();
}
pub(crate) async fn update_erasure_type(setup_type: ecstore_layout::SetupType) {
ecstore_global::update_erasure_type(setup_type).await
}
pub(crate) async fn new_global_notification_sys(eps: EndpointServerPools) -> EcstoreResult<()> {
ecstore_notification::new_global_notification_sys(eps).await
}
pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> EcstoreResult<()> {
ecstore_storage::init_local_disks(endpoint_pools).await
}
pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) {
ecstore_storage::init_lock_clients(endpoint_pools);
}
pub(crate) async fn prewarm_local_disk_id_map() {
ecstore_storage::prewarm_local_disk_id_map().await;
}
rustfs/src/table_catalog.rs
+31 -24
View File
@@ -27,11 +27,6 @@ use std::{
time::{Duration as StdDuration, Instant},
};
use super::table_catalog_storage_compat::{
BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
BUCKET_TABLE_RESERVED_PREFIX, EcstoreError, RUSTFS_META_BUCKET, StorageError, get_bucket_metadata, get_lock_acquire_timeout,
table_catalog_path_hash,
};
use bytes::Bytes;
use datafusion::{
arrow::datatypes::SchemaRef,
@@ -39,6 +34,10 @@ use datafusion::{
};
use http::HeaderMap;
use metrics::{counter, histogram};
use rustfs_ecstore::api::{
bucket::{metadata as ecstore_metadata, metadata_sys as ecstore_metadata_sys},
disk as ecstore_disk, error as ecstore_error, set_disk as ecstore_set_disk,
};
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{
HTTPPreconditions, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
@@ -56,7 +55,8 @@ use crate::storage::{
StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader,
};
pub(crate) const TABLE_BUCKET_MARKER_CONFIG: &str = BUCKET_TABLE_CONFIG;
pub(crate) const TABLE_BUCKET_MARKER_CONFIG: &str = ecstore_metadata::BUCKET_TABLE_CONFIG;
const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET;
pub(crate) const RESERVED_CATALOG_OBJECT_MESSAGE: &str = "Object key is reserved for the table catalog";
pub(crate) const TABLE_BUCKET_CATALOG_TYPE: &str = "iceberg-rest";
pub(crate) const TABLE_BUCKET_CONFIG_VERSION: u16 = 1;
@@ -69,7 +69,7 @@ pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1;
pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1;
pub(crate) const TABLE_CATALOG_BACKING_MANIFEST_VERSION: u16 = 1;
pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128;
pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX;
pub const TABLE_RESERVED_PREFIX: &str = ecstore_metadata::BUCKET_TABLE_RESERVED_PREFIX;
const WAREHOUSE_ROOT: &str = "warehouses";
const NAMESPACE_ROOT: &str = "namespaces";
const TABLE_ROOT: &str = "tables";
@@ -85,8 +85,8 @@ const TABLE_BUCKET_ENTRY_FILE: &str = "table-bucket.json";
const NAMESPACE_ENTRY_FILE: &str = "namespace-entry.json";
const TABLE_ENTRY_FILE: &str = "table-entry.json";
const VIEW_ENTRY_FILE: &str = "view-entry.json";
const INTERNAL_CATALOG_ROOT: &str = BUCKET_TABLE_CATALOG_META_PREFIX;
const TABLE_BUCKET_ROOT: &str = BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX;
const INTERNAL_CATALOG_ROOT: &str = ecstore_metadata::BUCKET_TABLE_CATALOG_META_PREFIX;
const TABLE_BUCKET_ROOT: &str = ecstore_metadata::BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX;
const COMMIT_LOG_ROOT: &str = "commits";
const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency";
const EXTERNAL_CATALOG_ROOT: &str = "external-catalog";
@@ -115,8 +115,10 @@ const ICEBERG_REF_MAX_REF_AGE_MS_FIELD: &str = "max-ref-age-ms";
type CatalogListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type CatalogListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type EcstoreError = ecstore_error::Error;
type CatalogObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, EcstoreError>;
type CatalogWalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
type StorageError = ecstore_error::StorageError;
pub(crate) trait TableCatalogStorage:
StorageObjectIO<
@@ -1461,7 +1463,7 @@ impl TableCatalogObjectPaths {
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CONFIG_FILE}",
self.table_entries_prefix(table_bucket, namespace),
table.as_str(),
table_catalog_path_hash(table_id)
ecstore_metadata::table_catalog_path_hash(table_id)
)
}
@@ -1477,8 +1479,8 @@ impl TableCatalogObjectPaths {
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_JOB_ROOT}/{}.json",
self.table_entries_prefix(table_bucket, namespace),
table.as_str(),
table_catalog_path_hash(table_id),
table_catalog_path_hash(job_id)
ecstore_metadata::table_catalog_path_hash(table_id),
ecstore_metadata::table_catalog_path_hash(job_id)
)
}
@@ -1493,7 +1495,7 @@ impl TableCatalogObjectPaths {
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_LATEST_JOB_FILE}",
self.table_entries_prefix(table_bucket, namespace),
table.as_str(),
table_catalog_path_hash(table_id)
ecstore_metadata::table_catalog_path_hash(table_id)
)
}
@@ -1508,7 +1510,7 @@ impl TableCatalogObjectPaths {
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CURRENT_JOB_FILE}",
self.table_entries_prefix(table_bucket, namespace),
table.as_str(),
table_catalog_path_hash(table_id)
ecstore_metadata::table_catalog_path_hash(table_id)
)
}
@@ -1517,8 +1519,8 @@ impl TableCatalogObjectPaths {
"{}{}/{}/{}.json",
self.table_bucket_root_prefix(table_bucket),
COMMIT_LOG_ROOT,
table_catalog_path_hash(table_id),
table_catalog_path_hash(commit_id)
ecstore_metadata::table_catalog_path_hash(table_id),
ecstore_metadata::table_catalog_path_hash(commit_id)
)
}
@@ -1527,7 +1529,7 @@ impl TableCatalogObjectPaths {
"{}{}/{}/",
self.table_bucket_root_prefix(table_bucket),
COMMIT_LOG_ROOT,
table_catalog_path_hash(table_id)
ecstore_metadata::table_catalog_path_hash(table_id)
)
}
@@ -1536,8 +1538,8 @@ impl TableCatalogObjectPaths {
"{}{}/{}/{}.json",
self.table_bucket_root_prefix(table_bucket),
COMMIT_IDEMPOTENCY_ROOT,
table_catalog_path_hash(table_id),
table_catalog_path_hash(idempotency_key)
ecstore_metadata::table_catalog_path_hash(table_id),
ecstore_metadata::table_catalog_path_hash(idempotency_key)
)
}
@@ -1546,12 +1548,17 @@ impl TableCatalogObjectPaths {
"{}{}/{}/",
self.table_bucket_root_prefix(table_bucket),
COMMIT_IDEMPOTENCY_ROOT,
table_catalog_path_hash(table_id)
ecstore_metadata::table_catalog_path_hash(table_id)
)
}
fn table_bucket_root_prefix(&self, table_bucket: &str) -> String {
format!("{}/{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT, table_catalog_path_hash(table_bucket))
format!(
"{}/{}/{}/",
self.catalog_root,
TABLE_BUCKET_ROOT,
ecstore_metadata::table_catalog_path_hash(table_bucket)
)
}
}
@@ -4031,7 +4038,7 @@ where
.await
.map_err(|err| storage_error_to_catalog("create catalog table lock", err))?;
let guard = lock
.get_write_lock(get_lock_acquire_timeout())
.get_write_lock(ecstore_set_disk::get_lock_acquire_timeout())
.await
.map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog table lock: {err}")))?;
Ok(Box::new(guard))
@@ -7002,7 +7009,7 @@ pub fn validate_object_mutation(table_bucket_enabled: bool, object_key: &str) ->
}
pub(crate) async fn validate_bucket_object_mutation(bucket: &str, object_key: &str) -> Result<(), TableObjectMutationError> {
let table_bucket_enabled = get_bucket_metadata(bucket)
let table_bucket_enabled = ecstore_metadata_sys::get(bucket)
.await
.is_ok_and(|metadata| metadata.table_bucket_enabled());
@@ -7410,7 +7417,7 @@ mod tests {
let bucket = "analytics";
let namespace = Namespace::parse("analytics.daily_events").unwrap();
let table = IdentifierSegment::parse("events").unwrap();
let bucket_root = format!("s3tables/catalog/table-buckets/{}/", table_catalog_path_hash(bucket));
let bucket_root = format!("s3tables/catalog/table-buckets/{}/", ecstore_metadata::table_catalog_path_hash(bucket));
assert_eq!(paths.table_bucket_entry_path(bucket), format!("{bucket_root}table-bucket.json"));
assert_eq!(
rustfs/src/table_catalog_storage_compat.rs
-41
View File
@@ -1,41 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{sync::Arc, time::Duration};
use rustfs_ecstore::api::bucket as ecstore_bucket;
use rustfs_ecstore::api::disk as ecstore_disk;
use rustfs_ecstore::api::error as ecstore_error;
use rustfs_ecstore::api::set_disk as ecstore_set_disk;
pub(crate) const BUCKET_TABLE_CATALOG_META_PREFIX: &str = ecstore_bucket::metadata::BUCKET_TABLE_CATALOG_META_PREFIX;
pub(crate) const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str =
ecstore_bucket::metadata::BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX;
pub(crate) const BUCKET_TABLE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_TABLE_CONFIG;
pub(crate) const BUCKET_TABLE_RESERVED_PREFIX: &str = ecstore_bucket::metadata::BUCKET_TABLE_RESERVED_PREFIX;
pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET;
pub(crate) type EcstoreError = ecstore_error::Error;
pub(crate) type StorageError = ecstore_error::StorageError;
pub(crate) fn table_catalog_path_hash(value: &str) -> String {
ecstore_bucket::metadata::table_catalog_path_hash(value)
}
pub(crate) async fn get_bucket_metadata(bucket: &str) -> ecstore_error::Result<Arc<ecstore_bucket::metadata::BucketMetadata>> {
ecstore_bucket::metadata_sys::get(bucket).await
}
pub(crate) fn get_lock_acquire_timeout() -> Duration {
ecstore_set_disk::get_lock_acquire_timeout()
}
rustfs/src/workload_admission.rs
+13 -5
View File
@@ -16,10 +16,8 @@ use rustfs_concurrency::{
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
WorkloadClass,
};
use rustfs_ecstore::api::bucket::{metadata_sys, replication};
use super::workload_admission_storage_compat::{
get_global_bucket_metadata_sys, get_global_replication_pool, replication_queue_current_count,
};
use crate::storage::concurrency::get_concurrency_manager;
const BUCKET_METADATA_RUNTIME_NOT_INITIALIZED: &str = "bucket metadata runtime not initialized";
@@ -77,7 +75,7 @@ pub fn foreground_read_workload_admission_snapshot() -> WorkloadAdmissionSnapsho
}
pub fn metadata_workload_admission_snapshot() -> WorkloadAdmissionSnapshot {
metadata_workload_admission_snapshot_from_initialized(get_global_bucket_metadata_sys().is_some())
metadata_workload_admission_snapshot_from_initialized(metadata_sys::get_global_bucket_metadata_sys().is_some())
}
fn metadata_workload_admission_snapshot_from_initialized(runtime_initialized: bool) -> WorkloadAdmissionSnapshot {
@@ -153,7 +151,7 @@ fn repair_workload_admission_snapshot_from_counts(
}
pub fn replication_workload_admission_snapshot() -> WorkloadAdmissionSnapshot {
let Some(pool) = get_global_replication_pool() else {
let Some(pool) = replication::get_global_replication_pool() else {
return replication_workload_admission_snapshot_from_counts(false, None, None);
};
@@ -200,6 +198,16 @@ fn i32_to_usize_saturated(value: i32) -> usize {
usize::try_from(value.max(0)).unwrap_or(usize::MAX)
}
fn replication_queue_current_count() -> Option<i64> {
replication::GLOBAL_REPLICATION_STATS.get().and_then(|stats| {
stats
.q_cache
.try_lock()
.ok()
.map(|cache| cache.sr_queue_stats.curr.get_current_count())
})
}
#[cfg(test)]
mod tests {
use super::*;
rustfs/src/workload_admission_storage_compat.rs
-36
View File
@@ -1,36 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use rustfs_ecstore::api::bucket as ecstore_bucket;
pub(crate) fn get_global_bucket_metadata_sys() -> Option<Arc<tokio::sync::RwLock<ecstore_bucket::metadata_sys::BucketMetadataSys>>>
{
ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys()
}
pub(crate) fn get_global_replication_pool() -> Option<Arc<ecstore_bucket::replication::DynReplicationPool>> {
ecstore_bucket::replication::get_global_replication_pool()
}
pub(crate) fn replication_queue_current_count() -> Option<i64> {
ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get().and_then(|stats| {
stats
.q_cache
.try_lock()
.ok()
.map(|cache| cache.sr_queue_stats.curr.get_current_count())
})
}
scripts/check_architecture_migration_rules.sh
+63 -6
View File
@@ -107,6 +107,8 @@ RUSTFS_STORAGE_SECONDARY_COMPAT_BRIDGE_HITS_FILE="${TMP_DIR}/rustfs_storage_seco
RUSTFS_NESTED_SECONDARY_COMPAT_BRIDGE_HITS_FILE="${TMP_DIR}/rustfs_nested_secondary_compat_bridge_hits.txt"
RUSTFS_STORAGE_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_storage_local_compat_relative_consumer_hits.txt"
RUSTFS_RUNTIME_LOCAL_COMPAT_BRIDGE_HITS_FILE="${TMP_DIR}/rustfs_runtime_local_compat_bridge_hits.txt"
RUSTFS_ROOT_ONE_OFF_COMPAT_BRIDGE_HITS_FILE="${TMP_DIR}/rustfs_root_one_off_compat_bridge_hits.txt"
RUSTFS_STARTUP_COMPAT_BRIDGE_HITS_FILE="${TMP_DIR}/rustfs_startup_compat_bridge_hits.txt"
RUSTFS_ADMIN_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_admin_local_compat_relative_consumer_hits.txt"
RUSTFS_APP_SERVER_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_app_server_local_compat_relative_consumer_hits.txt"
RUSTFS_HEAL_TEST_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_heal_test_local_compat_relative_consumer_hits.txt"
@@ -116,6 +118,7 @@ NOTIFY_STORAGE_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/notify_storage_compat_module_
OBS_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE="${TMP_DIR}/obs_storage_compat_passthrough_hits.txt"
E2E_STORAGE_COMPAT_RPC_PASSTHROUGH_HITS_FILE="${TMP_DIR}/e2e_storage_compat_rpc_passthrough_hits.txt"
TEST_STORAGE_COMPAT_PASSTHROUGH_HITS_FILE="${TMP_DIR}/test_storage_compat_passthrough_hits.txt"
TEST_FUZZ_COMPAT_BRIDGE_HITS_FILE="${TMP_DIR}/test_fuzz_compat_bridge_hits.txt"
PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt"
BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt"
NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt"
@@ -710,7 +713,7 @@ fi
--glob '!**/storage_compat.rs' \
--glob '!**/*storage_compat.rs' \
--glob '!target/**' \
| rg -v '^(rustfs/src/capacity/service\.rs|rustfs/src/server/(event|http|module_switch|readiness)\.rs|rustfs/src/storage/s3_api/(bucket|multipart)\.rs):' || true
| rg -v '^(rustfs/src/(capacity/service|config/config_test|error|init|runtime_capabilities|startup_(background|bucket_metadata|fs_guard|iam|lifecycle|notification|server|services|shutdown|storage)|table_catalog|workload_admission)\.rs|rustfs/src/server/(event|http|module_switch|readiness)\.rs|rustfs/src/storage/s3_api/(bucket|multipart)\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true
) |
cat >"$DIRECT_ECSTORE_IMPORT_HITS_FILE"
@@ -989,10 +992,6 @@ fi
crates/protocols/src/swift/storage_compat.rs \
crates/s3select-api/src/storage_compat.rs \
crates/scanner/src/storage_compat.rs \
crates/heal/tests/common/storage_compat.rs \
crates/scanner/tests/common/storage_compat.rs \
fuzz/fuzz_targets/path_containment/storage_compat.rs \
fuzz/fuzz_targets/bucket_validation/storage_compat.rs \
| rg -v '^[^:]+:[0-9]+:use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' || true
) >"$OUTER_CONSUMER_COMPAT_RAW_FACADE_PATH_HITS_FILE"
@@ -1058,7 +1057,7 @@ fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'crate::(?:startup_storage_compat|runtime_capabilities_storage_compat|workload_admission_storage_compat|table_catalog_storage_compat|error_storage_compat)' \
rg -n --with-filename 'crate::startup_storage_compat' \
rustfs/src/startup_*.rs \
rustfs/src/init.rs \
rustfs/src/runtime_capabilities.rs \
@@ -1193,6 +1192,41 @@ if [[ -s "$RUSTFS_RUNTIME_LOCAL_COMPAT_BRIDGE_HITS_FILE" ]]; then
report_failure "RustFS capacity/server/S3 API runtime consumers must use direct owner APIs instead of local compatibility bridge modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_LOCAL_COMPAT_BRIDGE_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
{
for file in \
rustfs/src/config_storage_compat.rs \
rustfs/src/error_storage_compat.rs \
rustfs/src/runtime_capabilities_storage_compat.rs \
rustfs/src/table_catalog_storage_compat.rs \
rustfs/src/workload_admission_storage_compat.rs; do
[[ -e "$file" ]] && printf '%s:1:root one-off bridge file exists\n' "$file"
done
rg -n --with-filename '\bmod\s+(?:config_storage_compat|error_storage_compat|runtime_capabilities_storage_compat|table_catalog_storage_compat|workload_admission_storage_compat)|(?:crate::|self::|super::)(?:config_storage_compat|error_storage_compat|runtime_capabilities_storage_compat|table_catalog_storage_compat|workload_admission_storage_compat)|(?:config_storage_compat|error_storage_compat|runtime_capabilities_storage_compat|table_catalog_storage_compat|workload_admission_storage_compat)::' \
rustfs/src \
-g '*.rs' || true
}
) >"$RUSTFS_ROOT_ONE_OFF_COMPAT_BRIDGE_HITS_FILE"
if [[ -s "$RUSTFS_ROOT_ONE_OFF_COMPAT_BRIDGE_HITS_FILE" ]]; then
report_failure "RustFS root one-off consumers must use direct ECStore owner APIs instead of compatibility bridge modules: $(paste -sd '; ' "$RUSTFS_ROOT_ONE_OFF_COMPAT_BRIDGE_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
{
[[ -e rustfs/src/startup_storage_compat.rs ]] && printf '%s:1:startup bridge file exists\n' "rustfs/src/startup_storage_compat.rs"
rg -n --with-filename '\bmod\s+startup_storage_compat|(?:crate::|self::|super::)startup_storage_compat|startup_storage_compat::' \
rustfs/src \
-g '*.rs' || true
}
) >"$RUSTFS_STARTUP_COMPAT_BRIDGE_HITS_FILE"
if [[ -s "$RUSTFS_STARTUP_COMPAT_BRIDGE_HITS_FILE" ]]; then
report_failure "RustFS startup consumers must use direct ECStore owner APIs instead of startup compatibility bridge modules: $(paste -sd '; ' "$RUSTFS_STARTUP_COMPAT_BRIDGE_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
{
@@ -1246,6 +1280,29 @@ if [[ -s "$RUSTFS_HEAL_TEST_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE" ]]; then
report_failure "RustFS config/heal/scanner compatibility consumers must use relative owner paths instead of crate-qualified local compatibility paths: $(paste -sd '; ' "$RUSTFS_HEAL_TEST_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
{
for file in \
crates/heal/tests/common/storage_compat.rs \
crates/scanner/tests/common/storage_compat.rs \
fuzz/fuzz_targets/bucket_validation/storage_compat.rs \
fuzz/fuzz_targets/path_containment/storage_compat.rs; do
[[ -e "$file" ]] && printf '%s:1:test/fuzz bridge file exists\n' "$file"
done
rg -n --with-filename 'common::storage_compat|storage_compat::|\bmod\s+storage_compat|#\[path\s*=\s*"[^"]*storage_compat\.rs"\]' \
crates/heal/tests \
crates/scanner/tests/lifecycle_integration_test.rs \
fuzz/fuzz_targets/bucket_validation.rs \
fuzz/fuzz_targets/path_containment.rs \
-g '*.rs' || true
}
) >"$TEST_FUZZ_COMPAT_BRIDGE_HITS_FILE"
if [[ -s "$TEST_FUZZ_COMPAT_BRIDGE_HITS_FILE" ]]; then
report_failure "heal/scanner test and fuzz targets must import ECStore owner APIs directly instead of local storage compatibility bridges: $(paste -sd '; ' "$TEST_FUZZ_COMPAT_BRIDGE_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'crate::storage_compat' \