diff --git a/.config/make/pre-commit.mak b/.config/make/pre-commit.mak index 7665b5229..4914a2613 100644 --- a/.config/make/pre-commit.mak +++ b/.config/make/pre-commit.mak @@ -9,11 +9,11 @@ setup-hooks: ## Set up git hooks @echo "✅ Git hooks setup complete!" .PHONY: pre-commit -pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check quick-check ## Run fast pre-commit checks +pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check quick-check ## Run fast pre-commit checks without clippy/full tests @echo "✅ All pre-commit checks passed!" .PHONY: pre-pr -pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check clippy-check test ## Run full pre-PR checks +pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check clippy-check test ## Run full pre-PR checks with clippy and tests @echo "✅ All pre-PR checks passed!" .PHONY: dev-check diff --git a/AGENTS.md b/AGENTS.md index 821e82375..194667992 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -78,6 +78,10 @@ surface and use the faster local gate when a broad smoke check is needed: make pre-commit ``` +For migration batches, do not run the full `make pre-pr` gate before every +intermediate commit. Use focused tests and `make pre-commit` during +development, then reserve `make pre-pr` for the final PR-ready branch. + Before pushing code changes, make sure formatting is clean: - Run `cargo fmt --all`. diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index da445a219..bb3d7527e 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -123,6 +123,14 @@ lazy_static! { pub static ref GLOBAL_TransitionState: Arc = TransitionState::new(); } +pub fn get_global_expiry_state() -> Arc> { + GLOBAL_ExpiryState.clone() +} + +pub fn get_global_transition_state() -> Arc { + GLOBAL_TransitionState.clone() +} + fn resolve_transition_worker_count() -> (i64, i64, i64) { let fallback = std::cmp::min(num_cpus::get() as i64, DEFAULT_TRANSITION_WORKERS_CAP); let configured = env::var(ENV_TRANSITION_WORKERS) diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 13012476e..d610c7101 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -21,11 +21,12 @@ )] use http::HeaderMap; +use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config as config_get_global_server_config}; use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys; use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc; use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{ - GLOBAL_ExpiryState as ECSTORE_GLOBAL_EXPIRY_STATE, apply_expiry_rule as ecstore_apply_expiry_rule, - apply_transition_rule as ecstore_apply_transition_rule, + apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule, + get_global_expiry_state as ecstore_get_global_expiry_state, }; use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator; use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{ @@ -67,7 +68,7 @@ use rustfs_ecstore::api::disk::{ use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk}; use rustfs_ecstore::api::error::{Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError}; use rustfs_ecstore::api::global::{ - GLOBAL_TierConfigMgr as ECSTORE_GLOBAL_TIER_CONFIG_MGR, is_erasure as ecstore_is_erasure, + get_global_tier_config_mgr as ecstore_get_global_tier_config_mgr, is_erasure as ecstore_is_erasure, is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle, }; use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks; @@ -271,21 +272,25 @@ pub(crate) async fn apply_expiry_rule(event: &Event, src: &LcEventSrc, oi: &Scan ecstore_apply_expiry_rule(event, src, oi).await } -pub(crate) async fn list_global_tiers() -> Vec { - ECSTORE_GLOBAL_TIER_CONFIG_MGR.read().await.list_tiers() +pub(crate) fn resolve_scanner_server_config() -> Option { + config_get_global_server_config() } -pub(crate) async fn enqueue_global_free_version(oi: ScannerObjectInfo) { - ECSTORE_GLOBAL_EXPIRY_STATE.write().await.enqueue_free_version(oi).await; +pub(crate) async fn list_runtime_tiers() -> Vec { + ecstore_get_global_tier_config_mgr().read().await.list_tiers() } -pub(crate) async fn enqueue_global_newer_noncurrent( +pub(crate) async fn enqueue_runtime_free_version(oi: ScannerObjectInfo) { + ecstore_get_global_expiry_state().write().await.enqueue_free_version(oi).await; +} + +pub(crate) async fn enqueue_runtime_newer_noncurrent( bucket: &str, to_delete_objs: Vec, event: Event, src: &LcEventSrc, ) -> bool { - ECSTORE_GLOBAL_EXPIRY_STATE + ecstore_get_global_expiry_state() .write() .await .enqueue_by_newer_noncurrent(bucket, to_delete_objs, event, src) @@ -317,11 +322,11 @@ pub(crate) fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> ecstore_path2_bucket_object_with_base_path(base_path, path) } -pub(crate) async fn is_erasure() -> bool { +pub(crate) async fn scanner_is_erasure() -> bool { ecstore_is_erasure().await } -pub(crate) async fn is_erasure_sd() -> bool { +pub(crate) async fn scanner_is_erasure_sd() -> bool { ecstore_is_erasure_sd().await } diff --git a/crates/scanner/src/runtime_config.rs b/crates/scanner/src/runtime_config.rs index cdb481021..eea497898 100644 --- a/crates/scanner/src/runtime_config.rs +++ b/crates/scanner/src/runtime_config.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::resolve_scanner_server_config; use crate::scanner_budget::ScannerCycleBudgetConfig; use crate::sleeper::{SCANNER_SLEEPER, scanner_default_speed}; use rustfs_config::{ @@ -674,7 +675,7 @@ fn apply_resolved_runtime_config(config: ScannerRuntimeConfig) { } fn current_server_config() -> Option { - rustfs_config::server_config::get_global_server_config() + resolve_scanner_server_config() } pub(crate) fn resolve_scanner_runtime_config_from_global() -> ScannerRuntimeConfig { diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index a737443df..d1051dcc0 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -47,7 +47,8 @@ use tracing::{debug, error, info, instrument, warn}; use crate::{ ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, - get_lifecycle_config, get_replication_config, is_erasure_sd, read_config, replace_bucket_usage_memory_from_info, save_config, + get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config, + scanner_is_erasure_sd, }; const LOG_COMPONENT_SCANNER: &str = "scanner"; @@ -419,7 +420,7 @@ async fn detect_scanner_maintenance_features(storeapi: &Arc) -> Scanner } async fn configure_scanner_defaults(storeapi: &Arc) -> ScannerMaintenanceFeatures { - if is_erasure_sd().await { + if scanner_is_erasure_sd().await { let features = detect_scanner_maintenance_features(storeapi).await; let default_cycle_secs = single_disk_default_cycle_secs(features); set_scanner_default_speed(single_disk_default_speed()); @@ -568,7 +569,7 @@ pub struct BackgroundHealInfo { /// Read background healing information from storage pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHealInfo { // Skip for ErasureSD setup - if is_erasure_sd().await { + if scanner_is_erasure_sd().await { return BackgroundHealInfo::default(); } @@ -610,7 +611,7 @@ pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHeal #[instrument(skip(storeapi))] pub async fn save_background_heal_info(storeapi: Arc, info: BackgroundHealInfo) { // Skip for ErasureSD setup - if is_erasure_sd().await { + if scanner_is_erasure_sd().await { return; } diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 3e6d427e5..307f3e87d 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -54,8 +54,8 @@ use crate::{ BucketVersioningSys, Disk, DiskError, DiskInfoOptions, Evaluator, Event, LcEventSrc, ListPathRawOptions, ObjectOpts, ReplicationConfig, ReplicationQueueAdmission, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, StorageError, apply_expiry_rule, apply_transition_rule, - enqueue_global_newer_noncurrent, is_erasure, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object, - path2_bucket_object_with_base_path, queue_replication_heal_internal, + enqueue_runtime_newer_noncurrent, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object, + path2_bucket_object_with_base_path, queue_replication_heal_internal, scanner_is_erasure, }; use crate::{ScannerObjectInfo as ObjectInfo, ScannerObjectToDelete as ObjectToDelete}; @@ -910,7 +910,7 @@ impl ScannerItem { let action = event.action; let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX); let done_ilm = Metrics::time_ilm(action); - let queued = enqueue_global_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await; + let queued = enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await; if record_scanner_ilm_action_if_queued(global_metrics(), action, count, queued) { done_ilm(count)(); remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len()); @@ -1711,7 +1711,7 @@ impl FolderScanner { return Err(ScannerError::Other("Operation cancelled".to_string())); } - if found_objects && is_erasure().await { + if found_objects && scanner_is_erasure().await { // If we found an object in erasure mode, we skip subdirs (only datadirs)... debug!( target: "rustfs::scanner::folder", @@ -2310,7 +2310,7 @@ pub async fn scan_data_folder( let (update_current_path, close_disk) = current_path_updater(&base_path, &cache.info.name); // Create skip_heal flag - let is_erasure_mode = is_erasure().await; + let is_erasure_mode = scanner_is_erasure().await; let skip_heal = Arc::new(std::sync::atomic::AtomicBool::new(!is_erasure_mode || cache.info.skip_healing)); // Create heal_object_select flag diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index c6cdc8f54..ffc8b59cf 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -46,9 +46,9 @@ use crate::ScannerObjectInfo as ObjectInfo; use crate::{ BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, - ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_global_free_version, - get_lifecycle_config, get_object_lock_config, get_replication_config, list_global_tiers, resolve_scanner_object_store_handle, - storageclass, + ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_runtime_free_version, + get_lifecycle_config, get_object_lock_config, get_replication_config, list_runtime_tiers, + resolve_scanner_object_store_handle, storageclass, }; pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file"; @@ -1350,7 +1350,7 @@ impl ScannerIODisk for Disk { let mut size_summary = SizeSummary::default(); - let tiers = list_global_tiers().await; + let tiers = list_runtime_tiers().await; for tier in tiers.iter() { size_summary.tier_stats.insert(tier.name.clone(), TierStats::default()); @@ -1373,7 +1373,7 @@ impl ScannerIODisk for Disk { if !free_version_infos.is_empty() { for oi in free_version_infos { - enqueue_global_free_version(oi).await; + enqueue_runtime_free_version(oi).await; } } diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 7e68fbe33..c1e4c0a1e 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -26,7 +26,7 @@ use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys; use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path; use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl}; use rustfs_ecstore::api::disk::{DiskAPI as _, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk}; -use rustfs_ecstore::api::global::GLOBAL_TierConfigMgr; +use rustfs_ecstore::api::global::get_global_tier_config_mgr; use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType}; @@ -423,7 +423,8 @@ async fn create_test_tier(server: u32) { }) }, }; - let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + let tier_config_mgr_handle = get_global_tier_config_mgr(); + let mut tier_config_mgr = tier_config_mgr_handle.write().await; if let Err(err) = tier_config_mgr.add(args, false).await { println!("tier_config_mgr add failed, e: {err:?}"); panic!("tier add failed. {err}"); @@ -761,7 +762,8 @@ impl ScannerWarmBackend for MockWarmBackend { async fn register_mock_tier(tier_name: &str) -> MockWarmBackend { let backend = MockWarmBackend::default(); - let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + let tier_config_mgr_handle = get_global_tier_config_mgr(); + let mut tier_config_mgr = tier_config_mgr_handle.write().await; tier_config_mgr.tiers.insert( tier_name.to_string(), TierConfig { diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index ca64ec4ef..200b671ff 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-ecstore-rpc-test-runtime-sources` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198`. -- Based on: stacked on API-197 PR branch while PR #3817 is pending. +- Branch: `overtrue/arch-scanner-runtime-source-resolvers` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200`. +- Based on: stacked on API-199 PR branch while PR #3826 is pending. - PR type for this branch: `consumer-migration` - Runtime behavior changes: none. - Rust code changes: route replication pool, outbound TLS generation, runtime @@ -33,8 +33,13 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block notification, bucket-metadata, endpoint, region, tier-config, server-address, object-store publication, lock-client publication, and local-node publication paths, plus ECStore batch processor, dynamic storage-class publication, - RustFS cluster snapshot facade alias paths, and ECStore RPC test runtime - global helpers, + RustFS cluster snapshot facade alias paths, ECStore RPC test runtime global + helpers, ECStore lifecycle queue and transition state handle facades, RustFS + AppContext lifecycle expiry-state resolver paths, RustFS app/admin test + runtime-source helpers, scanner lifecycle/tier runtime source reads, and the + stale RustFS tier-config and expiry-state test compat shims, plus scanner + runtime config, erasure-mode, lifecycle queue, and tier runtime source helper + names, through AppContext-first or owner-crate resolver boundaries. - CI/script changes: lock completed owner and test/fuzz boundaries against bare/glob imports, scattered raw ECStore facade subpaths, and startup @@ -44,7 +49,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block and storage owner thin bridge regressions, plus app context and notify event-bridge thin module regressions; accept the reviewed AppContext resolver reverse dependencies in the layer baseline. -- Docs changes: record the API-136 through API-198 owner facade cleanup. +- Docs changes: record the API-136 through API-200 owner facade and lifecycle + runtime-source cleanup. ## Phase 0 Tasks @@ -4894,6 +4900,21 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block global scan, migration/layer guards, fast PR gate, and full PR gate before PR. +- [x] `API-199` Centralize RustFS test runtime-source helpers. + - Do: route lifecycle transition tier-config test mutation and site + replication outbound TLS generation test mutation through AppContext-owned + runtime-source helpers, then retire the stale RustFS tier-config test compat + shim. + - Acceptance: lifecycle transition and site replication tests no longer + import or mutate the ECStore tier-config global or outbound TLS generation + global directly, and no app-level tier-config test global alias remains. + - Must preserve: transition tier registration, peer-client cache rebuild on + TLS generation changes, cache restore behavior, and AppContext-first + resolver precedence. + - Verification: focused RustFS tests, formatting, diff hygiene, residual test + global scan, migration/layer guards, fast PR gate, and full PR gate before + PR. + ## Next PRs 1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries. @@ -4902,6 +4923,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| +| Quality/architecture | pass | API-199 keeps RustFS test tier-config and TLS-generation mutation behind AppContext-owned runtime-source helpers and retires the stale tier-config test compat shim. | +| Migration preservation | pass | Lifecycle transition tier registration and site replication peer-client generation-cache behavior preserve existing semantics. | +| Testing/verification | pass | Focused RustFS tests, formatting, migration/layer guards, residual scan, fast PR gate, and full PR gate are planned before PR. | | Quality/architecture | pass | API-198 keeps RPC test connection-cache and RPC-secret globals behind the ECStore runtime-source test boundary. | | Migration preservation | pass | Cached-channel eviction assertions, timeout behavior, signature generation behavior, and test-only secret initialization preserve existing semantics. | | Testing/verification | pass | Focused RPC tests, formatting, migration/layer guards, residual scan, fast PR gate, and full PR gate are planned before PR. | diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index bce6e7785..7d1469b97 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -4603,8 +4603,8 @@ mod tests { use super::super::super::Endpoint; use super::super::super::{EndpointServerPools, Endpoints, PoolEndpoints}; use super::*; + use crate::app::context::{resolve_outbound_tls_generation, set_test_outbound_tls_generation}; use http::{HeaderMap, HeaderValue, Uri}; - use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation}; use rustfs_policy::policy::action::S3Action; use serial_test::serial; use temp_env::with_var; @@ -5864,7 +5864,7 @@ mod tests { #[tokio::test] #[serial] async fn test_site_replication_peer_client_rebuilds_when_generation_changes() { - let previous_generation = get_global_outbound_tls_generation(); + let previous_generation = resolve_outbound_tls_generation().0; let previous_cache = { let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await; let snapshot = cache.clone(); @@ -5872,7 +5872,7 @@ mod tests { snapshot }; - set_global_outbound_tls_generation(101); + set_test_outbound_tls_generation(101); site_replication_peer_client() .await .expect("initial client build should succeed"); @@ -5882,7 +5882,7 @@ mod tests { assert!(matches!(cached.entry, SiteReplicationPeerClientCacheEntry::Ready(_))); drop(cache); - set_global_outbound_tls_generation(102); + set_test_outbound_tls_generation(102); site_replication_peer_client() .await .expect("new generation should rebuild client"); @@ -5892,7 +5892,7 @@ mod tests { assert!(matches!(cached.entry, SiteReplicationPeerClientCacheEntry::Ready(_))); drop(cache); - set_global_outbound_tls_generation(previous_generation); + set_test_outbound_tls_generation(previous_generation); let mut cache = SITE_REPLICATION_PEER_CLIENT.lock().await; *cache = previous_cache; } diff --git a/rustfs/src/app/context.rs b/rustfs/src/app/context.rs index 5948cc6a8..f31fcbd6c 100644 --- a/rustfs/src/app/context.rs +++ b/rustfs/src/app/context.rs @@ -33,7 +33,7 @@ use super::StorageClassConfig; use super::TierConfigMgr; use super::metadata_sys::BucketMetadataSys; use super::new_object_layer_fn; -use super::{BucketBandwidthMonitor, DynReplicationPool, NotificationSys, ReplicationStats}; +use super::{BucketBandwidthMonitor, DynReplicationPool, ExpiryState, NotificationSys, ReplicationStats}; use crate::config::RustFSBufferConfig; use rustfs_config::server_config::Config; use rustfs_credentials::Credentials; @@ -67,6 +67,11 @@ pub fn resolve_outbound_tls_generation() -> TlsGeneration { resolve_outbound_tls_generation_with(get_global_app_context(), || default_outbound_tls_runtime_interface().generation()) } +#[cfg(test)] +pub(crate) fn set_test_outbound_tls_generation(generation: u64) { + rustfs_common::set_global_outbound_tls_generation(generation); +} + /// Resolve outbound TLS state using AppContext-first precedence. pub async fn resolve_outbound_tls_state() -> GlobalPublishedOutboundTlsState { resolve_outbound_tls_state_with(get_global_app_context()).await @@ -230,6 +235,11 @@ pub fn resolve_tier_config_handle() -> Arc> { resolve_tier_config_handle_with(get_global_app_context(), || default_tier_config_interface().handle()) } +/// Resolve lifecycle expiry state using AppContext-first precedence. +pub fn resolve_expiry_state_handle() -> Arc> { + resolve_expiry_state_handle_with(get_global_app_context(), || default_expiry_state_interface().handle()) +} + /// Resolve server config using AppContext-first precedence. pub fn resolve_server_config() -> Option { resolve_server_config_with(get_global_app_context(), || default_server_config_interface().get()) @@ -510,6 +520,15 @@ fn resolve_tier_config_handle_with( context.map(|context| context.tier_config().handle()).unwrap_or_else(fallback) } +fn resolve_expiry_state_handle_with( + context: Option>, + fallback: impl FnOnce() -> Arc>, +) -> Arc> { + context + .map(|context| context.expiry_state().handle()) + .unwrap_or_else(fallback) +} + fn resolve_server_config_with(context: Option>, fallback: impl FnOnce() -> Option) -> Option { context.map_or_else(fallback, |context| context.server_config().get()) } @@ -847,6 +866,16 @@ mod tests { } } + struct TestExpiryStateInterface { + expiry_state: Arc>, + } + + impl ExpiryStateInterface for TestExpiryStateInterface { + fn handle(&self) -> Arc> { + self.expiry_state.clone() + } + } + struct TestServerConfigInterface { config: Option, published: Arc, @@ -974,6 +1003,8 @@ mod tests { ..Default::default() }; let tier_config = TierConfigMgr::new(); + let context_expiry_state = ExpiryState::new(); + let fallback_expiry_state = ExpiryState::new(); let server_config = Config::new(); let context_server_config_published = Arc::new(AtomicUsize::new(0)); let fallback_server_config_published = Arc::new(AtomicUsize::new(0)); @@ -1101,6 +1132,9 @@ mod tests { tier_config: Arc::new(TestTierConfigInterface { tier_config: tier_config.clone(), }), + expiry_state: Arc::new(TestExpiryStateInterface { + expiry_state: context_expiry_state.clone(), + }), server_config: Arc::new(TestServerConfigInterface { config: Some(server_config.clone()), published: context_server_config_published.clone(), @@ -1222,6 +1256,10 @@ mod tests { &resolve_tier_config_handle_with(Some(context.clone()), TierConfigMgr::new), &tier_config )); + assert!(Arc::ptr_eq( + &resolve_expiry_state_handle_with(Some(context.clone()), || fallback_expiry_state.clone()), + &context_expiry_state + )); assert_eq!( resolve_server_config_with(Some(context.clone()), || None).expect("context server config"), server_config @@ -1334,6 +1372,10 @@ mod tests { fallback_region ); assert!(Arc::ptr_eq(&resolve_tier_config_handle_with(None, || tier_config.clone()), &tier_config)); + assert!(Arc::ptr_eq( + &resolve_expiry_state_handle_with(None, || fallback_expiry_state.clone()), + &fallback_expiry_state + )); assert_eq!( resolve_server_config_with(None, || Some(server_config.clone())).expect("fallback server config"), server_config diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index f4efe5296..2ba380d08 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -16,9 +16,9 @@ use super::super::{ECStore, set_object_store_resolver}; use super::handles::{ IamHandle, KmsHandle, default_action_credential_interface, default_boot_time_interface, default_bucket_metadata_interface, default_bucket_monitor_interface, default_buffer_config_interface, default_deployment_id_interface, - default_endpoints_interface, default_internode_metrics_interface, default_kms_runtime_interface, - default_local_node_name_interface, default_lock_client_interface, default_lock_clients_interface, - default_notification_system_interface, default_notify_interface, default_oidc_interface, + default_endpoints_interface, default_expiry_state_interface, default_internode_metrics_interface, + default_kms_runtime_interface, default_local_node_name_interface, default_lock_client_interface, + default_lock_clients_interface, default_notification_system_interface, default_notify_interface, default_oidc_interface, default_outbound_tls_runtime_interface, default_performance_metrics_interface, default_region_interface, default_replication_pool_interface, default_replication_stats_interface, default_runtime_port_interface, default_s3select_db_interface, default_scanner_metrics_interface, default_server_config_interface, @@ -26,11 +26,11 @@ use super::handles::{ }; use super::interfaces::{ ActionCredentialInterface, BootTimeInterface, BucketMetadataInterface, BucketMonitorInterface, BufferConfigInterface, - DeploymentIdInterface, EndpointsInterface, IamInterface, InternodeMetricsInterface, KmsInterface, KmsRuntimeInterface, - LocalNodeNameInterface, LockClientInterface, LockClientsInterface, NotificationSystemInterface, NotifyInterface, - OidcInterface, OutboundTlsRuntimeInterface, PerformanceMetricsInterface, RegionInterface, ReplicationPoolInterface, - ReplicationStatsInterface, RuntimePortInterface, S3SelectDbInterface, ScannerMetricsInterface, ServerConfigInterface, - StorageClassInterface, TierConfigInterface, TierStatsInterface, + DeploymentIdInterface, EndpointsInterface, ExpiryStateInterface, IamInterface, InternodeMetricsInterface, KmsInterface, + KmsRuntimeInterface, LocalNodeNameInterface, LockClientInterface, LockClientsInterface, NotificationSystemInterface, + NotifyInterface, OidcInterface, OutboundTlsRuntimeInterface, PerformanceMetricsInterface, RegionInterface, + ReplicationPoolInterface, ReplicationStatsInterface, RuntimePortInterface, S3SelectDbInterface, ScannerMetricsInterface, + ServerConfigInterface, StorageClassInterface, TierConfigInterface, TierStatsInterface, }; use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; use rustfs_kms::KmsServiceManager; @@ -67,6 +67,7 @@ pub struct AppContext { action_credentials: Arc, region: Arc, tier_config: Arc, + expiry_state: Arc, server_config: Arc, storage_class: Arc, buffer_config: Arc, @@ -102,6 +103,7 @@ impl AppContext { action_credentials: default_action_credential_interface(), region: default_region_interface(), tier_config: default_tier_config_interface(), + expiry_state: default_expiry_state_interface(), server_config: default_server_config_interface(), storage_class: default_storage_class_interface(), buffer_config: default_buffer_config_interface(), @@ -225,6 +227,10 @@ impl AppContext { self.tier_config.clone() } + pub fn expiry_state(&self) -> Arc { + self.expiry_state.clone() + } + pub fn server_config(&self) -> Arc { self.server_config.clone() } @@ -266,6 +272,7 @@ pub(super) struct AppContextTestInterfaces { pub(super) action_credentials: Arc, pub(super) region: Arc, pub(super) tier_config: Arc, + pub(super) expiry_state: Arc, pub(super) server_config: Arc, pub(super) storage_class: Arc, pub(super) buffer_config: Arc, @@ -302,6 +309,7 @@ impl AppContext { action_credentials: interfaces.action_credentials, region: interfaces.region, tier_config: interfaces.tier_config, + expiry_state: interfaces.expiry_state, server_config: interfaces.server_config, storage_class: interfaces.storage_class, buffer_config: interfaces.buffer_config, diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index 47aec04eb..55d41701b 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -18,17 +18,17 @@ use super::super::TierConfigMgr; use super::super::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys}; use super::super::{ collect_scanner_metrics_report, get_daily_all_tier_stats, get_global_boot_time, get_global_bucket_monitor, - get_global_deployment_id, get_global_endpoints_opt, get_global_lock_client, get_global_lock_clients, + get_global_deployment_id, get_global_endpoints_opt, get_global_expiry_state, get_global_lock_client, get_global_lock_clients, get_global_notification_sys, get_global_region, get_global_replication_pool, get_global_replication_stats, get_global_tier_config_mgr, global_rustfs_port, set_global_storage_class, }; use super::interfaces::{ ActionCredentialInterface, BootTimeInterface, BucketMetadataInterface, BucketMonitorInterface, BufferConfigInterface, - DeploymentIdInterface, EndpointsInterface, IamInterface, InternodeMetricsInterface, KmsInterface, KmsRuntimeInterface, - LocalNodeNameInterface, LockClientInterface, LockClientsInterface, NotificationSystemInterface, NotifyInterface, - OidcInterface, OutboundTlsRuntimeInterface, PerformanceMetricsInterface, RegionInterface, ReplicationPoolInterface, - ReplicationStatsInterface, RuntimePortInterface, S3SelectDbInterface, ScannerMetricsInterface, ServerConfigInterface, - StorageClassInterface, TierConfigInterface, TierStatsInterface, + DeploymentIdInterface, EndpointsInterface, ExpiryStateInterface, IamInterface, InternodeMetricsInterface, KmsInterface, + KmsRuntimeInterface, LocalNodeNameInterface, LockClientInterface, LockClientsInterface, NotificationSystemInterface, + NotifyInterface, OidcInterface, OutboundTlsRuntimeInterface, PerformanceMetricsInterface, RegionInterface, + ReplicationPoolInterface, ReplicationStatsInterface, RuntimePortInterface, S3SelectDbInterface, ScannerMetricsInterface, + ServerConfigInterface, StorageClassInterface, TierConfigInterface, TierStatsInterface, }; use crate::config::{RustFSBufferConfig, get_global_buffer_config}; use async_trait::async_trait; @@ -368,6 +368,16 @@ impl TierConfigInterface for TierConfigHandle { } } +/// Default lifecycle expiry state interface adapter. +#[derive(Default)] +pub struct ExpiryStateHandle; + +impl ExpiryStateInterface for ExpiryStateHandle { + fn handle(&self) -> Arc> { + get_global_expiry_state() + } +} + /// Default server config interface adapter. #[derive(Default)] pub struct ServerConfigHandle; @@ -498,6 +508,10 @@ pub fn default_tier_config_interface() -> Arc { Arc::new(TierConfigHandle) } +pub fn default_expiry_state_interface() -> Arc { + Arc::new(ExpiryStateHandle) +} + pub fn default_server_config_interface() -> Arc { Arc::new(ServerConfigHandle) } diff --git a/rustfs/src/app/context/interfaces.rs b/rustfs/src/app/context/interfaces.rs index 81e9cd164..794b0a1b1 100644 --- a/rustfs/src/app/context/interfaces.rs +++ b/rustfs/src/app/context/interfaces.rs @@ -18,7 +18,7 @@ use super::super::ScannerMetricsReport; use super::super::StorageClassConfig; use super::super::TierConfigMgr; use super::super::metadata_sys::BucketMetadataSys; -use super::super::{BucketBandwidthMonitor, DynReplicationPool, NotificationSys, ReplicationStats}; +use super::super::{BucketBandwidthMonitor, DynReplicationPool, ExpiryState, NotificationSys, ReplicationStats}; use crate::config::RustFSBufferConfig; use async_trait::async_trait; use rustfs_config::server_config::Config; @@ -193,6 +193,11 @@ pub trait TierConfigInterface: Send + Sync { fn handle(&self) -> Arc>; } +/// Lifecycle expiry state interface for transition cleanup queues. +pub trait ExpiryStateInterface: Send + Sync { + fn handle(&self) -> Arc>; +} + /// Server config interface for application-layer and server modules. pub trait ServerConfigInterface: Send + Sync { fn get(&self) -> Option; diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index 921487832..08ef627d2 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -13,8 +13,7 @@ // limitations under the License. use super::{ - AppWarmBackend, ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, TierConfig, TierType, - WarmBackendGetOpts, + AppWarmBackend, ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, TierConfig, TierType, WarmBackendGetOpts, metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG}, metadata_sys, object_api_utils::to_s3s_etag, @@ -22,6 +21,7 @@ use super::{ }; use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase}; use crate::app::bucket_usecase::DefaultBucketUsecase; +use crate::app::context::resolve_tier_config_handle; use crate::storage::ecfs::FS; use crate::storage::{ StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader, @@ -337,7 +337,8 @@ impl AppWarmBackend for MockWarmBackend { async fn register_mock_tier(tier_name: &str) -> MockWarmBackend { let backend = MockWarmBackend::default(); - let mut tier_config_mgr = GLOBAL_TierConfigMgr.write().await; + let tier_config_mgr_handle = resolve_tier_config_handle(); + let mut tier_config_mgr = tier_config_mgr_handle.write().await; tier_config_mgr.tiers.insert( tier_name.to_string(), TierConfig { diff --git a/rustfs/src/app/mod.rs b/rustfs/src/app/mod.rs index 06c8b6345..fbdf6b9e0 100644 --- a/rustfs/src/app/mod.rs +++ b/rustfs/src/app/mod.rs @@ -80,11 +80,6 @@ mod ecstore_data_usage { }; } -#[cfg(test)] -mod ecstore_global { - pub(crate) use crate::storage::ecstore_global::GLOBAL_TierConfigMgr; -} - #[allow(unused_imports)] mod ecstore_tier { pub(crate) use crate::storage::ecstore_tier::{tier, tier_config, warm_backend}; @@ -97,6 +92,7 @@ pub(crate) type DynReader = crate::storage::DynReader; pub(crate) type DynReplicationPool = crate::storage::DynReplicationPool; pub(crate) type ECStore = crate::storage::ECStore; pub(crate) type EndpointServerPools = crate::storage::EndpointServerPools; +pub(crate) type ExpiryState = crate::storage::ExpiryState; pub(crate) type HashReader = crate::storage::HashReader; pub(crate) type NotificationSys = crate::storage::NotificationSys; pub(crate) type BucketBandwidthMonitor = crate::storage::BucketBandwidthMonitor; @@ -208,27 +204,11 @@ pub(crate) mod lifecycle { } pub(crate) mod bucket_lifecycle_ops { - use std::ops::Deref; use std::sync::Arc; use super::ECStore; use super::bucket_lifecycle_audit::LcEventSrc; - pub(crate) type ExpiryState = super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ExpiryState; - - pub(crate) struct GlobalExpiryStateCompat; - - #[allow(non_upper_case_globals)] - pub(crate) static GLOBAL_ExpiryState: GlobalExpiryStateCompat = GlobalExpiryStateCompat; - - impl Deref for GlobalExpiryStateCompat { - type Target = Arc>; - - fn deref(&self) -> &Self::Target { - &super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState - } - } - #[cfg(test)] pub(crate) async fn init_background_expiry(api: Arc) { super::super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(api).await; @@ -623,22 +603,6 @@ pub(crate) fn is_err_version_not_found(err: &Error) -> bool { crate::storage::is_err_version_not_found(err) } -#[cfg(test)] -pub(crate) struct GlobalTierConfigMgrCompat; - -#[cfg(test)] -#[allow(non_upper_case_globals)] -pub(crate) static GLOBAL_TierConfigMgr: GlobalTierConfigMgrCompat = GlobalTierConfigMgrCompat; - -#[cfg(test)] -impl std::ops::Deref for GlobalTierConfigMgrCompat { - type Target = Arc>; - - fn deref(&self) -> &Self::Target { - &ecstore_global::GLOBAL_TierConfigMgr - } -} - pub(crate) fn get_global_endpoints_opt() -> Option { crate::storage::get_global_endpoints_opt() } @@ -667,6 +631,10 @@ pub(crate) fn get_global_tier_config_mgr() -> Arc Arc> { + crate::storage::get_global_expiry_state() +} + pub(crate) fn new_object_layer_fn() -> Option> { crate::storage::new_object_layer_fn() } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index d72f4606c..7057416d2 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -45,7 +45,8 @@ use super::{ versioning_sys::BucketVersioningSys, }; use crate::app::context::{ - AppContext, get_global_app_context, resolve_notify_interface_for_context, resolve_object_store_handle_for_context, + AppContext, get_global_app_context, resolve_expiry_state_handle, resolve_notify_interface_for_context, + resolve_object_store_handle_for_context, }; use crate::config::RustFSBufferConfig; use crate::delete_tail_activity::{DeleteTailActivityGuard, DeleteTailStage}; @@ -294,7 +295,8 @@ async fn enqueue_transitioned_delete_cleanup( super::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?; - let mut expiry_state = super::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState.write().await; + let expiry_state = resolve_expiry_state_handle(); + let mut expiry_state = expiry_state.write().await; if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await { warn!( bucket, diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index d023e0ef0..a63693856 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -235,6 +235,7 @@ pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint; pub(crate) type Endpoints = ecstore_layout::Endpoints; pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools; pub(crate) type EventArgs = ecstore_event::EventArgs; +pub(crate) type ExpiryState = ecstore_bucket::lifecycle::bucket_lifecycle_ops::ExpiryState; pub(crate) type FileInfoVersions = ecstore_disk::FileInfoVersions; pub(crate) type FileReader = ecstore_disk::FileReader; pub(crate) type FileWriter = ecstore_disk::FileWriter; @@ -316,7 +317,11 @@ pub(crate) fn get_global_boot_time() -> Option { } pub(crate) fn get_daily_all_tier_stats() -> DailyAllTierStats { - ecstore_bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState.get_daily_all_tier_stats() + ecstore_bucket::lifecycle::bucket_lifecycle_ops::get_global_transition_state().get_daily_all_tier_stats() +} + +pub(crate) fn get_global_expiry_state() -> Arc> { + ecstore_bucket::lifecycle::bucket_lifecycle_ops::get_global_expiry_state() } pub(crate) async fn try_migrate_bucket_metadata(store: Arc) {