// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //! Regression test for rustfs#4304: IAM bootstrap must not depend on the //! distributed namespace-lock quorum. //! //! During a sequential cluster restart the peer lock RPC endpoints are //! unreachable, so every namespace-locked read fails with //! `QuorumNotReached` even though the storage read quorum is already //! satisfiable. The bulk snapshot load (`Store::load_all`) therefore has to //! read with `no_lock = true` (startup contract rustfs#4056). //! //! The test builds a real 4-disk ECStore over temp dirs, seeds IAM data in //! single-node mode, then flips the runtime into distributed-erasure mode. //! In that mode `SetDisks::new_ns_lock` builds a distributed lock over the //! set's lock clients — which are empty for a locally-built store — so every //! locked read fails exactly like the sequential-restart scenario, while //! plain storage reads keep working. The old (locked) `load_group` path must //! fail and the lock-free `load_all` path must succeed. mod ecstore_test_compat; use ecstore_test_compat::fixture::{ ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, SetupType, init_local_disks, update_erasure_type, }; use rustfs_iam::cache::Cache; use rustfs_iam::store::object::ObjectStore; use rustfs_iam::store::{GroupInfo, Store}; use serial_test::serial; use std::collections::HashMap; use std::sync::Arc; use tokio_util::sync::CancellationToken; const TEST_GROUP: &str = "seq-restart-group"; const TEST_MEMBERS: [&str; 2] = ["alice", "bob"]; async fn build_local_ecstore(temp_dir: &std::path::Path) -> Arc { let disk_paths: Vec<_> = (1..=4).map(|i| temp_dir.join(format!("disk{i}"))).collect(); for disk_path in &disk_paths { tokio::fs::create_dir_all(disk_path).await.unwrap(); } let mut endpoints = Vec::new(); for (i, disk_path) in disk_paths.iter().enumerate() { let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap(); endpoint.set_pool_index(0); endpoint.set_set_index(0); endpoint.set_disk_index(i); endpoints.push(endpoint); } let pool_endpoints = PoolEndpoints { legacy: false, set_count: 1, drives_per_set: 4, endpoints: Endpoints::from(endpoints), cmd_line: "test".to_string(), platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), }; let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints]); init_local_disks(endpoint_pools.clone()).await.unwrap(); // Port 0 keeps this integration binary parallel-safe alongside other // ECStore-backed tests. let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) .await .unwrap() } /// Restores single-node erasure mode even when an assertion panics, so a /// failing run cannot poison later `#[serial]` tests in this process. struct ErasureModeGuard; impl Drop for ErasureModeGuard { fn drop(&mut self) { pollster::block_on(update_erasure_type(SetupType::Erasure)); } } #[tokio::test(flavor = "multi_thread")] #[serial] async fn load_all_bypasses_namespace_lock_quorum() { // The lock acquire timeout is latched into a OnceLock on first use, so it // must be shortened before the first locked operation of this process. temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async { let temp_dir = tempfile::TempDir::with_prefix("rustfs_iam_no_lock_test_").unwrap(); let ecstore = build_local_ecstore(temp_dir.path()).await; let store = ObjectStore::new(ecstore); // Seed IAM data while namespace locks still work (single-node mode). store .save_group_info(TEST_GROUP, GroupInfo::new(TEST_MEMBERS.iter().map(|m| m.to_string()).collect())) .await .expect("seeding group info in single-node mode must succeed"); let mut baseline = HashMap::new(); store .load_group(TEST_GROUP, &mut baseline) .await .expect("locked load_group must succeed in single-node mode"); assert_eq!(baseline[TEST_GROUP].members, TEST_MEMBERS, "seeded group must round-trip"); // Flip into distributed-erasure mode: new_ns_lock now builds a // distributed lock over the set's (empty) lock-client list, so every // locked read fails — the sequential-restart failure mode of // rustfs#4304 (lock quorum unavailable, storage quorum healthy). update_erasure_type(SetupType::DistErasure).await; let _mode_guard = ErasureModeGuard; let mut locked_read = HashMap::new(); let locked_err = store .load_group(TEST_GROUP, &mut locked_read) .await .expect_err("locked load_group must fail while the lock quorum is unavailable"); assert!(locked_read.is_empty(), "failed locked read must not populate results"); // The P0 fix: the bulk snapshot load reads with no_lock = true, so it // must succeed in exactly the state where the locked path fails. let cache = Cache::default(); store .load_all(&cache) .await .unwrap_or_else(|err| panic!("load_all must bypass namespace locks (locked path failed with: {locked_err}): {err}")); // P3 step 1: notification-path cache refreshes use the same lock-free // reads, so a cross-node notification must also be able to refresh // this group while the lock quorum is unavailable. let mut notification_read = HashMap::new(); store .load_group_no_lock(TEST_GROUP, &mut notification_read) .await .expect("lock-free notification-path load_group must succeed while the lock quorum is unavailable"); assert_eq!(notification_read[TEST_GROUP].members, TEST_MEMBERS); // Back in single-node mode the locked path works again; verify the // data survived the whole exercise intact (fail-closed integrity). drop(_mode_guard); let mut recovered = HashMap::new(); store .load_group(TEST_GROUP, &mut recovered) .await .expect("locked load_group must succeed again after restoring single-node mode"); assert_eq!(recovered[TEST_GROUP].members, TEST_MEMBERS, "group data must be intact"); }) .await; }