gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-17 10:17:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into reatang/fix-replication-switches

Browse Source
This commit is contained in:
Zhengchao An
2026-07-30 07:27:05 +08:00
committed by GitHub
parent 310ff96460 422e0ad768
commit 4bd765a2c0
62 changed files with 5702 additions and 474 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/config/src/constants/internode.rs
+31 -9
View File
@@ -158,17 +158,33 @@ pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
/// one-time consumption of body-bound v2 signatures.
/// Require the replay-scoped internode RPC signature after the fleet has converged on it.
///
/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
/// shared secret) — and increments
/// The default keeps v1/v2 peers available during a rolling upgrade. Operators may set this only
/// after `rustfs_system_network_internode_replay_scope_fallback_total` remains zero for a full
/// release window. The node still accepts a v2-authenticated `Ping` carrying an epoch challenge:
/// that narrowly scoped bootstrap lets an upgraded client learn the receiving process epoch and
/// immediately retry with the replay-scoped signature after a peer restart.
pub const ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT: &str = "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT";
pub const DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT: bool = false;
// Compile-time invariant: mixed-version clusters must remain available until operators make the
// observed fallback counter an explicit strictness decision.
const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
/// one-time consumption of authenticated RPC signatures.
///
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
/// the shared secret) — and increments
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
/// counter means this capacity is undersized for the node's peak mutation rate.
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
@@ -354,6 +370,12 @@ mod tests {
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
}
#[test]
fn internode_replay_scope_strict_env_name_is_stable() {
// The fail-open default invariant is asserted at compile time next to the definition.
assert_eq!(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT");
}
#[test]
fn internode_replay_cache_capacity_defaults_and_env_name() {
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
crates/data-usage/Cargo.toml
-1
View File
@@ -29,7 +29,6 @@ workspace = true
[dependencies]
serde = { workspace = true, features = ["derive"] }
path-clean = { workspace = true }
rmp-serde = { workspace = true }
async-trait = { workspace = true }
rustfs-filemeta = { workspace = true }
crates/data-usage/src/data_usage.rs
+94 -6
View File
@@ -12,12 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use path_clean::PathClean;
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, HashSet},
hash::{DefaultHasher, Hash, Hasher},
path::Path,
time::{Duration, SystemTime},
};
@@ -334,7 +332,7 @@ impl<'de> Deserialize<'de> for SizeHistogram {
impl SizeHistogram {
pub fn add(&mut self, size: u64) {
let intervals = [
(0, 1024), // LESS_THAN_1024_B
(0, 1024 - 1), // LESS_THAN_1024_B
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
@@ -362,7 +360,7 @@ impl SizeHistogram {
// the sub-ranges in [1 KiB, 512 KiB).
const ONE_MIB: u64 = 1024 * 1024;
let intervals = [
(0, 1024), // LESS_THAN_1024_B
(0, 1024 - 1), // LESS_THAN_1024_B
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
@@ -1110,9 +1108,39 @@ fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String
}
}
/// Hash a path for data usage caching
fn clean_data_usage_path(data: &str) -> String {
let rooted = data.starts_with('/');
let mut parts = Vec::new();
for part in data.split('/') {
match part {
"" | "." => {}
".." => {
if parts.last().is_some_and(|last| *last != "..") {
parts.pop();
} else if !rooted {
parts.push(part);
}
}
_ => parts.push(part),
}
}
let clean = parts.join("/");
match (rooted, clean.is_empty()) {
(true, true) => "/".to_string(),
(true, false) => format!("/{clean}"),
(false, true) => ".".to_string(),
(false, false) => clean,
}
}
/// Hash a slash-separated path for data usage caching.
///
/// Cache identifiers are persisted and exchanged across nodes, so their
/// normalization must not depend on the host operating system.
pub fn hash_path(data: &str) -> DataUsageHash {
DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
DataUsageHash(clean_data_usage_path(data))
}
impl DataUsageInfo {
@@ -1497,6 +1525,23 @@ mod tests {
buckets_count: u64,
}
#[test]
fn hash_path_uses_portable_slash_semantics() {
for (input, expected) in [
("", "."),
(".", "."),
("/", "/"),
("//bucket///prefix/", "/bucket/prefix"),
("bucket/./prefix//object", "bucket/prefix/object"),
("bucket/a/../b", "bucket/b"),
("../bucket/..", ".."),
("/../../bucket", "/bucket"),
("bucket\\prefix/object", "bucket\\prefix/object"),
] {
assert_eq!(hash_path(input).key(), expected, "unexpected portable cache key for {input:?}");
}
}
#[test]
fn completeness_marker_is_additive_for_legacy_named_readers() {
let current = DataUsageInfo {
@@ -1601,6 +1646,49 @@ mod tests {
assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1);
}
#[test]
fn test_size_histogram_classifies_adjacent_boundaries_once() {
let cases = [
(1023, 0),
(1024, 1),
(64 * 1024 - 1, 1),
(64 * 1024, 2),
(256 * 1024 - 1, 2),
(256 * 1024, 3),
(512 * 1024 - 1, 3),
(512 * 1024, 4),
(1024 * 1024 - 1, 4),
(1024 * 1024, 6),
(10 * 1024 * 1024 - 1, 6),
(10 * 1024 * 1024, 7),
(64 * 1024 * 1024 - 1, 7),
(64 * 1024 * 1024, 8),
(128 * 1024 * 1024 - 1, 8),
(128 * 1024 * 1024, 9),
(512 * 1024 * 1024 - 1, 9),
(512 * 1024 * 1024, 10),
];
for (size, expected_bucket) in cases {
let mut hist = SizeHistogram::default();
hist.add(size);
assert_eq!(hist.0.iter().sum::<u64>(), 1, "size {size} must have exactly one physical bucket");
assert_eq!(hist.0[expected_bucket], 1, "size {size} must select the expected bucket");
}
}
#[test]
fn test_size_histogram_1024_bytes_contributes_to_compat_rollup() {
let mut hist = SizeHistogram::default();
hist.add(1024);
let map = hist.to_map();
assert_eq!(map["LESS_THAN_1024_B"], 0);
assert_eq!(map["BETWEEN_1024_B_AND_64_KB"], 1);
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], 1);
}
#[test]
fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() {
let mut hist = SizeHistogram::default();
crates/e2e_test/src/internode_rpc_signature_e2e_test.rs
+217 -14
View File
@@ -13,8 +13,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Cross-process replay / tamper acceptance for the internode NodeService v2 RPC
//! signature (<https://github.com/rustfs/backlog/issues/1327>).
//! Cross-process replay / tamper acceptance for internode NodeService RPC
//! signatures (<https://github.com/rustfs/backlog/issues/1327>,
//! <https://github.com/rustfs/backlog/issues/1542>).
//!
//! # Why this exists on top of the in-process tests
//!
@@ -78,6 +79,8 @@
//! | mixed version: legacy-only still served, not blocked | [`legacy_only_signature_is_accepted_in_default_posture`] |
//! | strict flip closes the signature downgrade | [`signature_strict_rejects_legacy_only_downgrade`] |
//! | strict flip closes the body-digest downgrade, incl. v1 | [`body_digest_strict_rejects_digestless_mutation`] |
//! | replay scope binds every RPC and rejects restart replay | [`replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e`] |
//! | strict replay scope allows only Ping bootstrap before v3 | [`replay_scope_strict_requires_v3_after_ping_bootstrap_e2e`] |
//!
//! Two acceptance items are deliberately left to the in-process tests. A stale
//! timestamp cannot be forged from outside — it is inside the HMAC — so
@@ -88,18 +91,20 @@
use crate::common::{RustFSTestEnvironment, init_logging};
use crate::storage_api::internode_rpc_signature::{
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
};
use http::{HeaderMap, Method};
use rustfs_config::{
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_SIGNATURE_STRICT,
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
ENV_INTERNODE_RPC_SIGNATURE_STRICT,
};
use rustfs_protos::canonical_make_volume_request_body;
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse};
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::error::Error;
use tonic::{Code, Request, Status};
use tonic::{Code, Request, Response, Status};
use uuid::Uuid;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
@@ -115,12 +120,14 @@ const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
/// clears authentication stops harmlessly at `find_disk`.
const ABSENT_DISK: &str = "/nonexistent/rustfs-signature-e2e-disk";
/// Wire names of the two v2 headers these tests edit. They are `pub(crate)` in
/// ecstore, so they are repeated here rather than imported — [`overwrite_header`]
/// asserts the header it replaces was actually present, which turns a rename
/// into a loud failure instead of silently reducing an attack to a no-op.
/// Wire names of the v2 and replay-scope headers these black-box tests edit. They are
/// `pub(crate)` in ecstore, so they are repeated here rather than imported.
/// [`overwrite_header`] asserts the header it replaces was actually present, which turns a
/// rename into a loud failure instead of silently reducing an attack to a no-op.
const CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
const NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
const BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
/// gRPC service name carried in the signed scope, i.e. `TONIC_RPC_PREFIX`
/// without its leading `/`.
@@ -155,19 +162,28 @@ fn align_rpc_secret_with_server() {
///
/// Uses the no-cleanup spawn so a `pkill` pattern cannot reap servers belonging
/// to other tests running in the same binary.
async fn start_server(extra_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
let mut env = RustFSTestEnvironment::new().await?;
fn server_env(extra_env: &[(&'static str, &'static str)]) -> Vec<(&'static str, &'static str)> {
let mut child_env = vec![
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "false"),
(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "false"),
(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "false"),
(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "1048576"),
];
child_env.extend_from_slice(extra_env);
env.start_rustfs_server_without_cleanup_with_env(&child_env).await?;
child_env
}
async fn start_server_with_env(child_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_without_cleanup_with_env(child_env).await?;
Ok(env)
}
async fn start_server(extra_env: &[(&'static str, &'static str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
start_server_with_env(&server_env(extra_env)).await
}
/// Stop the child and drop the cached gRPC channel for its address.
///
/// `node_service_time_out_client_no_auth` memoises channels in a process-global
@@ -238,12 +254,83 @@ fn overwrite_header(headers: &mut HeaderMap, name: &'static str, value: &str) {
/// and nothing else — no interceptor adds or rewrites auth metadata, so the
/// bytes on the wire are the ones the test chose.
async fn call_make_volume(url: &str, request: MakeVolumeRequest, headers: HeaderMap) -> Result<MakeVolumeResponse, Status> {
call_make_volume_response(url, request, headers)
.await
.map(Response::into_inner)
}
async fn call_make_volume_response(
url: &str,
request: MakeVolumeRequest,
headers: HeaderMap,
) -> Result<Response<MakeVolumeResponse>, Status> {
let mut client = node_service_time_out_client_no_auth(&url.to_string())
.await
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
let mut rpc_request = Request::new(request);
rpc_request.metadata_mut().as_mut().extend(headers);
client.make_volume(rpc_request).await.map(|response| response.into_inner())
client.make_volume(rpc_request).await
}
async fn call_ping_response(url: &str, headers: HeaderMap) -> Result<Response<PingResponse>, Status> {
let mut client = node_service_time_out_client_no_auth(&url.to_string())
.await
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
let mut rpc_request = Request::new(PingRequest {
version: 1,
body: bytes::Bytes::new(),
});
rpc_request.metadata_mut().as_mut().extend(headers);
client.ping(rpc_request).await
}
fn attach_boot_epoch_challenge(headers: &mut HeaderMap) -> Uuid {
let challenge = Uuid::new_v4();
headers.insert(
BOOT_EPOCH_CHALLENGE_HEADER,
challenge.to_string().parse().expect("UUID must be a valid header value"),
);
challenge
}
fn mint_replay_scope_headers(audience: &str, path: &str, content_sha256: &str, boot_epoch: Uuid) -> HeaderMap {
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(content_sha256));
let timestamp = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 headers must carry a timestamp")
.to_string();
headers.extend(
gen_tonic_replay_scope_headers(audience, path, &timestamp, content_sha256, boot_epoch)
.expect("replay-scope headers must mint with the aligned RPC secret"),
);
headers
}
async fn learn_boot_epoch_from_make_volume(url: &str, audience: &str) -> Uuid {
let request = make_volume_request("signature-e2e-epoch-bootstrap");
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(&canonical_digest(&request)));
let challenge = attach_boot_epoch_challenge(&mut headers);
let response = call_make_volume_response(url, request, headers)
.await
.expect("v2 request with epoch challenge must clear default authentication");
let boot_epoch = verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
.expect("server must HMAC-authenticate the advertised boot epoch");
assert_authenticated(
Ok(response.into_inner()),
"a v2 epoch-challenge request in the default replay-scope posture",
);
boot_epoch
}
async fn learn_boot_epoch_from_ping(url: &str, audience: &str) -> Uuid {
let mut headers = mint_v2_headers(audience, "Ping", None);
let challenge = attach_boot_epoch_challenge(&mut headers);
let response = call_ping_response(url, headers)
.await
.expect("v2 Ping with an epoch challenge must bootstrap strict replay scope");
verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
.expect("strict replay-scope Ping must return a valid boot epoch proof")
}
/// Assert a call cleared authentication.
@@ -332,6 +419,122 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
Ok(())
}
/// A replay-scoped signature is usable exactly once against the exact gRPC path and the server
/// process epoch that minted it. This crosses the child-process boundary twice: the HMAC-protected
/// epoch is learned from a real response, then the same server is restarted in place to prove its
/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
#[tokio::test]
#[serial]
async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
init_logging();
align_rpc_secret_with_server();
let child_env = server_env(&[]);
let mut env = start_server_with_env(&child_env).await?;
let url = env.url.clone();
let audience = audience_of(&env);
let boot_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
let request = make_volume_request("replay-scope-e2e-once");
let captured = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
boot_epoch,
);
assert_authenticated(
call_make_volume(&url, request.clone(), captured.clone()).await,
"the first replay-scoped mutation delivery",
);
assert_rejected(
call_make_volume(&url, request.clone(), captured).await,
Code::Unauthenticated,
None,
"the same replay-scoped mutation delivered twice",
);
let transplanted =
mint_replay_scope_headers(&audience, &format!("{TONIC_RPC_PREFIX}/Ping"), &canonical_digest(&request), boot_epoch);
assert_rejected(
call_make_volume(&url, request.clone(), transplanted).await,
Code::Unauthenticated,
None,
"a replay-scoped Ping signature transplanted onto MakeVolume",
);
let stale_epoch = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
boot_epoch,
);
env.restart_server_preserving_data(Vec::new(), &child_env).await?;
rustfs_protos::evict_failed_connection(&url).await;
assert_rejected(
call_make_volume(&url, request.clone(), stale_epoch).await,
Code::Unauthenticated,
None,
"a replay-scoped signature captured before the receiving process restart",
);
let restarted_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
assert_ne!(boot_epoch, restarted_epoch, "a restarted child process must advertise a new boot epoch");
let fresh_epoch = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
restarted_epoch,
);
assert_authenticated(
call_make_volume(&url, request, fresh_epoch).await,
"a replay-scoped mutation signed with the replacement process epoch",
);
stop_server(env, &url).await;
Ok(())
}
/// Strict replay scope leaves one authenticated v2 bootstrap: `Ping` carrying a fresh challenge.
/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
#[tokio::test]
#[serial]
async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
init_logging();
align_rpc_secret_with_server();
let env = start_server(&[(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "true")]).await?;
let url = env.url.clone();
let audience = audience_of(&env);
let v2_request = make_volume_request("replay-scope-e2e-strict-v2");
assert_rejected(
call_make_volume(
&url,
v2_request.clone(),
mint_v2_headers(&audience, "MakeVolume", Some(&canonical_digest(&v2_request))),
)
.await,
Code::Unauthenticated,
None,
"a v2 mutation after replay-scope strictness is enabled",
);
let boot_epoch = learn_boot_epoch_from_ping(&url, &audience).await;
let request = make_volume_request("replay-scope-e2e-strict-v3");
let replay_scoped = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
boot_epoch,
);
assert_authenticated(
call_make_volume(&url, request, replay_scoped).await,
"a replay-scoped mutation after Ping bootstrap under strict replay scope",
);
stop_server(env, &url).await;
Ok(())
}
/// Baseline: correctly signed mutations are accepted, both with and without a
/// body digest.
///
crates/e2e_test/src/reliant/grpc_lock_client.rs
+2 -2
View File
@@ -24,7 +24,7 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
use tonic::Request;
use tracing::{info, warn};
use crate::storage_api::grpc_lock::{TonicInterceptor, node_service_time_out_client_no_auth};
use crate::storage_api::grpc_lock::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
/// gRPC lock client without authentication for testing
/// Similar to RemoteClient but uses no_auth client
@@ -42,7 +42,7 @@ impl GrpcLockClient {
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, TonicInterceptor>,
tonic::service::interceptor::InterceptedService<AuthenticatedChannel, TonicInterceptor>,
>,
> {
node_service_time_out_client_no_auth(&self.addr)
crates/e2e_test/src/storage_api.rs
+8 -4
View File
@@ -16,9 +16,12 @@
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers};
pub(crate) use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
pub(crate) use rustfs_ecstore::api::rpc::{
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
verify_tonic_boot_epoch_response,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
@@ -30,7 +33,7 @@ pub(crate) mod node_interact {
}
pub(crate) mod grpc_lock {
pub(crate) use super::{TonicInterceptor, node_service_time_out_client_no_auth};
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
}
/// Signing/transport surface used by the cross-process internode RPC signature
@@ -40,7 +43,8 @@ pub(crate) mod grpc_lock {
#[cfg(test)]
pub(crate) mod internode_rpc_signature {
pub(crate) use super::{
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
};
}
crates/ecstore/Cargo.toml
+4
View File
@@ -153,6 +153,10 @@ metrics = { workspace = true }
[target.'cfg(target_os = "linux")'.dependencies]
rustfs-uring = "0.2.1"
[target.'cfg(windows)'.dependencies]
winapi-util.workspace = true
windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }
[dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
criterion = { workspace = true, features = ["html_reports"] }
crates/ecstore/src/api/mod.rs
+18 -7
View File
@@ -67,6 +67,14 @@ pub mod bucket {
};
}
pub mod transition_transaction {
pub use crate::bucket::lifecycle::transition_transaction::{
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
inspect_transition_transaction_for_operator,
};
}
pub mod evaluator {
pub use crate::bucket::lifecycle::evaluator::Evaluator;
}
@@ -377,6 +385,7 @@ pub mod metrics {
pub mod notification {
pub use crate::services::notification_sys::{
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
start_remote_version_state_fleet_probe,
};
}
@@ -407,13 +416,15 @@ pub mod rio {
pub mod rpc {
pub use crate::cluster::rpc::{
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature,
AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
};
}
crates/ecstore/src/bucket/lifecycle/transition_transaction.rs
+216 -1
View File
@@ -23,6 +23,7 @@ use uuid::Uuid;
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::bucket::lifecycle::tier_sweeper::{
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
};
@@ -616,6 +617,199 @@ pub enum TransitionTransactionRecoveryOutcome {
Retained,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TransitionOperatorProbe {
Missing,
UnversionedPresent,
VersionedPresent(String),
Ambiguous,
Unsupported,
}
impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
fn from(value: TransitionCandidateProbe) -> Self {
match value {
TransitionCandidateProbe::Missing => Self::Missing,
TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
TransitionCandidateProbe::Unsupported => Self::Unsupported,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct TransitionOperatorStatus {
pub transaction_id: Uuid,
pub state: TransitionTransactionState,
pub tier_name: String,
pub remote_object: String,
pub not_after_unix_nanos: i64,
pub probe: TransitionOperatorProbe,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct TransitionOperatorDeleteResult {
pub status: TransitionOperatorStatus,
pub journal_observed_after_delete: bool,
}
#[derive(Debug, thiserror::Error)]
pub enum TransitionOperatorError {
#[error("transition transaction was not found")]
NotFound,
#[error("transition transaction is still inside its active ownership window")]
NotExpired,
#[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
InvalidState(TransitionTransactionState),
#[error("an exact non-empty remote version is required")]
RemoteVersionRequired,
#[error("remote candidate is not proven missing: {0:?}")]
CandidateNotMissing(TransitionOperatorProbe),
#[error("remote candidate version does not match requested exact version: expected {expected}, observed {actual:?}")]
CandidateVersionMismatch {
expected: String,
actual: TransitionOperatorProbe,
},
#[error("transition transaction store failed: {0}")]
Store(#[source] Error),
#[error("remote tier reconciliation failed: {0}")]
Remote(#[source] std::io::Error),
}
type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
fn validate_operator_reconcile_transaction(
transaction: &TransitionTransaction,
now_unix_nanos: i128,
) -> TransitionOperatorResult<()> {
transaction
.validate()
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
return Err(TransitionOperatorError::InvalidState(transaction.state));
}
if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
return Err(TransitionOperatorError::NotExpired);
}
Ok(())
}
async fn load_operator_reconcile_transaction(
api: Arc<ECStore>,
transaction_id: Uuid,
) -> TransitionOperatorResult<TransitionTransaction> {
match load_transition_transaction_record(api, transaction_id).await {
Ok(transaction) => Ok(transaction),
Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
Err(err) => Err(TransitionOperatorError::Store(err)),
}
}
async fn operator_probe_transition_candidate(
api: Arc<ECStore>,
transaction: &TransitionTransaction,
) -> TransitionOperatorResult<TransitionOperatorProbe> {
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
&api.tier_config_mgr(),
&transaction.tier_name,
transaction.backend_fingerprint,
)
.await
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
lease
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
.await
.map(TransitionOperatorProbe::from)
.map_err(TransitionOperatorError::Remote)
}
pub async fn inspect_transition_transaction_for_operator(
api: Arc<ECStore>,
transaction_id: Uuid,
) -> TransitionOperatorResult<TransitionOperatorStatus> {
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
let probe = operator_probe_transition_candidate(api, &transaction).await?;
Ok(TransitionOperatorStatus {
transaction_id,
state: transaction.state,
tier_name: transaction.tier_name,
remote_object: transaction.remote_object,
not_after_unix_nanos: transaction.not_after_unix_nanos,
probe,
})
}
pub async fn delete_transition_candidate_for_operator(
api: Arc<ECStore>,
transaction_id: Uuid,
remote_version_id: &str,
) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
if remote_version_id.is_empty() {
return Err(TransitionOperatorError::RemoteVersionRequired);
}
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
&api.tier_config_mgr(),
&transaction.tier_name,
transaction.backend_fingerprint,
)
.await
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
lease
.validate_remote_version_id(remote_version_id)
.map_err(TransitionOperatorError::Remote)?;
let before_delete_probe = lease
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
.await
.map(TransitionOperatorProbe::from)
.map_err(TransitionOperatorError::Remote)?;
if !matches!(&before_delete_probe, TransitionOperatorProbe::VersionedPresent(version_id) if version_id == remote_version_id) {
return Err(TransitionOperatorError::CandidateVersionMismatch {
expected: remote_version_id.to_string(),
actual: before_delete_probe,
});
}
delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
.await
.map_err(TransitionOperatorError::Remote)?;
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
Ok(_) => true,
Err(Error::ConfigNotFound) => false,
Err(err) => return Err(TransitionOperatorError::Store(err)),
};
Ok(TransitionOperatorDeleteResult {
status: TransitionOperatorStatus {
transaction_id,
state: transaction.state,
tier_name: transaction.tier_name,
remote_object: transaction.remote_object,
not_after_unix_nanos: transaction.not_after_unix_nanos,
probe,
},
journal_observed_after_delete,
})
}
pub async fn finalize_missing_transition_transaction_for_operator(
api: Arc<ECStore>,
transaction_id: Uuid,
) -> TransitionOperatorResult<()> {
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
if probe != TransitionOperatorProbe::Missing {
return Err(TransitionOperatorError::CandidateNotMissing(probe));
}
delete_transition_transaction_record(api, transaction_id)
.await
.map_err(TransitionOperatorError::Store)
}
pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
let transaction_id = transition_transaction_id_from_record_object_name(object)?;
TransitionTransaction::decode(transaction_id, data)
@@ -700,7 +894,7 @@ async fn recover_unknown_upload_outcome(
.map_err(Error::other)?;
match lease
.probe_transition_candidate(&transaction.remote_object)
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
.await
.map_err(Error::other)?
{
@@ -1097,6 +1291,27 @@ mod tests {
.expect("upload state change should succeed")
}
#[test]
fn operator_reconcile_requires_expired_unknown_upload_outcome() {
let mut transaction = new_transaction();
let active_deadline = transaction.not_after_unix_nanos;
assert!(matches!(
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
));
transaction
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
.expect("unknown upload outcome should be recorded");
assert!(matches!(
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
Err(TransitionOperatorError::NotExpired)
));
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
.expect("expired unknown upload outcome should be eligible");
}
fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
TransitionCleanupProof {
transaction_id: transaction.transaction_id,
crates/ecstore/src/cluster/rpc/background_monitor.rs
-38
View File
@@ -84,44 +84,6 @@ where
)
}
/// Stop a sibling test thread from poisoning tracing's process-global callsite
/// interest cache while this test asserts on span or event context.
///
/// `tracing` caches every callsite's `Interest` in **process-global** state, and
/// the first thread to reach a callsite fixes that value. While at most one
/// dispatcher is registered, tracing-core takes a fast path
/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
/// newly-registered callsite's interest from whichever subscriber is current
/// *on the registering thread*.
///
/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
/// reaches a production callsite first, from a thread with no subscriber
/// installed: the interest is derived from that thread's `NoSubscriber` and
/// cached as `Interest::never()` for the whole process. From then on the
/// callsite is dead for *every* later caller, including a test that installed
/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`
/// (so the monitor's log line lands under the caller's span instead of
/// `recovery-monitor`), and a `warn!`/`info!` event never fires at all.
///
/// Registering a second, inert dispatcher closes both halves of that race:
///
/// * constructing it rebuilds every *already-registered* callsite's interest
/// against the live dispatcher set — which includes the caller's subscriber —
/// repairing whatever a sibling may already have poisoned; and
/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
/// a callsite registered *later* by any thread is resolved against that live
/// set instead of the registering thread's `NoSubscriber`.
///
/// Call this **after** installing the test's subscriber, and hold the returned
/// guard for the rest of the test. Only the `cargo test` fallback needs it:
/// nextest runs each test in its own process, where there is no sibling thread
/// to lose the race to (see `docs/testing/README.md`).
#[cfg(test)]
#[must_use = "callsite interest is only pinned while the returned guard is held"]
pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
}
#[cfg(test)]
mod tests {
use super::*;
crates/ecstore/src/cluster/rpc/client.rs
+214 -9
View File
@@ -12,11 +12,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
#[cfg(test)]
use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
use crate::cluster::rpc::http_auth::{
RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
};
use crate::cluster::rpc::{
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
};
#[cfg(test)]
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
use crate::runtime::sources as runtime_sources;
use http::Uri;
use http::{Request as HttpRequest, Response as HttpResponse, Uri};
use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class,
proto_gen::node_service::{
@@ -24,9 +33,19 @@ use rustfs_protos::{
tier_mutation_control_service_client::TierMutationControlServiceClient,
},
};
use std::{error::Error, io::ErrorKind};
use std::{
collections::HashMap,
error::Error,
future::Future,
io::ErrorKind,
pin::Pin,
sync::{LazyLock, Mutex},
task::{Context, Poll},
};
use tonic::{service::interceptor::InterceptedService, transport::Channel};
use tower::Service;
use tracing::debug;
use uuid::Uuid;
use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
@@ -35,7 +54,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c
pub async fn node_service_time_out_client(
addr: &String,
interceptor: TonicInterceptor,
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
// Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
// `_for_class` variant below (grpc-optimization P1).
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
@@ -44,13 +63,14 @@ pub async fn node_service_time_out_client(
pub async fn heal_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr).await?,
};
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
@@ -59,13 +79,14 @@ pub async fn heal_control_time_out_client(
pub async fn tier_mutation_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr).await?,
};
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
@@ -81,7 +102,7 @@ pub async fn node_service_time_out_client_for_class(
addr: &String,
interceptor: TonicInterceptor,
class: ChannelClass,
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match class {
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
@@ -96,6 +117,7 @@ pub async fn node_service_time_out_client_for_class(
};
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
@@ -103,7 +125,7 @@ pub async fn node_service_time_out_client_for_class(
pub async fn node_service_time_out_client_no_auth(
addr: &String,
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
}
@@ -199,6 +221,104 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
}
}
/// The transport service that learns an authenticated peer boot epoch and adds the replay-scoped
/// signature only after one has been observed. The v1/v2 interceptor stays inside this wrapper so
/// old servers continue receiving precisely the metadata they understand.
#[derive(Clone, Debug)]
pub struct ReplayScopeChannel<S> {
inner: S,
audience: Option<String>,
}
/// The channel type used by internode clients after v2 authentication and replay-scope handling.
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
impl<S> ReplayScopeChannel<S> {
fn new(inner: S, audience: Option<String>) -> Self {
Self { inner, audience }
}
}
fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
}
fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
epochs.insert(audience, epoch);
}
}
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
where
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
S::Error: Send + 'static,
S::Future: Send + 'static,
ReqBody: Send + 'static,
ResBody: Send + 'static,
{
type Response = HttpResponse<ResBody>;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
let authenticated = self.audience.as_ref().is_some_and(|_| {
request
.headers()
.get(RPC_AUTH_VERSION_HEADER)
.and_then(|value| value.to_str().ok())
== Some(RPC_AUTH_VERSION_V2)
});
let challenge = authenticated.then(Uuid::new_v4);
if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
// The challenge is independently HMAC-authenticated by the response proof. It is not
// part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
request.headers_mut().insert(
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
challenge.to_string().parse().expect("UUID must be a valid header value"),
);
if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
cached_peer_boot_epoch(audience),
request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
request
.headers()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok()),
) {
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
Ok(headers) => request.headers_mut().extend(headers),
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
}
}
}
let audience = self.audience.clone();
let future = self.inner.call(request);
Box::pin(async move {
let response = future.await?;
if let (Some(audience), Some(challenge)) = (audience, challenge) {
match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
Err(error)
if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
|| response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
{
debug!(error = %error, "peer boot epoch response proof was rejected")
}
Err(_) => {}
}
}
Ok(response)
})
}
}
pub struct TonicSignatureInterceptor {
audience: Option<String>,
}
@@ -257,6 +377,13 @@ impl TonicInterceptor {
}
Ok(self)
}
fn replay_scope_audience(&self) -> Option<String> {
match self {
Self::Signature(interceptor) => interceptor.audience.clone(),
Self::NoOp(_) => None,
}
}
}
impl tonic::service::Interceptor for TonicInterceptor {
@@ -279,6 +406,38 @@ mod tests {
use tracing_opentelemetry::OpenTelemetrySpanExt;
use tracing_subscriber::{Registry, layer::SubscriberExt};
#[derive(Clone)]
struct EpochProofService {
audience: String,
seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
}
impl Service<HttpRequest<()>> for EpochProofService {
type Response = HttpResponse<()>;
type Error = std::convert::Infallible;
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
self.seen_headers
.lock()
.expect("test header capture lock must not be poisoned")
.push(request.headers().clone());
let challenge = tonic_boot_epoch_challenge(request.headers())
.expect("client challenge must be syntactically valid")
.expect("authenticated client request must carry a boot epoch challenge");
let mut response = HttpResponse::new(());
response.headers_mut().extend(
tonic_boot_epoch_response_headers(&self.audience, challenge)
.expect("test server must be able to sign an epoch proof"),
);
std::future::ready(Ok(response))
}
}
fn ensure_test_rpc_secret() {
runtime_sources::ensure_test_rpc_secret();
}
@@ -420,6 +579,52 @@ mod tests {
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
}
#[test]
fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
ensure_test_rpc_secret();
let audience = "replay-scope-client-test:9000";
PEER_BOOT_EPOCHS
.lock()
.expect("peer epoch cache lock must not be poisoned")
.remove(audience);
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
let service = EpochProofService {
audience: audience.to_string(),
seen_headers: seen_headers.clone(),
};
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
let make_request = || {
let mut request = HttpRequest::builder()
.uri("/node_service.NodeService/Ping")
.body(())
.expect("test RPC request must build");
request.headers_mut().extend(
gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
.expect("v2 test headers must mint"),
);
request
};
futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
assert_eq!(headers.len(), 2);
assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
assert!(
!headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
"the first request must remain v2-compatible until the peer proves its epoch"
);
assert!(
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
"the second request must carry the replay-scoped v3 signature"
);
PEER_BOOT_EPOCHS
.lock()
.expect("peer epoch cache lock must not be poisoned")
.remove(audience);
}
#[test]
fn test_signature_interceptor_requires_generated_method_metadata() {
ensure_test_rpc_secret();
crates/ecstore/src/cluster/rpc/http_auth.rs
+435 -9
View File
@@ -20,8 +20,8 @@
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
//! below, plus the broader negative-signature suite. The advisory class is: a
//! node must never accept an RPC whose auth is missing, malformed, or signed
//! with the default/empty shared secret. Body-bound v2 requests additionally
//! receive process-local replay protection. See
//! with the default/empty shared secret. Body-bound v2 requests and all replay-scoped v3
//! requests additionally receive process-local replay protection. See
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
//!
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
@@ -50,13 +50,22 @@ use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
const RPC_AUTH_VERSION_V2: &str = "2";
pub(crate) const RPC_AUTH_VERSION_V2: &str = "2";
pub const RPC_REPLAY_SCOPE_VERSION_HEADER: &str = "x-rustfs-rpc-replay-scope-version";
pub const RPC_REPLAY_SCOPE_SIGNATURE_HEADER: &str = "x-rustfs-rpc-signature-v3";
pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
@@ -75,9 +84,15 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
)
});
// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
// cannot disable replay protection by rejecting every body-bound request.
static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
get_env_bool(
rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
)
});
// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
rustfs_utils::get_env_usize(
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
@@ -86,6 +101,7 @@ static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
.max(1)
});
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
#[derive(Default)]
struct RpcNonceCache {
@@ -313,6 +329,189 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
mac.verify_slice(&signature).is_ok()
}
#[derive(Clone, Copy)]
struct ReplayScope<'a> {
audience: &'a str,
path: &'a str,
timestamp: &'a str,
nonce: Uuid,
content_sha256: &'a str,
boot_epoch: Uuid,
}
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
for part in [
scope.audience.as_bytes(),
b"|",
scope.path.as_bytes(),
b"|POST|",
scope.timestamp.as_bytes(),
b"|",
scope.nonce.as_bytes(),
b"|",
scope.content_sha256.as_bytes(),
b"|",
scope.boot_epoch.as_bytes(),
] {
mac.update(part);
}
}
fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std::io::Result<String> {
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_replay_scope(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
}
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
return false;
};
update_replay_scope(&mut mac, scope);
mac.verify_slice(&signature).is_ok()
}
fn update_boot_epoch_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
mac.update(RPC_BOOT_EPOCH_PROOF_DOMAIN);
mac.update(audience.as_bytes());
mac.update(b"|");
mac.update(challenge.as_bytes());
mac.update(boot_epoch.as_bytes());
}
fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid) -> std::io::Result<String> {
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
}
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
}
let proof = general_purpose::STANDARD
.decode(proof)
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
mac.verify_slice(&proof)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
}
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
(!value.is_nil())
.then_some(value)
.ok_or_else(|| std::io::Error::other(format!("Invalid {name}")))
}
fn parse_tonic_rpc_path(path: &str) -> std::io::Result<(&str, &str)> {
path.strip_prefix('/')
.and_then(|path| path.split_once('/'))
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))
}
/// The process-unique epoch included in every replay-scoped server verification.
///
/// A fresh process gets a fresh value, so a signature captured before a server restart cannot be
/// admitted even though the bounded in-memory nonce cache necessarily starts empty again.
pub fn tonic_rpc_boot_epoch() -> Uuid {
*RPC_BOOT_EPOCH
}
/// Build the additive replay-scope headers for a request that already carries rolling-upgrade-safe
/// v1/v2 metadata. `timestamp` and `content_sha256` are deliberately reused from the v2 scope so
/// old servers can continue validating the same request unchanged.
pub fn gen_tonic_replay_scope_headers(
audience: &str,
path: &str,
timestamp: &str,
content_sha256: &str,
boot_epoch: Uuid,
) -> std::io::Result<HeaderMap> {
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
}
parse_tonic_rpc_path(path)?;
timestamp
.parse::<i64>()
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
let nonce = Uuid::new_v4();
let signature = generate_replay_scope_signature(
&get_shared_secret()?,
ReplayScope {
audience,
path,
timestamp,
nonce,
content_sha256,
boot_epoch,
},
)?;
let mut headers = HeaderMap::new();
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
headers.insert(
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
header_value(&signature, RPC_REPLAY_SCOPE_SIGNATURE_HEADER)?,
);
headers.insert(
RPC_REPLAY_SCOPE_NONCE_HEADER,
header_value(&nonce.to_string(), RPC_REPLAY_SCOPE_NONCE_HEADER)?,
);
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
Ok(headers)
}
/// Parse the optional client challenge used to authenticate a server boot-epoch advertisement.
pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option<Uuid>> {
headers
.get(RPC_BOOT_EPOCH_CHALLENGE_HEADER)
.map(|value| {
value
.to_str()
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch challenge"))
.and_then(|value| non_nil_uuid(value, "RPC boot epoch challenge"))
})
.transpose()
}
/// Build the authenticated response headers for a client boot-epoch challenge.
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
let boot_epoch = tonic_rpc_boot_epoch();
let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
let mut headers = HeaderMap::new();
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
Ok(headers)
}
/// Verify the server boot-epoch response for a challenge generated by this client.
pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
let boot_epoch = headers
.get(RPC_BOOT_EPOCH_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
let proof = headers
.get(RPC_BOOT_EPOCH_PROOF_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
Ok(boot_epoch)
}
fn valid_content_sha256(value: &str) -> bool {
value == UNSIGNED_PAYLOAD
|| (value.len() == 64
@@ -531,6 +730,17 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
.any(|name| headers.contains_key(*name))
}
fn has_replay_scope_headers(headers: &HeaderMap) -> bool {
[
RPC_REPLAY_SCOPE_VERSION_HEADER,
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
RPC_REPLAY_SCOPE_NONCE_HEADER,
RPC_BOOT_EPOCH_HEADER,
]
.iter()
.any(|name| headers.contains_key(*name))
}
/// Whether the server requires target-bound v2 authentication on every internode gRPC request,
/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
/// lever gated on the v1-fallback counter reading zero fleet-wide; see
@@ -540,9 +750,127 @@ fn internode_rpc_signature_strict() -> bool {
*INTERNODE_RPC_SIGNATURE_STRICT
}
fn internode_rpc_replay_scope_strict() -> bool {
*INTERNODE_RPC_REPLAY_SCOPE_STRICT
}
fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
if audience.is_empty() {
return Err(std::io::Error::other("Missing RPC audience"));
}
parse_tonic_rpc_path(path)?;
let version = headers
.get(RPC_REPLAY_SCOPE_VERSION_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope version"))?;
if version != RPC_REPLAY_SCOPE_VERSION_V3 {
return Err(std::io::Error::other("Unsupported RPC replay scope version"));
}
let signature = headers
.get(RPC_REPLAY_SCOPE_SIGNATURE_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope signature"))?;
let timestamp = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
let signed_at = timestamp
.parse::<i64>()
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
check_timestamp(signed_at)?;
let nonce = headers
.get(RPC_REPLAY_SCOPE_NONCE_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope nonce"))
.and_then(|value| non_nil_uuid(value, "RPC replay scope nonce"))?;
let content_sha256 = headers
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
if !valid_content_sha256(content_sha256) {
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
}
let boot_epoch = headers
.get(RPC_BOOT_EPOCH_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
let secret = get_shared_secret()?;
if !verify_replay_scope_signature(
&secret,
ReplayScope {
audience,
path,
timestamp,
nonce,
content_sha256,
boot_epoch,
},
signature,
) {
return Err(std::io::Error::other("Invalid RPC replay scope signature"));
}
if boot_epoch != tonic_rpc_boot_epoch() {
return Err(std::io::Error::other("RPC boot epoch is stale"));
}
check_and_record_nonce(nonce, signed_at)
}
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
verify_tonic_rpc_signature_with_policy(
audience,
path,
headers,
internode_rpc_signature_strict(),
internode_rpc_replay_scope_strict(),
false,
)
}
/// Verify gRPC authentication while allowing the narrowly scoped v2 `Ping` bootstrap used to
/// obtain an authenticated server boot epoch when replay-scope strictness is enabled.
pub fn verify_tonic_rpc_signature_with_bootstrap(
audience: &str,
path: &str,
headers: &HeaderMap,
allow_replay_scope_bootstrap: bool,
) -> std::io::Result<()> {
verify_tonic_rpc_signature_with_policy(
audience,
path,
headers,
internode_rpc_signature_strict(),
internode_rpc_replay_scope_strict(),
allow_replay_scope_bootstrap,
)
}
fn verify_tonic_rpc_signature_with_policy(
audience: &str,
path: &str,
headers: &HeaderMap,
signature_strict: bool,
replay_scope_strict: bool,
allow_replay_scope_bootstrap: bool,
) -> std::io::Result<()> {
if has_replay_scope_headers(headers) {
return verify_tonic_replay_scope_signature(audience, path, headers);
}
// Only a method-bound v2 Ping with a syntactically valid challenge may bootstrap a strict
// client after its peer restarts. Legacy metadata never gets this exception.
let bootstrap = allow_replay_scope_bootstrap
&& has_v2_auth_headers(headers)
&& tonic_boot_epoch_challenge(headers).is_ok_and(|challenge| challenge.is_some());
if replay_scope_strict && !bootstrap {
return Err(std::io::Error::other("RPC replay-scoped authentication required"));
}
verify_tonic_rpc_signature_with_strictness(audience, path, headers, signature_strict)?;
global_internode_metrics().record_replay_scope_fallback();
Ok(())
}
/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
@@ -1273,6 +1601,104 @@ mod tests {
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
}
#[test]
fn replay_scope_binds_path_epoch_and_random_nonce() {
ensure_test_rpc_secret();
let path = "/node_service.NodeService/Ping";
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("v2 compatibility headers should build");
let timestamp = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 timestamp")
.to_string();
let content_sha256 = headers
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 content digest")
.to_string();
headers.extend(
gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, tonic_rpc_boot_epoch())
.expect("replay-scope headers should build"),
);
assert!(
verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false).is_ok(),
"the first replay-scoped request must be accepted"
);
let replay = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false)
.expect_err("the random replay-scope nonce must be single-use");
assert_eq!(replay.to_string(), "RPC request replay detected");
let path_error = verify_tonic_replay_scope_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
.expect_err("a replay-scoped signature must not move to another method");
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
}
#[test]
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
ensure_test_rpc_secret();
let path = "/node_service.NodeService/Ping";
let mut partial = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("v2 compatibility headers should build");
partial.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
let error = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
.expect_err("partial replay-scope metadata must never downgrade to v2");
assert_eq!(error.to_string(), "Missing RPC replay scope signature");
let timestamp = partial
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 timestamp")
.to_string();
let content_sha256 = partial
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 content digest")
.to_string();
let stale_epoch = Uuid::new_v4();
partial.extend(
gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, stale_epoch)
.expect("replay-scope headers should build"),
);
let stale = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
.expect_err("a signature from a prior server boot epoch must be rejected");
assert_eq!(stale.to_string(), "RPC boot epoch is stale");
}
#[test]
fn replay_scope_strictness_allows_only_authenticated_ping_bootstrap() {
ensure_test_rpc_secret();
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("v2 compatibility headers should build");
let rejected =
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, false)
.expect_err("strict replay scope must reject stripped new metadata");
assert_eq!(rejected.to_string(), "RPC replay-scoped authentication required");
headers.insert(
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
HeaderValue::from_str(&Uuid::new_v4().to_string()).expect("UUID header"),
);
assert!(
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, true,)
.is_ok(),
"only the signed Ping bootstrap may obtain a new server epoch in strict mode"
);
}
#[test]
fn boot_epoch_response_proof_binds_audience_challenge_and_epoch() {
ensure_test_rpc_secret();
let challenge = Uuid::new_v4();
let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("proof headers should build");
let epoch =
verify_tonic_boot_epoch_response("node-a:9000", challenge, &headers).expect("matching proof headers should verify");
assert_eq!(epoch, tonic_rpc_boot_epoch());
assert!(verify_tonic_boot_epoch_response("node-b:9000", challenge, &headers).is_err());
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
}
#[test]
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
ensure_test_rpc_secret();
crates/ecstore/src/cluster/rpc/mod.rs
+10 -7
View File
@@ -23,18 +23,21 @@ pub(crate) mod remote_disk;
pub(crate) mod remote_locker;
pub(crate) mod runtime_sources;
#[cfg(test)]
pub(crate) use background_monitor::pin_callsite_interest_for_test;
pub use background_monitor::shutdown_background_monitors;
pub(crate) use background_monitor::spawn_background_monitor;
pub use client::{
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth,
};
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
#[allow(unused_imports)]
pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+53 -7
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::cluster::rpc::client::{
TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
};
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
@@ -66,7 +66,6 @@ use std::{
use tokio::{net::TcpStream, time::Duration};
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::{debug, info, warn};
use uuid::Uuid;
@@ -222,6 +221,21 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
}
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
if topology_member != expected_member {
return Err(Error::other(
"peer returned a remote version state capability for a different topology member",
));
}
let server_epoch =
Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
if server_epoch.is_nil() {
return Err(Error::other("peer returned a nil remote version state epoch"));
}
Ok(server_epoch)
}
#[derive(Clone, Debug)]
pub struct PeerLiveEventsBatch {
pub events: Vec<u8>,
@@ -233,6 +247,7 @@ pub struct PeerLiveEventsBatch {
pub struct PeerRestClient {
pub host: XHost,
pub grid_host: String,
topology_member: String,
offline: Arc<AtomicBool>,
recovery_running: Arc<AtomicBool>,
}
@@ -325,9 +340,11 @@ impl PeerRestClient {
}
pub fn new(host: XHost, grid_host: String) -> Self {
let topology_member = host.to_string();
Self {
host,
grid_host,
topology_member,
offline: Arc::new(AtomicBool::new(false)),
recovery_running: Arc::new(AtomicBool::new(false)),
}
@@ -347,7 +364,11 @@ impl PeerRestClient {
let client = match grid_host {
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
Ok(host) => Some(PeerRestClient::new(host, grid_host)),
Ok(host) => {
let mut client = PeerRestClient::new(host, grid_host);
client.topology_member = peer_host_port.clone();
Some(client)
}
Err(err) => {
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
None
@@ -390,7 +411,7 @@ impl PeerRestClient {
(remote, all, remote_topology_hosts)
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
if self.offline.load(Ordering::Acquire) {
self.mark_offline_and_spawn_recovery();
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -411,7 +432,7 @@ impl PeerRestClient {
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
InterceptedService<Channel, TonicInterceptor>,
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
>,
> {
if self.offline.load(Ordering::Acquire) {
@@ -432,7 +453,7 @@ impl PeerRestClient {
async fn get_tier_mutation_control_client(
&self,
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
if self.offline.load(Ordering::Acquire) {
self.mark_offline_and_spawn_recovery();
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -1154,6 +1175,15 @@ impl PeerRestClient {
validate_heal_control_capability_proof(&canonical_ack, &proof)
}
pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
let result = self
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
.await?;
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
Ok((self.topology_member.clone(), epoch))
}
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
self.finalize_result(
async {
@@ -2470,6 +2500,22 @@ mod tests {
}
}
#[test]
fn remote_version_state_capability_decoder_fails_closed() {
let epoch = Uuid::new_v4();
let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
.expect("small capability response should encode");
assert_eq!(
decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
epoch
);
assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
.expect("small capability response should encode");
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
}
struct TierMutationResponseFixture<'a> {
version: u32,
phase: TierMutationRpcPhase,
@@ -2745,7 +2791,7 @@ mod tests {
// `mark_offline_and_spawn_recovery` path that sibling tests exercise from
// subscriber-less threads; without this the span can be cached as
// `Interest::never()` and silently degrade to `Span::none()`.
let _callsite_pin = crate::cluster::rpc::pin_callsite_interest_for_test();
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
let client = test_peer_client();
let span = tracing::info_span!("request-span", request_id = "req-peer-rest");
crates/ecstore/src/cluster/rpc/peer_s3_client.rs
+180 -13
View File
@@ -14,7 +14,8 @@
use crate::bucket::metadata_sys;
use crate::cluster::rpc::client::{
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
node_service_time_out_client,
};
use crate::cluster::rpc::set_tonic_mutation_body_digest;
use crate::disk::error::DiskError;
@@ -40,16 +41,84 @@ use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClie
use rustfs_protos::proto_gen::node_service::{
DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
};
#[cfg(test)]
use std::sync::{
Mutex as StdMutex,
atomic::{AtomicBool, Ordering},
};
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
#[cfg(test)]
use tokio::sync::Notify;
use tokio::{net::TcpStream, sync::RwLock, time};
use tokio_util::sync::CancellationToken;
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::{debug, info, warn};
type Client = Arc<Box<dyn PeerS3Client>>;
#[cfg(test)]
#[derive(Default)]
pub(crate) struct DeleteBucketEmptyScanBarrier {
arrived: AtomicBool,
arrived_notify: Notify,
released: AtomicBool,
release_notify: Notify,
}
#[cfg(test)]
impl DeleteBucketEmptyScanBarrier {
pub(crate) async fn wait_until_paused(&self) {
loop {
let notified = self.arrived_notify.notified();
if self.arrived.load(Ordering::Acquire) {
return;
}
notified.await;
}
}
pub(crate) fn release(&self) {
self.released.store(true, Ordering::Release);
self.release_notify.notify_waiters();
}
async fn pause(&self) {
self.arrived.store(true, Ordering::Release);
self.arrived_notify.notify_waiters();
loop {
let notified = self.release_notify.notified();
if self.released.load(Ordering::Acquire) {
return;
}
notified.await;
}
}
}
#[cfg(test)]
static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
#[cfg(test)]
pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
*DELETE_BUCKET_EMPTY_SCAN_BARRIER
.lock()
.expect("empty scan barrier lock should not be poisoned") = Some(barrier.clone());
barrier
}
#[cfg(test)]
async fn pause_after_delete_bucket_empty_scan() {
let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
.lock()
.expect("empty scan barrier lock should not be poisoned")
.take();
if let Some(barrier) = barrier {
barrier.pause().await;
}
}
#[derive(Clone, Debug)]
pub struct ScannerBucketListing {
pub buckets: Vec<BucketInfo>,
@@ -650,24 +719,22 @@ impl PeerS3Client for LocalPeerS3Client {
return Err(Error::ErasureWriteQuorum);
}
let force = if opts.force_if_empty && !opts.force {
if opts.force_if_empty && !opts.force {
for disk in local_disks.iter() {
if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? {
return Err(Error::VolumeNotEmpty);
}
}
true
} else {
opts.force
};
#[cfg(test)]
pause_after_delete_bucket_empty_scan().await;
}
let mut futures = Vec::with_capacity(local_disks.len());
for disk in local_disks.iter() {
// Non-force delete refuses a non-empty bucket (VolumeNotEmpty), which
// the recreate loop below turns into BucketNotEmpty; only an explicit
// force delete removes recursively (backlog#799 B1).
futures.push(disk.delete_volume(bucket, force));
// `force_if_empty` is validation-only. Passing it as force would let
// a PutObject committed after the scan be removed recursively.
futures.push(disk.delete_volume(bucket, opts.force));
}
let results = join_all(futures).await;
@@ -730,6 +797,15 @@ pub struct RemotePeerS3Client {
}
impl RemotePeerS3Client {
fn encode_delete_bucket_options(opts: &DeleteBucketOptions) -> Result<String> {
let mut remote_opts = opts.clone();
// Older peers promote `force_if_empty` to recursive force after their
// metadata scan. Keep this coordinator-only hint off the wire so a
// mixed-version delete fails closed on non-empty directory remnants.
remote_opts.force_if_empty = false;
serde_json::to_string(&remote_opts).map_err(Into::into)
}
fn recovery_monitor_span(addr: &str) -> tracing::Span {
tracing::info_span!(
"recovery-monitor",
@@ -756,7 +832,7 @@ impl RemotePeerS3Client {
client
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
@@ -1040,7 +1116,7 @@ impl PeerS3Client for RemotePeerS3Client {
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
self.execute_with_timeout(
|| async {
let options = serde_json::to_string(opts)?;
let options = Self::encode_delete_bucket_options(opts)?;
let mut client = self.get_client().await?;
let mut request = Request::new(DeleteBucketRequest {
@@ -1414,6 +1490,49 @@ mod tests {
}
}
#[test]
fn remote_delete_bucket_options_fail_closed_for_legacy_peers() {
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
no_lock: true,
no_recreate: true,
force_if_empty: true,
..Default::default()
})
.expect("remote delete options should serialize");
let legacy_opts: DeleteBucketOptions =
serde_json::from_str(&encoded).expect("legacy peer should decode remote delete options");
assert!(legacy_opts.no_lock);
assert!(legacy_opts.no_recreate);
assert!(!legacy_opts.force);
assert!(!legacy_opts.force_if_empty);
let legacy_recursive_force = if legacy_opts.force_if_empty && !legacy_opts.force {
true
} else {
legacy_opts.force
};
assert!(
!legacy_recursive_force,
"legacy peer must not upgrade empty-only delete to recursive force"
);
}
#[test]
fn remote_delete_bucket_options_preserve_explicit_force() {
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
force: true,
force_if_empty: true,
..Default::default()
})
.expect("remote force-delete options should serialize");
let remote_opts: DeleteBucketOptions =
serde_json::from_str(&encoded).expect("remote peer should decode force-delete options");
assert!(remote_opts.force);
assert!(!remote_opts.force_if_empty);
}
#[tokio::test]
async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
let client = test_remote_peer("http://peer-network-error:9000");
@@ -1571,6 +1690,54 @@ mod tests {
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn local_peer_force_if_empty_preserves_unclassified_file_in_selected_pool() {
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for empty-only delete regression");
let disks = init_test_local_disks_for_pools(
&temp_dir,
&[(0, 1), (1, 1)],
"local-peer-force-if-empty-preserves-unclassified-file",
)
.await;
let bucket = "empty-only-delete-bucket";
let marker = "object/commit-marker";
let data = bytes::Bytes::from_static(b"committed object data");
disks[1]
.make_volume(bucket)
.await
.expect("bucket should be created in the selected pool");
disks[1]
.write_all(bucket, marker, data.clone())
.await
.expect("unclassified committed file should be written");
let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
.delete_bucket(
bucket,
&DeleteBucketOptions {
force_if_empty: true,
..Default::default()
},
)
.await
.expect_err("empty-only delete must not recursively remove an unclassified file");
assert_eq!(err, Error::VolumeNotEmpty);
assert_eq!(
disks[1]
.read_all(bucket, marker)
.await
.expect("unclassified committed file should be preserved"),
data
);
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn heal_bucket_local_recreates_missing_bucket_volumes() {
crates/ecstore/src/cluster/rpc/remote_disk.rs
+7 -7
View File
@@ -13,8 +13,8 @@
// limitations under the License.
use crate::cluster::rpc::client::{
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
};
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
use crate::cluster::rpc::internode_data_transport::{
@@ -71,7 +71,7 @@ use tokio::{
time::timeout,
};
use tokio_util::sync::CancellationToken;
use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
use tonic::{Code, Request, service::interceptor::InterceptedService};
use tracing::{debug, trace, warn};
use uuid::Uuid;
@@ -1083,7 +1083,7 @@ impl RemoteDisk {
internode_offline_bypass_reason(&self.addr).map(Error::other)
}
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
if let Some(err) = self.offline_bypass_error() {
return Err(err);
}
@@ -1096,7 +1096,7 @@ impl RemoteDisk {
/// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
/// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
/// is disabled.
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
if let Some(err) = self.offline_bypass_error() {
return Err(err);
}
@@ -5336,7 +5336,7 @@ mod tests {
// production callsites that sibling tests exercise from subscriber-less
// threads; without this they can be cached as `Interest::never()` and go
// silently missing here.
let _callsite_pin = crate::cluster::rpc::pin_callsite_interest_for_test();
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
let endpoint = Endpoint {
url: url::Url::parse("http://127.0.0.1:59996/data").expect("endpoint URL should parse"),
@@ -5395,7 +5395,7 @@ mod tests {
// production callsites that sibling tests exercise from subscriber-less
// threads; without this they can be cached as `Interest::never()` and go
// silently missing here.
let _callsite_pin = crate::cluster::rpc::pin_callsite_interest_for_test();
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
let addr = "http://127.0.0.1:59997".to_string();
let endpoint = Endpoint {
crates/ecstore/src/cluster/rpc/remote_locker.rs
+4 -3
View File
@@ -12,7 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use crate::cluster::rpc::client::{
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
};
use crate::cluster::rpc::set_tonic_mutation_body_digest;
use async_trait::async_trait;
use bytes::Bytes;
@@ -29,7 +31,6 @@ use std::time::Duration;
use tokio::time::timeout;
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::{debug, info, warn};
/// Remote lock client implementation
@@ -78,7 +79,7 @@ impl RemoteClient {
}
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
// P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
// offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
// change quorum; the self-healing re-probe keeps the peer recoverable.
crates/ecstore/src/disk/local.rs
+598 -15
View File
@@ -381,6 +381,291 @@ fn classify_delete_volume_error(err: std::io::Error) -> DiskError {
}
}
#[cfg(unix)]
struct EmptyDirectoryFrame {
path: PathBuf,
name_in_parent: std::ffi::CString,
entries: rustix::fs::Dir,
}
#[cfg(unix)]
fn empty_tree_io_error(err: rustix::io::Errno) -> std::io::Error {
match err {
rustix::io::Errno::NOTDIR | rustix::io::Errno::LOOP => std::io::Error::from(ErrorKind::DirectoryNotEmpty),
_ => err.into(),
}
}
#[cfg(unix)]
fn remove_empty_directory_tree_unix_with(
root: &Path,
mut before_descend: impl FnMut(&Path) -> std::io::Result<()>,
mut before_remove: impl FnMut(&Path) -> std::io::Result<()>,
) -> std::io::Result<()> {
use rustix::{
fs::{AtFlags, Dir, Mode, OFlags, fstat, open, openat, statat, unlinkat},
io::Errno,
};
use std::os::fd::AsFd;
use std::os::unix::ffi::OsStrExt;
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
let root_parent_path = root
.parent()
.ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?;
let root_name = root
.file_name()
.ok_or_else(|| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?
.as_bytes();
let root_name = std::ffi::CString::new(root_name).map_err(|_| std::io::Error::from(ErrorKind::DirectoryNotEmpty))?;
let root_parent = open(root_parent_path, flags, Mode::empty()).map_err(empty_tree_io_error)?;
let root_fd = openat(&root_parent, root_name.as_c_str(), flags, Mode::empty()).map_err(empty_tree_io_error)?;
// Each frame owns one directory iterator/FD, so memory and descriptors are
// bounded by path depth rather than by the number of empty remnants.
let mut stack = vec![EmptyDirectoryFrame {
path: root.to_path_buf(),
name_in_parent: root_name,
entries: Dir::new(root_fd).map_err(empty_tree_io_error)?,
}];
while let Some(mut frame) = stack.pop() {
let next_child = loop {
let Some(entry) = frame.entries.next() else {
break None;
};
let entry = entry.map_err(std::io::Error::from)?;
let name = entry.file_name();
if name.to_bytes() == b"." || name.to_bytes() == b".." {
continue;
}
let name = name.to_owned();
let child_path = frame.path.join(std::ffi::OsStr::from_bytes(name.as_bytes()));
before_descend(&child_path)?;
break Some((child_path, name));
};
if let Some((child_path, name)) = next_child {
let parent = frame.entries.fd().map_err(std::io::Error::from)?;
let child = match openat(parent, name.as_c_str(), flags, Mode::empty()) {
Ok(child) => child,
// A concurrent cleanup may remove an empty child after readdir
// returns it. Resume the parent instead of treating the whole
// bucket as missing and leaving its root behind.
Err(Errno::NOENT) => {
stack.push(frame);
continue;
}
Err(err) => return Err(empty_tree_io_error(err)),
};
stack.push(frame);
stack.push(EmptyDirectoryFrame {
path: child_path,
name_in_parent: name,
entries: Dir::new(child).map_err(empty_tree_io_error)?,
});
continue;
}
before_remove(&frame.path)?;
let parent = if let Some(parent) = stack.last() {
parent.entries.fd().map_err(std::io::Error::from)?
} else {
root_parent.as_fd()
};
let expected = fstat(frame.entries.fd().map_err(std::io::Error::from)?).map_err(empty_tree_io_error)?;
let current = statat(parent, frame.name_in_parent.as_c_str(), AtFlags::SYMLINK_NOFOLLOW).map_err(empty_tree_io_error)?;
if current.st_dev != expected.st_dev || current.st_ino != expected.st_ino {
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
}
match unlinkat(parent, frame.name_in_parent.as_c_str(), AtFlags::REMOVEDIR).map_err(empty_tree_io_error) {
Ok(()) => {}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
}
Ok(())
}
#[cfg(unix)]
async fn remove_empty_directory_tree_with(
root: &Path,
before_descend: impl FnMut(&Path) -> std::io::Result<()>,
before_remove: impl FnMut(&Path) -> std::io::Result<()>,
) -> std::io::Result<()> {
remove_empty_directory_tree_unix_with(root, before_descend, before_remove)
}
#[cfg(unix)]
async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> {
let root = root.to_path_buf();
tokio::task::spawn_blocking(move || remove_empty_directory_tree_unix_with(&root, |_| Ok(()), |_| Ok(()))).await?
}
#[cfg(windows)]
#[derive(Debug)]
struct LockedEmptyDirectory {
handle: winapi_util::Handle,
}
#[cfg(windows)]
fn validate_windows_empty_directory(file_attributes: u64) -> std::io::Result<()> {
const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
const FILE_ATTRIBUTE_REPARSE_POINT: u64 = 0x400;
if file_attributes & FILE_ATTRIBUTE_DIRECTORY == 0 || file_attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
}
Ok(())
}
#[cfg(windows)]
async fn lock_windows_empty_directory(path: &Path, canonical_root: Option<&Path>) -> std::io::Result<LockedEmptyDirectory> {
use std::os::windows::fs::OpenOptionsExt;
use windows_sys::Win32::{
Foundation::GENERIC_READ,
Storage::FileSystem::{DELETE, FILE_SHARE_READ},
};
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
let path = path.to_path_buf();
let canonical_root = canonical_root.map(Path::to_path_buf);
tokio::task::spawn_blocking(move || {
let file = std::fs::OpenOptions::new()
.access_mode(GENERIC_READ | DELETE)
.share_mode(FILE_SHARE_READ)
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
.open(&path)?;
let handle = winapi_util::Handle::from_file(file);
let info = winapi_util::file::information(&handle)?;
validate_windows_empty_directory(info.file_attributes())?;
if let Some(canonical_root) = canonical_root {
let canonical_path = std::fs::canonicalize(path)?;
if !canonical_path.starts_with(canonical_root) {
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
}
}
Ok::<_, std::io::Error>(LockedEmptyDirectory { handle })
})
.await?
}
#[cfg(windows)]
// SAFETY: This helper only passes an owned live handle and one initialized
// FILE_DISPOSITION_INFO to the synchronous Windows deletion API.
#[allow(unsafe_code)]
async fn remove_windows_empty_directory(directory: LockedEmptyDirectory) -> std::io::Result<()> {
tokio::task::spawn_blocking(move || {
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Storage::FileSystem::{FILE_DISPOSITION_INFO, FileDispositionInfo, SetFileInformationByHandle};
let disposition = FILE_DISPOSITION_INFO { DeleteFile: true };
let disposition_size = u32::try_from(std::mem::size_of_val(&disposition))
.map_err(|_| std::io::Error::other("FILE_DISPOSITION_INFO size exceeds the Win32 API limit"))?;
let handle = directory.handle.as_raw_handle();
// SAFETY: `handle` is owned by `directory` and stays live for this synchronous
// call. `disposition` is initialized with the exact structure and byte size
// required by `FileDispositionInfo`; Windows does not retain the pointer.
let deleted = unsafe {
SetFileInformationByHandle(handle, FileDispositionInfo, std::ptr::from_ref(&disposition).cast(), disposition_size)
};
if deleted == 0 {
Err(std::io::Error::last_os_error())
} else {
Ok(())
}
})
.await?
}
#[cfg(windows)]
struct WindowsEmptyDirectoryFrame {
path: PathBuf,
directory: LockedEmptyDirectory,
entries: fs::ReadDir,
}
#[cfg(windows)]
async fn remove_empty_directory_tree_with(
root: &Path,
mut before_descend: impl FnMut(&Path) -> std::io::Result<()>,
mut before_remove: impl FnMut(&Path) -> std::io::Result<()>,
) -> std::io::Result<()> {
let root_directory = lock_windows_empty_directory(root, None).await?;
let canonical_root = fs::canonicalize(root).await?;
let root_entries = fs::read_dir(root).await?;
// Holding each validated directory without delete sharing keeps its path
// generation stable until handle-relative deletion. State is O(depth).
let mut stack = vec![WindowsEmptyDirectoryFrame {
path: root.to_path_buf(),
directory: root_directory,
entries: root_entries,
}];
while let Some(mut frame) = stack.pop() {
match frame.entries.next_entry().await {
Ok(Some(entry)) => {
let child = entry.path();
before_descend(&child)?;
let child_directory = match lock_windows_empty_directory(&child, Some(&canonical_root)).await {
Ok(directory) => directory,
Err(err) if err.kind() == ErrorKind::NotFound => {
stack.push(frame);
continue;
}
Err(err) => return Err(err),
};
let child_entries = match fs::read_dir(&child).await {
Ok(entries) => entries,
Err(err) if err.kind() == ErrorKind::NotFound => {
stack.push(frame);
continue;
}
Err(err) if err.kind() == ErrorKind::NotADirectory => {
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
}
Err(err) => return Err(err),
};
stack.push(frame);
stack.push(WindowsEmptyDirectoryFrame {
path: child,
directory: child_directory,
entries: child_entries,
});
}
Ok(None) => {
before_remove(&frame.path)?;
drop(frame.entries);
match remove_windows_empty_directory(frame.directory).await {
Ok(()) => {}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) if err.kind() == ErrorKind::NotADirectory => {
return Err(std::io::Error::from(ErrorKind::DirectoryNotEmpty));
}
Err(err) => return Err(err),
}
}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
}
Ok(())
}
#[cfg(windows)]
async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> {
remove_empty_directory_tree_with(root, |_| Ok(()), |_| Ok(())).await
}
#[cfg(all(not(unix), not(windows)))]
async fn remove_empty_directory_tree(root: &Path) -> std::io::Result<()> {
fs::remove_dir(root).await
}
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_DISK_LOCAL: &str = "disk_local";
const EVENT_DISK_LOCAL_STARTUP_CLEANUP: &str = "disk_local_startup_cleanup";
@@ -1556,6 +1841,8 @@ static RENAME_DATA_FAIL_COMMIT_RENAME: std::sync::Mutex<Option<String>> = std::s
#[cfg(test)]
static LOCAL_INLINE_ROLLBACK_HARDLINK_FAILURE: std::sync::Mutex<Option<PathBuf>> = std::sync::Mutex::new(None);
#[cfg(test)]
static RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT: std::sync::Mutex<Option<(String, PathBuf)>> = std::sync::Mutex::new(None);
#[cfg(test)]
static DELETE_VERSION_FAIL_AFTER_DATA_STAGED: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());
#[cfg(test)]
static DELETE_VERSION_FAIL_AFTER_COMMIT: std::sync::Mutex<Vec<(PathBuf, String)>> = std::sync::Mutex::new(Vec::new());
@@ -1588,6 +1875,13 @@ fn set_local_inline_rollback_hardlink_failure(dst_path: &Path) {
.expect("test failpoint lock should not be poisoned") = Some(dst_path.to_path_buf());
}
#[cfg(test)]
fn set_rename_data_remove_dst_base_before_commit(dst_path: &str, dst_base: &Path) {
*RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT
.lock()
.expect("test failpoint lock should not be poisoned") = Some((dst_path.to_string(), dst_base.to_path_buf()));
}
#[cfg(test)]
fn set_delete_version_fail_after_data_staged(path: &str) {
DELETE_VERSION_FAIL_AFTER_DATA_STAGED
@@ -1656,6 +1950,19 @@ fn should_fail_local_inline_rollback_hardlink(dst_path: &Path) -> bool {
}
}
#[cfg(test)]
fn remove_dst_base_before_commit(dst_path: &str) -> std::io::Result<()> {
let mut target = RENAME_DATA_REMOVE_DST_BASE_BEFORE_COMMIT
.lock()
.expect("test failpoint lock should not be poisoned");
let Some((_, base)) = target.as_ref().filter(|(target_path, _)| target_path == dst_path) else {
return Ok(());
};
std::fs::remove_dir_all(base)?;
target.take();
Ok(())
}
#[cfg(test)]
fn should_fail_after_delete_data_staged(path: &str) -> bool {
let mut targets = DELETE_VERSION_FAIL_AFTER_DATA_STAGED
@@ -1705,6 +2012,11 @@ fn should_fail_local_inline_rollback_hardlink(_dst_path: &Path) -> bool {
false
}
#[cfg(not(test))]
fn remove_dst_base_before_commit(_dst_path: &str) -> std::io::Result<()> {
Ok(())
}
#[cfg(not(test))]
fn should_fail_after_delete_data_staged(_path: &str) -> bool {
false
@@ -4243,9 +4555,11 @@ impl LocalDisk {
let tmp_path = Self::meta_path(root, RUSTFS_META_TMP_BUCKET);
let tmp_old_path = Self::meta_path(root, RUSTFS_META_TMP_OLD_BUCKET).join(Uuid::new_v4().to_string());
rename_all(&tmp_path, &tmp_old_path, root).await.inspect_err(|err| {
log_startup_disk_error("cleanup_tmp_rename_all", &tmp_path, err);
})?;
rename_all_ignore_missing_source(&tmp_path, &tmp_old_path, root)
.await
.inspect_err(|err| {
log_startup_disk_error("cleanup_tmp_rename_all", &tmp_path, err);
})?;
let tmp_deleted_path = Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET);
tokio::fs::create_dir_all(&tmp_deleted_path).await.inspect_err(|err| {
@@ -7359,6 +7673,10 @@ impl DiskAPI for LocalDisk {
dst_path: &str,
) -> Result<RenameDataResp> {
crate::hp_guard!("LocalDisk::rename_data");
// A non-force DeleteBucket must not remove a directory while a local
// object commit is publishing into it. The peer's empty scan remains
// optimistic; this guard establishes the local commit/delete order.
let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, dst_volume).read_owned().await;
if fi.is_legacy_indexed_delete_marker() {
fi.erasure.index = 0;
}
@@ -7555,6 +7873,7 @@ impl DiskAPI for LocalDisk {
// sequential version did.
tmp_meta_res?;
shard_sync_res?;
remove_dst_base_before_commit(dst_path).map_err(to_file_error)?;
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref()
&& let Err(err) = rename_all(src_data_path, dst_data_path, &skip_parent).await
@@ -7814,6 +8133,7 @@ impl DiskAPI for LocalDisk {
xlmeta.add_version(fi)?;
let version_signature = rename_data_versions_signature(&xlmeta);
let new_buf = xlmeta.marshal_msg()?;
remove_dst_base_before_commit(&dst_path_for_failpoint).map_err(to_file_error)?;
// Write new xl.meta + rename. Inline objects carry their data
// inside xl.meta, so this whole sequence is a metadata commit:
@@ -7838,9 +8158,11 @@ impl DiskAPI for LocalDisk {
{
let old_path = dst_parent.join(old_dir.to_string()).join(STORAGE_FORMAT_FILE_BACKUP);
let old_parent = old_path.parent().map(|p| p.to_path_buf());
if let Some(ref old_parent) = old_parent {
std::fs::create_dir_all(old_parent)?;
}
let _old_parent_guard = old_parent
.as_deref()
.map(|parent| os::mkdir_all_below_existing_base_std(parent, &bucket_dir))
.transpose()
.map_err(to_file_error)?;
// This rollback backup is the sole restore source for a later
// undo_write when the set-level write quorum fails. Persist it as
// durably as the new xl.meta written above (and as the non-inline
@@ -7866,6 +8188,12 @@ impl DiskAPI for LocalDisk {
local_rollback_path = Some(create_local_inline_rollback_backup(&dst, &src, old_metadata)?);
}
#[cfg(windows)]
let _commit_parent_guard = if let Some(parent) = dst.parent() {
Some(os::mkdir_all_below_existing_base_std(parent, &bucket_dir).map_err(to_file_error)?)
} else {
None
};
let commit_result = if should_fail_commit_rename(&dst_path_for_failpoint) {
Err(std::io::Error::other("test fail during metadata commit rename"))
} else {
@@ -7874,7 +8202,8 @@ impl DiskAPI for LocalDisk {
Err(err) if err.kind() == ErrorKind::NotFound && !src.exists() => Ok(()),
Err(err) if err.kind() == ErrorKind::NotFound => {
if let Some(parent) = dst.parent() {
std::fs::create_dir_all(parent)?;
let _parent_guard =
os::mkdir_all_below_existing_base_std(parent, &bucket_dir).map_err(to_file_error)?;
}
std::fs::rename(&src, &dst).map_err(to_file_error)?;
Ok(())
@@ -8791,18 +9120,16 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn delete_volume(&self, volume: &str, force_delete: bool) -> Result<()> {
let p = self.get_bucket_path(volume)?;
let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, volume).write_owned().await;
// Non-force is non-recursive: `remove_dir` (rmdir) fails atomically with
// `DirectoryNotEmpty` -> VolumeNotEmpty if the bucket still holds any
// object data, so a misclassified "dangling" bucket on the heal path
// (or a non-force S3 DeleteBucket on a populated bucket) can never be
// recursively wiped. Only an explicit `force_delete` (e.g. S3 force
// bucket delete) removes recursively. Mirrors MinIO's
// xlStorage.DeleteVol (Remove vs RemoveAll). (backlog#799 B1)
// Non-force removes empty directory remnants children-first with
// non-recursive rmdir calls. A file that exists during the scan, or
// appears before its parent is removed, fails closed with
// VolumeNotEmpty. Only an explicit force delete removes recursively.
let res = if force_delete {
fs::remove_dir_all(&p).await
} else {
fs::remove_dir(&p).await
remove_empty_directory_tree(&p).await
};
if let Err(err) = res {
@@ -9061,6 +9388,35 @@ mod test {
}
}
#[cfg(windows)]
#[test]
fn windows_empty_tree_requires_non_reparse_directory() {
validate_windows_empty_directory(0x10).expect("ordinary directories should be accepted");
assert!(validate_windows_empty_directory(0).is_err());
assert!(validate_windows_empty_directory(0x10 | 0x400).is_err());
}
#[cfg(windows)]
#[tokio::test]
async fn windows_empty_tree_blocks_replacement_at_final_delete_boundary() {
let root = tempfile::tempdir().expect("temporary root should be created");
let bucket_path = root.path().join("bucket");
let child_path = bucket_path.join("child");
fs::create_dir_all(&child_path).await.expect("bucket child should be created");
let canonical_root = fs::canonicalize(&bucket_path).await.expect("bucket path should canonicalize");
let directory = lock_windows_empty_directory(&child_path, Some(&canonical_root))
.await
.expect("child directory should be locked");
std::fs::rename(&child_path, bucket_path.join("replacement"))
.expect_err("the locked directory must not be replaceable at the final deletion boundary");
remove_windows_empty_directory(directory)
.await
.expect("handle-relative deletion should remove the locked directory");
assert!(!child_path.exists(), "the exact locked directory should be removed");
}
async fn ensure_test_volume(disk: &LocalDisk, volume: &str) {
match disk.make_volume(volume).await {
Ok(()) | Err(DiskError::VolumeExists) => {}
@@ -10787,6 +11143,72 @@ mod test {
);
}
#[tokio::test]
#[serial_test::serial(rename_data_deleted_bucket)]
async fn rename_data_non_inline_does_not_recreate_bucket_deleted_before_commit() {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
let bucket = "deleted-before-non-inline-commit";
let object = "prefix/object";
let tmp_object = "tmp-non-inline-delete-race";
let data_dir = Uuid::parse_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee").expect("data dir should parse");
ensure_test_volume(&disk, bucket).await;
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
let staged_data = disk
.get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{data_dir}/part.1"))
.expect("staged data path should resolve");
fs::create_dir_all(staged_data.parent().expect("staged data should have a parent"))
.await
.expect("staged data parent should be created");
fs::write(&staged_data, b"payload")
.await
.expect("staged shard should be written");
let bucket_path = disk.get_bucket_path(bucket).expect("bucket path should resolve");
set_rename_data_remove_dst_base_before_commit(object, &bucket_path);
let fi = test_file_info(object, Uuid::new_v4(), Some(data_dir), None);
let err = disk
.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, fi, bucket, object)
.await
.expect_err("commit must fail after the destination bucket is deleted");
assert_eq!(err, DiskError::FileNotFound);
assert!(!bucket_path.exists(), "rename_data must not recreate the deleted bucket");
assert!(staged_data.exists(), "failed commit must preserve the staged shard");
}
#[tokio::test]
#[serial_test::serial(rename_data_deleted_bucket)]
async fn rename_data_inline_does_not_recreate_bucket_deleted_before_commit() {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
let bucket = "deleted-before-inline-commit";
let object = "prefix/object";
let tmp_object = "tmp-inline-delete-race";
ensure_test_volume(&disk, bucket).await;
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
let bucket_path = disk.get_bucket_path(bucket).expect("bucket path should resolve");
set_rename_data_remove_dst_base_before_commit(object, &bucket_path);
let fi = test_file_info(object, Uuid::new_v4(), None, Some(Bytes::from_static(b"inline payload")));
let err = disk
.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, fi, bucket, object)
.await
.expect_err("inline commit must fail after the destination bucket is deleted");
assert_eq!(err, DiskError::FileNotFound);
assert!(!bucket_path.exists(), "inline rename_data must not recreate the deleted bucket");
assert!(
disk.get_object_path(RUSTFS_META_TMP_BUCKET, &format!("{tmp_object}/{STORAGE_FORMAT_FILE}"))
.expect("staged metadata path should resolve")
.exists(),
"failed inline commit must preserve staged metadata"
);
}
#[test]
fn test_resolve_durability_mode_mapping() {
// Default: nothing set -> strict (current main behavior).
@@ -12645,6 +13067,19 @@ mod test {
assert!(format_info.last_check.is_none(), "cached format timestamp should be cleared");
}
#[tokio::test]
async fn cleanup_tmp_on_startup_allows_missing_tmp_directory() {
use tempfile::tempdir;
let dir = tempdir().expect("operation should succeed");
LocalDisk::cleanup_tmp_on_startup(dir.path(), Arc::new(AtomicU32::new(0)), Arc::new(Notify::new()))
.await
.expect("missing temporary directory should already be clean");
assert!(LocalDisk::meta_path(dir.path(), RUSTFS_META_TMP_DELETED_BUCKET).exists());
}
#[tokio::test]
async fn cleanup_tmp_on_startup_moves_existing_tmp_and_recreates_trash() {
use tempfile::tempdir;
@@ -14517,6 +14952,154 @@ mod test {
let _ = fs::remove_dir_all(&test_dir).await;
}
#[tokio::test]
async fn delete_volume_non_force_removes_nested_empty_directories() {
let root = tempfile::tempdir().expect("temporary disk root should be created");
let endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
let bucket = "nested-empty-bucket";
disk.make_volume(bucket).await.expect("bucket should be created");
fs::create_dir_all(disk.path().join(bucket).join("a/b/c"))
.await
.expect("nested empty directories should be created");
disk.delete_volume(bucket, false)
.await
.expect("non-force delete should remove an empty directory tree");
assert!(matches!(disk.stat_volume(bucket).await, Err(DiskError::VolumeNotFound)));
}
#[tokio::test]
async fn empty_tree_delete_preserves_xlmeta_published_after_scan() {
let root = tempfile::tempdir().expect("temporary disk root should be created");
let bucket_path = root.path().join("bucket");
let object_path = bucket_path.join("object").join(STORAGE_FORMAT_FILE);
fs::create_dir_all(object_path.parent().expect("object path should have a parent"))
.await
.expect("empty object directory should be created");
let data = b"committed object metadata";
let mut published = false;
let err = remove_empty_directory_tree_with(
&bucket_path,
|_| Ok(()),
|directory| {
if !published && directory == object_path.parent().expect("object path should have a parent") {
std::fs::write(&object_path, data)?;
published = true;
}
Ok(())
},
)
.await
.expect_err("rmdir should refuse metadata published after the directory scan");
assert!(published, "test barrier should publish metadata before rmdir");
assert!(matches!(classify_delete_volume_error(err), DiskError::VolumeNotEmpty));
assert_eq!(std::fs::read(&object_path).expect("object metadata should be preserved"), data);
}
#[cfg(unix)]
#[tokio::test]
async fn empty_tree_delete_rejects_child_replaced_with_external_symlink() {
use std::os::unix::fs::symlink;
let root = tempfile::tempdir().expect("temporary root should be created");
let bucket_path = root.path().join("bucket");
let child_path = bucket_path.join("child");
let outside_path = root.path().join("outside");
let outside_empty = outside_path.join("must-remain");
fs::create_dir_all(&child_path).await.expect("bucket child should be created");
fs::create_dir_all(&outside_empty)
.await
.expect("outside directory should be created");
let mut replaced = false;
let err = remove_empty_directory_tree_with(
&bucket_path,
|child| {
if !replaced && child == child_path {
std::fs::remove_dir(&child_path)?;
symlink(&outside_path, &child_path)?;
replaced = true;
}
Ok(())
},
|_| Ok(()),
)
.await
.expect_err("a replaced child must fail closed");
assert!(replaced, "test barrier should replace the child before it is opened");
assert!(matches!(classify_delete_volume_error(err), DiskError::VolumeNotEmpty));
assert!(outside_empty.exists(), "bucket deletion must not remove directories outside the bucket");
assert!(
std::fs::symlink_metadata(&child_path)
.expect("replacement symlink should remain")
.file_type()
.is_symlink()
);
}
#[cfg(unix)]
#[tokio::test]
async fn empty_tree_delete_rechecks_parent_after_child_disappears() {
let root = tempfile::tempdir().expect("temporary root should be created");
let bucket_path = root.path().join("bucket");
let child_path = bucket_path.join("child");
fs::create_dir_all(&child_path).await.expect("bucket child should be created");
let mut removed = false;
remove_empty_directory_tree_with(
&bucket_path,
|child| {
if !removed && child == child_path {
std::fs::remove_dir(&child_path)?;
removed = true;
}
Ok(())
},
|_| Ok(()),
)
.await
.expect("a vanished empty child should not leave the bucket root behind");
assert!(removed, "test hook should remove the child before openat");
assert!(!bucket_path.exists(), "parent should be rechecked and removed after the child disappears");
}
#[cfg(unix)]
#[tokio::test]
async fn empty_tree_delete_rejects_root_generation_replacement() {
let root = tempfile::tempdir().expect("temporary root should be created");
let bucket_path = root.path().join("bucket");
let moved_path = root.path().join("bucket-before-replacement");
fs::create_dir(&bucket_path).await.expect("bucket should be created");
let mut replaced = false;
let err = remove_empty_directory_tree_with(
&bucket_path,
|_| Ok(()),
|directory| {
if !replaced && directory == bucket_path {
std::fs::rename(&bucket_path, &moved_path)?;
std::fs::create_dir(&bucket_path)?;
replaced = true;
}
Ok(())
},
)
.await
.expect_err("a replacement root directory must fail closed");
assert!(replaced, "test hook should replace the root generation before rmdir");
assert!(matches!(classify_delete_volume_error(err), DiskError::VolumeNotEmpty));
assert!(moved_path.exists(), "the originally scanned root should not be removed by its old name");
assert!(bucket_path.exists(), "the replacement root should remain after identity validation fails");
}
#[tokio::test]
async fn test_local_disk_volume_operations() {
let test_dir = "./test_local_disk_volumes";
crates/ecstore/src/disk/os.rs
+321 -9
View File
@@ -25,7 +25,7 @@ use std::{
sync::{Arc, LazyLock, Weak},
};
use tokio::fs;
use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit};
use tracing::warn;
/// Check path length according to OS limits.
@@ -123,6 +123,8 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
let cpu_scaled = cpu_count
@@ -157,6 +159,24 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
limiter
}
/// Serialize a bucket's local metadata commits with physical bucket removal.
///
/// The key includes the canonical disk root, so independently reconnected
/// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
/// disconnected disks do not keep the registry alive.
pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
let key = root.join(volume);
let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
return lock;
}
let lock = Arc::new(RwLock::new(()));
locks.insert(key, Arc::downgrade(&lock));
lock
}
/// Always acquire the per-disk permit before the process-wide permit. Keeping
/// this order uniform prevents one slow disk from reserving global capacity
/// while it waits for its own concurrency slot.
@@ -572,15 +592,14 @@ async fn reliable_rename_inner(
base_dir: impl AsRef<Path>,
warn_on_missing_source: bool,
) -> io::Result<()> {
if let Some(parent) = dst_file_path.as_ref().parent()
&& !file_exists(parent)
{
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
}
let parent_guard = match dst_file_path.as_ref().parent() {
Some(parent) => Some(mkdir_all_below_existing_base(parent, base_dir.as_ref()).await?),
None => None,
};
let mut i = 0;
loop {
if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
if let Err(e) = rename_into_existing_parent(src_file_path.as_ref(), dst_file_path.as_ref(), parent_guard.as_ref()) {
if should_retry_rename(&e, i) {
i += 1;
continue;
@@ -597,6 +616,159 @@ async fn reliable_rename_inner(
Ok(())
}
#[cfg(unix)]
fn rename_into_existing_parent(
src_file_path: &Path,
dst_file_path: &Path,
parent_guard: Option<&ExistingBaseDirectoryGuard>,
) -> io::Result<()> {
use rustix::fs::{Mode, OFlags, open, renameat};
let Some(parent_guard) = parent_guard else {
return super::fs::rename_std(src_file_path, dst_file_path);
};
let src_parent = src_file_path
.parent()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
let src_name = src_file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a file name"))?;
let dst_name = dst_file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?;
let src_parent = open(
src_parent,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)
.map_err(io::Error::from)?;
let dst_parent = parent_guard
.last()
.ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?;
renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from)
}
#[cfg(not(unix))]
fn rename_into_existing_parent(
src_file_path: &Path,
dst_file_path: &Path,
_parent_guard: Option<&ExistingBaseDirectoryGuard>,
) -> io::Result<()> {
super::fs::rename_std(src_file_path, dst_file_path)
}
async fn mkdir_all_below_existing_base(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
let dir_path = dir_path.to_path_buf();
let base_dir = base_dir.to_path_buf();
tokio::task::spawn_blocking(move || mkdir_all_below_existing_base_std(&dir_path, &base_dir)).await?
}
#[cfg(windows)]
pub(crate) type ExistingBaseDirectoryGuard = Vec<winapi_util::Handle>;
#[cfg(unix)]
pub(crate) type ExistingBaseDirectoryGuard = Vec<std::os::fd::OwnedFd>;
#[cfg(all(not(unix), not(windows)))]
pub(crate) type ExistingBaseDirectoryGuard = ();
#[cfg(windows)]
fn lock_windows_directory(path: &Path) -> io::Result<winapi_util::Handle> {
use std::os::windows::fs::OpenOptionsExt;
const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
const FILE_SHARE_READ: u32 = 0x1;
let file = std::fs::OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ)
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
.open(path)?;
let handle = winapi_util::Handle::from_file(file);
let info = winapi_util::file::information(&handle)?;
if info.file_attributes() & FILE_ATTRIBUTE_DIRECTORY == 0
|| info.file_attributes() & u64::from(FILE_ATTRIBUTE_REPARSE_POINT) != 0
{
return Err(io::Error::from(io::ErrorKind::NotADirectory));
}
Ok(handle)
}
pub(crate) fn mkdir_all_below_existing_base_std(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
let relative = dir_path
.strip_prefix(base_dir)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?;
for component in relative.components() {
if !matches!(component, Component::Normal(_) | Component::CurDir) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"rename destination contains an invalid path component",
));
}
}
#[cfg(unix)]
{
use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
use rustix::io::Errno;
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
for component in relative.components() {
let Component::Normal(component) = component else {
continue;
};
let parent = parents
.last()
.expect("base directory guard should contain the base directory");
match mkdirat(parent, component, mode) {
Ok(()) => {}
Err(Errno::EXIST) => {}
Err(err) => return Err(err.into()),
}
parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
}
Ok(parents)
}
#[cfg(windows)]
{
let mut handles = vec![lock_windows_directory(base_dir)?];
let mut current = base_dir.to_path_buf();
for component in relative.components() {
let Component::Normal(component) = component else {
continue;
};
current.push(component);
match std::fs::create_dir(&current) {
Ok(()) => {}
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
Err(err) => return Err(err),
}
handles.push(lock_windows_directory(&current)?);
}
Ok(handles)
}
#[cfg(all(not(unix), not(windows)))]
{
let _ = relative;
Err(io::Error::new(
io::ErrorKind::Unsupported,
"safe recursive directory creation is unavailable on this platform",
))
}
}
fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
warn!(
"reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
@@ -727,6 +899,20 @@ mod tests {
Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
}
#[tokio::test]
async fn disk_volume_mutation_lock_is_shared_per_root_and_volume() {
let temp_dir = tempdir().expect("create temp dir");
let first = disk_volume_mutation_lock(temp_dir.path(), "bucket");
let second = disk_volume_mutation_lock(temp_dir.path(), "bucket");
let other = disk_volume_mutation_lock(temp_dir.path(), "other-bucket");
assert!(Arc::ptr_eq(&first, &second), "reconnected disks must share a bucket mutation lock");
assert!(!Arc::ptr_eq(&first, &other), "different buckets must not serialize each other");
let _write_guard = first.write().await;
assert!(second.try_read().is_err(), "a bucket delete lock must exclude local commits");
}
#[derive(Clone, Default)]
struct CapturedLogs {
buffer: Arc<Mutex<Vec<u8>>>,
@@ -771,9 +957,26 @@ mod tests {
}
}
/// Holds a `warn_capture()` capture alive: the thread-local subscriber, plus
/// the pin that keeps tracing's process-global callsite-interest cache from
/// being decided by some other test's thread.
struct WarnCaptureGuard {
_subscriber: tracing::subscriber::DefaultGuard,
_callsite_pin: tracing::Dispatch,
}
/// Capture WARN-level output on the current thread; tokio tests here run on
/// the current-thread runtime, so the guard covers the whole test body.
fn warn_capture() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
///
/// The callsite pin matters because `warn_reliable_rename_failure` is a
/// single production callsite shared with tests that call `rename_all`
/// *without* installing a subscriber — `rename_all_missing_source_returns_file_not_found`
/// is one. Whichever thread reaches it first fixes its `Interest`
/// process-wide, so without the pin that sibling can cache
/// `Interest::never()` and the WARN never fires here at all, leaving the
/// "must keep the WARN" assertions staring at empty output. See
/// [`crate::test_tracing::pin_callsite_interest_for_test`].
fn warn_capture() -> (CapturedLogs, WarnCaptureGuard) {
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
.with_max_level(tracing::Level::WARN)
@@ -781,7 +984,10 @@ mod tests {
.with_ansi(false)
.without_time()
.finish();
let guard = tracing::subscriber::set_default(subscriber);
let guard = WarnCaptureGuard {
_subscriber: tracing::subscriber::set_default(subscriber),
_callsite_pin: crate::test_tracing::pin_callsite_interest_for_test(),
};
(logs, guard)
}
@@ -968,6 +1174,112 @@ mod tests {
assert_eq!(std::fs::read(dst.join("nested").join("part.1")).expect("read moved part"), b"payload");
}
#[tokio::test]
async fn rename_all_does_not_recreate_missing_base_directory() {
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
std::fs::create_dir(&base).expect("create destination base");
let src = temp_dir.path().join("staged-object");
std::fs::write(&src, b"payload").expect("write staged object");
let dst = base.join("object").join("xl.meta");
std::fs::remove_dir(&base).expect("delete destination base before commit");
let err = rename_all(&src, &dst, &base)
.await
.expect_err("rename must not recreate a deleted destination base");
assert!(matches!(err, DiskError::FileNotFound));
assert!(src.exists(), "failed commit must preserve the staged source");
assert!(!base.exists(), "failed commit must not recreate the deleted bucket");
}
#[cfg(unix)]
#[tokio::test]
async fn rename_all_rejects_a_replaced_base_with_an_existing_parent() {
use std::os::unix::fs::symlink;
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
let outside = temp_dir.path().join("outside");
std::fs::create_dir(&base).expect("create destination base");
std::fs::create_dir_all(outside.join("object")).expect("create outside destination parent");
let src = temp_dir.path().join("staged-object");
std::fs::write(&src, b"payload").expect("write staged object");
let dst = base.join("object").join("xl.meta");
std::fs::remove_dir(&base).expect("remove destination base before replacement");
symlink(&outside, &base).expect("replace destination base with a symlink");
rename_all(&src, &dst, &base)
.await
.expect_err("rename must reject an existing destination parent below a replaced base");
assert!(src.exists(), "rejected rename must preserve the staged source");
assert!(
!outside.join("object/xl.meta").exists(),
"rename must not publish through the replacement symlink"
);
}
#[cfg(windows)]
#[test]
fn windows_parent_guard_blocks_base_and_intermediate_replacement() {
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
std::fs::create_dir(&base).expect("create destination base");
let parent = base.join("object").join("nested");
let guard = mkdir_all_below_existing_base_std(&parent, &base).expect("create and lock destination parents");
std::fs::rename(&base, temp_dir.path().join("replacement-base"))
.expect_err("the locked base must not be replaceable before commit");
std::fs::rename(base.join("object"), base.join("replacement-object"))
.expect_err("a locked intermediate directory must not be replaceable before commit");
drop(guard);
std::fs::rename(base.join("object"), base.join("replacement-object"))
.expect("replacement should succeed after the commit guard is released");
}
#[cfg(unix)]
#[tokio::test]
async fn rename_parent_creation_rejects_symlinked_base() {
use std::os::unix::fs::symlink;
let temp_dir = tempdir().expect("create temp dir");
let outside = temp_dir.path().join("outside");
std::fs::create_dir(&outside).expect("create outside directory");
let base = temp_dir.path().join("bucket");
symlink(&outside, &base).expect("create symlinked base");
mkdir_all_below_existing_base(&base.join("object"), &base)
.await
.expect_err("symlinked base must be rejected");
assert!(!outside.join("object").exists(), "parent creation must remain confined to the base");
}
#[cfg(unix)]
#[tokio::test]
async fn rename_parent_creation_rejects_symlink_below_base() {
use std::os::unix::fs::symlink;
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
let outside = temp_dir.path().join("outside");
std::fs::create_dir(&base).expect("create destination base");
std::fs::create_dir(&outside).expect("create outside directory");
symlink(&outside, base.join("linked")).expect("create symlink below base");
mkdir_all_below_existing_base(&base.join("linked/object"), &base)
.await
.expect_err("symlink below base must be rejected");
assert!(
!outside.join("object").exists(),
"parent creation must not follow a symlink outside the base"
);
}
#[tokio::test]
async fn fsync_dir_succeeds_on_directory() {
let temp_dir = tempdir().expect("create temp dir");
crates/ecstore/src/lib.rs
+3
View File
@@ -93,3 +93,6 @@ pub(crate) mod ecstore_validation_blackbox;
#[cfg(test)]
pub(crate) mod test_metrics;
#[cfg(test)]
pub(crate) mod test_tracing;
crates/ecstore/src/services/notification_sys.rs
+384 -2
View File
@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_madmin::net::NetInfo;
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
use rustfs_utils::XHost;
use std::collections::{HashMap, hash_map::DefaultHasher};
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
use std::future::Future;
use std::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, SystemTime};
use std::time::{Duration, Instant, SystemTime};
use tokio::time::{sleep, timeout};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
@@ -47,6 +47,9 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
/// Cached result from the last successful admin call to a peer.
struct PeerAdminCache {
@@ -91,6 +94,180 @@ lazy_static! {
pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
}
#[derive(Clone)]
struct RemoteVersionStateFleetProof {
topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>,
expires_at: Instant,
}
impl RemoteVersionStateFleetProof {
fn token(&self) -> RemoteVersionStateFleetProofToken {
RemoteVersionStateFleetProofToken {
topology_fingerprint: self.topology_fingerprint.clone(),
peer_epochs: self.peer_epochs.clone(),
}
}
}
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct RemoteVersionStateFleetProofToken {
topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>,
}
#[derive(Default)]
struct RemoteVersionStateFleetProofState {
proof: Option<RemoteVersionStateFleetProof>,
topology_conflict: bool,
}
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
}
fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
}
fn replace_remote_version_state_fleet_proof_in(
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
proof: Option<RemoteVersionStateFleetProof>,
) {
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
}
fn publish_remote_version_state_probe_result(
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
topology_fingerprint: &str,
result: Result<BTreeMap<String, Uuid>>,
observed_at: Instant,
) -> Option<Error> {
match result {
Ok(peer_epochs) => {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
let peer_epochs = state
.proof
.as_ref()
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
.map(|proof| Arc::clone(&proof.peer_epochs))
.unwrap_or_else(|| Arc::new(peer_epochs));
state.proof = Some(RemoteVersionStateFleetProof {
topology_fingerprint: topology_fingerprint.to_string(),
peer_epochs,
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
});
None
}
Err(err) => {
replace_remote_version_state_fleet_proof_in(slot, None);
Some(err)
}
}
}
pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
let state = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
}
fn acquire_remote_version_state_fleet_proof_from(
state: &RemoteVersionStateFleetProofState,
expected_topology: &str,
now: Instant,
) -> Option<RemoteVersionStateFleetProofToken> {
if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
return None;
}
state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
}
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
return false;
};
let state = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.topology_conflict {
return false;
}
state.proof.as_ref().is_some_and(|current| {
current.topology_fingerprint == *expected_topology
&& current.topology_fingerprint == proof.topology_fingerprint
&& Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
&& Instant::now() < current.expires_at
})
}
fn remote_version_state_fleet_proof_valid_at(
proof: Option<&RemoteVersionStateFleetProof>,
expected_topology: &str,
now: Instant,
) -> bool {
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
}
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
return Err(Error::other("remote version state capability peer identity is invalid"));
}
Ok(())
}
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
let mut state = remote_version_state_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.topology_conflict = true;
state.proof = None;
}
return;
}
tokio::spawn(async move {
loop {
let result = match get_global_notification_sys() {
Some(notification_sys) => {
match timeout(
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
)
.await
{
Ok(result) => result,
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
}
}
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
};
let topology_conflict = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.topology_conflict;
if topology_conflict {
replace_remote_version_state_fleet_proof(None);
} else if let Some(err) = publish_remote_version_state_probe_result(
remote_version_state_fleet_proof_slot(),
&topology_fingerprint,
result,
Instant::now(),
) {
debug!(error = %err, "remote version state fleet capability probe failed closed");
}
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
}
});
}
pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
let _ = GLOBAL_NOTIFICATION_SYS
.set(Arc::new(NotificationSys::new(eps).await))
@@ -115,7 +292,17 @@ pub struct NotificationSys {
impl NotificationSys {
pub async fn new(eps: EndpointServerPools) -> Self {
let expected_remote_hosts = eps
.peer_grid_host_slots_sorted()
.into_iter()
.filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
.collect::<Vec<_>>();
let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
let peer_topology_hosts = if peer_topology_hosts.is_empty() {
expected_remote_hosts
} else {
peer_topology_hosts
};
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
Self {
peer_clients,
@@ -125,6 +312,24 @@ impl NotificationSys {
tier_config_reload_workers: Default::default(),
}
}
async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
if self.peer_clients.len() != self.peer_topology_hosts.len() {
return Err(Error::other("remote version state capability fleet membership is incomplete"));
}
let probes = self.peer_clients.iter().map(|client| async {
let client = client
.as_ref()
.ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
client.probe_remote_version_state(topology_fingerprint.to_string()).await
});
let mut peer_epochs = BTreeMap::new();
for result in join_all(probes).await {
let (peer, epoch) = result?;
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
}
Ok(peer_epochs)
}
}
pub struct NotificationPeerErr {
@@ -1890,6 +2095,183 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
mod tests {
use super::*;
#[test]
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
let now = Instant::now();
let mut peer_epochs = BTreeMap::new();
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(peer_epochs),
expires_at: now + Duration::from_secs(1),
};
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
}
#[test]
fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
let mut peer_epochs = BTreeMap::new();
assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
assert!(peer_epochs.is_empty());
}
#[test]
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
let now = Instant::now();
let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
};
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
}
#[test]
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
let now = Instant::now();
let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: now + Duration::from_secs(1),
};
let captured = proof.token();
let restarted = RemoteVersionStateFleetProof {
topology_fingerprint: proof.topology_fingerprint.clone(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: proof.expires_at,
};
assert!(captured != restarted.token());
}
#[test]
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
let now = Instant::now();
let epoch = Uuid::new_v4();
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
let original = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("successful probe should publish proof")
.token();
assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
);
let renewed = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("renewal should retain proof")
.token();
assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
.is_none()
);
let replaced = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("restarted peer should publish a new proof")
.token();
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
}
#[test]
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
let now = Instant::now();
let mut state = RemoteVersionStateFleetProofState {
proof: Some(RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
}),
topology_conflict: false,
};
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
state.topology_conflict = true;
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
}
#[test]
fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
let epoch = Uuid::new_v4();
let mut peer_epochs = BTreeMap::new();
insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
.expect("first member should be admitted");
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
}
#[test]
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
let now = Instant::now();
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
);
assert!(slot.read().expect("proof slot should not poison").proof.is_none());
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
}
#[tokio::test]
async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
let notification_sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: vec![None, None],
peer_topology_hosts: vec!["peer-a".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
tier_config_reload_workers: Default::default(),
};
let err = notification_sys
.probe_remote_version_state_fleet("topology-a")
.await
.expect_err("an unreachable configured member must fail the fleet proof");
assert!(err.to_string().contains("unreachable"));
}
#[tokio::test]
async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
let notification_sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: vec![None],
peer_topology_hosts: vec!["peer-a".to_string()],
peer_admin_caches: Vec::new(),
tier_config_reload_workers: Default::default(),
};
let err = notification_sys
.probe_remote_version_state_fleet("topology-a")
.await
.expect_err("a missing configured member slot must fail the fleet proof");
assert!(err.to_string().contains("incomplete"));
}
fn build_props(endpoint: &str) -> ServerProperties {
ServerProperties {
endpoint: endpoint.to_string(),
crates/ecstore/src/services/tier/tier.rs
+39
View File
@@ -236,6 +236,7 @@ struct TierPublishTransition {
struct PreparedTierDriver {
tier_name: String,
tier_config: TierConfig,
config_fingerprint: TierDriverFingerprint,
backend_identity: TierDestinationId,
exact_get_delete: bool,
@@ -1342,6 +1343,7 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
struct TierDriverGeneration {
tier_name: Arc<str>,
tier_config: TierConfig,
generation: DriverRevision,
// Process-local only: this may reflect credential changes and must never be persisted or logged.
config_fingerprint: TierDriverFingerprint,
@@ -1349,6 +1351,9 @@ struct TierDriverGeneration {
backend_identity: TierDestinationId,
exact_get_delete: bool,
driver: SharedWarmBackend,
reconciler: tokio::sync::OnceCell<
Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
>,
accepting: AtomicBool,
active_leases: AtomicUsize,
drained: tokio::sync::Notify,
@@ -1473,6 +1478,35 @@ impl TierOperationLease {
self.inner.backend_identity
}
pub(crate) async fn probe_transition_candidate_for(
&self,
object: &str,
transaction_id: uuid::Uuid,
) -> io::Result<TransitionCandidateProbe> {
let Some(reconciler) = self
.inner
.reconciler
.get_or_try_init(|| async {
crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
.await
.map(|reconciler| reconciler.map(Arc::from))
})
.await
.map_err(|err| io::Error::other(err.message))?
else {
return self.inner.driver.probe_transition_candidate(object).await;
};
reconciler
.probe_transition_candidate_for(
object,
crate::services::tier::warm_backend::TransitionCandidateIdentity {
transaction_id,
destination_id: self.backend_identity(),
},
)
.await
}
pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
self.inner.driver.validate_remote_version_id(remote_version_id)?;
if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
@@ -2833,6 +2867,7 @@ impl TierConfigMgr {
let exact_get_delete = tier_exact_get_delete(config);
Some(PreparedTierDriver {
tier_name: tier_name.to_string(),
tier_config: config.clone(),
config_fingerprint,
backend_identity,
exact_get_delete,
@@ -2861,11 +2896,13 @@ impl TierConfigMgr {
})?;
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(prepared.tier_name.as_str()),
tier_config: prepared.tier_config.clone(),
generation,
config_fingerprint: prepared.config_fingerprint,
backend_identity: prepared.backend_identity,
exact_get_delete: prepared.exact_get_delete,
driver: prepared.driver.clone(),
reconciler: tokio::sync::OnceCell::new(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
@@ -3609,11 +3646,13 @@ impl TierConfigMgr {
let driver: SharedWarmBackend = Arc::from(driver);
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(tier_name),
tier_config: config.clone(),
generation,
config_fingerprint,
backend_identity,
exact_get_delete,
driver: driver.clone(),
reconciler: tokio::sync::OnceCell::new(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
crates/ecstore/src/services/tier/warm_backend.rs
+105
View File
@@ -73,6 +73,21 @@ pub enum TransitionCandidateProbe {
Unsupported,
}
#[derive(Clone, Copy)]
pub(crate) struct TransitionCandidateIdentity {
pub transaction_id: uuid::Uuid,
pub destination_id: [u8; 32],
}
#[async_trait::async_trait]
pub(crate) trait TransitionCandidateReconciler {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error>;
}
#[async_trait::async_trait]
pub trait WarmBackend {
async fn validate(&self) -> Result<(), std::io::Error> {
@@ -189,6 +204,20 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
metadata.remove(key);
}
for suffix in [
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
] {
for key in [
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
] {
if let Some(value) = metadata.remove(&key) {
metadata.insert(format!("x-amz-meta-{key}"), value);
}
}
}
opts.user_metadata = metadata;
opts
}
@@ -446,6 +475,51 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
Ok(d)
}
pub(crate) async fn new_transition_candidate_reconciler(
tier: &TierConfig,
) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
TierType::S3 => Box::new(
WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
TierType::MinIO => Box::new(
WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
TierType::RustFS => Box::new(
WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
TierType::R2 => Box::new(
WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
_ => return Ok(None),
};
Ok(Some(reconciler))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -775,6 +849,37 @@ mod tests {
assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
}
#[test]
fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
let mut metadata = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
"5a".repeat(32),
);
let opts = build_transition_put_options("COLD".to_string(), metadata);
for suffix in [
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
] {
assert!(opts.user_metadata.contains_key(&format!(
"x-amz-meta-{}",
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
)));
assert!(opts.user_metadata.contains_key(&format!(
"x-amz-meta-{}{suffix}",
rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
)));
}
}
#[test]
fn build_transition_put_options_requests_no_checksum_and_content_md5() {
// Regression for rustfs/rustfs#4811: transition uploads must leave the
crates/ecstore/src/services/tier/warm_backend_minio.rs
+14
View File
@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendMinIO {
}
}
#[async_trait::async_trait]
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
&self.0, object, identity,
)
.await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
crates/ecstore/src/services/tier/warm_backend_r2.rs
+14
View File
@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendR2 {
}
}
#[async_trait::async_trait]
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
&self.0, object, identity,
)
.await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
crates/ecstore/src/services/tier/warm_backend_rustfs.rs
+14
View File
@@ -132,6 +132,20 @@ impl WarmBackend for WarmBackendRustFS {
}
}
#[async_trait::async_trait]
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
&self.0, object, identity,
)
.await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
crates/ecstore/src/services/tier/warm_backend_s3.rs
+157 -1
View File
@@ -37,7 +37,10 @@ use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::services::tier::{
tier_config::TierS3,
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options,
},
};
use http::HeaderMap;
use rustfs_utils::egress::validate_outbound_url;
@@ -219,6 +222,92 @@ impl WarmBackendS3 {
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
}
}
async fn probe_transition_candidate_identity(
&self,
object: &str,
identity: TransitionCandidateIdentity,
bucket_versioning: RemoteBucketVersioning,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let remote_object = self.get_dest(object);
let mut opts = ListObjectsOptions::default();
opts.set("prefix", &remote_object);
opts.set("max-keys", "1000");
let mut key_marker = String::new();
let mut version_id_marker = String::new();
let mut matched_version = None;
let mut saw_unproven_candidate = false;
loop {
let versions = self
.client
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
.await?;
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
let mut stat_opts = GetObjectOptions::default();
stat_opts.version_id.clone_from(&version.version_id);
let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
let mut metadata = info.user_metadata;
for (name, value) in &info.metadata {
if (name
.as_str()
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|| name
.as_str()
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
&& let Ok(value) = value.to_str()
{
metadata.insert(name.as_str().to_string(), value.to_string());
}
}
if transition_candidate_metadata_matches(&metadata, identity)? {
if matched_version.is_some() {
return Ok(TransitionCandidateProbe::Ambiguous);
}
matched_version = Some(version.version_id.clone());
} else {
saw_unproven_candidate = true;
}
}
if !versions.is_truncated {
if matched_version.is_none() && saw_unproven_candidate {
return Ok(TransitionCandidateProbe::Unsupported);
}
let candidates = TransitionCandidateVersions {
version_id: matched_version,
ambiguous: false,
};
return classify_transition_candidates(candidates, bucket_versioning);
}
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
}
}
}
fn transition_candidate_metadata_matches(
metadata: &HashMap<String, String>,
identity: TransitionCandidateIdentity,
) -> Result<bool, std::io::Error> {
use rustfs_utils::http::metadata_compat::{
SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
};
let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
if transaction_id.is_none() || destination_id.is_none() {
if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
|| contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
{
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"transition candidate identity metadata is empty or conflicting",
));
}
return Ok(false);
}
let expected_transaction_id = identity.transaction_id.to_string();
let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
}
fn classify_transition_candidates(
@@ -342,6 +431,60 @@ mod tests {
candidates.classify(bucket_versioning)
}
fn candidate_identity() -> TransitionCandidateIdentity {
TransitionCandidateIdentity {
transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
destination_id: [0x5a; 32],
}
}
fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
let mut metadata = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
identity.transaction_id.to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
rustfs_utils::crypto::hex(identity.destination_id),
);
metadata
}
#[test]
fn transition_candidate_identity_requires_exact_compatible_metadata() {
let identity = candidate_identity();
let metadata = candidate_metadata(identity);
assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
let mut adjacent = metadata;
rustfs_utils::http::metadata_compat::insert_str(
&mut adjacent,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
.unwrap()
.to_string(),
);
assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
}
#[test]
fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
let identity = candidate_identity();
let mut metadata = candidate_metadata(identity);
metadata.insert(
rustfs_utils::http::metadata_compat::internal_key_rustfs(
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
),
uuid::Uuid::new_v4().to_string(),
);
assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
}
#[test]
fn transition_candidate_probe_classifier_is_fail_closed() {
assert_eq!(
@@ -503,3 +646,16 @@ impl WarmBackend for WarmBackendS3 {
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
}
}
#[async_trait::async_trait]
impl TransitionCandidateReconciler for WarmBackendS3 {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let bucket_versioning = self.remote_bucket_versioning().await?;
self.probe_transition_candidate_identity(object, identity, bucket_versioning)
.await
}
}
crates/ecstore/src/set_disk/ops/multipart.rs
+88 -75
View File
@@ -3379,85 +3379,98 @@ mod tests {
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_holds_object_then_upload_lock_through_commit() {
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "multipart-layout-lock-order-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let create_opts = ObjectOptions {
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
..Default::default()
};
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
signaling.set_target(upload_resource.clone());
signaling.clear_observed();
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
temp_env::async_with_vars(
[
(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
],
async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "multipart-layout-lock-order-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let create_opts = ObjectOptions {
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
..Default::default()
};
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
signaling.set_target(upload_resource.clone());
signaling.clear_observed();
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
let observed = signaling.observed();
let object_position = observed
.iter()
.position(|resource| resource == &object_resource)
.expect("completion should acquire the object lock");
let upload_position = observed
.iter()
.position(|resource| resource == &upload_resource)
.expect("completion should acquire the upload lock");
assert!(object_position < upload_position, "completion must acquire object before upload");
let abort_store = set_disks.clone();
let abort_upload_id = upload_id.clone();
let abort = tokio::spawn(async move {
abort_store
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(2).await;
tokio::task::yield_now().await;
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
let list_store = set_disks.clone();
let list_upload_id = upload_id.clone();
let list = tokio::spawn(async move {
list_store
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(3).await;
tokio::task::yield_now().await;
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
barrier.release();
complete
.await
});
barrier.wait_until_paused().await;
let observed = signaling.observed();
let object_position = observed
.iter()
.position(|resource| resource == &object_resource)
.expect("completion should acquire the object lock");
let upload_position = observed
.iter()
.position(|resource| resource == &upload_resource)
.expect("completion should acquire the upload lock");
assert!(object_position < upload_position, "completion must acquire object before upload");
let abort_store = set_disks.clone();
let abort_upload_id = upload_id.clone();
let abort = tokio::spawn(async move {
abort_store
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
.expect("completion task should not panic")
.expect("completion should commit after the barrier is released");
let abort_err = abort
.await
});
signaling.wait_for_attempts(2).await;
tokio::task::yield_now().await;
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
let list_store = set_disks.clone();
let list_upload_id = upload_id.clone();
let list = tokio::spawn(async move {
list_store
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.expect("abort task should not panic")
.expect_err("the committed upload should no longer exist when abort acquires the lock");
assert!(
matches!(abort_err, StorageError::InvalidUploadID(..)),
"abort should return InvalidUploadID after completion, got {abort_err:?}"
);
let list_err = list
.await
});
signaling.wait_for_attempts(3).await;
tokio::task::yield_now().await;
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
barrier.release();
complete
.await
.expect("completion task should not panic")
.expect("completion should commit after the barrier is released");
let abort_err = abort
.await
.expect("abort task should not panic")
.expect_err("the committed upload should no longer exist when abort acquires the lock");
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
let list_err = list
.await
.expect("ListParts task should not panic")
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
})
.expect("ListParts task should not panic")
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
assert!(
matches!(list_err, StorageError::InvalidUploadID(..)),
"ListParts should return InvalidUploadID after completion, got {list_err:?}"
);
},
)
.await;
}
crates/ecstore/src/set_disk/ops/object.rs
+100 -25
View File
@@ -2320,13 +2320,25 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
}
}
#[cfg(test)]
fn persisted_transition_version(
remote_version: &str,
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
}
#[cfg(test)]
fn remote_version_state_writer_enabled() -> bool {
remote_version_state_writer_fleet_proof().is_some()
}
fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
remote_version_state_writer_requested()
.then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
.flatten()
}
fn remote_version_state_writer_requested() -> bool {
remote_version_state_writer_enabled_for(
rustfs_utils::get_env_bool(
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
@@ -2336,11 +2348,25 @@ fn remote_version_state_writer_enabled() -> bool {
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
),
true,
)
}
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
requested && fleet_confirmed
fn remote_version_state_writer_fleet_proof_matches(
proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
) -> bool {
remote_version_state_writer_fleet_proof_matches_for(
remote_version_state_writer_requested(),
crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
)
}
fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
requested && fleet_proof_matches
}
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
requested && fleet_confirmed && fleet_proof_valid
}
fn persisted_transition_version_with_gate(
@@ -2359,7 +2385,7 @@ fn persisted_transition_version_with_gate(
Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"opaque remote tier versions require the operator-attested fleet gate",
"opaque remote tier versions require the operator-attested live fleet capability gate",
)),
Err(_) if remote_version == "null" => {
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
@@ -2605,7 +2631,7 @@ mod transition_upload_completion_tests {
mod transition_version_id_tests {
use super::{
TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
remote_version_state_writer_enabled_for,
remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
};
use rustfs_filemeta::TransitionVersionState;
use uuid::Uuid;
@@ -2648,15 +2674,35 @@ mod transition_version_id_tests {
#[test]
fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
for (case, requested, fleet_confirmed, expected) in [
("old defaults", false, false, false),
("missing fleet confirmation", true, false, false),
("missing local opt-in", false, true, false),
("explicitly unconfirmed fleet", true, false, false),
("rolled-back writer", false, true, false),
("fully upgraded fleet", true, true, true),
for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
("old defaults", false, false, false, false),
("missing fleet confirmation", true, false, true, false),
("missing local opt-in", false, true, true, false),
("missing fleet proof", true, true, false, false),
("explicitly unconfirmed fleet", true, false, true, false),
("rolled-back writer", false, true, true, false),
("fully upgraded fleet", true, true, true, true),
] {
assert_eq!(remote_version_state_writer_enabled_for(requested, fleet_confirmed), expected, "{case}");
assert_eq!(
remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
expected,
"{case}"
);
}
}
#[test]
fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
for (case, requested, fleet_proof_matches, expected) in [
("operator gate closed", false, true, false),
("fleet proof changed", true, false, false),
("current authorization", true, true, true),
] {
assert_eq!(
remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
expected,
"{case}"
);
}
}
@@ -3841,6 +3887,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let dest_obj = transaction.remote_object.clone();
let mut transition_meta = (*oi.user_defined).clone();
transition_meta.insert("name".to_string(), object.to_string());
rustfs_utils::http::metadata_compat::insert_str(
&mut transition_meta,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
transaction.transaction_id.to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut transition_meta,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
rustfs_utils::crypto::hex(transaction.backend_fingerprint),
);
if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
@@ -3951,20 +4007,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
return Err(err.into());
}
let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) {
Ok(version) => version,
Err(err) => {
let cleanup_api = transition_cleanup_store(&self.ctx).await;
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
return Err(StorageError::Io(std::io::Error::other(format!(
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
))));
let fleet_proof = remote_version_state_writer_fleet_proof();
let remote_version_requires_fleet_proof =
!candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
let (transition_version_id, transition_version_state) =
match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
Ok(version) => version,
Err(err) => {
let cleanup_api = transition_cleanup_store(&self.ctx).await;
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
return Err(StorageError::Io(std::io::Error::other(format!(
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
))));
}
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
.await;
return Err(err.into());
}
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
.await;
return Err(err.into());
}
};
};
if let Err(err) = advance_and_save_transition_transaction(
transaction_api.as_ref(),
&mut transaction,
@@ -4080,6 +4140,21 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
#[cfg(test)]
pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
// This check is the fleet-proof lease linearization point. Revocation
// blocks later commits; an already-authorized local quorum commit is
// allowed to finish without holding a synchronous lock across I/O.
if remote_version_requires_fleet_proof
&& !fleet_proof
.as_ref()
.is_some_and(remote_version_state_writer_fleet_proof_matches)
{
drop(transition_lock_guard);
if upload_cleanup.cleanup().await.is_ok() {
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
.await;
}
return Err(Error::other("remote version state fleet capability changed during transition"));
}
if let Err(err) = advance_and_save_transition_transaction(
transaction_api.as_ref(),
&mut transaction,
crates/ecstore/src/store/bucket.rs
+52 -1
View File
@@ -568,6 +568,7 @@ mod tests {
};
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
use crate::bucket::metadata_sys;
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::error::StorageError;
use crate::object_api::{ObjectOptions, PutObjReader};
@@ -589,6 +590,7 @@ mod tests {
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, SystemTime};
use time::OffsetDateTime;
use tokio::io::AsyncReadExt;
use tokio::sync::{Notify, OnceCell};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
@@ -1048,7 +1050,7 @@ mod tests {
.await
.expect("bucket should be created");
for disk_path in &disk_paths {
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory"))
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
.await
.expect("empty directory remnant should be created");
}
@@ -1257,6 +1259,55 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn bucket_delete_preserves_put_committed_after_empty_scan() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
let object = "committed-after-empty-scan";
let payload = b"object committed after DeleteBucket empty scan".to_vec();
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let barrier = install_delete_bucket_empty_scan_barrier();
let delete_store = ecstore.clone();
let delete_bucket = bucket.clone();
let delete = tokio::spawn(async move {
delete_store
.delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
.await
});
barrier.wait_until_paused().await;
let mut put_reader = PutObjReader::from_vec(payload.clone());
ecstore
.put_object(&bucket, object, &mut put_reader, &ObjectOptions::default())
.await
.expect("PUT should commit while DeleteBucket is paused after its empty scan");
barrier.release();
let err = delete
.await
.expect("DeleteBucket task should join")
.expect_err("DeleteBucket must reject a PUT committed after its empty scan");
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
let mut reader = ecstore
.get_object_reader(&bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
.await
.expect("committed object should remain readable after DeleteBucket fails");
let mut restored = Vec::new();
reader
.stream
.read_to_end(&mut restored)
.await
.expect("object body should remain readable");
assert_eq!(restored, payload);
}
#[tokio::test]
#[serial]
async fn bucket_recreation_does_not_publish_unverified_usage() {
crates/ecstore/src/store/init.rs
+191 -6
View File
@@ -562,10 +562,12 @@ mod tests {
},
tier_sweeper::Jentry,
transition_transaction::{
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
save_transition_transaction_record,
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
inspect_transition_transaction_for_operator, load_transition_transaction_record,
recover_transition_transaction_records, save_transition_transaction_record,
},
},
client::transition_api::ReaderImpl,
@@ -575,6 +577,7 @@ mod tests {
services::tier::{
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
tier::{TIER_CONFIG_FILE, TierConfigMgr},
tier_config::{TierConfig, TierType, TierWasabi},
tier_mutation_intent::{
TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
@@ -1163,6 +1166,37 @@ mod tests {
.len()
}
#[cfg(feature = "test-util")]
async fn register_transition_reconcile_test_tier(
handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
tier_name: &str,
) -> MockWarmBackend {
let backend = MockWarmBackend::new();
let mut manager = handle.write().await;
manager.tiers.insert(
tier_name.to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::Wasabi,
name: tier_name.to_string(),
wasabi: Some(TierWasabi {
name: tier_name.to_string(),
endpoint: "https://s3.wasabisys.com".to_string(),
access_key: "test-access-key".to_string(),
secret_key: "test-secret-key".to_string(),
bucket: "mock-tier".to_string(),
prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
region: "us-east-1".to_string(),
}),
..Default::default()
},
);
manager
.install_test_driver(tier_name, Box::new(backend.clone()))
.expect("mock fallback tier driver should install");
backend
}
#[cfg(feature = "test-util")]
async fn wait_for_tier_delete_journal_recovery(
store: Arc<crate::store::ECStore>,
@@ -2738,7 +2772,7 @@ mod tests {
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
@@ -2809,6 +2843,157 @@ mod tests {
}
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXOPERATOR";
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
.backend_identity();
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
transaction_id: uuid::Uuid::new_v4(),
owner_epoch: uuid::Uuid::new_v4(),
write_id: uuid::Uuid::new_v4(),
source: TransitionSourceIdentity {
bucket: "source-bucket".to_string(),
object: "source-object".to_string(),
version_id: None,
data_dir: uuid::Uuid::new_v4(),
mod_time_unix_nanos: 1_770_000_000_000_000_000,
size: 42,
etag: "source-etag".to_string(),
version_mode: TransitionSourceVersionMode::Unversioned,
},
tier_name: tier_name.to_string(),
backend_fingerprint: backend_identity,
not_after_unix_nanos: 1,
})
.expect("transaction should build");
transaction
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
.expect("transaction should enter unknown upload outcome state");
let remote_version = uuid::Uuid::new_v4().to_string();
backend.set_put_remote_version(Some(remote_version.clone())).await;
let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
backend
.put(
&transaction.remote_object,
ReaderImpl::Body(candidate.clone()),
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
)
.await
.expect("mock backend should accept candidate");
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
.await
.expect("operator inspection should probe the candidate");
assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
let wrong_version = uuid::Uuid::new_v4().to_string();
let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
.await
.expect_err("a mismatched exact version must fail before deleting a candidate");
assert!(matches!(
err,
TransitionOperatorError::CandidateVersionMismatch {
expected,
actual: TransitionOperatorProbe::VersionedPresent(ref observed),
} if expected == wrong_version && observed == &remote_version
));
assert!(backend.contains(&transaction.remote_object).await);
assert_eq!(backend.exact_remove_count(), 0);
load_transition_transaction_record(store.clone(), transaction.transaction_id)
.await
.expect("an incorrect exact version must retain the transaction journal");
let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
.await
.expect("operator-confirmed exact candidate should be deleted");
assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
assert!(result.journal_observed_after_delete);
assert_eq!(backend.exact_remove_count(), 1);
assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
load_transition_transaction_record(store.clone(), transaction.transaction_id)
.await
.expect("candidate deletion must retain the transaction journal");
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
.await
.expect("a separately confirmed missing candidate should permit finalization");
assert!(matches!(
load_transition_transaction_record(store, transaction.transaction_id).await,
Err(Error::ConfigNotFound)
));
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn operator_finalize_retains_record_without_missing_proof() {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXOPERATORFAIL";
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
.backend_identity();
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
transaction_id: uuid::Uuid::new_v4(),
owner_epoch: uuid::Uuid::new_v4(),
write_id: uuid::Uuid::new_v4(),
source: TransitionSourceIdentity {
bucket: "source-bucket".to_string(),
object: "source-object".to_string(),
version_id: None,
data_dir: uuid::Uuid::new_v4(),
mod_time_unix_nanos: 1_770_000_000_000_000_000,
size: 42,
etag: "source-etag".to_string(),
version_mode: TransitionSourceVersionMode::Unversioned,
},
tier_name: tier_name.to_string(),
backend_fingerprint: backend_identity,
not_after_unix_nanos: 1,
})
.expect("transaction should build");
transaction
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
.expect("transaction should enter unknown upload outcome state");
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
backend
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
.await;
assert!(matches!(
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
));
load_transition_transaction_record(store, transaction.transaction_id)
.await
.expect("an unsupported probe must retain the transaction journal");
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
@@ -2819,7 +3004,7 @@ mod tests {
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXRESPONSELOSS";
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let bucket = "transition-response-loss-bucket";
let object = "source.bin";
store
crates/ecstore/src/store/list_objects.rs
+1
View File
@@ -6895,6 +6895,7 @@ mod test {
)
.await
.expect("local disk should be created");
disk.make_volume("bucket").await.expect("test bucket should be created");
let mut fi = FileInfo::new("object", 1, 1);
fi.volume = "bucket".to_owned();
fi.name = "object".to_owned();
crates/ecstore/src/test_tracing.rs
+43
View File
@@ -0,0 +1,43 @@
//! Test-only guard against tracing's process-global callsite-interest cache.
//!
//! Any test that installs its own subscriber and then asserts on span or event
//! context is asserting on process-global state. See
//! [`pin_callsite_interest_for_test`] for what goes wrong and why holding its
//! guard fixes it.
/// Stop a sibling test thread from poisoning tracing's process-global callsite
/// interest cache while this test asserts on span or event context.
///
/// `tracing` caches every callsite's `Interest` in **process-global** state, and
/// the first thread to reach a callsite fixes that value. While at most one
/// dispatcher is registered, tracing-core takes a fast path
/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
/// newly-registered callsite's interest from whichever subscriber is current
/// *on the registering thread*, and registration happens exactly once (guarded
/// by a CAS in `DefaultCallsite::register`).
///
/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
/// reaches a production callsite first, from a thread with no subscriber
/// installed: the interest is derived from that thread's `NoSubscriber` and
/// cached as `Interest::never()` for the whole process. From then on the
/// callsite is dead for *every* later caller, including a test that installed
/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`, and
/// a `warn!`/`info!` event never fires at all.
///
/// Registering a second, inert dispatcher closes both halves of that race:
///
/// * constructing it rebuilds every *already-registered* callsite's interest
/// against the live dispatcher set — which includes the caller's subscriber —
/// repairing whatever a sibling may already have poisoned; and
/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
/// a callsite registered *later* by any thread is resolved against that live
/// set instead of the registering thread's `NoSubscriber`.
///
/// Call this **after** installing the test's subscriber, and hold the returned
/// guard for the rest of the test. Only the `cargo test` fallback needs it:
/// nextest runs each test in its own process, where there is no sibling thread
/// to lose the race to (see `docs/testing/README.md`).
#[must_use = "callsite interest is only pinned while the returned guard is held"]
pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
}
crates/io-metrics/src/internode_metrics.rs
+25
View File
@@ -67,6 +67,7 @@ const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
@@ -159,6 +160,7 @@ pub struct InternodeMetricsSnapshot {
pub operation_write_shutdown_errors_total: u64,
pub signature_v1_fallback_total: u64,
pub body_digest_fallback_total: u64,
pub replay_scope_fallback_total: u64,
pub replay_cache_overflow_total: u64,
}
@@ -180,6 +182,7 @@ pub struct InternodeMetrics {
msgpack_json_decode_error_total: AtomicU64,
signature_v1_fallback_total: AtomicU64,
body_digest_fallback_total: AtomicU64,
replay_scope_fallback_total: AtomicU64,
replay_cache_overflow_total: AtomicU64,
}
@@ -431,6 +434,13 @@ impl InternodeMetrics {
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
}
/// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is
/// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`.
pub fn record_replay_scope_fallback(&self) {
self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1);
}
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
/// full. Overflow fails closed, so a sustained non-zero rate means
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
@@ -488,6 +498,7 @@ impl InternodeMetrics {
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
}
}
@@ -510,6 +521,7 @@ impl InternodeMetrics {
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
}
}
@@ -887,6 +899,19 @@ mod tests {
assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
}
#[test]
fn replay_scope_fallback_counter_updates_snapshot_and_resets() {
let metrics = InternodeMetrics::default();
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
metrics.record_replay_scope_fallback();
metrics.record_replay_scope_fallback();
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 2);
metrics.reset_for_test();
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
}
#[test]
fn cluster_peer_flips_offline_after_threshold_and_back_online() {
// Unique addr keeps this independent of the process-global registry / other tests.
crates/protos/src/lib.rs
+72 -3
View File
@@ -171,6 +171,7 @@ pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SI
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
@@ -197,6 +198,49 @@ pub fn is_heal_control_capability_probe(command: &[u8]) -> bool {
command.len() == HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.len() + 16 && command.starts_with(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX)
}
pub fn remote_version_state_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
let mut probe = Vec::with_capacity(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
probe.extend_from_slice(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX);
probe.extend_from_slice(nonce);
probe
}
pub fn is_remote_version_state_capability_probe(command: &[u8]) -> bool {
command.len() == REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + 16
&& command.starts_with(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX)
}
pub fn encode_remote_version_state_capability(
topology_member: &str,
process_epoch: &[u8; 16],
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let topology_member = topology_member.as_bytes();
let mut result = Vec::with_capacity(8 + topology_member.len() + process_epoch.len());
result.extend_from_slice(&u64::try_from(topology_member.len())?.to_be_bytes());
result.extend_from_slice(topology_member);
result.extend_from_slice(process_epoch);
Ok(result)
}
pub fn decode_remote_version_state_capability(result: &[u8]) -> Result<(&str, &[u8; 16]), &'static str> {
let member_len = result
.get(..8)
.and_then(|value| value.try_into().ok())
.map(u64::from_be_bytes)
.ok_or("remote version state capability is truncated")?;
let member_len = usize::try_from(member_len).map_err(|_| "remote version state member length cannot be represented")?;
let member_end = 8_usize
.checked_add(member_len)
.ok_or("remote version state member length overflow")?;
let topology_member = std::str::from_utf8(result.get(8..member_end).ok_or("remote version state member is truncated")?)
.map_err(|_| "remote version state member is not UTF-8")?;
let process_epoch = result
.get(member_end..)
.and_then(|value| value.try_into().ok())
.ok_or("remote version state process epoch has an invalid length")?;
Ok((topology_member, process_epoch))
}
/// Builds the stable byte representation authenticated for a heal-control request.
///
/// This deliberately does not reuse protobuf encoding: mixed-version peers may
@@ -1656,10 +1700,12 @@ mod scanner_activity_tests {
#[cfg(test)]
mod heal_control_tests {
use super::{
HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, canonical_heal_control_capability_ack,
canonical_heal_control_request_body, canonical_heal_control_response_body, heal_control_capability_probe,
HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX,
canonical_heal_control_capability_ack, canonical_heal_control_request_body, canonical_heal_control_response_body,
decode_remote_version_state_capability, encode_remote_version_state_capability, heal_control_capability_probe,
heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
internode_rpc_timeout, is_heal_control_capability_probe, normalize_internode_rpc_timeout,
internode_rpc_timeout, is_heal_control_capability_probe, is_remote_version_state_capability_probe,
normalize_internode_rpc_timeout, remote_version_state_capability_probe,
};
use crate::heal_control;
use std::time::Duration;
@@ -1716,6 +1762,29 @@ mod heal_control_tests {
assert!(!is_heal_control_capability_probe(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX));
}
#[test]
fn remote_version_state_capability_probe_requires_exact_nonce() {
let probe = remote_version_state_capability_probe(&[7; 16]);
assert!(is_remote_version_state_capability_probe(&probe));
assert!(!is_remote_version_state_capability_probe(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX));
}
#[test]
fn remote_version_state_capability_binds_member_and_process_epoch() {
let encoded =
encode_remote_version_state_capability("node-a:9000", &[7; 16]).expect("small capability response should encode");
assert_eq!(
decode_remote_version_state_capability(&encoded).expect("capability response should decode"),
("node-a:9000", &[7; 16])
);
assert!(decode_remote_version_state_capability(&encoded[..encoded.len() - 1]).is_err());
let mut invalid_utf8 =
encode_remote_version_state_capability("node-a", &[7; 16]).expect("small capability response should encode");
invalid_utf8[8] = 0xff;
assert!(decode_remote_version_state_capability(&invalid_utf8).is_err());
}
#[test]
fn canonical_response_binds_request_and_result() {
let baseline = canonical_heal_control_response_body(2, "abcdef", b"query", b"result").unwrap();
crates/scanner/src/data_usage_define.rs
+417 -17
View File
@@ -48,6 +48,7 @@ pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
pub(crate) const DATA_USAGE_CACHE_KEY_FORMAT: u16 = 1;
const DATA_USAGE_CACHE_SAVE_RETRIES: u32 = 2;
const DATA_USAGE_CACHE_BACKUP_SAVE_TIMEOUT_SECS_MAX: u64 = 5;
const DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES: u32 = 0;
@@ -437,6 +438,8 @@ pub struct DataUsageCacheInfo {
pub snapshot_complete: bool,
#[serde(default)]
pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
#[serde(default)]
pub cache_key_format: u16,
}
impl Serialize for DataUsageCacheInfo {
@@ -446,7 +449,7 @@ impl Serialize for DataUsageCacheInfo {
{
// Keep this metadata map-encoded so older readers can ignore fields
// appended by newer scanner versions during rolling upgrades.
let mut state = serializer.serialize_map(Some(15))?;
let mut state = serializer.serialize_map(Some(16))?;
state.serialize_entry("name", &self.name)?;
state.serialize_entry("next_cycle", &self.next_cycle)?;
state.serialize_entry("leader_epoch", &self.leader_epoch)?;
@@ -462,6 +465,7 @@ impl Serialize for DataUsageCacheInfo {
state.serialize_entry("source", &self.source)?;
state.serialize_entry("snapshot_complete", &self.snapshot_complete)?;
state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?;
state.serialize_entry("cache_key_format", &self.cache_key_format)?;
state.end()
}
}
@@ -500,19 +504,34 @@ impl DataUsageCache {
let source_matches = self.info.source == Some(source);
let plan_matches = self.info.scan_plan_digest == Some(scan_plan_digest);
let reusable = self.info.name == name
let metadata_is_reusable = self.info.name == name
&& self.info.leader_epoch == leader_epoch
&& plan_matches
&& (source_matches || (!require_source && self.info.source.is_none()));
&& (source_matches || (!require_source && self.info.source.is_none()))
&& self.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
let reusable = metadata_is_reusable
&& (self.cache.is_empty()
|| if name == DATA_USAGE_ROOT || self.info.snapshot_complete {
self.checked_flatten_complete_scope(name).is_some()
} else {
self.checked_flatten(name).is_some()
});
if !reusable {
let pending_heals = if self.info.name == name {
std::mem::take(&mut self.info.pending_heals)
} else {
Vec::new()
};
*self = Self::default();
self.info.name = name.to_string();
self.info.pending_heals = pending_heals;
}
self.info.next_cycle = next_cycle;
self.info.leader_epoch = leader_epoch;
self.info.source = Some(source);
self.info.scan_plan_digest = Some(scan_plan_digest);
self.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
self.info.snapshot_complete = false;
if reusable {
DataUsageCachePrepareOutcome::Reused
@@ -576,36 +595,60 @@ impl DataUsageCache {
}
pub(crate) fn checked_flatten(&self, path: &str) -> Option<DataUsageEntry> {
self.checked_flatten_inner(path).map(|(entry, _)| entry)
}
pub(crate) fn checked_flatten_complete(&self, path: &str) -> Option<DataUsageEntry> {
self.checked_flatten_inner(path)
.filter(|(_, visited)| *visited == self.cache.len())
.map(|(entry, _)| entry)
}
pub(crate) fn checked_flatten_complete_scope(&self, path: &str) -> Option<DataUsageEntry> {
if path == DATA_USAGE_ROOT {
return self.checked_flatten_complete(path);
}
let (entry, visited) = self.checked_flatten_inner(path)?;
let root_parent_only = {
let path_key = hash_path(path).key();
self.cache
.get(DATA_USAGE_ROOT)
.is_some_and(|root| root_is_parent_only(root, &path_key))
};
let expected_entries = self.cache.len().saturating_sub(usize::from(root_parent_only));
(visited == expected_entries).then_some(entry)
}
fn checked_flatten_inner(&self, path: &str) -> Option<(DataUsageEntry, usize)> {
let root_key = hash_path(path).key();
let root = self.cache.get(&root_key)?;
let mut visited = HashSet::from([root_key]);
let mut pending = root.children.iter().map(|child| (child.clone(), 1usize)).collect::<Vec<_>>();
let (root_key, root) = self.cache.get_key_value(&root_key)?;
if root.compacted && !root.children.is_empty() {
return None;
}
let mut visited = HashSet::from([root_key.as_str()]);
let mut pending = root.children.iter().map(|child| (child.as_str(), 1usize)).collect::<Vec<_>>();
let mut flattened = DataUsageEntry::default();
let mut root_entry = root.clone();
root_entry.children.clear();
if !flattened.checked_merge(&root_entry) {
if !flattened.checked_merge(root) {
return None;
}
flattened.compacted = root.compacted;
while let Some((key, depth)) = pending.pop() {
if depth > MAX_DATA_USAGE_CACHE_DEPTH || !visited.insert(key.clone()) {
if depth > MAX_DATA_USAGE_CACHE_DEPTH || !visited.insert(key) {
return None;
}
let entry = self.cache.get(&key)?;
if depth == MAX_DATA_USAGE_CACHE_DEPTH && !entry.children.is_empty() {
let entry = self.cache.get(key)?;
if (entry.compacted || depth == MAX_DATA_USAGE_CACHE_DEPTH) && !entry.children.is_empty() {
return None;
}
pending.extend(entry.children.iter().map(|child| (child.clone(), depth + 1)));
pending.extend(entry.children.iter().map(|child| (child.as_str(), depth + 1)));
let mut child_entry = entry.clone();
child_entry.children.clear();
if !flattened.checked_merge(&child_entry) {
if !flattened.checked_merge(entry) {
return None;
}
}
Some(flattened)
Some((flattened, visited.len()))
}
fn flatten_with_guard(&self, root: &DataUsageEntry, visited: &mut HashSet<String>, depth: usize) -> DataUsageEntry {
@@ -1524,6 +1567,18 @@ fn mark_with_depth(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut Has
}
}
fn root_is_parent_only(root: &DataUsageEntry, child: &str) -> bool {
root.children.len() == 1
&& root.children.contains(child)
&& root.size == 0
&& root.objects == 0
&& root.versions == 0
&& root.delete_markers == 0
&& root.replication_stats.is_none()
&& !root.compacted
&& root.failed_objects == 0
}
/// Trait for storage-specific operations on DataUsageCache
#[async_trait::async_trait]
pub trait DataUsageCacheStorage {
@@ -2287,6 +2342,7 @@ mod tests {
assert!(decoded.source.is_none());
assert!(!decoded.snapshot_complete);
assert!(decoded.scan_plan_digest.is_none());
assert_eq!(decoded.cache_key_format, 0);
}
#[test]
@@ -2328,6 +2384,7 @@ mod tests {
assert!(decoded.source.is_none());
assert!(!decoded.snapshot_complete);
assert!(decoded.scan_plan_digest.is_none());
assert_eq!(decoded.cache_key_format, 0);
}
#[test]
@@ -2363,6 +2420,7 @@ mod tests {
source: Some(DataUsageCacheSource::new(1, 2)),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -2381,6 +2439,7 @@ mod tests {
assert_eq!(current.info.source, Some(DataUsageCacheSource::new(1, 2)));
assert!(current.info.snapshot_complete);
assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
assert_eq!(current.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
assert_eq!(current.find("bucket").map(|entry| entry.objects), Some(3));
let decoded: OldDataUsageCache = rmp_serde::from_slice(&buf).expect("Old reader failed to deserialize new cache");
@@ -2442,6 +2501,7 @@ mod tests {
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -2466,6 +2526,254 @@ mod tests {
assert!(!cache.info.snapshot_complete);
}
#[test]
fn data_usage_cache_prepare_for_scan_preserves_pending_heal_only_progress() {
let source = DataUsageCacheSource::new(1, 0);
let pending_heal = PendingScannerHeal {
kind: PendingScannerHealKind::Object,
bucket: "bucket".to_string(),
object: Some("prefix/object".to_string()),
version_id: Some("version-a".to_string()),
scan_mode: HealScanMode::Deep,
first_seen: 1,
last_attempt: 2,
attempts: 3,
last_admission_result: "full".to_string(),
last_admission_reason: "queue_full".to_string(),
};
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
pending_heals: vec![pending_heal.clone()],
..Default::default()
},
..Default::default()
};
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reused);
assert_eq!(cache.info.pending_heals, vec![pending_heal]);
assert!(cache.cache.is_empty());
assert!(!cache.info.snapshot_complete);
}
#[test]
fn data_usage_cache_prepare_for_scan_preserves_namespace_pending_heals_during_key_format_rebuild() {
let source = DataUsageCacheSource::new(1, 0);
let pending_heal = PendingScannerHeal {
kind: PendingScannerHealKind::Object,
bucket: "bucket".to_string(),
object: Some("prefix/object".to_string()),
version_id: Some("version-a".to_string()),
scan_mode: HealScanMode::Deep,
first_seen: 1,
last_attempt: 2,
attempts: 3,
last_admission_result: "full".to_string(),
last_admission_reason: "queue_full".to_string(),
};
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
pending_heals: vec![pending_heal.clone()],
..Default::default()
},
..Default::default()
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let outcome = cache.prepare_for_scan(DATA_USAGE_ROOT, 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert_eq!(cache.info.pending_heals, vec![pending_heal]);
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn data_usage_cache_prepare_for_scan_drops_pending_heals_from_a_different_scope() {
let source = DataUsageCacheSource::new(1, 0);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "old-bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
pending_heals: vec![PendingScannerHeal {
kind: PendingScannerHealKind::Object,
bucket: "old-bucket".to_string(),
object: Some("prefix/object".to_string()),
version_id: None,
scan_mode: HealScanMode::Normal,
first_seen: 1,
last_attempt: 2,
attempts: 3,
last_admission_result: "full".to_string(),
last_admission_reason: "queue_full".to_string(),
}],
..Default::default()
},
..Default::default()
};
let outcome = cache.prepare_for_scan("new-bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert_eq!(cache.info.name, "new-bucket");
assert!(cache.info.pending_heals.is_empty());
}
#[test]
fn data_usage_cache_prepare_for_scan_resets_unknown_key_format() {
let source = DataUsageCacheSource::new(1, 0);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT + 1,
..Default::default()
},
..Default::default()
};
cache.replace(
"bucket",
"",
DataUsageEntry {
objects: 3,
..Default::default()
},
);
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn data_usage_cache_prepare_for_scan_resets_persisted_windows_key_mismatch() {
let source = DataUsageCacheSource::new(1, 0);
let root_key = hash_path("bucket").key();
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
..Default::default()
},
..Default::default()
};
cache.cache.insert(
root_key,
DataUsageEntry {
children: HashSet::from(["bucket/prefix".to_string()]),
..Default::default()
},
);
cache.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 3,
..Default::default()
},
);
let encoded = cache.marshal_msg().expect("legacy Windows cache should serialize");
let mut decoded = DataUsageCache::unmarshal(&encoded).expect("legacy Windows cache should deserialize");
let outcome = decoded.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert!(decoded.cache.is_empty());
assert_eq!(decoded.info.name, "bucket");
assert_eq!(decoded.info.next_cycle, 8);
assert_eq!(decoded.info.source, Some(source));
assert!(!decoded.info.snapshot_complete);
}
#[test]
fn data_usage_cache_prepare_for_scan_resets_current_cache_with_dangling_child() {
let source = DataUsageCacheSource::new(1, 0);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
hash_path("bucket").key(),
DataUsageEntry {
children: HashSet::from([hash_path("bucket/missing").key()]),
..Default::default()
},
);
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn data_usage_cache_prepare_for_scan_resets_complete_bucket_cache_with_detached_entry() {
let source = DataUsageCacheSource::new(1, 0);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
hash_path("bucket").key(),
DataUsageEntry {
objects: 1,
..Default::default()
},
);
cache.cache.insert(
hash_path("bucket/detached").key(),
DataUsageEntry {
objects: 2,
..Default::default()
},
);
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
assert!(
cache.checked_flatten_complete("bucket").is_none(),
"complete bucket cache reuse must reject detached entries"
);
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn data_usage_cache_prepare_for_scan_rejects_legacy_cache_without_a_bucket_plan() {
let source = DataUsageCacheSource::new(0, 0);
@@ -2836,6 +3144,98 @@ mod tests {
);
}
#[test]
fn checked_flatten_complete_rejects_detached_entries() {
let mut cache = DataUsageCache::default();
cache.cache.insert(
hash_path("bucket").key(),
DataUsageEntry {
objects: 1,
..Default::default()
},
);
cache.cache.insert(
hash_path("bucket/detached").key(),
DataUsageEntry {
objects: 2,
..Default::default()
},
);
assert_eq!(
cache.checked_flatten("bucket").map(|entry| entry.objects),
Some(1),
"subtree flattening may ignore entries outside the requested subtree"
);
assert!(
cache.checked_flatten_complete("bucket").is_none(),
"an authoritative cache root must reach every persisted entry"
);
}
#[test]
fn checked_flatten_rejects_compacted_entries_with_children() {
let root_key = hash_path("bucket").key();
let child_key = hash_path("bucket/prefix").key();
let mut cache = DataUsageCache::default();
cache.cache.insert(
root_key,
DataUsageEntry {
children: HashSet::from([child_key.clone()]),
compacted: true,
..Default::default()
},
);
cache.cache.insert(
child_key,
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(
cache.checked_flatten_complete("bucket").is_none(),
"a compacted entry cannot retain child links without double-counting"
);
}
#[test]
fn checked_flatten_rejects_compacted_descendants_with_children() {
let root_key = hash_path("bucket").key();
let child_key = hash_path("bucket/prefix").key();
let grandchild_key = hash_path("bucket/prefix/object").key();
let mut cache = DataUsageCache::default();
cache.cache.insert(
root_key,
DataUsageEntry {
children: HashSet::from([child_key.clone()]),
..Default::default()
},
);
cache.cache.insert(
child_key,
DataUsageEntry {
objects: 1,
children: HashSet::from([grandchild_key.clone()]),
compacted: true,
..Default::default()
},
);
cache.cache.insert(
grandchild_key,
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(
cache.checked_flatten("bucket").is_none(),
"a compacted descendant cannot retain child links without double-counting"
);
}
#[test]
fn checked_flatten_accepts_depth_limit_and_rejects_deeper_tree() {
let root_key = hash_path("bucket").key();
crates/scanner/src/remote_scanner.rs
+34 -30
View File
@@ -14,8 +14,8 @@
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig};
use crate::scanner_io::{
ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, cache_snapshot_is_current, scanner_cache_lock_resource,
scanner_cache_lock_timeout, scanner_set_disk_inventory,
DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, current_cache_root_or_prepare,
scanner_cache_lock_resource, scanner_cache_lock_timeout, scanner_set_disk_inventory,
};
use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
use crate::storage_api::scan::NamespaceLocking as _;
@@ -966,32 +966,34 @@ async fn scan_and_persist_local_bucket(
let revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
})?;
if cache_snapshot_is_current(&cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest) {
if guard.is_lock_lost() {
return Err(RemoteScannerServerError::worker(
"remote namespace scanner cache lock was lost before reusing the current snapshot",
));
let scan_state = current_cache_root_or_prepare(&mut cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest, true);
match scan_state {
DataUsageCacheScanState::Current(usage) => {
if guard.is_lock_lost() {
return Err(RemoteScannerServerError::worker(
"remote namespace scanner cache lock was lost before reusing the current snapshot",
));
}
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
source,
scan_plan_digest,
usage: *usage,
pending_maintenance_work: !cache.info.pending_heals.is_empty(),
})));
}
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
source,
scan_plan_digest,
usage: cache_root_entry_info(&cache)
.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache is corrupt: {err}")))?,
pending_maintenance_work: !cache.info.pending_heals.is_empty(),
})));
}
match cache.prepare_for_scan(&bucket, next_cycle, leader_epoch, source, scan_plan_digest, true) {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
return Ok(RemoteScannerFrameResult::CycleAhead {
required_cycle: cache.info.next_cycle,
});
}
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
return Err(RemoteScannerServerError::worker(
"remote namespace scanner rejected work from an older leader epoch",
));
}
DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
DataUsageCacheScanState::Prepared { outcome, .. } => match outcome {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
return Ok(RemoteScannerFrameResult::CycleAhead {
required_cycle: cache.info.next_cycle,
});
}
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
return Err(RemoteScannerServerError::worker(
"remote namespace scanner rejected work from an older leader epoch",
));
}
DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
},
}
cache.info.skip_healing = skip_healing;
@@ -1971,6 +1973,7 @@ mod tests {
#[test]
fn request_rejects_empty_truncated_oversized_and_wrong_version_payloads() {
assert_eq!(NS_SCANNER_PROTOCOL_VERSION, 3);
assert!(decode_remote_scanner_request(&[]).is_err());
let mut body = rmp_serde::to_vec_named(&test_request(Uuid::new_v4())).expect("request should encode");
@@ -1980,7 +1983,7 @@ mod tests {
let oversized = vec![0_u8; NS_SCANNER_MAX_REQUEST_BODY_SIZE + 1];
assert!(decode_remote_scanner_request(&oversized).is_err());
for version in [NS_SCANNER_PROTOCOL_VERSION - 1, NS_SCANNER_PROTOCOL_VERSION + 1] {
for version in [2, 4] {
let mut wrong_version = test_request(Uuid::new_v4());
wrong_version.version = version;
let body = rmp_serde::to_vec_named(&wrong_version).expect("request should encode");
@@ -2885,14 +2888,15 @@ mod tests {
#[tokio::test]
async fn wrong_frame_version_and_sequence_are_rejected() {
assert_eq!(NS_SCANNER_PROTOCOL_VERSION, 3);
let request_id = Uuid::new_v4();
let auth = FrameAuthenticator::for_test(request_id);
let frame = RemoteScannerFrame::progress(RemoteScannerProgress::default());
let payload = rmp_serde::to_vec_named(&frame).expect("frame should encode");
for (version, sequence, expected_error) in [
(NS_SCANNER_PROTOCOL_VERSION - 1, 0, "unsupported remote namespace scanner frame version"),
(NS_SCANNER_PROTOCOL_VERSION + 1, 0, "unsupported remote namespace scanner frame version"),
(2, 0, "unsupported remote namespace scanner frame version"),
(4, 0, "unsupported remote namespace scanner frame version"),
(NS_SCANNER_PROTOCOL_VERSION, 1, "frame sequence is invalid"),
] {
let envelope = RemoteScannerFrameEnvelope {
crates/scanner/src/scanner_folder.rs
+137 -26
View File
@@ -1508,6 +1508,9 @@ impl FolderScanner {
if entry.children.contains(&child) {
continue;
}
if !self.old_cache.cache.contains_key(&child) {
continue;
}
let child_hash = DataUsageHash(child.clone());
self.new_cache
@@ -2320,7 +2323,6 @@ impl FolderScanner {
}
tokio::task::yield_now().await;
let h = DataUsageHash(folder_item.name.clone());
into.add_child(&h);
self.record_scan_resume_hint(&folder_item.name);
// We scanned a folder, optionally send update.
@@ -2646,7 +2648,7 @@ impl FolderScanner {
tokio::task::yield_now().await;
} else {
let mut dst = DataUsageEntry::default();
let h = DataUsageHash(folder_item.name.clone());
let h = hash_path(&folder_item.name);
// Use Box::pin for recursive async call
let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), &mut dst));
@@ -2911,7 +2913,7 @@ mod tests {
use serial_test::serial;
#[cfg(unix)]
use std::os::unix::fs::{PermissionsExt, symlink};
use std::sync::atomic::AtomicBool;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use temp_env::{with_var, with_var_unset};
use uuid::Uuid;
@@ -3864,11 +3866,15 @@ mod tests {
}
async fn write_test_object_metadata(root: &std::path::Path, bucket: &str, object: &str) {
write_test_object_metadata_bytes(root, bucket, object, &metadata_for_object(bucket, object)).await;
}
async fn write_test_object_metadata_bytes(root: &std::path::Path, bucket: &str, object: &str, metadata: &[u8]) {
let object_dir = root.join(bucket).join(object);
tokio::fs::create_dir_all(&object_dir)
.await
.expect("failed to create test object directory");
tokio::fs::write(object_dir.join("xl.meta"), metadata_for_object(bucket, object))
tokio::fs::write(object_dir.join("xl.meta"), metadata)
.await
.expect("failed to write test object metadata");
}
@@ -4165,27 +4171,28 @@ mod tests {
async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let _heal_responder = rustfs_common::heal_channel::init_heal_channel().ok().map(|mut heal_rx| {
tokio::spawn(async move {
while let Some(command) = heal_rx.recv().await {
if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
}
let heal_starts = Arc::new(AtomicUsize::new(0));
let heal_starts_clone = heal_starts.clone();
let mut heal_rx =
rustfs_common::heal_channel::init_heal_channel().expect("heal channel should initialize once for scanner tests");
let _heal_responder = tokio::spawn(async move {
while let Some(command) = heal_rx.recv().await {
if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
heal_starts_clone.fetch_add(1, Ordering::Relaxed);
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
}
})
}
});
let bucket = "src-archive";
tokio::fs::create_dir_all(temp_dir.join(bucket))
.await
.expect("failed to create bucket directory");
let object = "snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4";
let metadata = metadata_for_object(bucket, object);
write_test_object_metadata_bytes(&temp_dir, bucket, object, &metadata).await;
let mut disks = vec![scanner.local_disk.clone()];
for disk_name in ["disk2", "disk3", "disk4"] {
let disk_root = temp_dir.join(disk_name);
tokio::fs::create_dir_all(disk_root.join(bucket))
.await
.expect("failed to create extra disk bucket directory");
write_test_object_metadata_bytes(&disk_root, bucket, object, &metadata).await;
let endpoint =
Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("failed to create extra disk endpoint");
let disk = new_disk(
@@ -4204,7 +4211,7 @@ mod tests {
scanner.disks = disks;
scanner.disks_quorum = 2;
scanner.old_cache.replace(
"src-archive/snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4",
&format!("{bucket}/{object}"),
bucket,
DataUsageEntry {
objects: 1,
@@ -4219,13 +4226,17 @@ mod tests {
object_heal_prob_div: 1,
};
tokio::time::timeout(
Duration::from_millis(200),
scanner.scan_folder(CancellationToken::new(), folder, &mut into),
)
.await
.expect("scan_folder should not hang after list_path_raw finishes")
.expect("scan_folder should finish successfully");
tokio::time::timeout(Duration::from_secs(2), scanner.scan_folder(CancellationToken::new(), folder, &mut into))
.await
.expect("scan_folder should not hang after list_path_raw finishes")
.expect("scan_folder should finish successfully");
let root = scanner
.new_cache
.checked_flatten(bucket)
.expect("healed cache must contain canonical child links");
assert_eq!(root.objects, 1);
assert!(heal_starts.load(Ordering::Relaxed) > 0, "test must execute the heal child-link path");
}
#[tokio::test]
@@ -4774,7 +4785,7 @@ mod tests {
.expect("unbounded scan should finish after partial progress");
let root = result
.size_recursive("bucket")
.checked_flatten("bucket")
.expect("completed cache should retain bucket usage");
assert_eq!(root.objects, 5);
assert!(result.info.snapshot_complete);
@@ -4828,6 +4839,106 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn test_partial_entry_does_not_carry_missing_old_child() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
let root_hash = hash_path("bucket");
scanner.old_cache.cache.insert(
root_hash.key(),
DataUsageEntry {
children: HashSet::from([hash_path("bucket/missing").key()]),
..Default::default()
},
);
let mut partial = DataUsageEntry {
objects: 2,
size: 2,
..Default::default()
};
scanner.carry_forward_old_children(&root_hash, &mut partial);
scanner.new_cache.replace_hashed(&root_hash, &None, &partial);
assert!(partial.children.is_empty());
let flattened = scanner
.new_cache
.checked_flatten("bucket")
.expect("a partial cache must not retain dangling child links");
assert_eq!(flattened.objects, 2);
assert_eq!(flattened.size, 2);
}
#[tokio::test]
#[serial]
async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir.clone()),
};
write_test_object_metadata(&temp_dir, "bucket", "prefix/object").await;
let source = crate::data_usage_define::DataUsageCacheSource::new(0, 0);
let scan_plan_digest = crate::data_usage_define::DataUsageScanPlanDigest([9; 32]);
let mut legacy = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(scan_plan_digest),
..Default::default()
},
..Default::default()
};
legacy.cache.insert(
"bucket".to_string(),
DataUsageEntry {
children: HashSet::from(["bucket\\prefix".to_string()]),
..Default::default()
},
);
legacy.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 1,
..Default::default()
},
);
let encoded = legacy.marshal_msg().expect("legacy cache should serialize");
let mut migrated = DataUsageCache::unmarshal(&encoded).expect("legacy cache should deserialize");
assert_eq!(
migrated.prepare_for_scan("bucket", 8, 0, source, scan_plan_digest, true),
crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
);
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let rebuilt = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk,
migrated,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await
.expect("portable cache rebuild should complete");
let persisted = rebuilt.marshal_msg().expect("rebuilt cache should serialize");
let decoded = DataUsageCache::unmarshal(&persisted).expect("rebuilt cache should deserialize");
assert_eq!(decoded.info.cache_key_format, crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT);
assert!(decoded.cache.keys().all(|key| !key.contains('\\')));
let root = decoded
.checked_flatten("bucket")
.expect("rebuilt persisted cache should have a complete root");
assert_eq!(root.objects, 1);
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_success_clears_resume_hint() {
crates/scanner/src/scanner_io.rs
+337 -59
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT;
use crate::scanner_budget::ScannerCycleBudget;
use crate::scanner_folder::{ScannerItem, scan_data_folder};
use crate::sleeper::SCANNER_SLEEPER;
@@ -830,7 +831,7 @@ pub(crate) fn cache_root_entry_info(cache: &DataUsageCache) -> std::result::Resu
return Err(ScannerError::Other("scanner cache root name is empty".to_string()));
}
let entry = cache
.checked_flatten(&cache.info.name)
.checked_flatten_complete_scope(&cache.info.name)
.ok_or_else(|| ScannerError::Other(format!("scanner cache root is missing or corrupt: {}", cache.info.name)))?;
Ok(DataUsageEntryInfo {
@@ -852,7 +853,7 @@ fn should_publish_completed_snapshot(completed_count: usize, total_count: usize,
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum NamespaceScannerWorkerMode {
Coordinator,
RemoteV3(uuid::Uuid),
RemoteV4(uuid::Uuid),
}
fn namespace_scanner_workers<T>(
@@ -868,7 +869,7 @@ fn namespace_scanner_workers<T>(
workers.extend(
remote_disks
.into_iter()
.map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV3(server_epoch))),
.map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV4(server_epoch))),
);
workers
}
@@ -958,7 +959,57 @@ where
let _ = tokio::time::timeout(SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan).await;
}
pub(crate) fn cache_snapshot_is_current(
pub(crate) fn current_cache_root_entry(
cache: &DataUsageCache,
name: &str,
source: DataUsageCacheSource,
next_cycle: u64,
leader_epoch: u64,
scan_plan_digest: DataUsageScanPlanDigest,
) -> std::result::Result<Option<DataUsageEntryInfo>, ScannerError> {
let metadata_is_current = cache.info.name == name
&& cache.info.source == Some(source)
&& cache.info.snapshot_complete
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
&& cache.info.last_update.is_some()
&& cache.info.next_cycle == next_cycle
&& cache.info.leader_epoch == leader_epoch
&& cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
if !metadata_is_current {
return Ok(None);
}
cache_root_entry_info(cache).map(Some)
}
pub(crate) enum DataUsageCacheScanState {
Current(Box<DataUsageEntryInfo>),
Prepared {
outcome: DataUsageCachePrepareOutcome,
invalid_current: Option<ScannerError>,
},
}
pub(crate) fn current_cache_root_or_prepare(
cache: &mut DataUsageCache,
name: &str,
source: DataUsageCacheSource,
next_cycle: u64,
leader_epoch: u64,
scan_plan_digest: DataUsageScanPlanDigest,
require_source: bool,
) -> DataUsageCacheScanState {
match current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest) {
Ok(Some(root)) => DataUsageCacheScanState::Current(Box::new(root)),
current => DataUsageCacheScanState::Prepared {
invalid_current: current.err(),
outcome: cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, require_source),
},
}
}
#[cfg(test)]
fn cache_snapshot_is_current(
cache: &DataUsageCache,
name: &str,
source: DataUsageCacheSource,
@@ -966,13 +1017,10 @@ pub(crate) fn cache_snapshot_is_current(
leader_epoch: u64,
scan_plan_digest: DataUsageScanPlanDigest,
) -> bool {
cache.info.name == name
&& cache.info.source == Some(source)
&& cache.info.snapshot_complete
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
&& cache.info.last_update.is_some()
&& cache.info.next_cycle == next_cycle
&& cache.info.leader_epoch == leader_epoch
matches!(
current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest),
Ok(Some(_))
)
}
fn completed_data_usage_info(
@@ -1061,6 +1109,7 @@ mod publish_gate_tests {
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -1102,6 +1151,7 @@ mod publish_gate_tests {
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -1423,17 +1473,145 @@ mod publish_gate_tests {
assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST));
}
#[test]
fn current_cache_snapshot_rejects_persisted_windows_key_mismatch() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 10,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
"bucket".to_string(),
DataUsageEntry {
children: HashSet::from(["bucket/prefix".to_string()]),
..Default::default()
},
);
cache.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 3,
..Default::default()
},
);
assert!(!cache_snapshot_is_current(&cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST));
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
invalid_current: Some(_),
} => {}
_ => panic!("an invalid current cache must enter the rebuild path"),
}
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn current_cache_snapshot_rejects_structurally_valid_legacy_key_format() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 10,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
..Default::default()
},
..Default::default()
};
cache.cache.insert(
"bucket".to_string(),
DataUsageEntry {
children: HashSet::from(["bucket\\prefix".to_string()]),
..Default::default()
},
);
cache.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 3,
..Default::default()
},
);
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(3));
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
invalid_current: None,
} => {}
_ => panic!("a legacy key format must enter the rebuild path"),
}
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn current_cache_snapshot_rejects_current_bucket_cache_with_detached_entry() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 10,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
"bucket".to_string(),
DataUsageEntry {
objects: 1,
..Default::default()
},
);
cache.cache.insert(
"bucket/detached".to_string(),
DataUsageEntry {
objects: 2,
..Default::default()
},
);
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
invalid_current: Some(_),
} => {}
_ => panic!("a detached complete bucket cache must enter the rebuild path"),
}
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn namespace_scanner_worker_selection_keeps_coordinator_fallback_disks() {
let server_epoch = uuid::Uuid::new_v4();
let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v3", server_epoch)]);
let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v4", server_epoch)]);
assert_eq!(
workers,
vec![
("local", NamespaceScannerWorkerMode::Coordinator),
("legacy-remote", NamespaceScannerWorkerMode::Coordinator),
("v3", NamespaceScannerWorkerMode::RemoteV3(server_epoch)),
("v4", NamespaceScannerWorkerMode::RemoteV4(server_epoch)),
]
);
assert!(namespace_scanner_workers::<()>(Vec::new(), Vec::new()).is_empty());
@@ -1507,6 +1685,7 @@ mod publish_gate_tests {
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(first),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -1595,6 +1774,15 @@ async fn send_cache_root_entry_info(
pending_maintenance_work: &AtomicBool,
) -> std::result::Result<(), ScannerError> {
let root = cache_root_entry_info(cache)?;
send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await
}
async fn send_cache_root_entry(
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
root: DataUsageEntryInfo,
cache: &DataUsageCache,
pending_maintenance_work: &AtomicBool,
) -> std::result::Result<(), ScannerError> {
record_bucket_pending_maintenance_work(cache, pending_maintenance_work);
bucket_result_tx
.send(root)
@@ -1690,13 +1878,16 @@ async fn persist_and_publish_cache_snapshot(
);
return None;
}
if cache_snapshot_is_current(
&persisted,
DATA_USAGE_ROOT,
source,
cache_snapshot.info.next_cycle,
cache_snapshot.info.leader_epoch,
scan_plan_digest,
if matches!(
current_cache_root_entry(
&persisted,
DATA_USAGE_ROOT,
source,
cache_snapshot.info.next_cycle,
cache_snapshot.info.leader_epoch,
scan_plan_digest,
),
Ok(Some(_))
) {
cache_snapshot = persisted;
} else {
@@ -2329,6 +2520,7 @@ impl ScannerIOCache for SetDisks {
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
@@ -2450,7 +2642,7 @@ impl ScannerIOCache for SetDisks {
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
v3_disks = remote_disk_count,
v4_disks = remote_disk_count,
unsupported_remote_disks,
state = "unsupported_remote_disks_using_coordinator",
"Scanner set assigned remote disks without namespace scanner support to coordinator-driven workers"
@@ -2547,6 +2739,7 @@ impl ScannerIOCache for SetDisks {
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
@@ -2638,7 +2831,7 @@ impl ScannerIOCache for SetDisks {
let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
let cache_cycle_floor_clone = cache_cycle_floor.clone();
let remote_server_epoch = match worker_mode {
NamespaceScannerWorkerMode::RemoteV3(server_epoch) => Some(server_epoch),
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
NamespaceScannerWorkerMode::Coordinator => None,
};
futs.push(tokio::spawn(async move {
@@ -2948,48 +3141,71 @@ impl ScannerIOCache for SetDisks {
continue;
}
};
if cache_snapshot_is_current(&cache, &bucket.name, source, want_cycle, leader_epoch, bucket_scan_plan_digest)
{
if cache_guard.is_lock_lost() {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "lock_lost_before_reuse",
"Current scanner bucket cache root publish skipped after lock loss"
);
continue;
}
if let Err(e) =
send_cache_root_entry_info(&bucket_result_tx_clone, &cache, &pending_maintenance_work_clone).await
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_current_root_failed",
error = %e,
"Current scanner bucket cache root entry publish failed"
);
}
continue;
}
match cache.prepare_for_scan(
let scan_state = current_cache_root_or_prepare(
&mut cache,
&bucket.name,
source,
want_cycle,
leader_epoch,
source,
bucket_scan_plan_digest,
require_cache_source,
) {
);
let outcome = match scan_state {
DataUsageCacheScanState::Current(root) => {
if cache_guard.is_lock_lost() {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "lock_lost_before_reuse",
"Current scanner bucket cache root publish skipped after lock loss"
);
continue;
}
if let Err(e) =
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_current_root_failed",
error = %e,
"Current scanner bucket cache root entry publish failed"
);
}
continue;
}
DataUsageCacheScanState::Prepared {
outcome,
invalid_current,
} => {
if let Some(e) = invalid_current {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "current_cache_invalid",
error = %e,
"Current scanner bucket cache is invalid; rebuilding"
);
}
outcome
}
};
match outcome {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
cache_cycle_floor_clone.fetch_max(cache.info.next_cycle, Ordering::AcqRel);
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
@@ -3361,6 +3577,7 @@ impl ScannerIOCache for SetDisks {
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
@@ -4668,6 +4885,67 @@ mod tests {
root.add_child(&crate::hash_path("bucket/missing"));
dangling.replace("bucket", DATA_USAGE_ROOT, root);
assert!(cache_root_entry_info(&dangling).is_err());
let mut detached = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
..Default::default()
},
..Default::default()
};
detached.replace("bucket", DATA_USAGE_ROOT, DataUsageEntry::default());
detached.replace(
"bucket/detached",
"",
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(cache_root_entry_info(&detached).is_err());
let mut detached_bucket = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
detached_bucket.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
objects: 1,
..Default::default()
},
);
detached_bucket.replace(
"bucket/detached",
"",
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(cache_root_entry_info(&detached_bucket).is_err());
let mut compacted_with_child = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
compacted_with_child.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
compacted: true,
..Default::default()
},
);
compacted_with_child.replace("bucket/prefix", "bucket", DataUsageEntry::default());
assert!(cache_root_entry_info(&compacted_with_child).is_err());
}
#[test]
crates/storage-api/src/bucket.rs
+1 -1
View File
@@ -53,7 +53,7 @@ pub struct DeleteBucketOptions {
pub no_recreate: bool,
/// Force deletion even if bucket is not empty.
pub force: bool,
/// Force deletion only after the local peer verifies the bucket is empty.
/// Delete empty directory remnants after the local peer verifies no object metadata exists.
pub force_if_empty: bool,
/// Site replication delete operation.
pub srdelete_op: SRBucketDeleteOp,
crates/utils/src/http/metadata_compat.rs
+1
View File
@@ -40,6 +40,7 @@ pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
pub const SUFFIX_TRANSITIONED_VERSION_STATE: &str = "transitioned-version-state";
pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier";
pub const SUFFIX_TRANSITION_TIER_DESTINATION_ID: &str = "transition-tier-destination-id";
pub const SUFFIX_TRANSITION_TRANSACTION_ID: &str = "transition-transaction-id";
pub const SUFFIX_RESTORE_OPERATION_ID: &str = "restore-operation-id";
pub const SUFFIX_FREE_VERSION: &str = "free-version";
pub const SUFFIX_PURGESTATUS: &str = "purgestatus";