gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-23 12:49:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into overtrue/fix-1905-activation-fence

Browse Source 
# Conflicts:
#	crates/ecstore/src/core/pools.rs
This commit is contained in:
overtrue
2026-08-23 02:36:20 +08:00
parent d5648b52a7 2d7120460b
commit 4d0ffa680a
48 changed files with 3880 additions and 1035 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/cluster/rpc/remote_disk.rs
+90 -7
View File
@@ -42,8 +42,9 @@ use futures::lock::Mutex;
use metrics::counter;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
};
use rustfs_protos::ChannelClass;
use rustfs_protos::evict_failed_connection;
@@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const BATCH_METADATA_RPC_OFF: &str = "off";
const BATCH_METADATA_RPC_AUTO: &str = "auto";
const BATCH_METADATA_RPC_ON: &str = "on";
@@ -202,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
}
fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
.or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
.as_deref()
.map(parse_batch_metadata_rpc_mode)
@@ -1826,6 +1829,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
}
}
fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
if let Some(started_at) = started_at {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
}
}
/// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
/// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
/// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
@@ -1936,6 +1945,27 @@ fn decode_batch_read_version_response_items(
Ok(batch_read_version_resps)
}
fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
req.items
.iter()
.fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
total
.saturating_add(item.org_volume.len())
.saturating_add(item.volume.len())
.saturating_add(item.path.len())
.saturating_add(item.version_id.len())
})
}
fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
response
.batch_read_version_resps
.iter()
.map(String::len)
.sum::<usize>()
.saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
}
fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
file_info.validate_for_metadata_read().map_err(Into::into)
}
@@ -2837,14 +2867,19 @@ impl DiskAPI for RemoteDisk {
state = "started",
"Remote disk RPC started"
);
let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let batch_read_version_req = compat_json(&req)?;
let batch_read_version_req_bin = encode_msgpack(&req)?;
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
let request_payload_bytes = batch_read_version_attribution_enabled
.then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
let batch_result = self
.execute_with_timeout_for_op(
"batch_read_version",
move || async move {
let disk = self.disk_ref().await;
let disk_len = disk.len();
let mut client = self
.get_bulk_client()
.await
@@ -2855,9 +2890,20 @@ impl DiskAPI for RemoteDisk {
batch_read_version_req_bin: batch_read_version_req_bin.into(),
});
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
if let Some(request_payload_bytes) = request_payload_bytes {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
request_payload_bytes.saturating_add(disk_len),
);
}
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let response = match client.batch_read_version(request).await {
Ok(response) => response.into_inner(),
Ok(response) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
response.into_inner()
}
Err(status) if status.code() == Code::Unimplemented => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
if mode.should_fallback_on_unimplemented() {
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
warn!(
@@ -2874,6 +2920,7 @@ impl DiskAPI for RemoteDisk {
}
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
@@ -2886,14 +2933,33 @@ impl DiskAPI for RemoteDisk {
);
return Err(Error::from(status));
}
Err(status) => return Err(Error::from(status)),
Err(status) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(Error::from(status));
}
};
if !response.success {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(response.error.unwrap_or_default().into());
}
decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
batch_read_version_response_payload_len(&response),
);
let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
match decode_batch_read_version_response_items(response, &self.endpoint) {
Ok(batch_read_version_resps) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
Ok(Some(batch_read_version_resps))
}
Err(err) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
Err(err)
}
}
},
get_max_timeout_duration(),
)
@@ -4621,6 +4687,7 @@ mod tests {
} else {
"file version not found".to_string()
},
error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
}
}
@@ -4740,6 +4807,7 @@ mod tests {
fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
@@ -4749,10 +4817,25 @@ mod tests {
);
}
#[test]
fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
],
|| {
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
},
);
}
#[test]
fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
crates/ecstore/src/cluster/rpc/runtime_sources.rs
+57 -3
View File
@@ -14,9 +14,10 @@
use rustfs_io_metrics::internode_metrics::{
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
global_internode_metrics,
};
use std::time::Duration;
@@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_stage_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
stage,
duration,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics()
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
}
pub(crate) fn record_remote_disk_grpc_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
crates/ecstore/src/core/pools.rs
+1013 -263
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/mod.rs
+69 -25
View File
@@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_DISK: &str = "disk";
const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;
pub fn part_transaction_path(part_path: &str) -> String {
match part_path.rsplit_once('/') {
@@ -62,6 +64,7 @@ use bytes::Bytes;
use endpoint::Endpoint;
use error::DiskError;
use error::{Error, Result};
use futures::stream::{self, StreamExt};
use local::LocalDisk;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_madmin::info_commands::DiskMetrics;
@@ -417,6 +420,14 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
match self {
Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
match self {
@@ -1028,36 +1039,47 @@ where
D: DiskAPI + ?Sized,
{
validate_batch_read_version_item_count(req.items.len())?;
let parallelism = batch_read_version_server_parallelism();
let mut responses = Vec::with_capacity(req.items.len());
for (index, item) in req.items.iter().enumerate() {
let response = match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: true,
file_info,
error: String::new(),
},
Err(err) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
},
};
responses.push(response);
}
let mut responses = stream::iter(req.items.into_iter().enumerate())
.map(|(index, item)| async move {
match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: true,
file_info,
error: String::new(),
error_code: 0,
},
Err(err) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
error_code: err.to_u32(),
},
}
})
.buffer_unordered(parallelism)
.collect::<Vec<_>>()
.await;
responses.sort_unstable_by_key(|response| response.index);
Ok(responses)
}
fn batch_read_version_server_parallelism() -> usize {
rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
.clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CheckPartsResp {
pub results: Vec<usize>,
@@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp {
pub success: bool,
pub file_info: FileInfo,
pub error: String,
#[serde(default)]
pub error_code: u32,
}
pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
@@ -1417,6 +1441,26 @@ mod tests {
assert!(!partial_valid_location.valid());
}
#[test]
fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
assert_eq!(batch_read_version_server_parallelism(), 4);
});
}
#[test]
fn batch_read_version_server_parallelism_honors_env_with_bounds() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
assert_eq!(batch_read_version_server_parallelism(), 8);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
assert_eq!(batch_read_version_server_parallelism(), 1);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
});
}
/// Test FileInfoVersions find_version_index
#[test]
fn test_file_info_versions_find_version_index() {
crates/ecstore/src/lib.rs
+8
View File
@@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() {
cluster::rpc::shutdown_background_monitors();
}
/// Publish that the process is ready to serve user-object GET traffic.
///
/// Experimental metadata coalescing is allowed to run only after this point so
/// startup and internal metadata reads keep the original per-disk path.
pub fn mark_get_metadata_read_version_coalescing_service_ready() {
runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
}
#[cfg(test)]
mod rio_tests {
#[test]
crates/ecstore/src/runtime/global.rs
+14 -1
View File
@@ -25,7 +25,10 @@ use lazy_static::lazy_static;
use rustfs_lock::client::LockClient;
use std::{
collections::HashMap,
sync::{Arc, OnceLock},
sync::{
Arc, OnceLock,
atomic::{AtomicBool, Ordering},
},
time::SystemTime,
};
use tokio::sync::{OnceCell, RwLock};
@@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000;
pub const DISK_FILL_FRACTION: f64 = 0.99;
pub const DISK_RESERVE_FRACTION: f64 = 0.15;
static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
}
pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
}
// Global singletons for backward compatibility with MinIO port.
// These should be migrated to AppContext over time.
// See issue #730 for migration plan.
crates/ecstore/src/runtime/instance.rs
+9
View File
@@ -160,6 +160,10 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Serializes decommission data-movement operations with cancellation and
/// a subsequent restart. Readers are held across one object side effect;
/// the transition path takes the writer after cancelling the routine.
decommission_operation_gate: Arc<RwLock<()>>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -200,6 +204,7 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
decommission_operation_gate: Arc::new(RwLock::new(())),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -218,6 +223,10 @@ impl InstanceContext {
self.lock_manager.clone()
}
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.decommission_operation_gate)
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
crates/ecstore/src/set_disk/core/io_primitives.rs
+565 -7
View File
@@ -53,11 +53,12 @@ use crate::diagnostics::get::{
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::disk::disk_store::DiskStoreRenameDataExt;
use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout};
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
use crate::disk::{
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
};
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::ShardReader;
@@ -75,7 +76,7 @@ use std::{
future::Future,
pin::Pin,
sync::{
OnceLock,
Arc, OnceLock,
atomic::{AtomicUsize, Ordering},
},
task::{Context, Poll},
@@ -94,6 +95,242 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String {
[bucket, object].join("/")
}
fn read_version_coalescing_enabled() -> bool {
let enabled = || {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on"))
};
#[cfg(test)]
{
enabled()
}
#[cfg(not(test))]
{
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(enabled)
}
}
fn read_version_coalescing_delay() -> Duration {
#[cfg(test)]
{
let micros = rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
);
Duration::from_micros(micros)
}
#[cfg(not(test))]
{
static DELAY: OnceLock<Duration> = OnceLock::new();
*DELAY.get_or_init(|| {
Duration::from_micros(rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
))
})
}
}
struct CoalescedReadVersionRequest {
item: BatchReadVersionItem,
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
struct ReadVersionCoalescerKey {
disk: usize,
incl_free_versions: bool,
read_data: bool,
healing: bool,
}
impl ReadVersionCoalescerKey {
fn new(disk: &DiskStore, opts: &ReadOptions) -> Self {
Self {
disk: Arc::as_ptr(disk) as usize,
incl_free_versions: opts.incl_free_versions,
read_data: opts.read_data,
healing: opts.healing,
}
}
}
#[derive(Default)]
struct ReadVersionCoalescer {
lanes: HashMap<ReadVersionCoalescerKey, Vec<CoalescedReadVersionRequest>>,
}
fn read_version_coalescer() -> &'static Mutex<ReadVersionCoalescer> {
static COALESCER: OnceLock<Mutex<ReadVersionCoalescer>> = OnceLock::new();
COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default()))
}
fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
counter!(
METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL,
"event" => event,
"item_count" => item_count.to_string()
)
.increment(1);
}
async fn read_version_via_coalescer(
disk: DiskStore,
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
opts: &ReadOptions,
allow_coalescing: bool,
) -> disk::error::Result<FileInfo> {
if !allow_coalescing || !read_version_coalescing_enabled() {
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
if !matches!(disk.as_ref(), Disk::Remote(_)) {
record_read_version_coalescer_event("bypass_non_remote", 1);
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
let (tx, rx) = oneshot::channel();
let item = BatchReadVersionItem {
org_volume: org_bucket.to_string(),
volume: bucket.to_string(),
path: object.to_string(),
version_id: version_id.to_string(),
};
let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
let lane = coalescer.lanes.entry(lane_key).or_default();
let schedule_delayed_flush = lane.is_empty();
lane.push(CoalescedReadVersionRequest { item, tx });
if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
coalescer.lanes.remove(&lane_key)
} else if schedule_delayed_flush {
let disk = disk.clone();
let task_opts = *opts;
tokio::spawn(async move {
tokio::time::sleep(read_version_coalescing_delay()).await;
flush_read_version_coalescer_lane(lane_key, disk, task_opts).await;
});
None
} else {
None
}
};
if let Some(pending) = pending {
flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
}
rx.await
.unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
}
async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
coalescer.lanes.remove(&lane_key).unwrap_or_default()
};
flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await;
}
async fn flush_read_version_coalescer_pending(
lane_key: ReadVersionCoalescerKey,
disk: DiskStore,
opts: ReadOptions,
pending: Vec<CoalescedReadVersionRequest>,
) {
if pending.is_empty() {
return;
}
#[cfg(test)]
{
let mut observed_paths = HashSet::new();
for request in &pending {
if observed_paths.insert(request.item.path.as_str()) {
disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk);
}
}
}
let mut senders = Vec::with_capacity(pending.len());
let mut items = Vec::with_capacity(pending.len());
for request in pending {
senders.push(request.tx);
items.push(request.item);
}
let expected_items = items.clone();
record_read_version_coalescer_event("attempted_batch", items.len());
let result =
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
.await
{
Ok(result) => result,
Err(_) => Err(DiskError::Timeout),
};
match result {
Ok(responses) => {
let results = map_batch_read_version_responses(&expected_items, responses);
for (tx, result) in senders.into_iter().zip(results) {
let _ = tx.send(result);
}
}
Err(err) => {
let message = err.to_string();
for tx in senders {
let _ = tx.send(Err(DiskError::other(message.clone())));
}
}
}
}
fn map_batch_read_version_responses(
expected_items: &[BatchReadVersionItem],
responses: Vec<BatchReadVersionResp>,
) -> Vec<crate::disk::error::Result<FileInfo>> {
let mut results = (0..expected_items.len())
.map(|_| Err(DiskError::other("coalesced read_version response missing")))
.collect::<Vec<_>>();
let mut seen = vec![false; expected_items.len()];
for response in responses {
let Some(expected) = expected_items.get(response.index) else {
continue;
};
let Some(slot) = results.get_mut(response.index) else {
continue;
};
if seen[response.index] {
*slot = Err(DiskError::other("coalesced read_version response duplicate index"));
continue;
}
seen[response.index] = true;
if response.path != expected.path || response.version_id != expected.version_id {
*slot = Err(DiskError::other("coalesced read_version response identity mismatch"));
} else {
*slot = if response.success {
Ok(response.file_info)
} else {
Err(batch_read_version_response_error(response.error_code, response.error))
};
}
}
results
}
fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError {
match DiskError::from_u32(error_code) {
Some(DiskError::Io(_)) | None => DiskError::other(error),
Some(error) => error,
}
}
pub(in crate::set_disk) fn bounded_metadata_fanout_order(
bucket: &str,
object: &str,
@@ -133,11 +370,15 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
order
}
use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::RwLock;
use tokio::sync::{Mutex, RwLock, oneshot};
use tokio::task::JoinSet;
pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
/// Default reader-setup strategy for the GET read path (rustfs/backlog#1215,
/// #1159, #923).
@@ -2356,6 +2597,7 @@ impl SetDisks {
false,
true,
0,
false,
)
.await?;
Ok((ress, errors))
@@ -2386,6 +2628,36 @@ impl SetDisks {
true,
caller_allows_early_stop,
default_parity_count,
false,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
incl_free_versions: bool,
caller_allows_early_stop: bool,
default_parity_count: usize,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
Self::read_all_fileinfo_inner(
disks,
org_bucket,
bucket,
object,
version_id,
read_data,
false,
incl_free_versions,
true,
caller_allows_early_stop,
default_parity_count,
true,
)
.await
}
@@ -2408,6 +2680,7 @@ impl SetDisks {
// subset would fail write quorum (backlog#872 regression).
caller_allows_early_stop: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let early_stop_enabled =
caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled());
@@ -2424,6 +2697,7 @@ impl SetDisks {
healing,
incl_free_versions,
default_parity_count,
allow_coalescing,
)
.await;
}
@@ -2446,6 +2720,7 @@ impl SetDisks {
healing,
incl_free_versions,
observe,
allow_coalescing,
)
.await
}
@@ -2461,6 +2736,7 @@ impl SetDisks {
healing: bool,
incl_free_versions: bool,
observe: bool,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = observe.then(Instant::now);
let mut ress = Vec::with_capacity(disks.len());
@@ -2492,7 +2768,7 @@ impl SetDisks {
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
@@ -2559,6 +2835,7 @@ impl SetDisks {
healing: bool,
incl_free_versions: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = Instant::now();
let mut ress = vec![FileInfo::default(); disks.len()];
@@ -2607,7 +2884,7 @@ impl SetDisks {
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
@@ -5737,6 +6014,7 @@ pub(crate) mod disk_call_counters {
/// Kind label for the per-disk `read_version` metadata RPC.
pub const KIND_READ_VERSION: &str = "read_version";
pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version";
/// Registry key: (object, kind, disk_index).
type CountKey = (String, String, usize);
@@ -6460,6 +6738,286 @@ mod tests {
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn metadata_read_version_coalescer_bypasses_local_disks() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-local-bypass-bucket";
let object_a = "coalesced-local-object-a";
let object_b = "coalesced-local-object-b";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await;
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")),
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")),
],
async {
let calls = disk_call_counters::observe(object_a);
let disks_a = disks.clone();
let disks_b = disks.clone();
let read_a = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_a, "", bucket, object_a, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
tokio::task::yield_now().await;
let read_b = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_b, "", bucket, object_b, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
let (metadata_a, errs_a) = read_a
.await
.expect("first read task should not panic")
.expect("first coalesced read should resolve");
let (metadata_b, errs_b) = read_b
.await
.expect("second read task should not panic")
.expect("second coalesced read should resolve");
assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS);
assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS);
assert!(errs_a.iter().all(Option::is_none));
assert!(errs_b.iter().all(Option::is_none));
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"local disks still execute the ordinary per-disk read_version path"
);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"GET coalescing targets internode RPC count only and must not batch local disk reads"
);
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn metadata_read_version_coalescer_requires_get_object_intent() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-default-bypass-bucket";
let object = "default-bypass-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async {
let calls = disk_call_counters::observe(object);
let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false)
.await
.expect("default metadata read should resolve");
assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"non-GET metadata paths must bypass coalescer even when the env gate is enabled"
);
})
.await;
drop(dirs);
}
#[test]
fn batch_read_version_response_mapping_preserves_index_and_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-c".to_string(),
version_id: "v-c".to_string(),
},
];
let ok_file_info = FileInfo {
name: "object-a".to_string(),
..Default::default()
};
let responses = vec![
BatchReadVersionResp {
index: 2,
path: "object-c".to_string(),
version_id: "v-c".to_string(),
success: false,
file_info: FileInfo::default(),
error: "disk read failed".to_string(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: ok_file_info,
error: String::new(),
error_code: 0,
},
];
let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
let first = results
.next()
.expect("slot 0 should exist")
.expect("slot 0 should map the success response by index");
assert_eq!(first.name, "object-a");
let missing = results
.next()
.expect("slot 1 should exist")
.expect_err("slot 1 should stay missing");
assert!(
missing.to_string().contains("response missing"),
"unexpected missing response error: {missing}"
);
let failed = results
.next()
.expect("slot 2 should exist")
.expect_err("slot 2 should map the response error");
assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}");
assert!(results.next().is_none());
}
#[test]
fn batch_read_version_response_mapping_preserves_typed_not_found_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
];
let results = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileNotFound.to_string(),
error_code: DiskError::FileNotFound.to_u32(),
},
BatchReadVersionResp {
index: 1,
path: "object-b".to_string(),
version_id: "v-b".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileVersionNotFound.to_string(),
error_code: DiskError::FileVersionNotFound.to_u32(),
},
],
);
assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound)));
assert!(matches!(
results.get(1).expect("slot 1 should exist"),
Err(DiskError::FileVersionNotFound)
));
}
#[test]
fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() {
let expected_items = vec![BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
}];
let mismatched = map_batch_read_version_responses(
&expected_items,
vec![BatchReadVersionResp {
index: 0,
path: "object-b".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-b".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
}],
)
.pop()
.expect("slot 0 should exist")
.expect_err("identity mismatch should fail closed");
assert!(
mismatched.to_string().contains("identity mismatch"),
"unexpected mismatch error: {mismatched}"
);
let duplicate = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
],
)
.pop()
.expect("slot 0 should exist")
.expect_err("duplicate response index should fail closed");
assert!(
duplicate.to_string().contains("duplicate index"),
"unexpected duplicate error: {duplicate}"
);
}
/// Isolation guard: unobserved objects record nothing (so parallel tests do
/// not inflate one another), and a scope clears its own counts on drop.
#[tokio::test]
crates/ecstore/src/set_disk/ops/object.rs
+1 -1
View File
@@ -1294,7 +1294,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
(prepared.snapshot, prepared.object_info)
} else {
match self
.get_object_fileinfo(
.get_object_fileinfo_for_get_object_reader(
bucket,
object,
opts,
crates/ecstore/src/set_disk/read.rs
+66 -14
View File
@@ -259,10 +259,33 @@ impl SetDisks {
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false)
.await
}
#[tracing::instrument(level = "debug", skip(self))]
#[hotpath::measure(impl_type = "SetDisks")]
pub(super) async fn get_object_fileinfo_for_get_object_reader(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<GetObjectFileInfo> {
let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket)
&& crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
self.get_object_fileinfo_gated_inner(
bucket,
object,
opts,
read_data,
caller_allows_early_stop,
allow_read_version_coalescing,
)
.await
}
/// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
/// quorum fanout. Read-before-write callers (object tagging) must use this:
/// the returned online-disk set is the write target, and the early-stop
@@ -275,6 +298,20 @@ impl SetDisks {
opts: &ObjectOptions,
read_data: bool,
allow_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false)
.await
}
#[allow(clippy::too_many_arguments)]
async fn get_object_fileinfo_gated_inner(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
read_data: bool,
allow_early_stop: bool,
allow_read_version_coalescing: bool,
) -> Result<GetObjectFileInfo> {
let vid = opts.version_id.clone().unwrap_or_default();
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
@@ -337,19 +374,34 @@ impl SetDisks {
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
// core/io_primitives.rs); unsafe requests and callers that opt out
// (allow_early_stop=false) fall back to full-wait.
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
false,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?;
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing {
Self::read_all_fileinfo_observed_for_get_object(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?
} else {
Self::read_all_fileinfo_observed(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
false,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?
};
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
GET_OBJECT_PATH_INTERNAL_META
} else {
crates/ecstore/src/store/rebalance.rs
+386 -1
View File
@@ -859,6 +859,7 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
use crate::disk::error::DiskError;
use crate::layout::endpoint::Endpoint;
@@ -1423,6 +1424,14 @@ mod tests {
}
}
fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
ObjectInfo {
version_id: Some(version_id),
etag,
..object_info_with_mod_time(unix_ts, delete_marker)
}
}
#[test]
fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
let candidates = vec![
@@ -1446,7 +1455,7 @@ mod tests {
}
#[test]
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() {
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_mod_time(10, false)),
@@ -1466,6 +1475,382 @@ mod tests {
assert_eq!(idx, 1);
}
#[test]
fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 2,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 7,
err: None,
},
];
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect("equivalent replicas must resolve deterministically");
assert_eq!(idx, 7);
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("divergent version ids must not silently resolve to the higher pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("divergent etags must not silently resolve to the higher pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("a delete marker tied with a live version must not be masked by the pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
let err = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(left),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(right),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect_err("equal-time identity divergence must fail closed");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut data_dir = base.clone();
data_dir.data_dir = Some(Uuid::from_u128(2));
assert_equal_time_identity_conflict(base.clone(), data_dir);
let mut size = base.clone();
size.size = 1;
assert_equal_time_identity_conflict(base.clone(), size);
let mut actual_size = base.clone();
actual_size.actual_size = 1;
assert_equal_time_identity_conflict(base.clone(), actual_size);
let mut checksum = base.clone();
checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
assert_equal_time_identity_conflict(base.clone(), checksum);
let mut parts = base.clone();
parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
etag: "part-etag".to_string(),
number: 1,
size: 1,
..Default::default()
}]);
assert_equal_time_identity_conflict(base.clone(), parts);
let mut transition = base;
transition.transitioned_object.tier = "tier-a".to_string();
assert_equal_time_identity_conflict(
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
transition,
);
}
#[test]
fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let mut minio_alias = base.clone();
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"X-MINIO-INTERNAL-COMPRESSION".to_string(),
"zstd".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(rustfs_alias),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(minio_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("same-value internal aliases should resolve");
assert_eq!(idx, 1);
let mut dual_alias = base.clone();
dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
("x-minio-internal-compression".to_string(), "zstd".to_string()),
]));
let mut single_alias = base;
single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(dual_alias),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(single_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("dual-key and single-key internal metadata should resolve");
assert_eq!(idx, 1);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let mut minio_alias = base;
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-minio-internal-compression".to_string(),
"snappy".to_string(),
)]));
assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
}
#[test]
fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!(
"X-RUSTFS-INTERNAL-{}{suffix}",
suffix_prefix.to_uppercase(),
suffix = "arn:aws:s3:::Bucket"
),
"value".to_string(),
)]));
let mut minio_alias = base.clone();
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
"value".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(rustfs_alias.clone()),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(minio_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("dynamic internal aliases with the same target should resolve");
assert_eq!(idx, 1);
let mut different_target_case = base;
different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
"value".to_string(),
)]));
assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
}
}
#[test]
fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut first = base.clone();
first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
("x-minio-internal-compression".to_string(), "snappy".to_string()),
]));
let mut second = base;
second.user_defined = first.user_defined.clone();
assert_equal_time_identity_conflict(first, second);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut replication = base.clone();
replication.replication_status_internal = Some("PENDING".to_string());
replication.replication_status = ReplicationStatusType::Pending;
assert_equal_time_identity_conflict(base.clone(), replication);
let mut purge = base.clone();
purge.version_purge_status_internal = Some("PENDING".to_string());
purge.version_purge_status = VersionPurgeStatusType::Pending;
assert_equal_time_identity_conflict(base.clone(), purge);
let mut decision = base;
decision.replication_decision = "replicate".to_string();
assert_equal_time_identity_conflict(
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
decision,
);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
without_mod_time.mod_time = None;
let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
}
#[test]
fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
older.data_dir = Some(Uuid::from_u128(2));
let (info, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(latest),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(older),
idx: 9,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("older identity divergence must not affect the latest candidate");
assert_eq!(idx, 0);
assert_eq!(
info.mod_time,
Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
);
}
#[test]
fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: None,
idx: 1,
err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
];
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect("not-found pools must not block resolution of found candidates");
assert_eq!(idx, 0);
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
}
#[test]
fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
let err = resolve_latest_object_info_candidates(
crates/ecstore/src/store/rebalance/support.rs
+146 -21
View File
@@ -12,10 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::cmp::Ordering;
use std::collections::HashMap;
use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{ObjectInfo, ObjectOptions};
use rustfs_utils::http::metadata_compat::{
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
strip_internal_prefix_preserving_case,
};
use rustfs_utils::path::decode_dir_object;
use time::OffsetDateTime;
@@ -137,37 +141,158 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p
))
}
fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option<OffsetDateTime> {
candidate
.info
.as_ref()
.map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
}
fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
left.transition_version_state == right.transition_version_state
&& left.transitioned_object.name == right.transitioned_object.name
&& left.transitioned_object.version_id == right.transitioned_object.version_id
&& left.transitioned_object.tier == right.transitioned_object.tier
&& left.transitioned_object.free_version == right.transitioned_object.free_version
&& left.transitioned_object.status == right.transitioned_object.status
}
#[derive(PartialEq, Eq)]
struct LatestUserDefinedIdentity {
internal: HashMap<String, String>,
other: HashMap<String, String>,
}
fn normalize_internal_identity_suffix(key: &str) -> Option<String> {
let suffix = strip_internal_prefix_preserving_case(key)?;
for dynamic_prefix in [
SUFFIX_REPLICATION_RESET_ARN_PREFIX,
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
] {
let prefix_len = dynamic_prefix.len();
if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..))
&& prefix.eq_ignore_ascii_case(dynamic_prefix)
{
return Some(format!("{dynamic_prefix}{remainder}"));
}
}
Some(suffix.to_lowercase())
}
fn normalize_user_defined_identity(user_defined: &HashMap<String, String>) -> Option<LatestUserDefinedIdentity> {
let mut identity = LatestUserDefinedIdentity {
internal: HashMap::with_capacity(user_defined.len()),
other: HashMap::with_capacity(user_defined.len()),
};
for (key, value) in user_defined {
if let Some(suffix) = normalize_internal_identity_suffix(key) {
if identity
.internal
.insert(suffix, value.clone())
.is_some_and(|previous| previous != *value)
{
return None;
}
} else {
identity.other.insert(key.clone(), value.clone());
}
}
Some(identity)
}
fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
match (
normalize_user_defined_identity(&left.user_defined),
normalize_user_defined_identity(&right.user_defined),
) {
(Some(left), Some(right)) => left == right,
_ => false,
}
}
/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
/// differ for the same object version while the selected winner still carries
/// the chosen pool's layout. `put_object_reader` is also intentionally
/// excluded because it is a transient request handle that `ObjectInfo::clone`
/// drops. Every other ObjectInfo field is part of the production-visible
/// identity and must agree before the pool index can provide a deterministic
/// tie-break.
fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
left.bucket == right.bucket
&& left.name == right.name
&& left.storage_class == right.storage_class
&& left.mod_time == right.mod_time
&& left.size == right.size
&& left.actual_size == right.actual_size
&& left.is_dir == right.is_dir
&& same_user_defined_identity(left, right)
&& left.user_tags == right.user_tags
&& left.version_id == right.version_id
&& left.data_dir == right.data_dir
&& left.delete_marker == right.delete_marker
&& same_transition_identity(left, right)
&& left.restore_ongoing == right.restore_ongoing
&& left.restore_expires == right.restore_expires
&& left.parts == right.parts
&& left.is_latest == right.is_latest
&& left.content_type == right.content_type
&& left.content_encoding == right.content_encoding
&& left.expires == right.expires
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.etag == right.etag
&& left.inlined == right.inlined
&& left.metadata_only == right.metadata_only
&& left.version_only == right.version_only
&& left.replication_status_internal == right.replication_status_internal
&& left.replication_status == right.replication_status
&& left.version_purge_status_internal == right.version_purge_status_internal
&& left.version_purge_status == right.version_purge_status
&& left.replication_decision == right.replication_decision
&& left.checksum == right.checksum
}
pub(super) fn resolve_latest_object_info_candidates(
mut candidates: Vec<LatestObjectInfoCandidate>,
candidates: Vec<LatestObjectInfoCandidate>,
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, usize)> {
candidates.sort_by(|a, b| {
let a_mod = if let Some(info) = &a.info {
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
OffsetDateTime::UNIX_EPOCH
let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max();
if let Some(latest_mod_time) = latest_mod_time {
let mut latest_candidates = candidates
.into_iter()
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
.collect::<Vec<_>>();
latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
let Some(winner) = latest_candidates.first() else {
return Err(Error::ErasureReadQuorum);
};
let Some(winner_info) = winner.info.as_ref() else {
return Err(Error::ErasureReadQuorum);
};
let b_mod = if let Some(info) = &b.info {
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
OffsetDateTime::UNIX_EPOCH
};
if a_mod == b_mod {
return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less };
if latest_candidates.iter().skip(1).any(|candidate| {
candidate
.info
.as_ref()
.is_none_or(|info| !same_latest_object_info_identity(winner_info, info))
}) {
return Err(Error::ErasureReadQuorum);
}
b_mod.cmp(&a_mod)
});
return Ok((winner_info.clone(), winner.idx));
}
for candidate in candidates {
if let Some(info) = candidate.info {
return Ok((info, candidate.idx));
}
if let Some(err) = candidate.err
&& !is_err_object_not_found(&err)
&& !is_err_version_not_found(&err)