gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-23 04:39:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into overtrue/fix-1905-activation-fence

Browse Source 
# Conflicts:
#	crates/ecstore/src/core/pools.rs
This commit is contained in:
overtrue
2026-08-23 02:36:20 +08:00
parent d5648b52a7 2d7120460b
commit 4d0ffa680a
48 changed files with 3880 additions and 1035 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/kms/mod.rs
-3
View File
@@ -39,9 +39,6 @@ mod kms_edge_cases_test;
#[cfg(test)]
mod kms_fault_recovery_test;
#[cfg(test)]
mod test_runner;
#[cfg(test)]
mod bucket_default_encryption_test;
crates/e2e_test/src/kms/test_runner.rs
-499
View File
@@ -1,499 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
#![allow(dead_code)]
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Unified KMS test suite runner
//!
//! This module provides a unified interface for running KMS tests with categorization,
//! filtering, and comprehensive reporting capabilities.
use crate::common::init_logging;
use std::time::Instant;
use tokio::time::{Duration, sleep};
use tracing::{debug, error, info, warn};
/// Test category for organization and filtering
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TestCategory {
CoreFunctionality,
MultipartEncryption,
EdgeCases,
FaultRecovery,
Comprehensive,
Performance,
}
impl TestCategory {
pub fn as_str(&self) -> &'static str {
match self {
TestCategory::CoreFunctionality => "core-functionality",
TestCategory::MultipartEncryption => "multipart-encryption",
TestCategory::EdgeCases => "edge-cases",
TestCategory::FaultRecovery => "fault-recovery",
TestCategory::Comprehensive => "comprehensive",
TestCategory::Performance => "performance",
}
}
}
/// Test definition with metadata
#[derive(Debug, Clone)]
pub struct TestDefinition {
pub name: String,
pub description: String,
pub category: TestCategory,
pub estimated_duration: Duration,
pub is_critical: bool,
}
impl TestDefinition {
pub fn new(
name: impl Into<String>,
description: impl Into<String>,
category: TestCategory,
estimated_duration: Duration,
is_critical: bool,
) -> Self {
Self {
name: name.into(),
description: description.into(),
category,
estimated_duration,
is_critical,
}
}
}
/// Test execution result
#[derive(Debug, Clone)]
pub struct TestResult {
pub test_name: String,
pub category: TestCategory,
pub success: bool,
pub duration: Duration,
pub error_message: Option<String>,
}
impl TestResult {
pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
Self {
test_name,
category,
success: true,
duration,
error_message: None,
}
}
pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
Self {
test_name,
category,
success: false,
duration,
error_message: Some(error),
}
}
}
/// Comprehensive test suite configuration
#[derive(Debug, Clone)]
pub struct TestSuiteConfig {
pub categories: Vec<TestCategory>,
pub include_critical_only: bool,
pub max_duration: Option<Duration>,
pub parallel_execution: bool,
}
impl Default for TestSuiteConfig {
fn default() -> Self {
Self {
categories: vec![
TestCategory::CoreFunctionality,
TestCategory::MultipartEncryption,
TestCategory::EdgeCases,
TestCategory::FaultRecovery,
TestCategory::Comprehensive,
],
include_critical_only: false,
max_duration: None,
parallel_execution: false,
}
}
}
/// Unified KMS test suite runner
pub struct KMSTestSuite {
tests: Vec<TestDefinition>,
config: TestSuiteConfig,
}
impl KMSTestSuite {
/// Create a new test suite with default configuration
pub fn new() -> Self {
let tests = vec![
// Core Functionality Tests
TestDefinition::new(
"test_local_kms_end_to_end",
"End-to-end KMS test with all encryption types",
TestCategory::CoreFunctionality,
Duration::from_secs(60),
true,
),
TestDefinition::new(
"test_local_kms_key_isolation",
"Test KMS key isolation and security",
TestCategory::CoreFunctionality,
Duration::from_secs(45),
true,
),
// Multipart Encryption Tests
TestDefinition::new(
"test_local_kms_multipart_upload",
"Test large file multipart upload with encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(120),
true,
),
TestDefinition::new(
"test_step1_basic_single_file_encryption",
"Basic single file encryption test",
TestCategory::MultipartEncryption,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_step2_basic_multipart_upload_without_encryption",
"Basic multipart upload without encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_step3_multipart_upload_with_sse_s3",
"Multipart upload with SSE-S3 encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(60),
true,
),
TestDefinition::new(
"test_step4_large_multipart_upload_with_encryption",
"Large file multipart upload with encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(90),
false,
),
TestDefinition::new(
"test_step5_all_encryption_types_multipart",
"All encryption types multipart test",
TestCategory::MultipartEncryption,
Duration::from_secs(120),
true,
),
// Edge Cases Tests
TestDefinition::new(
"test_kms_zero_byte_file_encryption",
"Test encryption of zero-byte files",
TestCategory::EdgeCases,
Duration::from_secs(20),
false,
),
TestDefinition::new(
"test_kms_single_byte_file_encryption",
"Test encryption of single-byte files",
TestCategory::EdgeCases,
Duration::from_secs(20),
false,
),
TestDefinition::new(
"test_kms_multipart_boundary_conditions",
"Test multipart upload boundary conditions",
TestCategory::EdgeCases,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_kms_invalid_key_scenarios",
"Test invalid key scenarios",
TestCategory::EdgeCases,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_kms_concurrent_encryption",
"Test concurrent encryption operations",
TestCategory::EdgeCases,
Duration::from_secs(60),
false,
),
TestDefinition::new(
"test_kms_key_validation_security",
"Test key validation security",
TestCategory::EdgeCases,
Duration::from_secs(30),
false,
),
// Fault Recovery Tests
TestDefinition::new(
"test_kms_key_directory_unavailable",
"Test KMS when key directory is unavailable",
TestCategory::FaultRecovery,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_kms_corrupted_key_files",
"Test KMS with corrupted key files",
TestCategory::FaultRecovery,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_kms_multipart_upload_interruption",
"Test multipart upload interruption recovery",
TestCategory::FaultRecovery,
Duration::from_secs(60),
false,
),
TestDefinition::new(
"test_kms_resource_constraints",
"Test KMS under resource constraints",
TestCategory::FaultRecovery,
Duration::from_secs(90),
false,
),
// Comprehensive Tests
TestDefinition::new(
"test_comprehensive_kms_full_workflow",
"Full KMS workflow comprehensive test",
TestCategory::Comprehensive,
Duration::from_secs(300),
true,
),
TestDefinition::new(
"test_comprehensive_stress_test",
"KMS stress test with large datasets",
TestCategory::Comprehensive,
Duration::from_secs(400),
false,
),
TestDefinition::new(
"test_comprehensive_key_isolation",
"Comprehensive key isolation test",
TestCategory::Comprehensive,
Duration::from_secs(180),
false,
),
TestDefinition::new(
"test_comprehensive_concurrent_operations",
"Comprehensive concurrent operations test",
TestCategory::Comprehensive,
Duration::from_secs(240),
false,
),
TestDefinition::new(
"test_comprehensive_performance_benchmark",
"KMS performance benchmark test",
TestCategory::Comprehensive,
Duration::from_secs(360),
false,
),
];
Self {
tests,
config: TestSuiteConfig::default(),
}
}
/// Configure the test suite
pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
self.config = config;
self
}
/// Filter tests based on category
pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
self.tests.iter().filter(|test| &test.category == category).collect()
}
/// Filter tests based on criticality
pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
self.tests.iter().filter(|test| test.is_critical).collect()
}
/// Get test summary by category
pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
let mut summary = std::collections::HashMap::new();
for test in &self.tests {
summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
}
summary
}
/// Run the complete test suite
pub async fn run_test_suite(&self) -> Vec<TestResult> {
init_logging();
info!("🚀 Starting unified KMS test suite");
let start_time = Instant::now();
let mut results = Vec::new();
// Filter tests based on configuration
let tests_to_run: Vec<&TestDefinition> = self
.tests
.iter()
.filter(|test| self.config.categories.contains(&test.category))
.filter(|test| !self.config.include_critical_only || test.is_critical)
.collect();
info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
for (i, test) in tests_to_run.iter().enumerate() {
info!(" {}. {} ({})", i + 1, test.name, test.category.as_str());
}
// Execute tests
for (i, test_def) in tests_to_run.iter().enumerate() {
info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
info!(" 📝 Description: {}", test_def.description);
info!(" 🏷️ Category: {}", test_def.category.as_str());
info!(" ⏱️ Estimated duration: {:?}", test_def.estimated_duration);
let test_start = Instant::now();
let result = self.run_single_test(test_def).await;
let test_duration = test_start.elapsed();
match result {
Ok(_) => {
info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
}
Err(e) => {
error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
results.push(TestResult::failure(
test_def.name.clone(),
test_def.category.clone(),
test_duration,
e.to_string(),
));
}
}
// Add delay between tests to avoid resource conflicts
if i < tests_to_run.len() - 1 {
debug!("⏸️ Waiting two seconds before the next test...");
sleep(Duration::from_secs(2)).await;
}
}
let total_duration = start_time.elapsed();
self.print_test_summary(&results, total_duration);
results
}
/// Run a single test by dispatching to the appropriate test function
async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// This is a placeholder for test dispatch logic
// In a real implementation, this would dispatch to actual test functions
warn!("⚠️ Test '{}' is not implemented in the unified runner; skipping", test_def.name);
Ok(())
}
/// Print comprehensive test summary
fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
info!("📊 KMS test suite summary");
info!("⏱️ Total duration: {:.2} seconds", total_duration.as_secs_f64());
info!("📈 Total tests: {}", results.len());
let passed = results.iter().filter(|r| r.success).count();
let failed = results.iter().filter(|r| !r.success).count();
info!("✅ Passed: {}", passed);
info!("❌ Failed: {}", failed);
info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
// Summary by category
let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
for result in results {
let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
*total += 1;
if result.success {
*passed_count += 1;
}
}
info!("📊 Category summary:");
for (category, (total, passed_count)) in category_summary {
info!(
" 🏷️ {}: {}/{} ({:.1}%)",
category.as_str(),
passed_count,
total,
(passed_count as f64 / total as f64) * 100.0
);
}
// List failed tests
if failed > 0 {
warn!("❌ Failing tests:");
for result in results.iter().filter(|r| !r.success) {
warn!(" - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
}
}
}
}
/// Quick test suite for critical tests only
#[tokio::test]
async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let config = TestSuiteConfig {
categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
include_critical_only: true,
max_duration: Some(Duration::from_secs(600)), // 10 minutes max
parallel_execution: false,
};
let suite = KMSTestSuite::new().with_config(config);
let results = suite.run_test_suite().await;
let failed_count = results.iter().filter(|r| !r.success).count();
if failed_count > 0 {
return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
}
info!("✅ All critical tests passed");
Ok(())
}
/// Full comprehensive test suite
#[tokio::test]
async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let suite = KMSTestSuite::new();
let results = suite.run_test_suite().await;
let total_tests = results.len();
let failed_count = results.iter().filter(|r| !r.success).count();
let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
info!("📊 Full suite success rate: {:.1}%", success_rate);
// Allow up to 10% failure rate for non-critical tests
if success_rate < 90.0 {
return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
}
info!("✅ Full test suite succeeded");
Ok(())
}
crates/e2e_test/src/object_lambda_test.rs
+25 -12
View File
@@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo
use aws_sdk_s3::primitives::ByteStream;
use http::header::{CONTENT_TYPE, HOST};
use reqwest::StatusCode;
use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
use rustfs_signer::pre_sign_v4;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use s3s::Body;
@@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
let bucket = "object-lambda-e2e-disabled-target";
let key = "input.txt";
@@ -992,17 +994,24 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
.send()
.await?;
configure_webhook_target_with_key_values(
&env,
"transformer",
vec![
("endpoint", "http://127.0.0.1:9/transform".to_string()),
("auth_token", "secret-token".to_string()),
("enable", "off".to_string()),
],
let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
tokio::fs::create_dir_all(&queue_dir).await?;
let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
let directive = format!(
"notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
);
let disable_response = signed_request(
http::Method::PUT,
&config_url,
&env.access_key,
&env.secret_key,
Some(directive.into_bytes()),
Some("text/plain"),
)
.await?;
wait_for_target_visibility(&env, "transformer").await?;
let disable_status = disable_response.status();
let disable_body = disable_response.text().await?;
assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");
let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
@@ -1021,7 +1030,8 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
let bucket = "object-lambda-e2e-invalid-endpoint";
@@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
let response = send_configure_webhook_target_request(
&env,
@@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
init_logging();
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
cluster.set_env(ENV_NOTIFY_ENABLE, "true");
cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
cluster.start().await?;
let bucket = "listen-notification-cluster";
crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs
+13 -24
View File
@@ -15,7 +15,6 @@
use crate::common::{RustFSTestClusterEnvironment, init_logging};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::CompletedMultipartUpload;
use tokio::time::{Duration, sleep};
use tracing::info;
use uuid::Uuid;
@@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload(
}
}
async fn complete_reports_missing_upload(
async fn multipart_listing_reports_missing_upload(
client: &aws_sdk_s3::Client,
bucket: &str,
key: &str,
upload_id: &str,
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
let result = client
.complete_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.multipart_upload(CompletedMultipartUpload::builder().build())
.send()
.await;
match result {
Ok(_) => Ok(false),
Err(SdkError::ServiceError(err)) => {
let code = err.err().meta().code().unwrap_or("");
if code == "NoSuchUpload" {
Ok(true)
} else {
Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
}
}
Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
}
let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
Ok(!result
.uploads()
.iter()
.any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
}
async fn wait_for_cleanup_on_all_nodes(
@@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes(
let mut all_cleaned = true;
for (idx, client) in clients.iter().enumerate() {
let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
if !(list_parts_missing && complete_missing) {
let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
if !(list_parts_missing && listing_missing) {
info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
all_cleaned = false;
break;
@@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
1,
"multipart upload should be visible before background cleanup"
);
assert!(
!multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
"multipart upload listing should contain the upload before background cleanup"
);
wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
crates/ecstore/src/cluster/rpc/remote_disk.rs
+90 -7
View File
@@ -42,8 +42,9 @@ use futures::lock::Mutex;
use metrics::counter;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
};
use rustfs_protos::ChannelClass;
use rustfs_protos::evict_failed_connection;
@@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const BATCH_METADATA_RPC_OFF: &str = "off";
const BATCH_METADATA_RPC_AUTO: &str = "auto";
const BATCH_METADATA_RPC_ON: &str = "on";
@@ -202,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
}
fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
.or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
.as_deref()
.map(parse_batch_metadata_rpc_mode)
@@ -1826,6 +1829,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
}
}
fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
if let Some(started_at) = started_at {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
}
}
/// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
/// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
/// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
@@ -1936,6 +1945,27 @@ fn decode_batch_read_version_response_items(
Ok(batch_read_version_resps)
}
fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
req.items
.iter()
.fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
total
.saturating_add(item.org_volume.len())
.saturating_add(item.volume.len())
.saturating_add(item.path.len())
.saturating_add(item.version_id.len())
})
}
fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
response
.batch_read_version_resps
.iter()
.map(String::len)
.sum::<usize>()
.saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
}
fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
file_info.validate_for_metadata_read().map_err(Into::into)
}
@@ -2837,14 +2867,19 @@ impl DiskAPI for RemoteDisk {
state = "started",
"Remote disk RPC started"
);
let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let batch_read_version_req = compat_json(&req)?;
let batch_read_version_req_bin = encode_msgpack(&req)?;
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
let request_payload_bytes = batch_read_version_attribution_enabled
.then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
let batch_result = self
.execute_with_timeout_for_op(
"batch_read_version",
move || async move {
let disk = self.disk_ref().await;
let disk_len = disk.len();
let mut client = self
.get_bulk_client()
.await
@@ -2855,9 +2890,20 @@ impl DiskAPI for RemoteDisk {
batch_read_version_req_bin: batch_read_version_req_bin.into(),
});
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
if let Some(request_payload_bytes) = request_payload_bytes {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
request_payload_bytes.saturating_add(disk_len),
);
}
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let response = match client.batch_read_version(request).await {
Ok(response) => response.into_inner(),
Ok(response) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
response.into_inner()
}
Err(status) if status.code() == Code::Unimplemented => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
if mode.should_fallback_on_unimplemented() {
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
warn!(
@@ -2874,6 +2920,7 @@ impl DiskAPI for RemoteDisk {
}
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
@@ -2886,14 +2933,33 @@ impl DiskAPI for RemoteDisk {
);
return Err(Error::from(status));
}
Err(status) => return Err(Error::from(status)),
Err(status) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(Error::from(status));
}
};
if !response.success {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(response.error.unwrap_or_default().into());
}
decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
batch_read_version_response_payload_len(&response),
);
let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
match decode_batch_read_version_response_items(response, &self.endpoint) {
Ok(batch_read_version_resps) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
Ok(Some(batch_read_version_resps))
}
Err(err) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
Err(err)
}
}
},
get_max_timeout_duration(),
)
@@ -4621,6 +4687,7 @@ mod tests {
} else {
"file version not found".to_string()
},
error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
}
}
@@ -4740,6 +4807,7 @@ mod tests {
fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
@@ -4749,10 +4817,25 @@ mod tests {
);
}
#[test]
fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
],
|| {
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
},
);
}
#[test]
fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
crates/ecstore/src/cluster/rpc/runtime_sources.rs
+57 -3
View File
@@ -14,9 +14,10 @@
use rustfs_io_metrics::internode_metrics::{
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
global_internode_metrics,
};
use std::time::Duration;
@@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_stage_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
stage,
duration,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics()
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
}
pub(crate) fn record_remote_disk_grpc_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
crates/ecstore/src/core/pools.rs
+1013 -263
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/mod.rs
+69 -25
View File
@@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_DISK: &str = "disk";
const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;
pub fn part_transaction_path(part_path: &str) -> String {
match part_path.rsplit_once('/') {
@@ -62,6 +64,7 @@ use bytes::Bytes;
use endpoint::Endpoint;
use error::DiskError;
use error::{Error, Result};
use futures::stream::{self, StreamExt};
use local::LocalDisk;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_madmin::info_commands::DiskMetrics;
@@ -417,6 +420,14 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
match self {
Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
match self {
@@ -1028,36 +1039,47 @@ where
D: DiskAPI + ?Sized,
{
validate_batch_read_version_item_count(req.items.len())?;
let parallelism = batch_read_version_server_parallelism();
let mut responses = Vec::with_capacity(req.items.len());
for (index, item) in req.items.iter().enumerate() {
let response = match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: true,
file_info,
error: String::new(),
},
Err(err) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
},
};
responses.push(response);
}
let mut responses = stream::iter(req.items.into_iter().enumerate())
.map(|(index, item)| async move {
match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: true,
file_info,
error: String::new(),
error_code: 0,
},
Err(err) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
error_code: err.to_u32(),
},
}
})
.buffer_unordered(parallelism)
.collect::<Vec<_>>()
.await;
responses.sort_unstable_by_key(|response| response.index);
Ok(responses)
}
fn batch_read_version_server_parallelism() -> usize {
rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
.clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CheckPartsResp {
pub results: Vec<usize>,
@@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp {
pub success: bool,
pub file_info: FileInfo,
pub error: String,
#[serde(default)]
pub error_code: u32,
}
pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
@@ -1417,6 +1441,26 @@ mod tests {
assert!(!partial_valid_location.valid());
}
#[test]
fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
assert_eq!(batch_read_version_server_parallelism(), 4);
});
}
#[test]
fn batch_read_version_server_parallelism_honors_env_with_bounds() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
assert_eq!(batch_read_version_server_parallelism(), 8);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
assert_eq!(batch_read_version_server_parallelism(), 1);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
});
}
/// Test FileInfoVersions find_version_index
#[test]
fn test_file_info_versions_find_version_index() {
crates/ecstore/src/lib.rs
+8
View File
@@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() {
cluster::rpc::shutdown_background_monitors();
}
/// Publish that the process is ready to serve user-object GET traffic.
///
/// Experimental metadata coalescing is allowed to run only after this point so
/// startup and internal metadata reads keep the original per-disk path.
pub fn mark_get_metadata_read_version_coalescing_service_ready() {
runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
}
#[cfg(test)]
mod rio_tests {
#[test]
crates/ecstore/src/runtime/global.rs
+14 -1
View File
@@ -25,7 +25,10 @@ use lazy_static::lazy_static;
use rustfs_lock::client::LockClient;
use std::{
collections::HashMap,
sync::{Arc, OnceLock},
sync::{
Arc, OnceLock,
atomic::{AtomicBool, Ordering},
},
time::SystemTime,
};
use tokio::sync::{OnceCell, RwLock};
@@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000;
pub const DISK_FILL_FRACTION: f64 = 0.99;
pub const DISK_RESERVE_FRACTION: f64 = 0.15;
static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
}
pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
}
// Global singletons for backward compatibility with MinIO port.
// These should be migrated to AppContext over time.
// See issue #730 for migration plan.
crates/ecstore/src/runtime/instance.rs
+9
View File
@@ -160,6 +160,10 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Serializes decommission data-movement operations with cancellation and
/// a subsequent restart. Readers are held across one object side effect;
/// the transition path takes the writer after cancelling the routine.
decommission_operation_gate: Arc<RwLock<()>>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -200,6 +204,7 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
decommission_operation_gate: Arc::new(RwLock::new(())),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -218,6 +223,10 @@ impl InstanceContext {
self.lock_manager.clone()
}
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.decommission_operation_gate)
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
crates/ecstore/src/set_disk/core/io_primitives.rs
+565 -7
View File
@@ -53,11 +53,12 @@ use crate::diagnostics::get::{
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::disk::disk_store::DiskStoreRenameDataExt;
use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout};
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
use crate::disk::{
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
};
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::ShardReader;
@@ -75,7 +76,7 @@ use std::{
future::Future,
pin::Pin,
sync::{
OnceLock,
Arc, OnceLock,
atomic::{AtomicUsize, Ordering},
},
task::{Context, Poll},
@@ -94,6 +95,242 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String {
[bucket, object].join("/")
}
fn read_version_coalescing_enabled() -> bool {
let enabled = || {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on"))
};
#[cfg(test)]
{
enabled()
}
#[cfg(not(test))]
{
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(enabled)
}
}
fn read_version_coalescing_delay() -> Duration {
#[cfg(test)]
{
let micros = rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
);
Duration::from_micros(micros)
}
#[cfg(not(test))]
{
static DELAY: OnceLock<Duration> = OnceLock::new();
*DELAY.get_or_init(|| {
Duration::from_micros(rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
))
})
}
}
struct CoalescedReadVersionRequest {
item: BatchReadVersionItem,
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
struct ReadVersionCoalescerKey {
disk: usize,
incl_free_versions: bool,
read_data: bool,
healing: bool,
}
impl ReadVersionCoalescerKey {
fn new(disk: &DiskStore, opts: &ReadOptions) -> Self {
Self {
disk: Arc::as_ptr(disk) as usize,
incl_free_versions: opts.incl_free_versions,
read_data: opts.read_data,
healing: opts.healing,
}
}
}
#[derive(Default)]
struct ReadVersionCoalescer {
lanes: HashMap<ReadVersionCoalescerKey, Vec<CoalescedReadVersionRequest>>,
}
fn read_version_coalescer() -> &'static Mutex<ReadVersionCoalescer> {
static COALESCER: OnceLock<Mutex<ReadVersionCoalescer>> = OnceLock::new();
COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default()))
}
fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
counter!(
METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL,
"event" => event,
"item_count" => item_count.to_string()
)
.increment(1);
}
async fn read_version_via_coalescer(
disk: DiskStore,
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
opts: &ReadOptions,
allow_coalescing: bool,
) -> disk::error::Result<FileInfo> {
if !allow_coalescing || !read_version_coalescing_enabled() {
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
if !matches!(disk.as_ref(), Disk::Remote(_)) {
record_read_version_coalescer_event("bypass_non_remote", 1);
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
let (tx, rx) = oneshot::channel();
let item = BatchReadVersionItem {
org_volume: org_bucket.to_string(),
volume: bucket.to_string(),
path: object.to_string(),
version_id: version_id.to_string(),
};
let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
let lane = coalescer.lanes.entry(lane_key).or_default();
let schedule_delayed_flush = lane.is_empty();
lane.push(CoalescedReadVersionRequest { item, tx });
if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
coalescer.lanes.remove(&lane_key)
} else if schedule_delayed_flush {
let disk = disk.clone();
let task_opts = *opts;
tokio::spawn(async move {
tokio::time::sleep(read_version_coalescing_delay()).await;
flush_read_version_coalescer_lane(lane_key, disk, task_opts).await;
});
None
} else {
None
}
};
if let Some(pending) = pending {
flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
}
rx.await
.unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
}
async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
coalescer.lanes.remove(&lane_key).unwrap_or_default()
};
flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await;
}
async fn flush_read_version_coalescer_pending(
lane_key: ReadVersionCoalescerKey,
disk: DiskStore,
opts: ReadOptions,
pending: Vec<CoalescedReadVersionRequest>,
) {
if pending.is_empty() {
return;
}
#[cfg(test)]
{
let mut observed_paths = HashSet::new();
for request in &pending {
if observed_paths.insert(request.item.path.as_str()) {
disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk);
}
}
}
let mut senders = Vec::with_capacity(pending.len());
let mut items = Vec::with_capacity(pending.len());
for request in pending {
senders.push(request.tx);
items.push(request.item);
}
let expected_items = items.clone();
record_read_version_coalescer_event("attempted_batch", items.len());
let result =
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
.await
{
Ok(result) => result,
Err(_) => Err(DiskError::Timeout),
};
match result {
Ok(responses) => {
let results = map_batch_read_version_responses(&expected_items, responses);
for (tx, result) in senders.into_iter().zip(results) {
let _ = tx.send(result);
}
}
Err(err) => {
let message = err.to_string();
for tx in senders {
let _ = tx.send(Err(DiskError::other(message.clone())));
}
}
}
}
fn map_batch_read_version_responses(
expected_items: &[BatchReadVersionItem],
responses: Vec<BatchReadVersionResp>,
) -> Vec<crate::disk::error::Result<FileInfo>> {
let mut results = (0..expected_items.len())
.map(|_| Err(DiskError::other("coalesced read_version response missing")))
.collect::<Vec<_>>();
let mut seen = vec![false; expected_items.len()];
for response in responses {
let Some(expected) = expected_items.get(response.index) else {
continue;
};
let Some(slot) = results.get_mut(response.index) else {
continue;
};
if seen[response.index] {
*slot = Err(DiskError::other("coalesced read_version response duplicate index"));
continue;
}
seen[response.index] = true;
if response.path != expected.path || response.version_id != expected.version_id {
*slot = Err(DiskError::other("coalesced read_version response identity mismatch"));
} else {
*slot = if response.success {
Ok(response.file_info)
} else {
Err(batch_read_version_response_error(response.error_code, response.error))
};
}
}
results
}
fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError {
match DiskError::from_u32(error_code) {
Some(DiskError::Io(_)) | None => DiskError::other(error),
Some(error) => error,
}
}
pub(in crate::set_disk) fn bounded_metadata_fanout_order(
bucket: &str,
object: &str,
@@ -133,11 +370,15 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
order
}
use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::RwLock;
use tokio::sync::{Mutex, RwLock, oneshot};
use tokio::task::JoinSet;
pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
/// Default reader-setup strategy for the GET read path (rustfs/backlog#1215,
/// #1159, #923).
@@ -2356,6 +2597,7 @@ impl SetDisks {
false,
true,
0,
false,
)
.await?;
Ok((ress, errors))
@@ -2386,6 +2628,36 @@ impl SetDisks {
true,
caller_allows_early_stop,
default_parity_count,
false,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
incl_free_versions: bool,
caller_allows_early_stop: bool,
default_parity_count: usize,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
Self::read_all_fileinfo_inner(
disks,
org_bucket,
bucket,
object,
version_id,
read_data,
false,
incl_free_versions,
true,
caller_allows_early_stop,
default_parity_count,
true,
)
.await
}
@@ -2408,6 +2680,7 @@ impl SetDisks {
// subset would fail write quorum (backlog#872 regression).
caller_allows_early_stop: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let early_stop_enabled =
caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled());
@@ -2424,6 +2697,7 @@ impl SetDisks {
healing,
incl_free_versions,
default_parity_count,
allow_coalescing,
)
.await;
}
@@ -2446,6 +2720,7 @@ impl SetDisks {
healing,
incl_free_versions,
observe,
allow_coalescing,
)
.await
}
@@ -2461,6 +2736,7 @@ impl SetDisks {
healing: bool,
incl_free_versions: bool,
observe: bool,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = observe.then(Instant::now);
let mut ress = Vec::with_capacity(disks.len());
@@ -2492,7 +2768,7 @@ impl SetDisks {
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
@@ -2559,6 +2835,7 @@ impl SetDisks {
healing: bool,
incl_free_versions: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = Instant::now();
let mut ress = vec![FileInfo::default(); disks.len()];
@@ -2607,7 +2884,7 @@ impl SetDisks {
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
@@ -5737,6 +6014,7 @@ pub(crate) mod disk_call_counters {
/// Kind label for the per-disk `read_version` metadata RPC.
pub const KIND_READ_VERSION: &str = "read_version";
pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version";
/// Registry key: (object, kind, disk_index).
type CountKey = (String, String, usize);
@@ -6460,6 +6738,286 @@ mod tests {
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn metadata_read_version_coalescer_bypasses_local_disks() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-local-bypass-bucket";
let object_a = "coalesced-local-object-a";
let object_b = "coalesced-local-object-b";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await;
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")),
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")),
],
async {
let calls = disk_call_counters::observe(object_a);
let disks_a = disks.clone();
let disks_b = disks.clone();
let read_a = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_a, "", bucket, object_a, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
tokio::task::yield_now().await;
let read_b = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_b, "", bucket, object_b, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
let (metadata_a, errs_a) = read_a
.await
.expect("first read task should not panic")
.expect("first coalesced read should resolve");
let (metadata_b, errs_b) = read_b
.await
.expect("second read task should not panic")
.expect("second coalesced read should resolve");
assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS);
assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS);
assert!(errs_a.iter().all(Option::is_none));
assert!(errs_b.iter().all(Option::is_none));
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"local disks still execute the ordinary per-disk read_version path"
);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"GET coalescing targets internode RPC count only and must not batch local disk reads"
);
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn metadata_read_version_coalescer_requires_get_object_intent() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-default-bypass-bucket";
let object = "default-bypass-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async {
let calls = disk_call_counters::observe(object);
let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false)
.await
.expect("default metadata read should resolve");
assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"non-GET metadata paths must bypass coalescer even when the env gate is enabled"
);
})
.await;
drop(dirs);
}
#[test]
fn batch_read_version_response_mapping_preserves_index_and_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-c".to_string(),
version_id: "v-c".to_string(),
},
];
let ok_file_info = FileInfo {
name: "object-a".to_string(),
..Default::default()
};
let responses = vec![
BatchReadVersionResp {
index: 2,
path: "object-c".to_string(),
version_id: "v-c".to_string(),
success: false,
file_info: FileInfo::default(),
error: "disk read failed".to_string(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: ok_file_info,
error: String::new(),
error_code: 0,
},
];
let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
let first = results
.next()
.expect("slot 0 should exist")
.expect("slot 0 should map the success response by index");
assert_eq!(first.name, "object-a");
let missing = results
.next()
.expect("slot 1 should exist")
.expect_err("slot 1 should stay missing");
assert!(
missing.to_string().contains("response missing"),
"unexpected missing response error: {missing}"
);
let failed = results
.next()
.expect("slot 2 should exist")
.expect_err("slot 2 should map the response error");
assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}");
assert!(results.next().is_none());
}
#[test]
fn batch_read_version_response_mapping_preserves_typed_not_found_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
];
let results = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileNotFound.to_string(),
error_code: DiskError::FileNotFound.to_u32(),
},
BatchReadVersionResp {
index: 1,
path: "object-b".to_string(),
version_id: "v-b".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileVersionNotFound.to_string(),
error_code: DiskError::FileVersionNotFound.to_u32(),
},
],
);
assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound)));
assert!(matches!(
results.get(1).expect("slot 1 should exist"),
Err(DiskError::FileVersionNotFound)
));
}
#[test]
fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() {
let expected_items = vec![BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
}];
let mismatched = map_batch_read_version_responses(
&expected_items,
vec![BatchReadVersionResp {
index: 0,
path: "object-b".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-b".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
}],
)
.pop()
.expect("slot 0 should exist")
.expect_err("identity mismatch should fail closed");
assert!(
mismatched.to_string().contains("identity mismatch"),
"unexpected mismatch error: {mismatched}"
);
let duplicate = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
],
)
.pop()
.expect("slot 0 should exist")
.expect_err("duplicate response index should fail closed");
assert!(
duplicate.to_string().contains("duplicate index"),
"unexpected duplicate error: {duplicate}"
);
}
/// Isolation guard: unobserved objects record nothing (so parallel tests do
/// not inflate one another), and a scope clears its own counts on drop.
#[tokio::test]
crates/ecstore/src/set_disk/ops/object.rs
+1 -1
View File
@@ -1294,7 +1294,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
(prepared.snapshot, prepared.object_info)
} else {
match self
.get_object_fileinfo(
.get_object_fileinfo_for_get_object_reader(
bucket,
object,
opts,
crates/ecstore/src/set_disk/read.rs
+66 -14
View File
@@ -259,10 +259,33 @@ impl SetDisks {
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false)
.await
}
#[tracing::instrument(level = "debug", skip(self))]
#[hotpath::measure(impl_type = "SetDisks")]
pub(super) async fn get_object_fileinfo_for_get_object_reader(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<GetObjectFileInfo> {
let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket)
&& crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
self.get_object_fileinfo_gated_inner(
bucket,
object,
opts,
read_data,
caller_allows_early_stop,
allow_read_version_coalescing,
)
.await
}
/// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
/// quorum fanout. Read-before-write callers (object tagging) must use this:
/// the returned online-disk set is the write target, and the early-stop
@@ -275,6 +298,20 @@ impl SetDisks {
opts: &ObjectOptions,
read_data: bool,
allow_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false)
.await
}
#[allow(clippy::too_many_arguments)]
async fn get_object_fileinfo_gated_inner(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
read_data: bool,
allow_early_stop: bool,
allow_read_version_coalescing: bool,
) -> Result<GetObjectFileInfo> {
let vid = opts.version_id.clone().unwrap_or_default();
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
@@ -337,19 +374,34 @@ impl SetDisks {
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
// core/io_primitives.rs); unsafe requests and callers that opt out
// (allow_early_stop=false) fall back to full-wait.
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
false,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?;
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing {
Self::read_all_fileinfo_observed_for_get_object(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?
} else {
Self::read_all_fileinfo_observed(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
false,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?
};
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
GET_OBJECT_PATH_INTERNAL_META
} else {
crates/ecstore/src/store/rebalance.rs
+386 -1
View File
@@ -859,6 +859,7 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
use crate::disk::error::DiskError;
use crate::layout::endpoint::Endpoint;
@@ -1423,6 +1424,14 @@ mod tests {
}
}
fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
ObjectInfo {
version_id: Some(version_id),
etag,
..object_info_with_mod_time(unix_ts, delete_marker)
}
}
#[test]
fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
let candidates = vec![
@@ -1446,7 +1455,7 @@ mod tests {
}
#[test]
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() {
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_mod_time(10, false)),
@@ -1466,6 +1475,382 @@ mod tests {
assert_eq!(idx, 1);
}
#[test]
fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 2,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 7,
err: None,
},
];
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect("equivalent replicas must resolve deterministically");
assert_eq!(idx, 7);
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("divergent version ids must not silently resolve to the higher pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("divergent etags must not silently resolve to the higher pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("a delete marker tied with a live version must not be masked by the pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
let err = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(left),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(right),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect_err("equal-time identity divergence must fail closed");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut data_dir = base.clone();
data_dir.data_dir = Some(Uuid::from_u128(2));
assert_equal_time_identity_conflict(base.clone(), data_dir);
let mut size = base.clone();
size.size = 1;
assert_equal_time_identity_conflict(base.clone(), size);
let mut actual_size = base.clone();
actual_size.actual_size = 1;
assert_equal_time_identity_conflict(base.clone(), actual_size);
let mut checksum = base.clone();
checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
assert_equal_time_identity_conflict(base.clone(), checksum);
let mut parts = base.clone();
parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
etag: "part-etag".to_string(),
number: 1,
size: 1,
..Default::default()
}]);
assert_equal_time_identity_conflict(base.clone(), parts);
let mut transition = base;
transition.transitioned_object.tier = "tier-a".to_string();
assert_equal_time_identity_conflict(
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
transition,
);
}
#[test]
fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let mut minio_alias = base.clone();
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"X-MINIO-INTERNAL-COMPRESSION".to_string(),
"zstd".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(rustfs_alias),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(minio_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("same-value internal aliases should resolve");
assert_eq!(idx, 1);
let mut dual_alias = base.clone();
dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
("x-minio-internal-compression".to_string(), "zstd".to_string()),
]));
let mut single_alias = base;
single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(dual_alias),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(single_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("dual-key and single-key internal metadata should resolve");
assert_eq!(idx, 1);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let mut minio_alias = base;
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-minio-internal-compression".to_string(),
"snappy".to_string(),
)]));
assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
}
#[test]
fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!(
"X-RUSTFS-INTERNAL-{}{suffix}",
suffix_prefix.to_uppercase(),
suffix = "arn:aws:s3:::Bucket"
),
"value".to_string(),
)]));
let mut minio_alias = base.clone();
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
"value".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(rustfs_alias.clone()),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(minio_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("dynamic internal aliases with the same target should resolve");
assert_eq!(idx, 1);
let mut different_target_case = base;
different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
"value".to_string(),
)]));
assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
}
}
#[test]
fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut first = base.clone();
first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
("x-minio-internal-compression".to_string(), "snappy".to_string()),
]));
let mut second = base;
second.user_defined = first.user_defined.clone();
assert_equal_time_identity_conflict(first, second);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut replication = base.clone();
replication.replication_status_internal = Some("PENDING".to_string());
replication.replication_status = ReplicationStatusType::Pending;
assert_equal_time_identity_conflict(base.clone(), replication);
let mut purge = base.clone();
purge.version_purge_status_internal = Some("PENDING".to_string());
purge.version_purge_status = VersionPurgeStatusType::Pending;
assert_equal_time_identity_conflict(base.clone(), purge);
let mut decision = base;
decision.replication_decision = "replicate".to_string();
assert_equal_time_identity_conflict(
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
decision,
);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
without_mod_time.mod_time = None;
let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
}
#[test]
fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
older.data_dir = Some(Uuid::from_u128(2));
let (info, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(latest),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(older),
idx: 9,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("older identity divergence must not affect the latest candidate");
assert_eq!(idx, 0);
assert_eq!(
info.mod_time,
Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
);
}
#[test]
fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: None,
idx: 1,
err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
];
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect("not-found pools must not block resolution of found candidates");
assert_eq!(idx, 0);
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
}
#[test]
fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
let err = resolve_latest_object_info_candidates(
crates/ecstore/src/store/rebalance/support.rs
+146 -21
View File
@@ -12,10 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::cmp::Ordering;
use std::collections::HashMap;
use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{ObjectInfo, ObjectOptions};
use rustfs_utils::http::metadata_compat::{
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
strip_internal_prefix_preserving_case,
};
use rustfs_utils::path::decode_dir_object;
use time::OffsetDateTime;
@@ -137,37 +141,158 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p
))
}
fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option<OffsetDateTime> {
candidate
.info
.as_ref()
.map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
}
fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
left.transition_version_state == right.transition_version_state
&& left.transitioned_object.name == right.transitioned_object.name
&& left.transitioned_object.version_id == right.transitioned_object.version_id
&& left.transitioned_object.tier == right.transitioned_object.tier
&& left.transitioned_object.free_version == right.transitioned_object.free_version
&& left.transitioned_object.status == right.transitioned_object.status
}
#[derive(PartialEq, Eq)]
struct LatestUserDefinedIdentity {
internal: HashMap<String, String>,
other: HashMap<String, String>,
}
fn normalize_internal_identity_suffix(key: &str) -> Option<String> {
let suffix = strip_internal_prefix_preserving_case(key)?;
for dynamic_prefix in [
SUFFIX_REPLICATION_RESET_ARN_PREFIX,
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
] {
let prefix_len = dynamic_prefix.len();
if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..))
&& prefix.eq_ignore_ascii_case(dynamic_prefix)
{
return Some(format!("{dynamic_prefix}{remainder}"));
}
}
Some(suffix.to_lowercase())
}
fn normalize_user_defined_identity(user_defined: &HashMap<String, String>) -> Option<LatestUserDefinedIdentity> {
let mut identity = LatestUserDefinedIdentity {
internal: HashMap::with_capacity(user_defined.len()),
other: HashMap::with_capacity(user_defined.len()),
};
for (key, value) in user_defined {
if let Some(suffix) = normalize_internal_identity_suffix(key) {
if identity
.internal
.insert(suffix, value.clone())
.is_some_and(|previous| previous != *value)
{
return None;
}
} else {
identity.other.insert(key.clone(), value.clone());
}
}
Some(identity)
}
fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
match (
normalize_user_defined_identity(&left.user_defined),
normalize_user_defined_identity(&right.user_defined),
) {
(Some(left), Some(right)) => left == right,
_ => false,
}
}
/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
/// differ for the same object version while the selected winner still carries
/// the chosen pool's layout. `put_object_reader` is also intentionally
/// excluded because it is a transient request handle that `ObjectInfo::clone`
/// drops. Every other ObjectInfo field is part of the production-visible
/// identity and must agree before the pool index can provide a deterministic
/// tie-break.
fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
left.bucket == right.bucket
&& left.name == right.name
&& left.storage_class == right.storage_class
&& left.mod_time == right.mod_time
&& left.size == right.size
&& left.actual_size == right.actual_size
&& left.is_dir == right.is_dir
&& same_user_defined_identity(left, right)
&& left.user_tags == right.user_tags
&& left.version_id == right.version_id
&& left.data_dir == right.data_dir
&& left.delete_marker == right.delete_marker
&& same_transition_identity(left, right)
&& left.restore_ongoing == right.restore_ongoing
&& left.restore_expires == right.restore_expires
&& left.parts == right.parts
&& left.is_latest == right.is_latest
&& left.content_type == right.content_type
&& left.content_encoding == right.content_encoding
&& left.expires == right.expires
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.etag == right.etag
&& left.inlined == right.inlined
&& left.metadata_only == right.metadata_only
&& left.version_only == right.version_only
&& left.replication_status_internal == right.replication_status_internal
&& left.replication_status == right.replication_status
&& left.version_purge_status_internal == right.version_purge_status_internal
&& left.version_purge_status == right.version_purge_status
&& left.replication_decision == right.replication_decision
&& left.checksum == right.checksum
}
pub(super) fn resolve_latest_object_info_candidates(
mut candidates: Vec<LatestObjectInfoCandidate>,
candidates: Vec<LatestObjectInfoCandidate>,
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, usize)> {
candidates.sort_by(|a, b| {
let a_mod = if let Some(info) = &a.info {
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
OffsetDateTime::UNIX_EPOCH
let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max();
if let Some(latest_mod_time) = latest_mod_time {
let mut latest_candidates = candidates
.into_iter()
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
.collect::<Vec<_>>();
latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
let Some(winner) = latest_candidates.first() else {
return Err(Error::ErasureReadQuorum);
};
let Some(winner_info) = winner.info.as_ref() else {
return Err(Error::ErasureReadQuorum);
};
let b_mod = if let Some(info) = &b.info {
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
OffsetDateTime::UNIX_EPOCH
};
if a_mod == b_mod {
return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less };
if latest_candidates.iter().skip(1).any(|candidate| {
candidate
.info
.as_ref()
.is_none_or(|info| !same_latest_object_info_identity(winner_info, info))
}) {
return Err(Error::ErasureReadQuorum);
}
b_mod.cmp(&a_mod)
});
return Ok((winner_info.clone(), winner.idx));
}
for candidate in candidates {
if let Some(info) = candidate.info {
return Ok((info, candidate.idx));
}
if let Some(err) = candidate.err
&& !is_err_object_not_found(&err)
&& !is_err_version_not_found(&err)
crates/io-metrics/src/internode_metrics.rs
+7
View File
@@ -54,6 +54,13 @@ pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_versio
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode";
pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip";
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE: &str = "batch_read_version_request_encode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE: &str = "batch_read_version_request_decode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ: &str = "batch_read_version_disk_read";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch_read_version_response_json_encode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip";
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode";
const OPERATION_LABEL: &str = "operation";
const BACKEND_LABEL: &str = "backend";
crates/policy/src/policy/policy.rs
+40 -2
View File
@@ -195,7 +195,8 @@ pub struct BucketPolicyArgs<'a> {
#[derive(Serialize, Deserialize, Clone, Default, Debug)]
#[serde(deny_unknown_fields)]
pub struct BucketPolicy {
#[serde(default, rename = "Id", skip_serializing_if = "ID::is_empty")]
// RUSTFS_COMPAT_TODO(rustfs-6339): accept bucket policies persisted with the legacy "ID" key. Remove after migration tooling rewrites every retained legacy bucket policy.
#[serde(default, rename = "Id", alias = "ID", skip_serializing_if = "ID::is_empty")]
pub id: ID,
#[serde(rename = "Version")]
pub version: String,
@@ -2786,7 +2787,7 @@ mod test {
let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse");
// Verify empty fields are omitted
assert!(!parsed.as_object().unwrap().contains_key("ID"), "Empty ID should be omitted");
assert!(parsed.get("Id").is_none(), "Empty ID should be omitted");
let statement = &parsed["Statement"][0];
assert!(!statement.as_object().unwrap().contains_key("Sid"), "Empty Sid should be omitted");
@@ -2809,6 +2810,43 @@ mod test {
assert_eq!(statement["Principal"]["AWS"], "*");
}
#[test]
fn test_bucket_policy_deserializes_legacy_id() {
let legacy_policy = br#"{"ID":"","Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"NotAction":[],"Resource":["arn:aws:s3:::bucket/*"],"NotResource":[],"Condition":{}}]}"#;
let policy: BucketPolicy =
serde_json::from_slice(legacy_policy).expect("bucket policy with legacy ID should deserialize");
assert!(policy.id.is_empty());
policy.is_valid().expect("legacy bucket policy should remain valid");
let policy: BucketPolicy = serde_json::from_str(r#"{"ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#)
.expect("non-empty legacy ID should deserialize");
assert_eq!(policy.id.0, "legacy-policy");
let serialized = serde_json::to_value(&policy).expect("bucket policy should serialize");
assert_eq!(serialized["Id"], "legacy-policy");
assert!(serialized.get("ID").is_none(), "legacy ID spelling should not be serialized");
}
#[test]
fn test_bucket_policy_legacy_id_alias_remains_strict() {
let unknown_field = r#"{"Version":"2012-10-17","Statement":[],"Unexpected":true}"#;
let error =
serde_json::from_str::<BucketPolicy>(unknown_field).expect_err("unrelated unknown fields should remain rejected");
assert!(
error.to_string().contains("unknown field `Unexpected`"),
"unexpected deserialization error: {error}"
);
let duplicate_id = r#"{"Id":"current-policy","ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#;
let error = serde_json::from_str::<BucketPolicy>(duplicate_id)
.expect_err("canonical and legacy ID fields should not be accepted together");
assert!(
error.to_string().contains("duplicate field `Id`"),
"unexpected deserialization error: {error}"
);
}
#[test]
fn test_existing_object_tag_condition_helpers() {
let identity_policy = Policy::parse_config(