gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-28 16:07:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ecstore): preserve multipart part quorum errors (#3920)

Browse Source 
* fix(ecstore): preserve multipart part quorum errors

* fix(ecstore): honor confirmed missing part quorum

* fix(ecstore): add multipart quorum diagnostics
This commit is contained in:
cxymds
2026-06-27 08:33:43 +08:00
committed by GitHub
parent 27bb9c75dc
commit 688b14b572
6 changed files with 497 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/mod.rs
+156 -7
View File
@@ -25,7 +25,8 @@ use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
use crate::disk::error_reduce::{
BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, count_errs, reduce_read_quorum_errs, reduce_write_quorum_errs,
BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs,
reduce_write_quorum_errs,
};
use crate::disk::{
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
@@ -152,6 +153,12 @@ type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk";
const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart";
const COMPLETE_MULTIPART_PART_MISSING: &str = "part_missing";
const COMPLETE_MULTIPART_PART_READ_QUORUM_UNAVAILABLE: &str = "read_quorum_unavailable";
const COMPLETE_MULTIPART_PART_ERROR: &str = "part_error";
const MULTIPART_WRITE_QUORUM_UPLOAD_METADATA: &str = "upload_metadata";
const MULTIPART_WRITE_QUORUM_WRITER_SETUP: &str = "writer_setup";
const MULTIPART_WRITE_QUORUM_RENAME_PART: &str = "rename_part";
const EVENT_SET_DISK_WRITE: &str = "set_disk_write";
const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
const EVENT_SET_DISK_COMMIT_TAIL_SLOW: &str = "set_disk_commit_tail_slow";
@@ -449,6 +456,28 @@ fn get_codec_streaming_reader_decision(
GetCodecStreamingDecision::Use
}
fn is_confirmed_complete_part_missing(err: &str) -> bool {
err.contains("file not found")
|| err.contains("Specified part could not be found")
|| (err.starts_with("part.") && err.ends_with(" not found"))
}
fn complete_multipart_part_error(part_number: usize, err: &str, bucket: &str, object: &str) -> Error {
if is_confirmed_complete_part_missing(err) {
return Error::InvalidPart(part_number, bucket.to_owned(), object.to_owned());
}
to_object_err(Error::ErasureReadQuorum, vec![bucket, object])
}
fn complete_multipart_part_error_result(err: &Error) -> &'static str {
match err {
Error::InvalidPart(_, _, _) => COMPLETE_MULTIPART_PART_MISSING,
Error::ErasureReadQuorum | Error::InsufficientReadQuorum(_, _) => COMPLETE_MULTIPART_PART_READ_QUORUM_UNAVAILABLE,
_ => COMPLETE_MULTIPART_PART_ERROR,
}
}
/// Record a lock acquisition for deadlock detection.
/// This records detailed lock information for deadlock analysis.
/// Returns the lock_id for later release tracking.
@@ -492,6 +521,47 @@ fn record_lock_release(bucket: &str, object: &str, lock_id: &str, lock_type: &st
);
}
#[derive(Clone, Copy, Debug)]
pub(super) struct MultipartWriteQuorumContext<'a> {
stage: &'static str,
bucket: &'a str,
object: &'a str,
upload_id: &'a str,
part_number: Option<usize>,
}
fn log_multipart_write_quorum_failure(
context: MultipartWriteQuorumContext<'_>,
errs: &[Option<DiskError>],
write_quorum: usize,
returned_error: &DiskError,
) {
let summary = build_write_quorum_failure_summary(errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
runtime_sources::record_erasure_write_quorum_failure(context.stage, summary.dominant_error_label);
warn!(
target: "rustfs_ecstore::set_disk",
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
op = "upload_part",
state = "write_quorum_unavailable",
stage = context.stage,
bucket = %context.bucket,
object = %context.object,
upload_id = %context.upload_id,
part_number = context.part_number,
required = summary.required,
achieved = summary.achieved,
failed = summary.failed,
total = summary.total,
offline_disks = summary.offline_disks,
retryable_failures = summary.retryable_failures,
dominant_error = summary.dominant_error_label,
returned_error = %returned_error,
"Set disk multipart write quorum unavailable"
);
}
fn issue3031_diag_enabled() -> bool {
rustfs_utils::get_env_bool(ENV_ISSUE3031_DIAG_ENABLE, false)
}
@@ -3702,6 +3772,18 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum {
if let Some(write_err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, write_quorum) {
log_multipart_write_quorum_failure(
MultipartWriteQuorumContext {
stage: MULTIPART_WRITE_QUORUM_WRITER_SETUP,
bucket,
object,
upload_id,
part_number: Some(part_id),
},
&errors,
write_quorum,
&write_err,
);
return Err(to_object_err(write_err.into(), vec![bucket, object]));
}
@@ -3803,6 +3885,13 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
&part_path,
part_info_buff.into(),
write_quorum,
Some(MultipartWriteQuorumContext {
stage: MULTIPART_WRITE_QUORUM_RENAME_PART,
bucket,
object,
upload_id,
part_number: Some(part_id),
}),
)
.await?;
@@ -4293,8 +4382,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let part_numbers = uploaded_parts.iter().map(|v| v.part_num).collect::<Vec<usize>>();
let object_parts =
Self::read_parts(&disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum).await?;
let object_parts = Self::read_parts(&disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
if object_parts.len() != uploaded_parts.len() {
return Err(Error::other("part result number err"));
@@ -4317,11 +4407,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
for (i, part) in object_parts.iter().enumerate() {
if let Some(err) = &part.error {
error!("complete_multipart_upload part error: {:?}", &err);
if issue3031_diag_enabled() {
warn!(
let mapped_err = complete_multipart_part_error(uploaded_parts[i].part_num, err, bucket, object);
let result = complete_multipart_part_error_result(&mapped_err);
if matches!(mapped_err, Error::InvalidPart(_, _, _)) {
debug!(
target: "rustfs_ecstore::set_disk",
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
op = "complete_multipart_upload",
result = result,
bucket = %bucket,
object = %object,
upload_id = %upload_id,
@@ -4330,9 +4425,28 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
read_quorum = read_quorum,
write_quorum = write_quorum,
error = %err,
"issue3031_complete_part_error"
"Set disk multipart part missing"
);
} else {
warn!(
target: "rustfs_ecstore::set_disk",
event = EVENT_SET_DISK_MULTIPART,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
op = "complete_multipart_upload",
result = result,
bucket = %bucket,
object = %object,
upload_id = %upload_id,
uploaded_part_num = uploaded_parts[i].part_num,
observed_part_num = part.number,
read_quorum = read_quorum,
write_quorum = write_quorum,
error = %err,
"Set disk multipart part resolution failed"
);
}
return Err(mapped_err);
}
if uploaded_parts[i].part_num != part.number {
@@ -5655,6 +5769,41 @@ mod tests {
use tempfile::TempDir;
use time::OffsetDateTime;
#[test]
fn complete_part_error_maps_confirmed_missing_to_invalid_part() {
for err in ["file not found", "Specified part could not be found", "part.7 not found"] {
let mapped = complete_multipart_part_error(7, err, "bucket", "object");
assert!(matches!(
mapped,
Error::InvalidPart(7, ref bucket, ref object) if bucket == "bucket" && object == "object"
));
assert_eq!(complete_multipart_part_error_result(&mapped), COMPLETE_MULTIPART_PART_MISSING);
}
}
#[test]
fn complete_part_error_maps_read_quorum_to_retryable_server_error() {
let mapped = complete_multipart_part_error(1, "erasure read quorum", "bucket", "object");
assert!(matches!(
mapped,
Error::InsufficientReadQuorum(ref bucket, ref object) if bucket == "bucket" && object == "object"
));
assert_eq!(
complete_multipart_part_error_result(&mapped),
COMPLETE_MULTIPART_PART_READ_QUORUM_UNAVAILABLE
);
}
#[test]
fn complete_part_error_maps_unknown_part_error_to_retryable_server_error() {
let mapped = complete_multipart_part_error(1, "metadata decode failed", "bucket", "object");
assert!(matches!(mapped, Error::InsufficientReadQuorum(_, _)));
assert_ne!(complete_multipart_part_error_result(&mapped), COMPLETE_MULTIPART_PART_MISSING);
}
#[derive(Debug, Default)]
struct FailingClient;