fix(ecstore): preserve CopyObject producer errors (#6090)

* fix(ecstore): preserve CopyObject producer errors

* fix(app): resume preserved relocation I/O errors

* fix(copy): preserve transformed source errors
This commit is contained in:
cxymds
2026-08-14 22:12:42 +08:00
committed by GitHub
parent 85be26b3c1
commit eca6bc1600
8 changed files with 666 additions and 56 deletions
+6 -4
View File
@@ -2241,10 +2241,11 @@ fn get_object_resume_control(ctx: GetObjectResumeContext) -> GetObjectResumeCont
/// disks" failures keep the existing fail-loud behavior.
fn is_object_relocation_error(err: &std::io::Error) -> bool {
let Some(inner) = err.get_ref() else { return false };
matches!(
inner.downcast_ref::<StorageError>(),
Some(StorageError::FileNotFound | StorageError::ObjectNotFound(..) | StorageError::InsufficientReadQuorum(..))
)
match inner.downcast_ref::<StorageError>() {
Some(StorageError::FileNotFound | StorageError::ObjectNotFound(..) | StorageError::InsufficientReadQuorum(..)) => true,
Some(StorageError::Io(source)) => source.kind() == std::io::ErrorKind::NotFound,
_ => false,
}
}
/// Resolve the S3 request-body inter-chunk read timeout from the environment.
@@ -13163,6 +13164,7 @@ mod tests {
StorageError::FileNotFound,
StorageError::ObjectNotFound("test-bucket".to_string(), "relocated-object".to_string()),
StorageError::InsufficientReadQuorum("test-bucket".to_string(), "relocated-object".to_string()),
StorageError::Io(std::io::Error::new(std::io::ErrorKind::NotFound, "relocated shard disappeared")),
] {
let reopen_count = Arc::new(AtomicUsize::new(0));
let control = counting_resume_control(Arc::clone(&reopen_count), |emitted| {
+49 -4
View File
@@ -326,7 +326,11 @@ impl From<StorageError> for ApiError {
_ => S3ErrorCode::InternalError,
};
let message = if matches!(&err, StorageError::QuotaExceeded { .. }) || code == S3ErrorCode::InternalError {
let message = if matches!(&err, StorageError::QuotaExceeded { .. }) {
err.to_string()
} else if code == S3ErrorCode::InternalError && matches!(&err, StorageError::Io(_)) {
ApiError::error_code_to_message(&code)
} else if code == S3ErrorCode::InternalError {
err.to_string()
} else if let StorageError::InvalidArgument(_, _, reason) = &err
&& !reason.is_empty()
@@ -525,6 +529,25 @@ mod tests {
assert!(api_error.source.is_some());
}
#[test]
fn storage_io_internal_error_redacts_public_message_and_retains_source() {
let sensitive_path = "/sensitive/storage/path";
let api_error = ApiError::from(StorageError::Io(IoError::new(
ErrorKind::PermissionDenied,
format!("permission denied: {sensitive_path}"),
)));
assert_eq!(api_error.code, S3ErrorCode::InternalError);
assert_eq!(api_error.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError));
assert!(!api_error.message.contains(sensitive_path));
let source = api_error
.source
.as_deref()
.and_then(|source| source.downcast_ref::<StorageError>())
.expect("API error should retain the storage error source");
assert!(matches!(source, StorageError::Io(io_error) if io_error.to_string().contains(sensitive_path)));
}
#[test]
fn test_kms_service_unavailable_maps_to_retryable_error() {
let api_error = ApiError::from(StorageError::other(KmsUnavailableError));
@@ -669,14 +692,36 @@ mod tests {
assert!(api_error.source.is_some());
}
#[test]
fn test_api_error_from_storage_io_copy_object_terminal_error_stays_internal() {
let io_error = IoError::other(StorageError::FileCorrupt);
let storage_error: StorageError = io_error.into();
assert!(matches!(storage_error, StorageError::FileCorrupt));
let api_error: ApiError = storage_error.into();
assert_eq!(api_error.code, S3ErrorCode::InternalError);
let source = api_error
.source
.as_deref()
.and_then(|source| source.downcast_ref::<StorageError>())
.expect("API error should retain the storage error source");
assert!(matches!(source, StorageError::FileCorrupt));
}
#[test]
fn test_api_error_from_iam_error() {
let iam_error = rustfs_iam::error::Error::other("IAM test error");
let api_error: ApiError = iam_error.into();
// IAM error is first converted to StorageError, then to ApiError
assert!(api_error.source.is_some());
assert!(api_error.message.contains("test error"));
assert_eq!(api_error.code, S3ErrorCode::InternalError);
assert_eq!(api_error.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError));
let source = api_error
.source
.as_deref()
.and_then(|source| source.downcast_ref::<StorageError>())
.expect("API error should retain the storage error source");
assert!(matches!(source, StorageError::Io(io_error) if io_error.to_string().contains("IAM test error")));
}
#[test]
+18 -3
View File
@@ -5336,7 +5336,16 @@ mod tests {
let error = TestSseDekProvider::decrypt_dek(&envelope, [0x55u8; 32])
.expect_err("unknown JSON envelope versions must fail closed");
assert!(error.message.contains("Unsupported encrypted DEK format version"));
assert_eq!(error.code, S3ErrorCode::InternalError);
assert_eq!(error.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError));
let source = error
.source
.as_deref()
.and_then(|source| source.downcast_ref::<StorageError>())
.expect("API error should retain the storage error source");
assert!(matches!(source, StorageError::Io(io_error) if io_error
.to_string()
.contains("Unsupported encrypted DEK format version")));
}
#[tokio::test]
@@ -5894,10 +5903,16 @@ mod tests {
}
#[test]
fn test_map_get_object_reader_error_leaves_non_ssec_errors_unchanged() {
fn test_map_get_object_reader_error_redacts_non_ssec_internal_errors() {
let err = map_get_object_reader_error(StorageError::other("plain io failure"));
assert_eq!(err.code, S3ErrorCode::InternalError);
assert_eq!(err.message, "Io error: plain io failure");
assert_eq!(err.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError));
let source = err
.source
.as_deref()
.and_then(|source| source.downcast_ref::<StorageError>())
.expect("API error should retain the storage error source");
assert!(matches!(source, StorageError::Io(io_error) if io_error.to_string().contains("plain io failure")));
}
#[test]