mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-08 21:25:59 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5fa789fce3 | |||
| e9cdd57a1f | |||
| af4430a6f1 | |||
| f391ab2cd8 |
@@ -627,7 +627,13 @@ impl From<tokio::task::JoinError> for DiskError {
|
||||
impl Clone for DiskError {
|
||||
fn clone(&self) -> Self {
|
||||
match self {
|
||||
DiskError::Io(io_error) => DiskError::Io(std::io::Error::new(io_error.kind(), io_error.to_string())),
|
||||
DiskError::Io(io_error) => DiskError::Io(
|
||||
rustfs_rio::clone_internode_http_io_error(io_error)
|
||||
.and_then(std::io::Error::into_inner)
|
||||
// The helper derives a kind from the source; Clone must retain the original outer kind.
|
||||
.map(|source| std::io::Error::new(io_error.kind(), source))
|
||||
.unwrap_or_else(|| std::io::Error::new(io_error.kind(), io_error.to_string())),
|
||||
),
|
||||
DiskError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
|
||||
DiskError::Unexpected => DiskError::Unexpected,
|
||||
DiskError::CorruptedFormat => DiskError::CorruptedFormat,
|
||||
@@ -1265,6 +1271,49 @@ mod tests {
|
||||
assert!(!bad_request.is_retryable_internode_write_failure());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_http_clone_preserves_retryability_status_and_context() {
|
||||
use http::StatusCode;
|
||||
use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, ConnectionReset, HttpStatus, Unknown};
|
||||
|
||||
for (kind, retryable) in [
|
||||
(ConnectionRefused, true),
|
||||
(ConnectionReset, true),
|
||||
(HttpStatus(StatusCode::TOO_MANY_REQUESTS), true),
|
||||
(HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
|
||||
(HttpStatus(StatusCode::CONFLICT), true),
|
||||
(Unknown, false),
|
||||
(HttpStatus(StatusCode::BAD_REQUEST), false),
|
||||
(HttpStatus(StatusCode::INTERNAL_SERVER_ERROR), false),
|
||||
] {
|
||||
let original = DiskError::from(rustfs_rio::new_test_internode_http_io_error(kind));
|
||||
assert_eq!(original.internode_http_error_kind(), Some(kind));
|
||||
assert_eq!(original.is_retryable_internode_write_failure(), retryable);
|
||||
|
||||
let cloned = original.clone();
|
||||
assert_eq!(cloned, original, "clone must preserve the error bucket for {kind:?}");
|
||||
assert_eq!(
|
||||
cloned.is_retryable_internode_write_failure(),
|
||||
retryable,
|
||||
"clone changed retryability for {kind:?}"
|
||||
);
|
||||
assert_eq!(cloned.internode_http_error_kind(), Some(kind));
|
||||
if let HttpStatus(status) = kind {
|
||||
assert!(cloned.is_internode_http_status(status.as_u16()));
|
||||
}
|
||||
let DiskError::Io(io_error) = &cloned else {
|
||||
panic!("unmarked internode error must remain Io: {cloned:?}");
|
||||
};
|
||||
let source = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.expect("clone must retain the structured internode error");
|
||||
assert_eq!(source.context().method(), "PUT");
|
||||
assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
|
||||
assert_eq!(source.context().operation(), Some(INTERNODE_OPERATION_PUT_FILE_STREAM));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_stream_conflict_is_not_a_retryable_put_file_failure() {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
@@ -1309,11 +1358,57 @@ mod tests {
|
||||
!error.is_retryable_internode_write_failure(),
|
||||
"read-operation 409 must not trigger put-file retry"
|
||||
);
|
||||
let cloned = error.clone();
|
||||
let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error)], &[], 1)
|
||||
.expect("the read conflict must remain the dominant error");
|
||||
for preserved in [&cloned, &reduced] {
|
||||
assert!(
|
||||
!preserved.is_retryable_internode_write_failure(),
|
||||
"cloning or reducing a read conflict must not turn it into a PUT retry"
|
||||
);
|
||||
assert!(preserved.is_internode_http_status(409));
|
||||
let DiskError::Io(io_error) = preserved else {
|
||||
panic!("read conflict must remain Io: {preserved:?}");
|
||||
};
|
||||
let source = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.expect("read conflict must retain its request context");
|
||||
assert_eq!(source.context().method(), "GET");
|
||||
assert_eq!(source.context().target(), "/rustfs/rpc/read_file_stream");
|
||||
assert_eq!(
|
||||
source.context().operation(),
|
||||
Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_READ_FILE_STREAM)
|
||||
);
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("isolated read-conflict test must finish within its budget");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_http_clone_preserves_outer_io_kind_and_message() {
|
||||
let source = rustfs_rio::new_test_internode_http_io_error(InternodeHttpErrorKind::ConnectionReset)
|
||||
.into_inner()
|
||||
.expect("the internode helper must provide a typed source");
|
||||
let original_io = io::Error::new(io::ErrorKind::InvalidData, source);
|
||||
let message = original_io.to_string();
|
||||
let original = DiskError::from(original_io);
|
||||
assert_eq!(original.internode_http_error_kind(), Some(InternodeHttpErrorKind::ConnectionReset));
|
||||
assert!(original.is_retryable_internode_write_failure());
|
||||
|
||||
let cloned = original.clone();
|
||||
let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(original)], &[], 1)
|
||||
.expect("the wrapped internode error must remain the dominant error");
|
||||
for preserved in [&cloned, &reduced] {
|
||||
let DiskError::Io(io_error) = preserved else {
|
||||
panic!("the wrapped error must remain Io: {preserved:?}");
|
||||
};
|
||||
assert_eq!(io_error.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(io_error.to_string(), message);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_missing_errors_preserve_disk_error_types() {
|
||||
let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
|
||||
@@ -1325,6 +1420,17 @@ mod tests {
|
||||
assert_eq!(file_missing, DiskError::FileNotFound);
|
||||
assert_eq!(volume_missing, DiskError::VolumeNotFound);
|
||||
assert!(matches!(unmarked_server_error, DiskError::Io(_)));
|
||||
for missing in [file_missing, volume_missing] {
|
||||
assert_eq!(missing.clone(), missing);
|
||||
assert_eq!(
|
||||
crate::disk::error_reduce::reduce_write_quorum_errs(
|
||||
&[Some(missing.clone()), Some(missing.clone()), None],
|
||||
&[],
|
||||
2
|
||||
),
|
||||
Some(missing)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -226,6 +226,78 @@ mod tests {
|
||||
assert_eq!(res, Some(quorum_err));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_quorum_reduction_preserves_internode_http_identity() {
|
||||
use http::StatusCode;
|
||||
use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, HttpStatus, Unknown};
|
||||
|
||||
for (kind, retryable) in [
|
||||
(ConnectionRefused, true),
|
||||
(HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
|
||||
(HttpStatus(StatusCode::CONFLICT), true),
|
||||
(Unknown, false),
|
||||
(HttpStatus(StatusCode::BAD_REQUEST), false),
|
||||
] {
|
||||
// Construct both producer errors independently: the reducer owns the first clone.
|
||||
let first = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
|
||||
let second = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
|
||||
assert_eq!(first.internode_http_error_kind(), Some(kind));
|
||||
assert_eq!(second.internode_http_error_kind(), Some(kind));
|
||||
assert_eq!(first.is_retryable_internode_write_failure(), retryable);
|
||||
let errors = [Some(first), Some(second), None];
|
||||
let reduced = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, 2)
|
||||
.expect("two equal producer errors must dominate one successful write");
|
||||
|
||||
assert_eq!(Some(&reduced), errors[0].as_ref());
|
||||
assert_eq!(
|
||||
reduced.is_retryable_internode_write_failure(),
|
||||
retryable,
|
||||
"quorum reduction changed retryability for {kind:?}"
|
||||
);
|
||||
assert_eq!(reduced.internode_http_error_kind(), Some(kind));
|
||||
if let HttpStatus(status) = kind {
|
||||
assert!(reduced.is_internode_http_status(status.as_u16()));
|
||||
}
|
||||
let Error::Io(io_error) = &reduced else {
|
||||
panic!("the dominant error must remain Io: {reduced:?}");
|
||||
};
|
||||
let source = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
|
||||
.expect("quorum reduction must retain the structured internode error");
|
||||
assert_eq!(source.context().method(), "PUT");
|
||||
assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
|
||||
assert_eq!(
|
||||
source.context().operation(),
|
||||
Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clone_and_write_quorum_do_not_promote_non_retryable_errors() {
|
||||
use http::StatusCode;
|
||||
use rustfs_rio::InternodeHttpErrorKind::{HttpStatus, Unknown};
|
||||
|
||||
for original in [
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(Unknown)),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::BAD_REQUEST))),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::FORBIDDEN))),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::NOT_FOUND))),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
))),
|
||||
err_io("internode connection reset: PUT /rustfs/rpc/put_file_stream"),
|
||||
] {
|
||||
assert!(!original.is_retryable_internode_write_failure());
|
||||
let cloned = original.clone();
|
||||
let reduced =
|
||||
reduce_write_quorum_errs(&[Some(original)], &[], 1).expect("a non-retryable error must remain an error");
|
||||
assert!(!cloned.is_retryable_internode_write_failure());
|
||||
assert!(!reduced.is_retryable_internode_write_failure());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_count_errs() {
|
||||
let e1 = err_io("a");
|
||||
|
||||
Reference in New Issue
Block a user