mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-09 22:59:59 +00:00
Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/observability-metrics
# Conflicts: # .github/workflows/build.yml # .github/workflows/ci.yml # Cargo.lock # Cargo.toml # appauth/src/token.rs # crates/config/src/config.rs # crates/event-notifier/examples/simple.rs # crates/event-notifier/src/global.rs # crates/event-notifier/src/lib.rs # crates/event-notifier/src/notifier.rs # crates/event-notifier/src/store.rs # crates/filemeta/src/filemeta.rs # crates/notify/examples/webhook.rs # crates/utils/Cargo.toml # ecstore/Cargo.toml # ecstore/src/cmd/bucket_replication.rs # ecstore/src/config/com.rs # ecstore/src/disk/error.rs # ecstore/src/disk/mod.rs # ecstore/src/set_disk.rs # ecstore/src/store_api.rs # ecstore/src/store_list_objects.rs # iam/Cargo.toml # iam/src/manager.rs # policy/Cargo.toml # rustfs/src/admin/rpc.rs # rustfs/src/main.rs # rustfs/src/storage/mod.rs
This commit is contained in:
houseme
@@ -0,0 +1,569 @@
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
pub enum Error {
|
||||
#[error("File not found")]
|
||||
FileNotFound,
|
||||
#[error("File version not found")]
|
||||
FileVersionNotFound,
|
||||
|
||||
#[error("Volume not found")]
|
||||
VolumeNotFound,
|
||||
|
||||
#[error("File corrupt")]
|
||||
FileCorrupt,
|
||||
|
||||
#[error("Done for now")]
|
||||
DoneForNow,
|
||||
|
||||
#[error("Method not allowed")]
|
||||
MethodNotAllowed,
|
||||
|
||||
#[error("Unexpected error")]
|
||||
Unexpected,
|
||||
|
||||
#[error("I/O error: {0}")]
|
||||
Io(std::io::Error),
|
||||
|
||||
#[error("rmp serde decode error: {0}")]
|
||||
RmpSerdeDecode(String),
|
||||
|
||||
#[error("rmp serde encode error: {0}")]
|
||||
RmpSerdeEncode(String),
|
||||
|
||||
#[error("Invalid UTF-8: {0}")]
|
||||
FromUtf8(String),
|
||||
|
||||
#[error("rmp decode value read error: {0}")]
|
||||
RmpDecodeValueRead(String),
|
||||
|
||||
#[error("rmp encode value write error: {0}")]
|
||||
RmpEncodeValueWrite(String),
|
||||
|
||||
#[error("rmp decode num value read error: {0}")]
|
||||
RmpDecodeNumValueRead(String),
|
||||
|
||||
#[error("rmp decode marker read error: {0}")]
|
||||
RmpDecodeMarkerRead(String),
|
||||
|
||||
#[error("time component range error: {0}")]
|
||||
TimeComponentRange(String),
|
||||
|
||||
#[error("uuid parse error: {0}")]
|
||||
UuidParse(String),
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub fn other<E>(error: E) -> Error
|
||||
where
|
||||
E: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
std::io::Error::other(error).into()
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for Error {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self, other) {
|
||||
(Error::FileCorrupt, Error::FileCorrupt) => true,
|
||||
(Error::DoneForNow, Error::DoneForNow) => true,
|
||||
(Error::MethodNotAllowed, Error::MethodNotAllowed) => true,
|
||||
(Error::FileNotFound, Error::FileNotFound) => true,
|
||||
(Error::FileVersionNotFound, Error::FileVersionNotFound) => true,
|
||||
(Error::VolumeNotFound, Error::VolumeNotFound) => true,
|
||||
(Error::Io(e1), Error::Io(e2)) => e1.kind() == e2.kind() && e1.to_string() == e2.to_string(),
|
||||
(Error::RmpSerdeDecode(e1), Error::RmpSerdeDecode(e2)) => e1 == e2,
|
||||
(Error::RmpSerdeEncode(e1), Error::RmpSerdeEncode(e2)) => e1 == e2,
|
||||
(Error::RmpDecodeValueRead(e1), Error::RmpDecodeValueRead(e2)) => e1 == e2,
|
||||
(Error::RmpEncodeValueWrite(e1), Error::RmpEncodeValueWrite(e2)) => e1 == e2,
|
||||
(Error::RmpDecodeNumValueRead(e1), Error::RmpDecodeNumValueRead(e2)) => e1 == e2,
|
||||
(Error::TimeComponentRange(e1), Error::TimeComponentRange(e2)) => e1 == e2,
|
||||
(Error::UuidParse(e1), Error::UuidParse(e2)) => e1 == e2,
|
||||
(Error::Unexpected, Error::Unexpected) => true,
|
||||
(a, b) => a.to_string() == b.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for Error {
|
||||
fn clone(&self) -> Self {
|
||||
match self {
|
||||
Error::FileNotFound => Error::FileNotFound,
|
||||
Error::FileVersionNotFound => Error::FileVersionNotFound,
|
||||
Error::FileCorrupt => Error::FileCorrupt,
|
||||
Error::DoneForNow => Error::DoneForNow,
|
||||
Error::MethodNotAllowed => Error::MethodNotAllowed,
|
||||
Error::VolumeNotFound => Error::VolumeNotFound,
|
||||
Error::Io(e) => Error::Io(std::io::Error::new(e.kind(), e.to_string())),
|
||||
Error::RmpSerdeDecode(s) => Error::RmpSerdeDecode(s.clone()),
|
||||
Error::RmpSerdeEncode(s) => Error::RmpSerdeEncode(s.clone()),
|
||||
Error::FromUtf8(s) => Error::FromUtf8(s.clone()),
|
||||
Error::RmpDecodeValueRead(s) => Error::RmpDecodeValueRead(s.clone()),
|
||||
Error::RmpEncodeValueWrite(s) => Error::RmpEncodeValueWrite(s.clone()),
|
||||
Error::RmpDecodeNumValueRead(s) => Error::RmpDecodeNumValueRead(s.clone()),
|
||||
Error::RmpDecodeMarkerRead(s) => Error::RmpDecodeMarkerRead(s.clone()),
|
||||
Error::TimeComponentRange(s) => Error::TimeComponentRange(s.clone()),
|
||||
Error::UuidParse(s) => Error::UuidParse(s.clone()),
|
||||
Error::Unexpected => Error::Unexpected,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for Error {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
match e.kind() {
|
||||
std::io::ErrorKind::UnexpectedEof => Error::Unexpected,
|
||||
_ => Error::Io(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Error> for std::io::Error {
|
||||
fn from(e: Error) -> Self {
|
||||
match e {
|
||||
Error::Unexpected => std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "Unexpected EOF"),
|
||||
Error::Io(e) => e,
|
||||
_ => std::io::Error::other(e.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rmp_serde::decode::Error> for Error {
|
||||
fn from(e: rmp_serde::decode::Error) -> Self {
|
||||
Error::RmpSerdeDecode(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rmp_serde::encode::Error> for Error {
|
||||
fn from(e: rmp_serde::encode::Error) -> Self {
|
||||
Error::RmpSerdeEncode(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::string::FromUtf8Error> for Error {
|
||||
fn from(e: std::string::FromUtf8Error) -> Self {
|
||||
Error::FromUtf8(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rmp::decode::ValueReadError> for Error {
|
||||
fn from(e: rmp::decode::ValueReadError) -> Self {
|
||||
Error::RmpDecodeValueRead(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rmp::encode::ValueWriteError> for Error {
|
||||
fn from(e: rmp::encode::ValueWriteError) -> Self {
|
||||
Error::RmpEncodeValueWrite(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rmp::decode::NumValueReadError> for Error {
|
||||
fn from(e: rmp::decode::NumValueReadError) -> Self {
|
||||
Error::RmpDecodeNumValueRead(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<time::error::ComponentRange> for Error {
|
||||
fn from(e: time::error::ComponentRange) -> Self {
|
||||
Error::TimeComponentRange(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<uuid::Error> for Error {
|
||||
fn from(e: uuid::Error) -> Self {
|
||||
Error::UuidParse(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rmp::decode::MarkerReadError> for Error {
|
||||
fn from(e: rmp::decode::MarkerReadError) -> Self {
|
||||
let serr = format!("{:?}", e);
|
||||
Error::RmpDecodeMarkerRead(serr)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_io_eof(e: &Error) -> bool {
|
||||
match e {
|
||||
Error::Io(e) => e.kind() == std::io::ErrorKind::UnexpectedEof,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Error as IoError, ErrorKind};
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_from_io_error() {
|
||||
let io_error = IoError::new(ErrorKind::PermissionDenied, "permission denied");
|
||||
let filemeta_error: Error = io_error.into();
|
||||
|
||||
match filemeta_error {
|
||||
Error::Io(inner_io) => {
|
||||
assert_eq!(inner_io.kind(), ErrorKind::PermissionDenied);
|
||||
assert!(inner_io.to_string().contains("permission denied"));
|
||||
}
|
||||
_ => panic!("Expected Io variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_other_function() {
|
||||
let custom_error = "Custom filemeta error";
|
||||
let filemeta_error = Error::other(custom_error);
|
||||
|
||||
match filemeta_error {
|
||||
Error::Io(io_error) => {
|
||||
assert!(io_error.to_string().contains(custom_error));
|
||||
assert_eq!(io_error.kind(), ErrorKind::Other);
|
||||
}
|
||||
_ => panic!("Expected Io variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_conversions() {
|
||||
// Test various error conversions
|
||||
let serde_decode_err =
|
||||
rmp_serde::decode::Error::InvalidMarkerRead(std::io::Error::new(ErrorKind::InvalidData, "invalid"));
|
||||
let filemeta_error: Error = serde_decode_err.into();
|
||||
assert!(matches!(filemeta_error, Error::RmpSerdeDecode(_)));
|
||||
|
||||
// Test with string-based error that we can actually create
|
||||
let encode_error_string = "test encode error";
|
||||
let filemeta_error = Error::RmpSerdeEncode(encode_error_string.to_string());
|
||||
assert!(matches!(filemeta_error, Error::RmpSerdeEncode(_)));
|
||||
|
||||
let utf8_err = std::string::String::from_utf8(vec![0xFF]).unwrap_err();
|
||||
let filemeta_error: Error = utf8_err.into();
|
||||
assert!(matches!(filemeta_error, Error::FromUtf8(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_clone() {
|
||||
let test_cases = vec![
|
||||
Error::FileNotFound,
|
||||
Error::FileVersionNotFound,
|
||||
Error::VolumeNotFound,
|
||||
Error::FileCorrupt,
|
||||
Error::DoneForNow,
|
||||
Error::MethodNotAllowed,
|
||||
Error::Unexpected,
|
||||
Error::Io(IoError::new(ErrorKind::NotFound, "test")),
|
||||
Error::RmpSerdeDecode("test decode error".to_string()),
|
||||
Error::RmpSerdeEncode("test encode error".to_string()),
|
||||
Error::FromUtf8("test utf8 error".to_string()),
|
||||
Error::RmpDecodeValueRead("test value read error".to_string()),
|
||||
Error::RmpEncodeValueWrite("test value write error".to_string()),
|
||||
Error::RmpDecodeNumValueRead("test num read error".to_string()),
|
||||
Error::RmpDecodeMarkerRead("test marker read error".to_string()),
|
||||
Error::TimeComponentRange("test time error".to_string()),
|
||||
Error::UuidParse("test uuid error".to_string()),
|
||||
];
|
||||
|
||||
for original_error in test_cases {
|
||||
let cloned_error = original_error.clone();
|
||||
assert_eq!(original_error, cloned_error);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_partial_eq() {
|
||||
// Test equality for simple variants
|
||||
assert_eq!(Error::FileNotFound, Error::FileNotFound);
|
||||
assert_ne!(Error::FileNotFound, Error::FileVersionNotFound);
|
||||
|
||||
// Test equality for Io variants
|
||||
let io1 = Error::Io(IoError::new(ErrorKind::NotFound, "test"));
|
||||
let io2 = Error::Io(IoError::new(ErrorKind::NotFound, "test"));
|
||||
let io3 = Error::Io(IoError::new(ErrorKind::PermissionDenied, "test"));
|
||||
assert_eq!(io1, io2);
|
||||
assert_ne!(io1, io3);
|
||||
|
||||
// Test equality for string variants
|
||||
let decode1 = Error::RmpSerdeDecode("error message".to_string());
|
||||
let decode2 = Error::RmpSerdeDecode("error message".to_string());
|
||||
let decode3 = Error::RmpSerdeDecode("different message".to_string());
|
||||
assert_eq!(decode1, decode2);
|
||||
assert_ne!(decode1, decode3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_display() {
|
||||
let test_cases = vec![
|
||||
(Error::FileNotFound, "File not found"),
|
||||
(Error::FileVersionNotFound, "File version not found"),
|
||||
(Error::VolumeNotFound, "Volume not found"),
|
||||
(Error::FileCorrupt, "File corrupt"),
|
||||
(Error::DoneForNow, "Done for now"),
|
||||
(Error::MethodNotAllowed, "Method not allowed"),
|
||||
(Error::Unexpected, "Unexpected error"),
|
||||
(Error::RmpSerdeDecode("test".to_string()), "rmp serde decode error: test"),
|
||||
(Error::RmpSerdeEncode("test".to_string()), "rmp serde encode error: test"),
|
||||
(Error::FromUtf8("test".to_string()), "Invalid UTF-8: test"),
|
||||
(Error::TimeComponentRange("test".to_string()), "time component range error: test"),
|
||||
(Error::UuidParse("test".to_string()), "uuid parse error: test"),
|
||||
];
|
||||
|
||||
for (error, expected_message) in test_cases {
|
||||
assert_eq!(error.to_string(), expected_message);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rmp_conversions() {
|
||||
// Test rmp value read error (this one works since it has the same signature)
|
||||
let value_read_err = rmp::decode::ValueReadError::InvalidMarkerRead(std::io::Error::new(ErrorKind::InvalidData, "test"));
|
||||
let filemeta_error: Error = value_read_err.into();
|
||||
assert!(matches!(filemeta_error, Error::RmpDecodeValueRead(_)));
|
||||
|
||||
// Test rmp num value read error
|
||||
let num_value_err =
|
||||
rmp::decode::NumValueReadError::InvalidMarkerRead(std::io::Error::new(ErrorKind::InvalidData, "test"));
|
||||
let filemeta_error: Error = num_value_err.into();
|
||||
assert!(matches!(filemeta_error, Error::RmpDecodeNumValueRead(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_time_and_uuid_conversions() {
|
||||
// Test time component range error
|
||||
use time::{Date, Month};
|
||||
let time_result = Date::from_calendar_date(2023, Month::January, 32); // Invalid day
|
||||
assert!(time_result.is_err());
|
||||
let time_error = time_result.unwrap_err();
|
||||
let filemeta_error: Error = time_error.into();
|
||||
assert!(matches!(filemeta_error, Error::TimeComponentRange(_)));
|
||||
|
||||
// Test UUID parse error
|
||||
let uuid_result = uuid::Uuid::parse_str("invalid-uuid");
|
||||
assert!(uuid_result.is_err());
|
||||
let uuid_error = uuid_result.unwrap_err();
|
||||
let filemeta_error: Error = uuid_error.into();
|
||||
assert!(matches!(filemeta_error, Error::UuidParse(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_marker_read_error_conversion() {
|
||||
// Test rmp marker read error conversion
|
||||
let marker_err = rmp::decode::MarkerReadError(std::io::Error::new(ErrorKind::InvalidData, "marker test"));
|
||||
let filemeta_error: Error = marker_err.into();
|
||||
assert!(matches!(filemeta_error, Error::RmpDecodeMarkerRead(_)));
|
||||
assert!(filemeta_error.to_string().contains("marker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_io_eof_function() {
|
||||
// Test is_io_eof helper function
|
||||
let eof_error = Error::Io(IoError::new(ErrorKind::UnexpectedEof, "eof"));
|
||||
assert!(is_io_eof(&eof_error));
|
||||
|
||||
let not_eof_error = Error::Io(IoError::new(ErrorKind::NotFound, "not found"));
|
||||
assert!(!is_io_eof(¬_eof_error));
|
||||
|
||||
let non_io_error = Error::FileNotFound;
|
||||
assert!(!is_io_eof(&non_io_error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_to_io_error_conversion() {
|
||||
// Test conversion from FileMeta Error to io::Error through other function
|
||||
let original_io_error = IoError::new(ErrorKind::InvalidData, "test data");
|
||||
let filemeta_error = Error::other(original_io_error);
|
||||
|
||||
match filemeta_error {
|
||||
Error::Io(io_err) => {
|
||||
assert_eq!(io_err.kind(), ErrorKind::Other);
|
||||
assert!(io_err.to_string().contains("test data"));
|
||||
}
|
||||
_ => panic!("Expected Io variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_roundtrip_conversion() {
|
||||
// Test roundtrip conversion: io::Error -> FileMeta Error -> io::Error
|
||||
let original_io_error = IoError::new(ErrorKind::PermissionDenied, "permission test");
|
||||
|
||||
// Convert to FileMeta Error
|
||||
let filemeta_error: Error = original_io_error.into();
|
||||
|
||||
// Extract the io::Error back
|
||||
match filemeta_error {
|
||||
Error::Io(extracted_io_error) => {
|
||||
assert_eq!(extracted_io_error.kind(), ErrorKind::PermissionDenied);
|
||||
assert!(extracted_io_error.to_string().contains("permission test"));
|
||||
}
|
||||
_ => panic!("Expected Io variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_io_error_kinds_preservation() {
|
||||
let io_error_kinds = vec![
|
||||
ErrorKind::NotFound,
|
||||
ErrorKind::PermissionDenied,
|
||||
ErrorKind::ConnectionRefused,
|
||||
ErrorKind::ConnectionReset,
|
||||
ErrorKind::ConnectionAborted,
|
||||
ErrorKind::NotConnected,
|
||||
ErrorKind::AddrInUse,
|
||||
ErrorKind::AddrNotAvailable,
|
||||
ErrorKind::BrokenPipe,
|
||||
ErrorKind::AlreadyExists,
|
||||
ErrorKind::WouldBlock,
|
||||
ErrorKind::InvalidInput,
|
||||
ErrorKind::InvalidData,
|
||||
ErrorKind::TimedOut,
|
||||
ErrorKind::WriteZero,
|
||||
ErrorKind::Interrupted,
|
||||
ErrorKind::UnexpectedEof,
|
||||
ErrorKind::Other,
|
||||
];
|
||||
|
||||
for kind in io_error_kinds {
|
||||
let io_error = IoError::new(kind, format!("test error for {:?}", kind));
|
||||
let filemeta_error: Error = io_error.into();
|
||||
|
||||
match filemeta_error {
|
||||
Error::Unexpected => {
|
||||
assert_eq!(kind, ErrorKind::UnexpectedEof);
|
||||
}
|
||||
Error::Io(extracted_io_error) => {
|
||||
assert_eq!(extracted_io_error.kind(), kind);
|
||||
assert!(extracted_io_error.to_string().contains("test error"));
|
||||
}
|
||||
_ => panic!("Expected Io variant for kind {:?}", kind),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_downcast_chain() {
|
||||
// Test error downcast chain functionality
|
||||
let original_io_error = IoError::new(ErrorKind::InvalidData, "original error");
|
||||
let filemeta_error = Error::other(original_io_error);
|
||||
|
||||
// The error should be wrapped as an Io variant
|
||||
if let Error::Io(io_err) = filemeta_error {
|
||||
// The wrapped error should be Other kind (from std::io::Error::other)
|
||||
assert_eq!(io_err.kind(), ErrorKind::Other);
|
||||
// But the message should still contain the original error information
|
||||
assert!(io_err.to_string().contains("original error"));
|
||||
} else {
|
||||
panic!("Expected Io variant");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_maintains_error_information() {
|
||||
let test_cases = vec![
|
||||
(ErrorKind::NotFound, "file not found"),
|
||||
(ErrorKind::PermissionDenied, "access denied"),
|
||||
(ErrorKind::InvalidData, "corrupt data"),
|
||||
(ErrorKind::TimedOut, "operation timed out"),
|
||||
];
|
||||
|
||||
for (kind, message) in test_cases {
|
||||
let io_error = IoError::new(kind, message);
|
||||
let error_message = io_error.to_string();
|
||||
let filemeta_error: Error = io_error.into();
|
||||
|
||||
match filemeta_error {
|
||||
Error::Io(extracted_io_error) => {
|
||||
assert_eq!(extracted_io_error.kind(), kind);
|
||||
assert_eq!(extracted_io_error.to_string(), error_message);
|
||||
}
|
||||
_ => panic!("Expected Io variant"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_complex_conversion_chain() {
|
||||
// Test conversion from string error types that we can actually create
|
||||
|
||||
// Test with UUID error conversion
|
||||
let uuid_result = uuid::Uuid::parse_str("invalid-uuid-format");
|
||||
assert!(uuid_result.is_err());
|
||||
let uuid_error = uuid_result.unwrap_err();
|
||||
let filemeta_error: Error = uuid_error.into();
|
||||
|
||||
match filemeta_error {
|
||||
Error::UuidParse(message) => {
|
||||
assert!(message.contains("invalid"));
|
||||
}
|
||||
_ => panic!("Expected UuidParse variant"),
|
||||
}
|
||||
|
||||
// Test with time error conversion
|
||||
use time::{Date, Month};
|
||||
let time_result = Date::from_calendar_date(2023, Month::January, 32); // Invalid day
|
||||
assert!(time_result.is_err());
|
||||
let time_error = time_result.unwrap_err();
|
||||
let filemeta_error2: Error = time_error.into();
|
||||
|
||||
match filemeta_error2 {
|
||||
Error::TimeComponentRange(message) => {
|
||||
assert!(message.contains("range"));
|
||||
}
|
||||
_ => panic!("Expected TimeComponentRange variant"),
|
||||
}
|
||||
|
||||
// Test with UTF8 error conversion
|
||||
let utf8_result = std::string::String::from_utf8(vec![0xFF]);
|
||||
assert!(utf8_result.is_err());
|
||||
let utf8_error = utf8_result.unwrap_err();
|
||||
let filemeta_error3: Error = utf8_error.into();
|
||||
|
||||
match filemeta_error3 {
|
||||
Error::FromUtf8(message) => {
|
||||
assert!(message.contains("utf"));
|
||||
}
|
||||
_ => panic!("Expected FromUtf8 variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_equality_with_io_errors() {
|
||||
// Test equality comparison for Io variants
|
||||
let io_error1 = IoError::new(ErrorKind::NotFound, "test message");
|
||||
let io_error2 = IoError::new(ErrorKind::NotFound, "test message");
|
||||
let io_error3 = IoError::new(ErrorKind::PermissionDenied, "test message");
|
||||
let io_error4 = IoError::new(ErrorKind::NotFound, "different message");
|
||||
|
||||
let filemeta_error1 = Error::Io(io_error1);
|
||||
let filemeta_error2 = Error::Io(io_error2);
|
||||
let filemeta_error3 = Error::Io(io_error3);
|
||||
let filemeta_error4 = Error::Io(io_error4);
|
||||
|
||||
// Same kind and message should be equal
|
||||
assert_eq!(filemeta_error1, filemeta_error2);
|
||||
|
||||
// Different kinds should not be equal
|
||||
assert_ne!(filemeta_error1, filemeta_error3);
|
||||
|
||||
// Different messages should not be equal
|
||||
assert_ne!(filemeta_error1, filemeta_error4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filemeta_error_clone_io_variants() {
|
||||
let io_error = IoError::new(ErrorKind::ConnectionReset, "connection lost");
|
||||
let original_error = Error::Io(io_error);
|
||||
let cloned_error = original_error.clone();
|
||||
|
||||
// Cloned error should be equal to original
|
||||
assert_eq!(original_error, cloned_error);
|
||||
|
||||
// Both should maintain the same properties
|
||||
match (original_error, cloned_error) {
|
||||
(Error::Io(orig_io), Error::Io(cloned_io)) => {
|
||||
assert_eq!(orig_io.kind(), cloned_io.kind());
|
||||
assert_eq!(orig_io.to_string(), cloned_io.to_string());
|
||||
}
|
||||
_ => panic!("Both should be Io variants"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
use crate::error::{Error, Result};
|
||||
use crate::headers::RESERVED_METADATA_PREFIX_LOWER;
|
||||
use crate::headers::RUSTFS_HEALING;
|
||||
use bytes::Bytes;
|
||||
use rmp_serde::Serializer;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
|
||||
pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
|
||||
|
||||
// Additional constants from Go version
|
||||
pub const NULL_VERSION_ID: &str = "null";
|
||||
// pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded";
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
|
||||
pub struct ObjectPartInfo {
|
||||
pub etag: String,
|
||||
pub number: usize,
|
||||
pub size: usize,
|
||||
pub actual_size: i64, // Original data size
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
// Index holds the index of the part in the erasure coding
|
||||
pub index: Option<Bytes>,
|
||||
// Checksums holds checksums of the part
|
||||
pub checksums: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)]
|
||||
// ChecksumInfo - carries checksums of individual scattered parts per disk.
|
||||
pub struct ChecksumInfo {
|
||||
pub part_number: usize,
|
||||
pub algorithm: HashAlgorithm,
|
||||
pub hash: Bytes,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)]
|
||||
pub enum ErasureAlgo {
|
||||
#[default]
|
||||
Invalid = 0,
|
||||
ReedSolomon = 1,
|
||||
}
|
||||
|
||||
impl ErasureAlgo {
|
||||
pub fn valid(&self) -> bool {
|
||||
*self > ErasureAlgo::Invalid
|
||||
}
|
||||
pub fn to_u8(&self) -> u8 {
|
||||
match self {
|
||||
ErasureAlgo::Invalid => 0,
|
||||
ErasureAlgo::ReedSolomon => 1,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_u8(u: u8) -> Self {
|
||||
match u {
|
||||
1 => ErasureAlgo::ReedSolomon,
|
||||
_ => ErasureAlgo::Invalid,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ErasureAlgo {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
ErasureAlgo::Invalid => write!(f, "Invalid"),
|
||||
ErasureAlgo::ReedSolomon => write!(f, "{}", ERASURE_ALGORITHM),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)]
|
||||
// ErasureInfo holds erasure coding and bitrot related information.
|
||||
pub struct ErasureInfo {
|
||||
// Algorithm is the String representation of erasure-coding-algorithm
|
||||
pub algorithm: String,
|
||||
// DataBlocks is the number of data blocks for erasure-coding
|
||||
pub data_blocks: usize,
|
||||
// ParityBlocks is the number of parity blocks for erasure-coding
|
||||
pub parity_blocks: usize,
|
||||
// BlockSize is the size of one erasure-coded block
|
||||
pub block_size: usize,
|
||||
// Index is the index of the current disk
|
||||
pub index: usize,
|
||||
// Distribution is the distribution of the data and parity blocks
|
||||
pub distribution: Vec<usize>,
|
||||
// Checksums holds all bitrot checksums of all erasure encoded blocks
|
||||
pub checksums: Vec<ChecksumInfo>,
|
||||
}
|
||||
|
||||
pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize {
|
||||
(block_size.div_ceil(data_shards) + 1) & !1
|
||||
}
|
||||
|
||||
impl ErasureInfo {
|
||||
pub fn get_checksum_info(&self, part_number: usize) -> ChecksumInfo {
|
||||
for sum in &self.checksums {
|
||||
if sum.part_number == part_number {
|
||||
return sum.clone();
|
||||
}
|
||||
}
|
||||
|
||||
ChecksumInfo {
|
||||
algorithm: HashAlgorithm::HighwayHash256S,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Calculate the size of each shard.
|
||||
pub fn shard_size(&self) -> usize {
|
||||
calc_shard_size(self.block_size, self.data_blocks)
|
||||
}
|
||||
/// Calculate the total erasure file size for a given original size.
|
||||
// Returns the final erasure size from the original size
|
||||
pub fn shard_file_size(&self, total_length: i64) -> i64 {
|
||||
if total_length == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if total_length < 0 {
|
||||
return total_length;
|
||||
}
|
||||
|
||||
let total_length = total_length as usize;
|
||||
|
||||
let num_shards = total_length / self.block_size;
|
||||
let last_block_size = total_length % self.block_size;
|
||||
let last_shard_size = calc_shard_size(last_block_size, self.data_blocks);
|
||||
(num_shards * self.shard_size() + last_shard_size) as i64
|
||||
}
|
||||
|
||||
/// Check if this ErasureInfo equals another ErasureInfo
|
||||
pub fn equals(&self, other: &ErasureInfo) -> bool {
|
||||
self.algorithm == other.algorithm
|
||||
&& self.data_blocks == other.data_blocks
|
||||
&& self.parity_blocks == other.parity_blocks
|
||||
&& self.block_size == other.block_size
|
||||
&& self.index == other.index
|
||||
&& self.distribution == other.distribution
|
||||
}
|
||||
}
|
||||
|
||||
// #[derive(Debug, Clone)]
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
|
||||
pub struct FileInfo {
|
||||
pub volume: String,
|
||||
pub name: String,
|
||||
pub version_id: Option<Uuid>,
|
||||
pub is_latest: bool,
|
||||
pub deleted: bool,
|
||||
// Transition related fields
|
||||
pub transition_status: Option<String>,
|
||||
pub transitioned_obj_name: Option<String>,
|
||||
pub transition_tier: Option<String>,
|
||||
pub transition_version_id: Option<String>,
|
||||
pub expire_restored: bool,
|
||||
pub data_dir: Option<Uuid>,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
pub size: i64,
|
||||
// File mode bits
|
||||
pub mode: Option<u32>,
|
||||
// WrittenByVersion is the unix time stamp of the version that created this version of the object
|
||||
pub written_by_version: Option<u64>,
|
||||
pub metadata: HashMap<String, String>,
|
||||
pub parts: Vec<ObjectPartInfo>,
|
||||
pub erasure: ErasureInfo,
|
||||
// MarkDeleted marks this version as deleted
|
||||
pub mark_deleted: bool,
|
||||
// ReplicationState - Internal replication state to be passed back in ObjectInfo
|
||||
// pub replication_state: Option<ReplicationState>, // TODO: implement ReplicationState
|
||||
pub data: Option<Bytes>,
|
||||
pub num_versions: usize,
|
||||
pub successor_mod_time: Option<OffsetDateTime>,
|
||||
pub fresh: bool,
|
||||
pub idx: usize,
|
||||
// Combined checksum when object was uploaded
|
||||
pub checksum: Option<Bytes>,
|
||||
pub versioned: bool,
|
||||
}
|
||||
|
||||
impl FileInfo {
|
||||
pub fn new(object: &str, data_blocks: usize, parity_blocks: usize) -> Self {
|
||||
let indexs = {
|
||||
let cardinality = data_blocks + parity_blocks;
|
||||
let mut nums = vec![0; cardinality];
|
||||
let key_crc = crc32fast::hash(object.as_bytes());
|
||||
|
||||
let start = key_crc as usize % cardinality;
|
||||
for i in 1..=cardinality {
|
||||
nums[i - 1] = 1 + ((start + i) % cardinality);
|
||||
}
|
||||
|
||||
nums
|
||||
};
|
||||
Self {
|
||||
erasure: ErasureInfo {
|
||||
algorithm: String::from(ERASURE_ALGORITHM),
|
||||
data_blocks,
|
||||
parity_blocks,
|
||||
block_size: BLOCK_SIZE_V2,
|
||||
distribution: indexs,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_valid(&self) -> bool {
|
||||
if self.deleted {
|
||||
return true;
|
||||
}
|
||||
|
||||
let data_blocks = self.erasure.data_blocks;
|
||||
let parity_blocks = self.erasure.parity_blocks;
|
||||
|
||||
(data_blocks >= parity_blocks)
|
||||
&& (data_blocks > 0)
|
||||
&& (self.erasure.index > 0
|
||||
&& self.erasure.index <= data_blocks + parity_blocks
|
||||
&& self.erasure.distribution.len() == (data_blocks + parity_blocks))
|
||||
}
|
||||
|
||||
pub fn get_etag(&self) -> Option<String> {
|
||||
self.metadata.get("etag").cloned()
|
||||
}
|
||||
|
||||
pub fn write_quorum(&self, quorum: usize) -> usize {
|
||||
if self.deleted {
|
||||
return quorum;
|
||||
}
|
||||
|
||||
if self.erasure.data_blocks == self.erasure.parity_blocks {
|
||||
return self.erasure.data_blocks + 1;
|
||||
}
|
||||
|
||||
self.erasure.data_blocks
|
||||
}
|
||||
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
|
||||
let mut buf = Vec::new();
|
||||
|
||||
self.serialize(&mut Serializer::new(&mut buf))?;
|
||||
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
|
||||
let t: FileInfo = rmp_serde::from_slice(buf)?;
|
||||
Ok(t)
|
||||
}
|
||||
|
||||
pub fn add_object_part(
|
||||
&mut self,
|
||||
num: usize,
|
||||
etag: String,
|
||||
part_size: usize,
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
actual_size: i64,
|
||||
index: Option<Bytes>,
|
||||
) {
|
||||
let part = ObjectPartInfo {
|
||||
etag,
|
||||
number: num,
|
||||
size: part_size,
|
||||
mod_time,
|
||||
actual_size,
|
||||
index,
|
||||
checksums: None,
|
||||
};
|
||||
|
||||
for p in self.parts.iter_mut() {
|
||||
if p.number == num {
|
||||
*p = part;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
self.parts.push(part);
|
||||
|
||||
self.parts.sort_by(|a, b| a.number.cmp(&b.number));
|
||||
}
|
||||
|
||||
// to_part_offset gets the part index where offset is located, returns part index and offset
|
||||
pub fn to_part_offset(&self, offset: usize) -> Result<(usize, usize)> {
|
||||
if offset == 0 {
|
||||
return Ok((0, 0));
|
||||
}
|
||||
|
||||
let mut part_offset = offset;
|
||||
for (i, part) in self.parts.iter().enumerate() {
|
||||
let part_index = i;
|
||||
if part_offset < part.size {
|
||||
return Ok((part_index, part_offset));
|
||||
}
|
||||
|
||||
part_offset -= part.size
|
||||
}
|
||||
|
||||
Err(Error::other("part not found"))
|
||||
}
|
||||
|
||||
pub fn set_healing(&mut self) {
|
||||
self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string());
|
||||
}
|
||||
|
||||
pub fn set_inline_data(&mut self) {
|
||||
self.metadata
|
||||
.insert(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned());
|
||||
}
|
||||
|
||||
pub fn set_data_moved(&mut self) {
|
||||
self.metadata
|
||||
.insert(format!("{}data-moved", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned());
|
||||
}
|
||||
|
||||
pub fn inline_data(&self) -> bool {
|
||||
self.metadata
|
||||
.contains_key(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).as_str())
|
||||
&& !self.is_remote()
|
||||
}
|
||||
|
||||
/// Check if the object is compressed
|
||||
pub fn is_compressed(&self) -> bool {
|
||||
self.metadata
|
||||
.contains_key(&format!("{}compression", RESERVED_METADATA_PREFIX_LOWER))
|
||||
}
|
||||
|
||||
/// Check if the object is remote (transitioned to another tier)
|
||||
pub fn is_remote(&self) -> bool {
|
||||
!self.transition_tier.as_ref().is_none_or(|s| s.is_empty())
|
||||
}
|
||||
|
||||
/// Get the data directory for this object
|
||||
pub fn get_data_dir(&self) -> String {
|
||||
if self.deleted {
|
||||
return "delete-marker".to_string();
|
||||
}
|
||||
self.data_dir.map_or("".to_string(), |dir| dir.to_string())
|
||||
}
|
||||
|
||||
/// Read quorum returns expected read quorum for this FileInfo
|
||||
pub fn read_quorum(&self, dquorum: usize) -> usize {
|
||||
if self.deleted {
|
||||
return dquorum;
|
||||
}
|
||||
self.erasure.data_blocks
|
||||
}
|
||||
|
||||
/// Create a shallow copy with minimal information for READ MRF checks
|
||||
pub fn shallow_copy(&self) -> Self {
|
||||
Self {
|
||||
volume: self.volume.clone(),
|
||||
name: self.name.clone(),
|
||||
version_id: self.version_id,
|
||||
deleted: self.deleted,
|
||||
erasure: self.erasure.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if this FileInfo equals another FileInfo
|
||||
pub fn equals(&self, other: &FileInfo) -> bool {
|
||||
// Check if both are compressed or both are not compressed
|
||||
if self.is_compressed() != other.is_compressed() {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check transition info
|
||||
if !self.transition_info_equals(other) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check mod time
|
||||
if self.mod_time != other.mod_time {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check erasure info
|
||||
self.erasure.equals(&other.erasure)
|
||||
}
|
||||
|
||||
/// Check if transition related information are equal
|
||||
pub fn transition_info_equals(&self, other: &FileInfo) -> bool {
|
||||
self.transition_status == other.transition_status
|
||||
&& self.transition_tier == other.transition_tier
|
||||
&& self.transitioned_obj_name == other.transitioned_obj_name
|
||||
&& self.transition_version_id == other.transition_version_id
|
||||
}
|
||||
|
||||
/// Check if metadata maps are equal
|
||||
pub fn metadata_equals(&self, other: &FileInfo) -> bool {
|
||||
if self.metadata.len() != other.metadata.len() {
|
||||
return false;
|
||||
}
|
||||
for (k, v) in &self.metadata {
|
||||
if other.metadata.get(k) != Some(v) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Check if replication related fields are equal
|
||||
pub fn replication_info_equals(&self, other: &FileInfo) -> bool {
|
||||
self.mark_deleted == other.mark_deleted
|
||||
// TODO: Add replication_state comparison when implemented
|
||||
// && self.replication_state == other.replication_state
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
pub struct FileInfoVersions {
|
||||
// Name of the volume.
|
||||
pub volume: String,
|
||||
|
||||
// Name of the file.
|
||||
pub name: String,
|
||||
|
||||
// Represents the latest mod time of the
|
||||
// latest version.
|
||||
pub latest_mod_time: Option<OffsetDateTime>,
|
||||
|
||||
pub versions: Vec<FileInfo>,
|
||||
pub free_versions: Vec<FileInfo>,
|
||||
}
|
||||
|
||||
impl FileInfoVersions {
|
||||
pub fn find_version_index(&self, v: &str) -> Option<usize> {
|
||||
if v.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let vid = Uuid::parse_str(v).unwrap_or_default();
|
||||
|
||||
self.versions.iter().position(|v| v.version_id == Some(vid))
|
||||
}
|
||||
|
||||
/// Calculate the total size of all versions for this object
|
||||
pub fn size(&self) -> i64 {
|
||||
self.versions.iter().map(|v| v.size).sum()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize, Deserialize)]
|
||||
pub struct RawFileInfo {
|
||||
pub buf: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
pub struct FilesInfo {
|
||||
pub files: Vec<FileInfo>,
|
||||
pub is_truncated: bool,
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,238 @@
|
||||
use crate::error::{Error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::io::{Cursor, Read};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct InlineData(Vec<u8>);
|
||||
|
||||
const INLINE_DATA_VER: u8 = 1;
|
||||
|
||||
impl InlineData {
|
||||
pub fn new() -> Self {
|
||||
Self(Vec::new())
|
||||
}
|
||||
pub fn update(&mut self, buf: &[u8]) {
|
||||
self.0 = buf.to_vec()
|
||||
}
|
||||
pub fn as_slice(&self) -> &[u8] {
|
||||
self.0.as_slice()
|
||||
}
|
||||
pub fn version_ok(&self) -> bool {
|
||||
if self.0.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER
|
||||
}
|
||||
|
||||
pub fn after_version(&self) -> &[u8] {
|
||||
if self.0.is_empty() { &self.0 } else { &self.0[1..] }
|
||||
}
|
||||
|
||||
pub fn find(&self, key: &str) -> Result<Option<Vec<u8>>> {
|
||||
if self.0.is_empty() || !self.version_ok() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let buf = self.after_version();
|
||||
|
||||
let mut cur = Cursor::new(buf);
|
||||
|
||||
let mut fields_len = rmp::decode::read_map_len(&mut cur)?;
|
||||
|
||||
while fields_len > 0 {
|
||||
fields_len -= 1;
|
||||
|
||||
let str_len = rmp::decode::read_str_len(&mut cur)?;
|
||||
|
||||
let mut field_buff = vec![0u8; str_len as usize];
|
||||
|
||||
cur.read_exact(&mut field_buff)?;
|
||||
|
||||
let field = String::from_utf8(field_buff)?;
|
||||
|
||||
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
|
||||
let start = cur.position() as usize;
|
||||
let end = start + bin_len;
|
||||
cur.set_position(end as u64);
|
||||
|
||||
if field.as_str() == key {
|
||||
let buf = &buf[start..end];
|
||||
return Ok(Some(buf.to_vec()));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.0.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut cur = Cursor::new(self.after_version());
|
||||
|
||||
let mut fields_len = rmp::decode::read_map_len(&mut cur)?;
|
||||
|
||||
while fields_len > 0 {
|
||||
fields_len -= 1;
|
||||
|
||||
let str_len = rmp::decode::read_str_len(&mut cur)?;
|
||||
|
||||
let mut field_buff = vec![0u8; str_len as usize];
|
||||
|
||||
cur.read_exact(&mut field_buff)?;
|
||||
|
||||
let field = String::from_utf8(field_buff)?;
|
||||
if field.is_empty() {
|
||||
return Err(Error::other("InlineData key empty"));
|
||||
}
|
||||
|
||||
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
|
||||
let start = cur.position() as usize;
|
||||
let end = start + bin_len;
|
||||
cur.set_position(end as u64);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn replace(&mut self, key: &str, value: Vec<u8>) -> Result<()> {
|
||||
if self.after_version().is_empty() {
|
||||
let mut keys = Vec::with_capacity(1);
|
||||
let mut values = Vec::with_capacity(1);
|
||||
|
||||
keys.push(key.to_owned());
|
||||
values.push(value);
|
||||
|
||||
return self.serialize(keys, values);
|
||||
}
|
||||
|
||||
let buf = self.after_version();
|
||||
let mut cur = Cursor::new(buf);
|
||||
|
||||
let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize;
|
||||
let mut keys = Vec::with_capacity(fields_len + 1);
|
||||
let mut values = Vec::with_capacity(fields_len + 1);
|
||||
|
||||
let mut replaced = false;
|
||||
|
||||
while fields_len > 0 {
|
||||
fields_len -= 1;
|
||||
|
||||
let str_len = rmp::decode::read_str_len(&mut cur)?;
|
||||
|
||||
let mut field_buff = vec![0u8; str_len as usize];
|
||||
|
||||
cur.read_exact(&mut field_buff)?;
|
||||
|
||||
let find_key = String::from_utf8(field_buff)?;
|
||||
|
||||
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
|
||||
let start = cur.position() as usize;
|
||||
let end = start + bin_len;
|
||||
cur.set_position(end as u64);
|
||||
|
||||
let find_value = &buf[start..end];
|
||||
|
||||
if find_key.as_str() == key {
|
||||
values.push(value.clone());
|
||||
replaced = true
|
||||
} else {
|
||||
values.push(find_value.to_vec());
|
||||
}
|
||||
|
||||
keys.push(find_key);
|
||||
}
|
||||
|
||||
if !replaced {
|
||||
keys.push(key.to_owned());
|
||||
values.push(value);
|
||||
}
|
||||
|
||||
self.serialize(keys, values)
|
||||
}
|
||||
pub fn remove(&mut self, remove_keys: Vec<Uuid>) -> Result<bool> {
|
||||
let buf = self.after_version();
|
||||
let mut cur = Cursor::new(buf);
|
||||
|
||||
let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize;
|
||||
let mut keys = Vec::with_capacity(fields_len + 1);
|
||||
let mut values = Vec::with_capacity(fields_len + 1);
|
||||
|
||||
let remove_key = |found_key: &str| {
|
||||
for key in remove_keys.iter() {
|
||||
if key.to_string().as_str() == found_key {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
};
|
||||
|
||||
let mut found = false;
|
||||
|
||||
while fields_len > 0 {
|
||||
fields_len -= 1;
|
||||
|
||||
let str_len = rmp::decode::read_str_len(&mut cur)?;
|
||||
|
||||
let mut field_buff = vec![0u8; str_len as usize];
|
||||
|
||||
cur.read_exact(&mut field_buff)?;
|
||||
|
||||
let find_key = String::from_utf8(field_buff)?;
|
||||
|
||||
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
|
||||
let start = cur.position() as usize;
|
||||
let end = start + bin_len;
|
||||
cur.set_position(end as u64);
|
||||
|
||||
let find_value = &buf[start..end];
|
||||
|
||||
if !remove_key(&find_key) {
|
||||
values.push(find_value.to_vec());
|
||||
keys.push(find_key);
|
||||
} else {
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if keys.is_empty() {
|
||||
self.0 = Vec::new();
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
self.serialize(keys, values)?;
|
||||
Ok(true)
|
||||
}
|
||||
fn serialize(&mut self, keys: Vec<String>, values: Vec<Vec<u8>>) -> Result<()> {
|
||||
assert_eq!(keys.len(), values.len(), "InlineData serialize: keys/values not match");
|
||||
|
||||
if keys.is_empty() {
|
||||
self.0 = Vec::new();
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut wr = Vec::new();
|
||||
|
||||
wr.push(INLINE_DATA_VER);
|
||||
|
||||
let map_len = keys.len();
|
||||
|
||||
rmp::encode::write_map_len(&mut wr, map_len as u32)?;
|
||||
|
||||
for i in 0..map_len {
|
||||
rmp::encode::write_str(&mut wr, keys[i].as_str())?;
|
||||
rmp::encode::write_bin(&mut wr, values[i].as_slice())?;
|
||||
}
|
||||
|
||||
self.0 = wr;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
pub const AMZ_META_UNENCRYPTED_CONTENT_LENGTH: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length";
|
||||
pub const AMZ_META_UNENCRYPTED_CONTENT_MD5: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5";
|
||||
|
||||
pub const AMZ_STORAGE_CLASS: &str = "x-amz-storage-class";
|
||||
|
||||
pub const RESERVED_METADATA_PREFIX: &str = "X-RustFS-Internal-";
|
||||
pub const RESERVED_METADATA_PREFIX_LOWER: &str = "x-rustfs-internal-";
|
||||
|
||||
pub const RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing";
|
||||
// pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
|
||||
|
||||
// pub const X_RUSTFS_INLINE_DATA: &str = "x-rustfs-inline-data";
|
||||
|
||||
pub const VERSION_PURGE_STATUS_KEY: &str = "X-Rustfs-Internal-purgestatus";
|
||||
|
||||
pub const X_RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing";
|
||||
pub const X_RUSTFS_DATA_MOV: &str = "X-Rustfs-Internal-data-mov";
|
||||
|
||||
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
||||
pub const AMZ_BUCKET_REPLICATION_STATUS: &str = "X-Amz-Replication-Status";
|
||||
pub const AMZ_DECODED_CONTENT_LENGTH: &str = "X-Amz-Decoded-Content-Length";
|
||||
|
||||
pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
|
||||
@@ -0,0 +1,14 @@
|
||||
mod error;
|
||||
mod fileinfo;
|
||||
mod filemeta;
|
||||
mod filemeta_inline;
|
||||
pub mod headers;
|
||||
mod metacache;
|
||||
|
||||
pub mod test_data;
|
||||
|
||||
pub use error::*;
|
||||
pub use fileinfo::*;
|
||||
pub use filemeta::*;
|
||||
pub use filemeta_inline::*;
|
||||
pub use metacache::*;
|
||||
@@ -0,0 +1,874 @@
|
||||
use crate::error::{Error, Result};
|
||||
use crate::{FileInfo, FileInfoVersions, FileMeta, FileMetaShallowVersion, VersionType, merge_file_meta_versions};
|
||||
use rmp::Marker;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::cmp::Ordering;
|
||||
use std::str::from_utf8;
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
ptr,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicPtr, AtomicU64, Ordering as AtomicOrdering},
|
||||
},
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||
use tokio::spawn;
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::warn;
|
||||
|
||||
const SLASH_SEPARATOR: &str = "/";
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct MetadataResolutionParams {
|
||||
pub dir_quorum: usize,
|
||||
pub obj_quorum: usize,
|
||||
pub requested_versions: usize,
|
||||
pub bucket: String,
|
||||
pub strict: bool,
|
||||
pub candidates: Vec<Vec<FileMetaShallowVersion>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct MetaCacheEntry {
|
||||
/// name is the full name of the object including prefixes
|
||||
pub name: String,
|
||||
/// Metadata. If none is present it is not an object but only a prefix.
|
||||
/// Entries without metadata will only be present in non-recursive scans.
|
||||
pub metadata: Vec<u8>,
|
||||
|
||||
/// cached contains the metadata if decoded.
|
||||
#[serde(skip)]
|
||||
pub cached: Option<FileMeta>,
|
||||
|
||||
/// Indicates the entry can be reused and only one reference to metadata is expected.
|
||||
pub reusable: bool,
|
||||
}
|
||||
|
||||
impl MetaCacheEntry {
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
|
||||
let mut wr = Vec::new();
|
||||
rmp::encode::write_bool(&mut wr, true)?;
|
||||
rmp::encode::write_str(&mut wr, &self.name)?;
|
||||
rmp::encode::write_bin(&mut wr, &self.metadata)?;
|
||||
Ok(wr)
|
||||
}
|
||||
|
||||
pub fn is_dir(&self) -> bool {
|
||||
self.metadata.is_empty() && self.name.ends_with('/')
|
||||
}
|
||||
|
||||
pub fn is_in_dir(&self, dir: &str, separator: &str) -> bool {
|
||||
if dir.is_empty() {
|
||||
let idx = self.name.find(separator);
|
||||
return idx.is_none() || idx.unwrap() == self.name.len() - separator.len();
|
||||
}
|
||||
|
||||
let ext = self.name.trim_start_matches(dir);
|
||||
|
||||
if ext.len() != self.name.len() {
|
||||
let idx = ext.find(separator);
|
||||
return idx.is_none() || idx.unwrap() == ext.len() - separator.len();
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
pub fn is_object(&self) -> bool {
|
||||
!self.metadata.is_empty()
|
||||
}
|
||||
|
||||
pub fn is_object_dir(&self) -> bool {
|
||||
!self.metadata.is_empty() && self.name.ends_with(SLASH_SEPARATOR)
|
||||
}
|
||||
|
||||
pub fn is_latest_delete_marker(&mut self) -> bool {
|
||||
if let Some(cached) = &self.cached {
|
||||
if cached.versions.is_empty() {
|
||||
return true;
|
||||
}
|
||||
return cached.versions[0].header.version_type == VersionType::Delete;
|
||||
}
|
||||
|
||||
if !FileMeta::is_xl2_v1_format(&self.metadata) {
|
||||
return false;
|
||||
}
|
||||
|
||||
match FileMeta::check_xl2_v1(&self.metadata) {
|
||||
Ok((meta, _, _)) => {
|
||||
if !meta.is_empty() {
|
||||
return FileMeta::is_latest_delete_marker(meta);
|
||||
}
|
||||
}
|
||||
Err(_) => return true,
|
||||
}
|
||||
|
||||
match self.xl_meta() {
|
||||
Ok(res) => {
|
||||
if res.versions.is_empty() {
|
||||
return true;
|
||||
}
|
||||
res.versions[0].header.version_type == VersionType::Delete
|
||||
}
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
pub fn to_fileinfo(&self, bucket: &str) -> Result<FileInfo> {
|
||||
if self.is_dir() {
|
||||
return Ok(FileInfo {
|
||||
volume: bucket.to_owned(),
|
||||
name: self.name.clone(),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
if self.cached.is_some() {
|
||||
let fm = self.cached.as_ref().unwrap();
|
||||
if fm.versions.is_empty() {
|
||||
return Ok(FileInfo {
|
||||
volume: bucket.to_owned(),
|
||||
name: self.name.clone(),
|
||||
deleted: true,
|
||||
is_latest: true,
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
|
||||
return Ok(fi);
|
||||
}
|
||||
|
||||
let mut fm = FileMeta::new();
|
||||
fm.unmarshal_msg(&self.metadata)?;
|
||||
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
|
||||
Ok(fi)
|
||||
}
|
||||
|
||||
pub fn file_info_versions(&self, bucket: &str) -> Result<FileInfoVersions> {
|
||||
if self.is_dir() {
|
||||
return Ok(FileInfoVersions {
|
||||
volume: bucket.to_string(),
|
||||
name: self.name.clone(),
|
||||
versions: vec![FileInfo {
|
||||
volume: bucket.to_string(),
|
||||
name: self.name.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
let mut fm = FileMeta::new();
|
||||
fm.unmarshal_msg(&self.metadata)?;
|
||||
fm.into_file_info_versions(bucket, self.name.as_str(), false)
|
||||
}
|
||||
|
||||
pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option<MetaCacheEntry>, bool) {
|
||||
if other.is_none() {
|
||||
return (None, false);
|
||||
}
|
||||
|
||||
let other = other.unwrap();
|
||||
if self.name != other.name {
|
||||
if self.name < other.name {
|
||||
return (Some(self.clone()), false);
|
||||
}
|
||||
return (Some(other.clone()), false);
|
||||
}
|
||||
|
||||
if other.is_dir() || self.is_dir() {
|
||||
if self.is_dir() {
|
||||
return (Some(self.clone()), other.is_dir() == self.is_dir());
|
||||
}
|
||||
return (Some(other.clone()), other.is_dir() == self.is_dir());
|
||||
}
|
||||
|
||||
let self_vers = match &self.cached {
|
||||
Some(file_meta) => file_meta.clone(),
|
||||
None => match FileMeta::load(&self.metadata) {
|
||||
Ok(meta) => meta,
|
||||
Err(_) => return (None, false),
|
||||
},
|
||||
};
|
||||
|
||||
let other_vers = match &other.cached {
|
||||
Some(file_meta) => file_meta.clone(),
|
||||
None => match FileMeta::load(&other.metadata) {
|
||||
Ok(meta) => meta,
|
||||
Err(_) => return (None, false),
|
||||
},
|
||||
};
|
||||
|
||||
if self_vers.versions.len() != other_vers.versions.len() {
|
||||
match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) {
|
||||
Ordering::Greater => return (Some(self.clone()), false),
|
||||
Ordering::Less => return (Some(other.clone()), false),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
if self_vers.versions.len() > other_vers.versions.len() {
|
||||
return (Some(self.clone()), false);
|
||||
}
|
||||
return (Some(other.clone()), false);
|
||||
}
|
||||
|
||||
let mut prefer = None;
|
||||
for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) {
|
||||
if s_version.header != o_version.header {
|
||||
if s_version.header.has_ec() != o_version.header.has_ec() {
|
||||
// One version has EC and the other doesn't - may have been written later.
|
||||
// Compare without considering EC.
|
||||
let (mut a, mut b) = (s_version.header.clone(), o_version.header.clone());
|
||||
(a.ec_n, a.ec_m, b.ec_n, b.ec_m) = (0, 0, 0, 0);
|
||||
if a == b {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if !strict && s_version.header.matches_not_strict(&o_version.header) {
|
||||
if prefer.is_none() {
|
||||
if s_version.header.sorts_before(&o_version.header) {
|
||||
prefer = Some(self.clone());
|
||||
} else {
|
||||
prefer = Some(other.clone());
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if prefer.is_some() {
|
||||
return (prefer, false);
|
||||
}
|
||||
|
||||
if s_version.header.sorts_before(&o_version.header) {
|
||||
return (Some(self.clone()), false);
|
||||
}
|
||||
|
||||
return (Some(other.clone()), false);
|
||||
}
|
||||
}
|
||||
|
||||
if prefer.is_none() {
|
||||
prefer = Some(self.clone());
|
||||
}
|
||||
|
||||
(prefer, true)
|
||||
}
|
||||
|
||||
pub fn xl_meta(&mut self) -> Result<FileMeta> {
|
||||
if self.is_dir() {
|
||||
return Err(Error::FileNotFound);
|
||||
}
|
||||
|
||||
if let Some(meta) = &self.cached {
|
||||
Ok(meta.clone())
|
||||
} else {
|
||||
if self.metadata.is_empty() {
|
||||
return Err(Error::FileNotFound);
|
||||
}
|
||||
|
||||
let meta = FileMeta::load(&self.metadata)?;
|
||||
self.cached = Some(meta.clone());
|
||||
Ok(meta)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MetaCacheEntries(pub Vec<Option<MetaCacheEntry>>);
|
||||
|
||||
impl MetaCacheEntries {
|
||||
#[allow(clippy::should_implement_trait)]
|
||||
pub fn as_ref(&self) -> &[Option<MetaCacheEntry>] {
|
||||
&self.0
|
||||
}
|
||||
|
||||
pub fn resolve(&self, mut params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
|
||||
if self.0.is_empty() {
|
||||
warn!("decommission_pool: entries resolve empty");
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut dir_exists = 0;
|
||||
let mut selected = None;
|
||||
|
||||
params.candidates.clear();
|
||||
let mut objs_agree = 0;
|
||||
let mut objs_valid = 0;
|
||||
|
||||
for entry in self.0.iter().flatten() {
|
||||
let mut entry = entry.clone();
|
||||
|
||||
warn!("decommission_pool: entries resolve entry {:?}", entry.name);
|
||||
if entry.name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if entry.is_dir() {
|
||||
dir_exists += 1;
|
||||
selected = Some(entry.clone());
|
||||
warn!("decommission_pool: entries resolve entry dir {:?}", entry.name);
|
||||
continue;
|
||||
}
|
||||
|
||||
let xl = match entry.xl_meta() {
|
||||
Ok(xl) => xl,
|
||||
Err(e) => {
|
||||
warn!("decommission_pool: entries resolve entry xl_meta {:?}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
objs_valid += 1;
|
||||
params.candidates.push(xl.versions.clone());
|
||||
|
||||
if selected.is_none() {
|
||||
selected = Some(entry.clone());
|
||||
objs_agree = 1;
|
||||
warn!("decommission_pool: entries resolve entry selected {:?}", entry.name);
|
||||
continue;
|
||||
}
|
||||
|
||||
if let (prefer, true) = entry.matches(selected.as_ref(), params.strict) {
|
||||
selected = prefer;
|
||||
objs_agree += 1;
|
||||
warn!("decommission_pool: entries resolve entry prefer {:?}", entry.name);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let Some(selected) = selected else {
|
||||
warn!("decommission_pool: entries resolve entry no selected");
|
||||
return None;
|
||||
};
|
||||
|
||||
if selected.is_dir() && dir_exists >= params.dir_quorum {
|
||||
warn!("decommission_pool: entries resolve entry dir selected {:?}", selected.name);
|
||||
return Some(selected);
|
||||
}
|
||||
|
||||
// If we would never be able to reach read quorum.
|
||||
if objs_valid < params.obj_quorum {
|
||||
warn!(
|
||||
"decommission_pool: entries resolve entry not enough objects {} < {}",
|
||||
objs_valid, params.obj_quorum
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
if objs_agree == objs_valid {
|
||||
warn!("decommission_pool: entries resolve entry all agree {} == {}", objs_agree, objs_valid);
|
||||
return Some(selected);
|
||||
}
|
||||
|
||||
let Some(cached) = selected.cached else {
|
||||
warn!("decommission_pool: entries resolve entry no cached");
|
||||
return None;
|
||||
};
|
||||
|
||||
let versions = merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, ¶ms.candidates);
|
||||
if versions.is_empty() {
|
||||
warn!("decommission_pool: entries resolve entry no versions");
|
||||
return None;
|
||||
}
|
||||
|
||||
let metadata = match cached.marshal_msg() {
|
||||
Ok(meta) => meta,
|
||||
Err(e) => {
|
||||
warn!("decommission_pool: entries resolve entry marshal_msg {:?}", e);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
// Merge if we have disagreement.
|
||||
// Create a new merged result.
|
||||
let new_selected = MetaCacheEntry {
|
||||
name: selected.name.clone(),
|
||||
cached: Some(FileMeta {
|
||||
meta_ver: cached.meta_ver,
|
||||
versions,
|
||||
..Default::default()
|
||||
}),
|
||||
reusable: true,
|
||||
metadata,
|
||||
};
|
||||
|
||||
warn!("decommission_pool: entries resolve entry selected {:?}", new_selected.name);
|
||||
Some(new_selected)
|
||||
}
|
||||
|
||||
pub fn first_found(&self) -> (Option<MetaCacheEntry>, usize) {
|
||||
(self.0.iter().find(|x| x.is_some()).cloned().unwrap_or_default(), self.0.len())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MetaCacheEntriesSortedResult {
|
||||
pub entries: Option<MetaCacheEntriesSorted>,
|
||||
pub err: Option<Error>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MetaCacheEntriesSorted {
|
||||
pub o: MetaCacheEntries,
|
||||
pub list_id: Option<String>,
|
||||
pub reuse: bool,
|
||||
pub last_skipped_entry: Option<String>,
|
||||
}
|
||||
|
||||
impl MetaCacheEntriesSorted {
|
||||
pub fn entries(&self) -> Vec<&MetaCacheEntry> {
|
||||
let entries: Vec<&MetaCacheEntry> = self.o.0.iter().flatten().collect();
|
||||
entries
|
||||
}
|
||||
|
||||
pub fn forward_past(&mut self, marker: Option<String>) {
|
||||
if let Some(val) = marker {
|
||||
if let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val) {
|
||||
self.o.0 = self.o.0.split_off(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const METACACHE_STREAM_VERSION: u8 = 2;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct MetacacheWriter<W> {
|
||||
wr: W,
|
||||
created: bool,
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl<W: AsyncWrite + Unpin> MetacacheWriter<W> {
|
||||
pub fn new(wr: W) -> Self {
|
||||
Self {
|
||||
wr,
|
||||
created: false,
|
||||
buf: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn flush(&mut self) -> Result<()> {
|
||||
self.wr.write_all(&self.buf).await?;
|
||||
self.buf.clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn init(&mut self) -> Result<()> {
|
||||
if !self.created {
|
||||
rmp::encode::write_u8(&mut self.buf, METACACHE_STREAM_VERSION).map_err(|e| Error::other(format!("{:?}", e)))?;
|
||||
self.flush().await?;
|
||||
self.created = true;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn write(&mut self, objs: &[MetaCacheEntry]) -> Result<()> {
|
||||
if objs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.init().await?;
|
||||
|
||||
for obj in objs.iter() {
|
||||
if obj.name.is_empty() {
|
||||
return Err(Error::other("metacacheWriter: no name"));
|
||||
}
|
||||
|
||||
self.write_obj(obj).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn write_obj(&mut self, obj: &MetaCacheEntry) -> Result<()> {
|
||||
self.init().await?;
|
||||
|
||||
rmp::encode::write_bool(&mut self.buf, true).map_err(|e| Error::other(format!("{:?}", e)))?;
|
||||
rmp::encode::write_str(&mut self.buf, &obj.name).map_err(|e| Error::other(format!("{:?}", e)))?;
|
||||
rmp::encode::write_bin(&mut self.buf, &obj.metadata).map_err(|e| Error::other(format!("{:?}", e)))?;
|
||||
self.flush().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn close(&mut self) -> Result<()> {
|
||||
rmp::encode::write_bool(&mut self.buf, false).map_err(|e| Error::other(format!("{:?}", e)))?;
|
||||
self.flush().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MetacacheReader<R> {
|
||||
rd: R,
|
||||
init: bool,
|
||||
err: Option<Error>,
|
||||
buf: Vec<u8>,
|
||||
offset: usize,
|
||||
current: Option<MetaCacheEntry>,
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
pub fn new(rd: R) -> Self {
|
||||
Self {
|
||||
rd,
|
||||
init: false,
|
||||
err: None,
|
||||
buf: Vec::new(),
|
||||
offset: 0,
|
||||
current: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn read_more(&mut self, read_size: usize) -> Result<&[u8]> {
|
||||
let ext_size = read_size + self.offset;
|
||||
|
||||
let extra = ext_size - self.offset;
|
||||
if self.buf.capacity() >= ext_size {
|
||||
// Extend the buffer if we have enough space.
|
||||
self.buf.resize(ext_size, 0);
|
||||
} else {
|
||||
self.buf.extend(vec![0u8; extra]);
|
||||
}
|
||||
|
||||
let pref = self.offset;
|
||||
|
||||
self.rd.read_exact(&mut self.buf[pref..ext_size]).await?;
|
||||
|
||||
self.offset += read_size;
|
||||
|
||||
let data = &self.buf[pref..ext_size];
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.buf.clear();
|
||||
self.offset = 0;
|
||||
}
|
||||
|
||||
async fn check_init(&mut self) -> Result<()> {
|
||||
if !self.init {
|
||||
let ver = match rmp::decode::read_u8(&mut self.read_more(2).await?) {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
self.err = Some(Error::other(format!("{:?}", err)));
|
||||
0
|
||||
}
|
||||
};
|
||||
match ver {
|
||||
1 | 2 => (),
|
||||
_ => {
|
||||
self.err = Some(Error::other("invalid version"));
|
||||
}
|
||||
}
|
||||
|
||||
self.init = true;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn read_str_len(&mut self) -> Result<u32> {
|
||||
let mark = match rmp::decode::read_marker(&mut self.read_more(1).await?) {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
let err: Error = err.into();
|
||||
self.err = Some(err.clone());
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
match mark {
|
||||
Marker::FixStr(size) => Ok(u32::from(size)),
|
||||
Marker::Str8 => Ok(u32::from(self.read_u8().await?)),
|
||||
Marker::Str16 => Ok(u32::from(self.read_u16().await?)),
|
||||
Marker::Str32 => Ok(self.read_u32().await?),
|
||||
_marker => Err(Error::other("str marker err")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_bin_len(&mut self) -> Result<u32> {
|
||||
let mark = match rmp::decode::read_marker(&mut self.read_more(1).await?) {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
let err: Error = err.into();
|
||||
self.err = Some(err.clone());
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
match mark {
|
||||
Marker::Bin8 => Ok(u32::from(self.read_u8().await?)),
|
||||
Marker::Bin16 => Ok(u32::from(self.read_u16().await?)),
|
||||
Marker::Bin32 => Ok(self.read_u32().await?),
|
||||
_ => Err(Error::other("bin marker err")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_u8(&mut self) -> Result<u8> {
|
||||
let buf = self.read_more(1).await?;
|
||||
Ok(u8::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
|
||||
}
|
||||
|
||||
async fn read_u16(&mut self) -> Result<u16> {
|
||||
let buf = self.read_more(2).await?;
|
||||
Ok(u16::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
|
||||
}
|
||||
|
||||
async fn read_u32(&mut self) -> Result<u32> {
|
||||
let buf = self.read_more(4).await?;
|
||||
Ok(u32::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
|
||||
}
|
||||
|
||||
pub async fn skip(&mut self, size: usize) -> Result<()> {
|
||||
self.check_init().await?;
|
||||
|
||||
if let Some(err) = &self.err {
|
||||
return Err(err.clone());
|
||||
}
|
||||
|
||||
let mut n = size;
|
||||
|
||||
if self.current.is_some() {
|
||||
n -= 1;
|
||||
self.current = None;
|
||||
}
|
||||
|
||||
while n > 0 {
|
||||
match rmp::decode::read_bool(&mut self.read_more(1).await?) {
|
||||
Ok(res) => {
|
||||
if !res {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let err: Error = err.into();
|
||||
self.err = Some(err.clone());
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
let l = self.read_str_len().await?;
|
||||
let _ = self.read_more(l as usize).await?;
|
||||
let l = self.read_bin_len().await?;
|
||||
let _ = self.read_more(l as usize).await?;
|
||||
|
||||
n -= 1;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn peek(&mut self) -> Result<Option<MetaCacheEntry>> {
|
||||
self.check_init().await?;
|
||||
|
||||
if let Some(err) = &self.err {
|
||||
return Err(err.clone());
|
||||
}
|
||||
|
||||
match rmp::decode::read_bool(&mut self.read_more(1).await?) {
|
||||
Ok(res) => {
|
||||
if !res {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let err: Error = err.into();
|
||||
self.err = Some(err.clone());
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
let l = self.read_str_len().await?;
|
||||
|
||||
let buf = self.read_more(l as usize).await?;
|
||||
let name_buf = buf.to_vec();
|
||||
let name = match from_utf8(&name_buf) {
|
||||
Ok(decoded) => decoded.to_owned(),
|
||||
Err(err) => {
|
||||
self.err = Some(Error::other(err.to_string()));
|
||||
return Err(Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let l = self.read_bin_len().await?;
|
||||
|
||||
let buf = self.read_more(l as usize).await?;
|
||||
|
||||
let metadata = buf.to_vec();
|
||||
|
||||
self.reset();
|
||||
|
||||
let entry = Some(MetaCacheEntry {
|
||||
name,
|
||||
metadata,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
});
|
||||
self.current = entry.clone();
|
||||
|
||||
Ok(entry)
|
||||
}
|
||||
|
||||
pub async fn read_all(&mut self) -> Result<Vec<MetaCacheEntry>> {
|
||||
let mut ret = Vec::new();
|
||||
|
||||
loop {
|
||||
if let Some(entry) = self.peek().await? {
|
||||
ret.push(entry);
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
}
|
||||
|
||||
pub type UpdateFn<T> = Box<dyn Fn() -> Pin<Box<dyn Future<Output = std::io::Result<T>> + Send>> + Send + Sync + 'static>;
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Opts {
|
||||
pub return_last_good: bool,
|
||||
pub no_wait: bool,
|
||||
}
|
||||
|
||||
pub struct Cache<T: Clone + Debug + Send> {
|
||||
update_fn: UpdateFn<T>,
|
||||
ttl: Duration,
|
||||
opts: Opts,
|
||||
val: AtomicPtr<T>,
|
||||
last_update_ms: AtomicU64,
|
||||
updating: Arc<Mutex<bool>>,
|
||||
}
|
||||
|
||||
impl<T: Clone + Debug + Send + 'static> Cache<T> {
|
||||
pub fn new(update_fn: UpdateFn<T>, ttl: Duration, opts: Opts) -> Self {
|
||||
let val = AtomicPtr::new(ptr::null_mut());
|
||||
Self {
|
||||
update_fn,
|
||||
ttl,
|
||||
opts,
|
||||
val,
|
||||
last_update_ms: AtomicU64::new(0),
|
||||
updating: Arc::new(Mutex::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unsafe_code)]
|
||||
pub async fn get(self: Arc<Self>) -> std::io::Result<T> {
|
||||
let v_ptr = self.val.load(AtomicOrdering::SeqCst);
|
||||
let v = if v_ptr.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(unsafe { (*v_ptr).clone() })
|
||||
};
|
||||
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Time went backwards")
|
||||
.as_secs();
|
||||
if now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() {
|
||||
if let Some(v) = v {
|
||||
return Ok(v);
|
||||
}
|
||||
}
|
||||
|
||||
if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() * 2 {
|
||||
if self.updating.try_lock().is_ok() {
|
||||
let this = Arc::clone(&self);
|
||||
spawn(async move {
|
||||
let _ = this.update().await;
|
||||
});
|
||||
}
|
||||
|
||||
return Ok(v.unwrap());
|
||||
}
|
||||
|
||||
let _ = self.updating.lock().await;
|
||||
|
||||
if let Ok(duration) =
|
||||
SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(AtomicOrdering::SeqCst)))
|
||||
{
|
||||
if duration < self.ttl {
|
||||
return Ok(v.unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
match self.update().await {
|
||||
Ok(_) => {
|
||||
let v_ptr = self.val.load(AtomicOrdering::SeqCst);
|
||||
let v = if v_ptr.is_null() {
|
||||
None
|
||||
} else {
|
||||
Some(unsafe { (*v_ptr).clone() })
|
||||
};
|
||||
Ok(v.unwrap())
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
async fn update(&self) -> std::io::Result<()> {
|
||||
match (self.update_fn)().await {
|
||||
Ok(val) => {
|
||||
self.val.store(Box::into_raw(Box::new(val)), AtomicOrdering::SeqCst);
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Time went backwards")
|
||||
.as_secs();
|
||||
self.last_update_ms.store(now, AtomicOrdering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => {
|
||||
let v_ptr = self.val.load(AtomicOrdering::SeqCst);
|
||||
if self.opts.return_last_good && !v_ptr.is_null() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_writer() {
|
||||
let mut f = Cursor::new(Vec::new());
|
||||
let mut w = MetacacheWriter::new(&mut f);
|
||||
|
||||
let mut objs = Vec::new();
|
||||
for i in 0..10 {
|
||||
let info = MetaCacheEntry {
|
||||
name: format!("item{}", i),
|
||||
metadata: vec![0u8, 10],
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
objs.push(info);
|
||||
}
|
||||
|
||||
w.write(&objs).await.unwrap();
|
||||
w.close().await.unwrap();
|
||||
|
||||
let data = f.into_inner();
|
||||
let nf = Cursor::new(data);
|
||||
|
||||
let mut r = MetacacheReader::new(nf);
|
||||
let nobjs = r.read_all().await.unwrap();
|
||||
|
||||
assert_eq!(objs, nobjs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
use crate::error::Result;
|
||||
use crate::filemeta::*;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// 创建一个真实的 xl.meta 文件数据用于测试
|
||||
pub fn create_real_xlmeta() -> Result<Vec<u8>> {
|
||||
let mut fm = FileMeta::new();
|
||||
|
||||
// 创建一个真实的对象版本
|
||||
let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef")?;
|
||||
let data_dir = Uuid::parse_str("fedcba98-7654-3210-fedc-ba9876543210")?;
|
||||
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("Content-Type".to_string(), "text/plain".to_string());
|
||||
metadata.insert("X-Amz-Meta-Author".to_string(), "test-user".to_string());
|
||||
metadata.insert("X-Amz-Meta-Created".to_string(), "2024-01-15T10:30:00Z".to_string());
|
||||
|
||||
let object_version = MetaObject {
|
||||
version_id: Some(version_id),
|
||||
data_dir: Some(data_dir),
|
||||
erasure_algorithm: crate::fileinfo::ErasureAlgo::ReedSolomon,
|
||||
erasure_m: 4,
|
||||
erasure_n: 2,
|
||||
erasure_block_size: 1024 * 1024, // 1MB
|
||||
erasure_index: 1,
|
||||
erasure_dist: vec![0, 1, 2, 3, 4, 5],
|
||||
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
|
||||
part_numbers: vec![1],
|
||||
part_etags: vec!["d41d8cd98f00b204e9800998ecf8427e".to_string()],
|
||||
part_sizes: vec![1024],
|
||||
part_actual_sizes: vec![1024],
|
||||
part_indices: Vec::new(),
|
||||
size: 1024,
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312200)?), // 2024-01-15 10:30:00 UTC
|
||||
meta_sys: HashMap::new(),
|
||||
meta_user: metadata,
|
||||
};
|
||||
|
||||
let file_version = FileMetaVersion {
|
||||
version_type: VersionType::Object,
|
||||
object: Some(object_version),
|
||||
delete_marker: None,
|
||||
write_version: 1,
|
||||
};
|
||||
|
||||
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
|
||||
fm.versions.push(shallow_version);
|
||||
|
||||
// 添加一个删除标记版本
|
||||
let delete_version_id = Uuid::parse_str("11111111-2222-3333-4444-555555555555")?;
|
||||
let delete_marker = MetaDeleteMarker {
|
||||
version_id: Some(delete_version_id),
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312260)?), // 1分钟后
|
||||
meta_sys: None,
|
||||
};
|
||||
|
||||
let delete_file_version = FileMetaVersion {
|
||||
version_type: VersionType::Delete,
|
||||
object: None,
|
||||
delete_marker: Some(delete_marker),
|
||||
write_version: 2,
|
||||
};
|
||||
|
||||
let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?;
|
||||
fm.versions.push(delete_shallow_version);
|
||||
|
||||
// 添加一个 Legacy 版本用于测试
|
||||
let legacy_version_id = Uuid::parse_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")?;
|
||||
let legacy_version = FileMetaVersion {
|
||||
version_type: VersionType::Legacy,
|
||||
object: None,
|
||||
delete_marker: None,
|
||||
write_version: 3,
|
||||
};
|
||||
|
||||
let mut legacy_shallow = FileMetaShallowVersion::try_from(legacy_version)?;
|
||||
legacy_shallow.header.version_id = Some(legacy_version_id);
|
||||
legacy_shallow.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1705312140)?); // 更早的时间
|
||||
fm.versions.push(legacy_shallow);
|
||||
|
||||
// 按修改时间排序（最新的在前）
|
||||
fm.versions.sort_by(|a, b| b.header.mod_time.cmp(&a.header.mod_time));
|
||||
|
||||
fm.marshal_msg()
|
||||
}
|
||||
|
||||
/// 创建一个包含多个版本的复杂 xl.meta 文件
|
||||
pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
|
||||
let mut fm = FileMeta::new();
|
||||
|
||||
// 创建10个版本的对象
|
||||
for i in 0i64..10i64 {
|
||||
let version_id = Uuid::new_v4();
|
||||
let data_dir = if i % 3 == 0 { Some(Uuid::new_v4()) } else { None };
|
||||
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("Content-Type".to_string(), "application/octet-stream".to_string());
|
||||
metadata.insert("X-Amz-Meta-Version".to_string(), i.to_string());
|
||||
metadata.insert("X-Amz-Meta-Test".to_string(), format!("test-value-{}", i));
|
||||
|
||||
let object_version = MetaObject {
|
||||
version_id: Some(version_id),
|
||||
data_dir,
|
||||
erasure_algorithm: crate::fileinfo::ErasureAlgo::ReedSolomon,
|
||||
erasure_m: 4,
|
||||
erasure_n: 2,
|
||||
erasure_block_size: 1024 * 1024,
|
||||
erasure_index: (i % 6) as usize,
|
||||
erasure_dist: vec![0, 1, 2, 3, 4, 5],
|
||||
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
|
||||
part_numbers: vec![1],
|
||||
part_etags: vec![format!("etag-{:08x}", i)],
|
||||
part_sizes: vec![1024 * (i + 1) as usize],
|
||||
part_actual_sizes: vec![1024 * (i + 1)],
|
||||
part_indices: Vec::new(),
|
||||
size: 1024 * (i + 1),
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312200 + i * 60)?),
|
||||
meta_sys: HashMap::new(),
|
||||
meta_user: metadata,
|
||||
};
|
||||
|
||||
let file_version = FileMetaVersion {
|
||||
version_type: VersionType::Object,
|
||||
object: Some(object_version),
|
||||
delete_marker: None,
|
||||
write_version: (i + 1) as u64,
|
||||
};
|
||||
|
||||
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
|
||||
fm.versions.push(shallow_version);
|
||||
|
||||
// 每隔3个版本添加一个删除标记
|
||||
if i % 3 == 2 {
|
||||
let delete_version_id = Uuid::new_v4();
|
||||
let delete_marker = MetaDeleteMarker {
|
||||
version_id: Some(delete_version_id),
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312200 + i * 60 + 30)?),
|
||||
meta_sys: None,
|
||||
};
|
||||
|
||||
let delete_file_version = FileMetaVersion {
|
||||
version_type: VersionType::Delete,
|
||||
object: None,
|
||||
delete_marker: Some(delete_marker),
|
||||
write_version: (i + 100) as u64,
|
||||
};
|
||||
|
||||
let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?;
|
||||
fm.versions.push(delete_shallow_version);
|
||||
}
|
||||
}
|
||||
|
||||
// 按修改时间排序（最新的在前）
|
||||
fm.versions.sort_by(|a, b| b.header.mod_time.cmp(&a.header.mod_time));
|
||||
|
||||
fm.marshal_msg()
|
||||
}
|
||||
|
||||
/// 创建一个损坏的 xl.meta 文件用于错误处理测试
|
||||
pub fn create_corrupted_xlmeta() -> Vec<u8> {
|
||||
let mut data = vec![
|
||||
// 正确的文件头
|
||||
b'X', b'L', b'2', b' ', // 版本号
|
||||
1, 0, 3, 0, // 版本号
|
||||
0xc6, 0x00, 0x00, 0x00, 0x10, // 正确的 bin32 长度标记，但数据长度不匹配
|
||||
];
|
||||
|
||||
// 添加不足的数据（少于声明的长度）
|
||||
data.extend_from_slice(&[0x42; 8]); // 只有8字节，但声明了16字节
|
||||
|
||||
data
|
||||
}
|
||||
|
||||
/// 创建一个空的 xl.meta 文件
|
||||
pub fn create_empty_xlmeta() -> Result<Vec<u8>> {
|
||||
let fm = FileMeta::new();
|
||||
fm.marshal_msg()
|
||||
}
|
||||
|
||||
/// 验证解析结果的辅助函数
|
||||
pub fn verify_parsed_metadata(fm: &FileMeta, expected_versions: usize) -> Result<()> {
|
||||
assert_eq!(fm.versions.len(), expected_versions, "版本数量不匹配");
|
||||
assert_eq!(fm.meta_ver, crate::filemeta::XL_META_VERSION, "元数据版本不匹配");
|
||||
|
||||
// 验证版本是否按修改时间排序
|
||||
for i in 1..fm.versions.len() {
|
||||
let prev_time = fm.versions[i - 1].header.mod_time;
|
||||
let curr_time = fm.versions[i].header.mod_time;
|
||||
|
||||
if let (Some(prev), Some(curr)) = (prev_time, curr_time) {
|
||||
assert!(prev >= curr, "版本未按修改时间正确排序");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 创建一个包含内联数据的 xl.meta 文件
|
||||
pub fn create_xlmeta_with_inline_data() -> Result<Vec<u8>> {
|
||||
let mut fm = FileMeta::new();
|
||||
|
||||
// 添加内联数据
|
||||
let inline_data = b"This is inline data for testing purposes";
|
||||
let version_id = Uuid::new_v4();
|
||||
fm.data.replace(&version_id.to_string(), inline_data.to_vec())?;
|
||||
|
||||
let object_version = MetaObject {
|
||||
version_id: Some(version_id),
|
||||
data_dir: None,
|
||||
erasure_algorithm: crate::fileinfo::ErasureAlgo::ReedSolomon,
|
||||
erasure_m: 1,
|
||||
erasure_n: 1,
|
||||
erasure_block_size: 64 * 1024,
|
||||
erasure_index: 0,
|
||||
erasure_dist: vec![0, 1],
|
||||
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
|
||||
part_numbers: vec![1],
|
||||
part_etags: Vec::new(),
|
||||
part_sizes: vec![inline_data.len()],
|
||||
part_actual_sizes: Vec::new(),
|
||||
part_indices: Vec::new(),
|
||||
size: inline_data.len() as i64,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
meta_sys: HashMap::new(),
|
||||
meta_user: HashMap::new(),
|
||||
};
|
||||
|
||||
let file_version = FileMetaVersion {
|
||||
version_type: VersionType::Object,
|
||||
object: Some(object_version),
|
||||
delete_marker: None,
|
||||
write_version: 1,
|
||||
};
|
||||
|
||||
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
|
||||
fm.versions.push(shallow_version);
|
||||
|
||||
fm.marshal_msg()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_create_real_xlmeta() {
|
||||
let data = create_real_xlmeta().expect("创建测试数据失败");
|
||||
assert!(!data.is_empty(), "生成的数据不应为空");
|
||||
|
||||
// 验证文件头
|
||||
assert_eq!(&data[0..4], b"XL2 ", "文件头不正确");
|
||||
|
||||
// 尝试解析
|
||||
let fm = FileMeta::load(&data).expect("解析失败");
|
||||
verify_parsed_metadata(&fm, 3).expect("验证失败");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_complex_xlmeta() {
|
||||
let data = create_complex_xlmeta().expect("创建复杂测试数据失败");
|
||||
assert!(!data.is_empty(), "生成的数据不应为空");
|
||||
|
||||
let fm = FileMeta::load(&data).expect("解析失败");
|
||||
assert!(fm.versions.len() >= 10, "应该有至少10个版本");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_xlmeta_with_inline_data() {
|
||||
let data = create_xlmeta_with_inline_data().expect("创建内联数据测试失败");
|
||||
assert!(!data.is_empty(), "生成的数据不应为空");
|
||||
|
||||
let fm = FileMeta::load(&data).expect("解析失败");
|
||||
assert_eq!(fm.versions.len(), 1, "应该有1个版本");
|
||||
assert!(!fm.data.as_slice().is_empty(), "应该包含内联数据");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_corrupted_xlmeta_handling() {
|
||||
let data = create_corrupted_xlmeta();
|
||||
let result = FileMeta::load(&data);
|
||||
assert!(result.is_err(), "损坏的数据应该解析失败");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_xlmeta() {
|
||||
let data = create_empty_xlmeta().expect("创建空测试数据失败");
|
||||
let fm = FileMeta::load(&data).expect("解析空数据失败");
|
||||
assert_eq!(fm.versions.len(), 0, "空文件应该没有版本");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user