mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 08:36:54 +00:00
refactor: move HTTP range helper contracts (#3533)
This commit is contained in:
@@ -38,8 +38,7 @@ use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deploy
|
||||
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{
|
||||
GetObjectReader, HTTPRangeSpec, ListOperations, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions,
|
||||
ObjectToDelete,
|
||||
GetObjectReader, ListOperations, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete,
|
||||
};
|
||||
use crate::tier::warm_backend::WarmBackendGetOpts;
|
||||
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
|
||||
@@ -61,6 +60,7 @@ use rustfs_filemeta::{
|
||||
VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
|
||||
|
||||
@@ -34,8 +34,8 @@ use crate::global::get_global_bucket_monitor;
|
||||
use crate::resolve_object_store_handle;
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::store_api::{
|
||||
DeletedObject, HTTPRangeSpec, ListOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions,
|
||||
ObjectToDelete, WalkOptions,
|
||||
DeletedObject, ListOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete,
|
||||
WalkOptions,
|
||||
};
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
|
||||
@@ -61,6 +61,7 @@ use rustfs_filemeta::{
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
|
||||
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
|
||||
|
||||
@@ -25,9 +25,10 @@ use std::{collections::HashMap, io::Cursor, sync::Arc};
|
||||
use tokio::io::BufReader;
|
||||
|
||||
use crate::error::ErrorResponse;
|
||||
use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions};
|
||||
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use rustfs_filemeta::ObjectPartInfo;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
//#[derive(Clone)]
|
||||
|
||||
@@ -1217,7 +1217,7 @@ mod tests {
|
||||
use crate::error::{Error, Result};
|
||||
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::store_api::{GetObjectReader, HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use http::HeaderMap;
|
||||
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::notify::{
|
||||
@@ -1231,7 +1231,7 @@ mod tests {
|
||||
use rustfs_lock::client::LockClient;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
|
||||
use rustfs_storage_api::StorageAdminApi;
|
||||
use rustfs_storage_api::{HTTPRangeSpec, StorageAdminApi};
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::bucket::error::BucketMetadataError;
|
||||
use crate::disk::error::DiskError;
|
||||
use rustfs_storage_api::StorageErrorCode;
|
||||
use rustfs_storage_api::{HTTPRangeError, StorageErrorCode};
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
|
||||
@@ -218,6 +218,14 @@ impl StorageError {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<HTTPRangeError> for StorageError {
|
||||
fn from(err: HTTPRangeError) -> Self {
|
||||
match err {
|
||||
HTTPRangeError::InvalidRangeSpec(message) => Self::InvalidRangeSpec(message),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<DiskError> for StorageError {
|
||||
fn from(e: DiskError) -> Self {
|
||||
match e {
|
||||
|
||||
@@ -23,11 +23,11 @@ use crate::global::get_global_endpoints;
|
||||
use crate::pools::ListCallback;
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{GetObjectReader, HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions};
|
||||
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions};
|
||||
use http::HeaderMap;
|
||||
use rand::RngExt as _;
|
||||
use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||
use rustfs_storage_api::StorageAdminApi;
|
||||
use rustfs_storage_api::{HTTPRangeSpec, StorageAdminApi};
|
||||
use rustfs_utils::path::encode_dir_object;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
@@ -3167,10 +3167,10 @@ mod rebalance_unit_tests {
|
||||
use super::percent_free_ratio;
|
||||
use super::rebalance_goal_reached;
|
||||
use super::{
|
||||
DiskError, GetObjectReader, HTTPRangeSpec, MigrationBackend, MigrationVersionResult, ObjectInfo, ObjectOptions,
|
||||
RebalSaveOpt, RebalStatus, RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo,
|
||||
RebalanceMeta, RebalanceStats, RebalanceTerminalEvent, apply_rebalance_save_option, apply_rebalance_terminal_event,
|
||||
apply_stopped_at, classify_rebalance_terminal_event, clone_arc_by_index, clone_first_arc, clone_rebalance_pool_stats,
|
||||
DiskError, GetObjectReader, MigrationBackend, MigrationVersionResult, ObjectInfo, ObjectOptions, RebalSaveOpt,
|
||||
RebalStatus, RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta,
|
||||
RebalanceStats, RebalanceTerminalEvent, apply_rebalance_save_option, apply_rebalance_terminal_event, apply_stopped_at,
|
||||
classify_rebalance_terminal_event, clone_arc_by_index, clone_first_arc, clone_rebalance_pool_stats,
|
||||
complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, defer_bucket_in_rebalance_queue,
|
||||
ensure_rebalance_listing_disks_available, ensure_rebalance_not_decommissioning, ensure_valid_rebalance_pool_index,
|
||||
has_deferred_rebalance_error, is_rebalance_stopped_terminal_event, is_transient_rebalance_error,
|
||||
@@ -3197,6 +3197,7 @@ mod rebalance_unit_tests {
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_filemeta::TRANSITION_COMPLETE;
|
||||
use rustfs_rio::Index;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use serde::Serialize;
|
||||
use std::io::Cursor;
|
||||
|
||||
@@ -52,8 +52,8 @@ use crate::{
|
||||
event_notification::{EventArgs, send_event},
|
||||
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure},
|
||||
store_api::{
|
||||
DeletedObject, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations,
|
||||
NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, PutObjReader,
|
||||
DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations, NamespaceLocking,
|
||||
ObjectIO, ObjectInfo, ObjectOperations, PutObjReader,
|
||||
},
|
||||
store_init::load_format_erasure,
|
||||
};
|
||||
@@ -85,6 +85,7 @@ use rustfs_object_capacity::capacity_scope::{
|
||||
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo,
|
||||
MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo,
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::{
|
||||
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
|
||||
set_disk::SetDisks,
|
||||
store_api::{
|
||||
DeletedObject, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations,
|
||||
DeletedObject, GetObjectReader, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations,
|
||||
MultipartOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete,
|
||||
PutObjReader,
|
||||
},
|
||||
@@ -49,6 +49,7 @@ use rustfs_lock::NamespaceLockWrapper;
|
||||
use rustfs_lock::client::LockClient;
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
|
||||
use rustfs_storage_api::CompletePart;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, MakeBucketOptions,
|
||||
MultipartInfo, MultipartUploadResult, PartInfo,
|
||||
|
||||
@@ -63,8 +63,8 @@ use crate::{
|
||||
rpc::S3PeerSys,
|
||||
sets::Sets,
|
||||
store_api::{
|
||||
DeletedObject, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations,
|
||||
NamespaceLocking, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
|
||||
DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations, NamespaceLocking,
|
||||
ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
|
||||
},
|
||||
store_init,
|
||||
};
|
||||
@@ -78,6 +78,7 @@ use rustfs_config::server_config::{Config, get_global_server_config, set_global_
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo,
|
||||
MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo,
|
||||
|
||||
@@ -34,6 +34,7 @@ use rustfs_filemeta::{
|
||||
use rustfs_lock::NamespaceLockWrapper;
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_rio::Checksum;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
|
||||
|
||||
@@ -99,6 +99,10 @@ fn part_plaintext_size(part: &ObjectPartInfo) -> i64 {
|
||||
}
|
||||
}
|
||||
|
||||
fn http_range_spec_from_object_info(oi: &ObjectInfo, part_number: usize) -> Option<HTTPRangeSpec> {
|
||||
HTTPRangeSpec::from_part_sizes(oi.size, part_number, oi.parts.iter().map(part_plaintext_size))
|
||||
}
|
||||
|
||||
fn restore_request_active(opts: &ObjectOptions) -> bool {
|
||||
let restore = &opts.transition.restore_request;
|
||||
restore.type_.is_some() || restore.days.is_some() || restore.output_location.is_some() || restore.select_parameters.is_some()
|
||||
@@ -301,7 +305,7 @@ impl ReadPlan {
|
||||
if let Some(part_number) = opts.part_number
|
||||
&& rs.is_none()
|
||||
{
|
||||
rs = HTTPRangeSpec::from_object_info(oi, part_number);
|
||||
rs = http_range_spec_from_object_info(oi, part_number);
|
||||
}
|
||||
|
||||
let mut is_encrypted = oi.is_encrypted();
|
||||
@@ -719,105 +723,6 @@ impl<R: AsyncRead + Unpin + Send + Sync> AsyncRead for SkipReader<R> {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HTTPRangeSpec {
|
||||
pub is_suffix_length: bool,
|
||||
pub start: i64,
|
||||
pub end: i64,
|
||||
}
|
||||
|
||||
impl HTTPRangeSpec {
|
||||
pub fn from_object_info(oi: &ObjectInfo, part_number: usize) -> Option<Self> {
|
||||
if oi.size == 0 || oi.parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if part_number == 0 || part_number > oi.parts.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut start = 0_i64;
|
||||
let mut end = -1_i64;
|
||||
for i in 0..part_number {
|
||||
let part = &oi.parts[i];
|
||||
start = end + 1;
|
||||
end = start + part_plaintext_size(part) - 1;
|
||||
}
|
||||
|
||||
Some(HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start,
|
||||
end,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_offset_length(&self, res_size: i64) -> Result<(usize, i64)> {
|
||||
let len = self.get_length(res_size)?;
|
||||
|
||||
let mut start = self.start;
|
||||
if self.is_suffix_length {
|
||||
let suffix_len = if self.start < 0 {
|
||||
self.start
|
||||
.checked_neg()
|
||||
.ok_or_else(|| Error::InvalidRangeSpec("range value invalid: suffix length overflow".to_string()))?
|
||||
} else {
|
||||
self.start
|
||||
};
|
||||
start = res_size - suffix_len;
|
||||
if start < 0 {
|
||||
start = 0;
|
||||
}
|
||||
}
|
||||
Ok((start as usize, len))
|
||||
}
|
||||
pub fn get_length(&self, res_size: i64) -> Result<i64> {
|
||||
if res_size < 0 {
|
||||
return Err(Error::InvalidRangeSpec("The requested range is not satisfiable".to_string()));
|
||||
}
|
||||
|
||||
if self.is_suffix_length {
|
||||
let specified_len = if self.start < 0 {
|
||||
self.start
|
||||
.checked_neg()
|
||||
.ok_or_else(|| Error::InvalidRangeSpec("range value invalid: suffix length overflow".to_string()))?
|
||||
} else {
|
||||
self.start
|
||||
};
|
||||
let mut range_length = specified_len;
|
||||
|
||||
if specified_len > res_size {
|
||||
range_length = res_size;
|
||||
}
|
||||
|
||||
return Ok(range_length);
|
||||
}
|
||||
|
||||
if self.start >= res_size {
|
||||
return Err(Error::InvalidRangeSpec("The requested range is not satisfiable".to_string()));
|
||||
}
|
||||
|
||||
if self.end > -1 {
|
||||
let mut end = self.end;
|
||||
if res_size <= end {
|
||||
end = res_size - 1;
|
||||
}
|
||||
|
||||
let range_length = end - self.start + 1;
|
||||
return Ok(range_length);
|
||||
}
|
||||
|
||||
if self.end == -1 {
|
||||
let range_length = res_size - self.start;
|
||||
return Ok(range_length);
|
||||
}
|
||||
|
||||
Err(Error::InvalidRangeSpec(format!(
|
||||
"range value invalid: start={}, end={}, expected start <= end and end >= -1",
|
||||
self.start, self.end
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
/// A streaming decompression reader that supports range requests by skipping data in the decompressed stream.
|
||||
/// This implementation acknowledges that compressed streams (like LZ4) must be decompressed sequentially
|
||||
/// from the beginning, so it streams and discards data until reaching the target offset.
|
||||
@@ -1818,12 +1723,12 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let spec = HTTPRangeSpec::from_object_info(&object_info, 2).unwrap();
|
||||
let spec = http_range_spec_from_object_info(&object_info, 2).unwrap();
|
||||
assert_eq!(spec.start, 100);
|
||||
assert_eq!(spec.end, 199);
|
||||
|
||||
assert!(HTTPRangeSpec::from_object_info(&object_info, 0).is_none());
|
||||
assert!(HTTPRangeSpec::from_object_info(&object_info, 4).is_none());
|
||||
assert!(http_range_spec_from_object_info(&object_info, 0).is_none());
|
||||
assert!(http_range_spec_from_object_info(&object_info, 4).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1856,7 +1761,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let spec = HTTPRangeSpec::from_object_info(&object_info, 2).unwrap();
|
||||
let spec = http_range_spec_from_object_info(&object_info, 2).unwrap();
|
||||
assert_eq!(spec.start, 30);
|
||||
assert_eq!(spec.end, 69);
|
||||
}
|
||||
@@ -1891,7 +1796,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let spec = HTTPRangeSpec::from_object_info(&object_info, 3).unwrap();
|
||||
let spec = http_range_spec_from_object_info(&object_info, 3).unwrap();
|
||||
assert_eq!(spec.start, 60);
|
||||
assert_eq!(spec.end, 99);
|
||||
}
|
||||
|
||||
@@ -300,7 +300,7 @@ async fn create_dlo_stream(
|
||||
|
||||
async move {
|
||||
let range_spec = if byte_start > 0 || byte_end < segment.size as u64 - 1 {
|
||||
Some(rustfs_ecstore::store_api::HTTPRangeSpec {
|
||||
Some(rustfs_storage_api::HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: byte_start as i64,
|
||||
end: byte_end as i64,
|
||||
|
||||
@@ -533,7 +533,7 @@ pub async fn get_object(
|
||||
container: &str,
|
||||
object: &str,
|
||||
credentials: &Credentials,
|
||||
range: Option<rustfs_ecstore::store_api::HTTPRangeSpec>,
|
||||
range: Option<rustfs_storage_api::HTTPRangeSpec>,
|
||||
) -> SwiftResult<rustfs_ecstore::store_api::GetObjectReader> {
|
||||
use rustfs_ecstore::store_api::GetObjectReader;
|
||||
|
||||
@@ -1032,8 +1032,8 @@ pub fn parse_copy_from_header(copy_from: &str) -> SwiftResult<(String, String)>
|
||||
/// assert_eq!(range.end, 1023);
|
||||
/// ```
|
||||
#[allow(dead_code)] // Handler integration: Range header
|
||||
pub fn parse_range_header(range_str: &str) -> SwiftResult<rustfs_ecstore::store_api::HTTPRangeSpec> {
|
||||
use rustfs_ecstore::store_api::HTTPRangeSpec;
|
||||
pub fn parse_range_header(range_str: &str) -> SwiftResult<rustfs_storage_api::HTTPRangeSpec> {
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
|
||||
if !range_str.starts_with("bytes=") {
|
||||
return Err(SwiftError::BadRequest("Range header must start with 'bytes='".to_string()));
|
||||
|
||||
@@ -420,7 +420,7 @@ async fn create_slo_stream(
|
||||
|
||||
// Fetch segment with range
|
||||
let range_spec = if byte_start > 0 || byte_end < segment.size_bytes - 1 {
|
||||
Some(rustfs_ecstore::store_api::HTTPRangeSpec {
|
||||
Some(rustfs_storage_api::HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: byte_start as i64,
|
||||
end: byte_end as i64,
|
||||
|
||||
@@ -33,6 +33,7 @@ chrono.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
datafusion = { workspace = true }
|
||||
rustfs-ecstore.workspace = true
|
||||
rustfs-storage-api.workspace = true
|
||||
futures = { workspace = true }
|
||||
futures-core = { workspace = true }
|
||||
http.workspace = true
|
||||
|
||||
@@ -29,7 +29,8 @@ use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object
|
||||
use rustfs_ecstore::resolve_object_store_handle;
|
||||
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
use rustfs_ecstore::store::ECStore;
|
||||
use rustfs_ecstore::store_api::{GetObjectReader, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectOptions};
|
||||
use rustfs_ecstore::store_api::{GetObjectReader, ObjectIO, ObjectOperations, ObjectOptions};
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use s3s::S3Result;
|
||||
use s3s::dto::SelectObjectContentInput;
|
||||
use s3s::header::{
|
||||
|
||||
@@ -1124,7 +1124,7 @@ mod tests {
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
_range: Option<rustfs_ecstore::store_api::HTTPRangeSpec>,
|
||||
_range: Option<rustfs_storage_api::HTTPRangeSpec>,
|
||||
_h: http::HeaderMap,
|
||||
_opts: &ObjectOptions,
|
||||
) -> rustfs_ecstore::error::Result<GetObjectReader> {
|
||||
|
||||
@@ -24,4 +24,4 @@ pub use admin::{DiskSetSelector, StorageAdminApi};
|
||||
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use object::{HTTPPreconditions, ObjectLockRetentionOptions};
|
||||
pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
@@ -39,6 +40,120 @@ pub struct ObjectLockRetentionOptions {
|
||||
pub bypass_governance: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum HTTPRangeError {
|
||||
InvalidRangeSpec(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for HTTPRangeError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::InvalidRangeSpec(message) => f.write_str(message),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for HTTPRangeError {}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HTTPRangeSpec {
|
||||
pub is_suffix_length: bool,
|
||||
pub start: i64,
|
||||
pub end: i64,
|
||||
}
|
||||
|
||||
impl HTTPRangeSpec {
|
||||
pub fn from_part_sizes(object_size: i64, part_number: usize, part_sizes: impl IntoIterator<Item = i64>) -> Option<Self> {
|
||||
if object_size == 0 || part_number == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut start = 0_i64;
|
||||
let mut end = -1_i64;
|
||||
let mut parts = part_sizes.into_iter();
|
||||
for _ in 0..part_number {
|
||||
let part_size = parts.next()?;
|
||||
start = end.checked_add(1)?;
|
||||
end = start.checked_add(part_size)?.checked_sub(1)?;
|
||||
}
|
||||
|
||||
Some(Self {
|
||||
is_suffix_length: false,
|
||||
start,
|
||||
end,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_offset_length(&self, res_size: i64) -> Result<(usize, i64), HTTPRangeError> {
|
||||
let len = self.get_length(res_size)?;
|
||||
|
||||
let mut start = self.start;
|
||||
if self.is_suffix_length {
|
||||
let suffix_len = if self.start < 0 {
|
||||
self.start
|
||||
.checked_neg()
|
||||
.ok_or_else(|| HTTPRangeError::InvalidRangeSpec("range value invalid: suffix length overflow".to_string()))?
|
||||
} else {
|
||||
self.start
|
||||
};
|
||||
start = res_size - suffix_len;
|
||||
if start < 0 {
|
||||
start = 0;
|
||||
}
|
||||
}
|
||||
let offset = usize::try_from(start)
|
||||
.map_err(|_| HTTPRangeError::InvalidRangeSpec("range value invalid: start offset overflow".to_string()))?;
|
||||
Ok((offset, len))
|
||||
}
|
||||
|
||||
pub fn get_length(&self, res_size: i64) -> Result<i64, HTTPRangeError> {
|
||||
if res_size < 0 {
|
||||
return Err(HTTPRangeError::InvalidRangeSpec("The requested range is not satisfiable".to_string()));
|
||||
}
|
||||
|
||||
if self.is_suffix_length {
|
||||
let specified_len = if self.start < 0 {
|
||||
self.start
|
||||
.checked_neg()
|
||||
.ok_or_else(|| HTTPRangeError::InvalidRangeSpec("range value invalid: suffix length overflow".to_string()))?
|
||||
} else {
|
||||
self.start
|
||||
};
|
||||
let mut range_length = specified_len;
|
||||
|
||||
if specified_len > res_size {
|
||||
range_length = res_size;
|
||||
}
|
||||
|
||||
return Ok(range_length);
|
||||
}
|
||||
|
||||
if self.start >= res_size {
|
||||
return Err(HTTPRangeError::InvalidRangeSpec("The requested range is not satisfiable".to_string()));
|
||||
}
|
||||
|
||||
if self.end > -1 {
|
||||
let mut end = self.end;
|
||||
if res_size <= end {
|
||||
end = res_size - 1;
|
||||
}
|
||||
|
||||
let range_length = end - self.start + 1;
|
||||
return Ok(range_length);
|
||||
}
|
||||
|
||||
if self.end == -1 {
|
||||
let range_length = res_size - self.start;
|
||||
return Ok(range_length);
|
||||
}
|
||||
|
||||
Err(HTTPRangeError::InvalidRangeSpec(format!(
|
||||
"range value invalid: start={}, end={}, expected start <= end and end >= -1",
|
||||
self.start, self.end
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn non_empty_condition_value(value: Option<&str>) -> Option<&str> {
|
||||
value.map(str::trim).filter(|value| !value.is_empty())
|
||||
}
|
||||
@@ -67,4 +182,33 @@ mod tests {
|
||||
assert!(opts.retain_until.is_none());
|
||||
assert!(!opts.bypass_governance);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_range_spec_offset_length_handles_suffix_and_bounds() {
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 5,
|
||||
end: 10,
|
||||
};
|
||||
|
||||
assert_eq!(range.get_offset_length(20).expect("range should fit"), (5, 6));
|
||||
|
||||
let suffix = HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: -5,
|
||||
end: -1,
|
||||
};
|
||||
|
||||
assert_eq!(suffix.get_offset_length(20).expect("suffix range should fit"), (15, 5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_range_spec_from_part_sizes_keeps_part_boundaries() {
|
||||
let spec = HTTPRangeSpec::from_part_sizes(100, 3, [10, 15, 20]).expect("third part should exist");
|
||||
|
||||
assert_eq!(spec.start, 25);
|
||||
assert_eq!(spec.end, 44);
|
||||
assert!(HTTPRangeSpec::from_part_sizes(100, 0, [10]).is_none());
|
||||
assert!(HTTPRangeSpec::from_part_sizes(100, 2, [10]).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user