mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-16 18:08:21 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ea40a41c9b | |||
| 33eff4c3c4 |
@@ -2446,6 +2446,7 @@ impl SetDisks {
|
||||
let bucket: Arc<str> = Arc::from(bucket);
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
||||
let disk = disk.clone();
|
||||
let task_opts = opts;
|
||||
@@ -2453,10 +2454,14 @@ impl SetDisks {
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let slowtail_fault = slowtail_fault.clone();
|
||||
tokio::spawn(async move {
|
||||
let response_start = observe.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, disk_index);
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
} else {
|
||||
@@ -2552,6 +2557,7 @@ impl SetDisks {
|
||||
let mut scheduled_count = 0usize;
|
||||
let mut force_full_wait = false;
|
||||
let mut final_miss_reason_override = None;
|
||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||
let spawn_read_version =
|
||||
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
||||
let task_opts = opts;
|
||||
@@ -2559,6 +2565,7 @@ impl SetDisks {
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let slowtail_fault = slowtail_fault.clone();
|
||||
join_set.spawn(async move {
|
||||
let response_start = Instant::now();
|
||||
let result = if let Some(disk) = disk {
|
||||
@@ -2567,6 +2574,9 @@ impl SetDisks {
|
||||
Self::record_read_version_call(&object, index);
|
||||
#[cfg(test)]
|
||||
Self::read_version_fanout_barrier(&object, index).await;
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
} else {
|
||||
@@ -5744,6 +5754,130 @@ mod tests {
|
||||
(dirs, disks)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_slowtail_fault_delay_parses_and_filters_request() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,3")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some("bench-bucket")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||
],
|
||||
|| {
|
||||
assert_eq!(
|
||||
get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, true),
|
||||
Some(Duration::from_millis(25))
|
||||
);
|
||||
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 2, true).is_none());
|
||||
assert!(get_metadata_slowtail_fault_delay("other-bucket", "objects/000001", 3, true).is_none());
|
||||
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "other/000001", 3, true).is_none());
|
||||
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, false).is_none());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_slowtail_fault_delay_disables_invalid_disk_list() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,nope")),
|
||||
],
|
||||
|| {
|
||||
assert!(get_metadata_slowtail_fault_delay("bucket", "object", 1, true).is_none());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn metadata_slowtail_fault_delays_only_data_read_metadata_task() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "metadata-slowtail-fault-bucket";
|
||||
let object = "objects/metadata-slowtail-fault-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("false")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let read_without_data =
|
||||
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", false, false, false, true, 2);
|
||||
tokio::time::timeout(Duration::from_millis(100), read_without_data)
|
||||
.await
|
||||
.expect("non-data metadata fanout must not be delayed by the data-read slowtail hook")
|
||||
.expect("metadata fanout without read_data should resolve");
|
||||
|
||||
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
||||
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
||||
));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
||||
.await
|
||||
.is_err(),
|
||||
"data-read metadata fanout must wait for the injected slow read_version response"
|
||||
);
|
||||
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
||||
.await
|
||||
.expect("injected slowtail should eventually complete")
|
||||
.expect("data-read metadata fanout should resolve");
|
||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn metadata_slowtail_fault_delays_early_stop_metadata_task() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "metadata-slowtail-early-stop-bucket";
|
||||
let object = "objects/metadata-slowtail-early-stop-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
|
||||
&disks, bucket, bucket, object, "", true, false, false, true, 2,
|
||||
));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
|
||||
.await
|
||||
.is_err(),
|
||||
"early-stop metadata fanout must still wait for the injected slow response after fallback to full wait"
|
||||
);
|
||||
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
|
||||
.await
|
||||
.expect("injected early-stop slowtail should eventually complete")
|
||||
.expect("early-stop metadata fanout should resolve");
|
||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
/// Demo / regression guard for the backlog#1325 per-disk call counters.
|
||||
///
|
||||
/// The metadata fan-out issues each `read_version` inside its own
|
||||
|
||||
@@ -174,15 +174,14 @@ use std::future::Future;
|
||||
use std::hash::{BuildHasher, Hash, Hasher};
|
||||
use std::mem::{self};
|
||||
use std::pin::Pin;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
io::{Cursor, Write},
|
||||
path::Path,
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
@@ -717,6 +716,11 @@ const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true;
|
||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT";
|
||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = false;
|
||||
|
||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS";
|
||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS";
|
||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET";
|
||||
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX";
|
||||
|
||||
// --- Multipart Reader-Setup Prefetch Configuration (backlog#870) ---
|
||||
|
||||
const ENV_RUSTFS_GET_MULTIPART_READER_SETUP_PREFETCH: &str = "RUSTFS_GET_MULTIPART_READER_SETUP_PREFETCH";
|
||||
@@ -1695,6 +1699,95 @@ fn is_get_metadata_early_stop_bounded_fanout_enabled() -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GetMetadataSlowtailFaultConfig {
|
||||
delay: Duration,
|
||||
disks: Arc<[usize]>,
|
||||
bucket: Option<String>,
|
||||
object_prefix: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct GetMetadataSlowtailFaultRequest {
|
||||
delay: Duration,
|
||||
disks: Arc<[usize]>,
|
||||
}
|
||||
|
||||
impl GetMetadataSlowtailFaultRequest {
|
||||
fn delay_for_disk(&self, disk_index: usize) -> Option<Duration> {
|
||||
self.disks.contains(&disk_index).then_some(self.delay)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_get_metadata_slowtail_fault_disks(raw: &str) -> Option<Vec<usize>> {
|
||||
let mut disks = Vec::new();
|
||||
for item in raw.split(',').map(str::trim).filter(|item| !item.is_empty()) {
|
||||
let Ok(index) = item.parse::<usize>() else {
|
||||
return None;
|
||||
};
|
||||
if !disks.contains(&index) {
|
||||
disks.push(index);
|
||||
}
|
||||
}
|
||||
(!disks.is_empty()).then_some(disks)
|
||||
}
|
||||
|
||||
fn load_get_metadata_slowtail_fault_config() -> Option<GetMetadataSlowtailFaultConfig> {
|
||||
let delay_ms = rustfs_utils::get_env_u64(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, 0);
|
||||
if delay_ms == 0 {
|
||||
return None;
|
||||
}
|
||||
let disks = parse_get_metadata_slowtail_fault_disks(&std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS).ok()?)?;
|
||||
let bucket = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET)
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty());
|
||||
let object_prefix = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX)
|
||||
.ok()
|
||||
.filter(|value| !value.is_empty());
|
||||
Some(GetMetadataSlowtailFaultConfig {
|
||||
delay: Duration::from_millis(delay_ms),
|
||||
disks: Arc::from(disks.into_boxed_slice()),
|
||||
bucket,
|
||||
object_prefix,
|
||||
})
|
||||
}
|
||||
|
||||
fn get_metadata_slowtail_fault_request(bucket: &str, object: &str, read_data: bool) -> Option<GetMetadataSlowtailFaultRequest> {
|
||||
if !read_data {
|
||||
return None;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let config = load_get_metadata_slowtail_fault_config();
|
||||
#[cfg(test)]
|
||||
let config = config.as_ref()?;
|
||||
#[cfg(not(test))]
|
||||
let config = ({
|
||||
static CACHED: OnceLock<Option<GetMetadataSlowtailFaultConfig>> = OnceLock::new();
|
||||
CACHED.get_or_init(load_get_metadata_slowtail_fault_config).as_ref()
|
||||
})?;
|
||||
|
||||
if let Some(expected_bucket) = &config.bucket
|
||||
&& expected_bucket != bucket
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if let Some(expected_prefix) = &config.object_prefix
|
||||
&& !object.starts_with(expected_prefix)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(GetMetadataSlowtailFaultRequest {
|
||||
delay: config.delay,
|
||||
disks: config.disks.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn get_metadata_slowtail_fault_delay(bucket: &str, object: &str, disk_index: usize, read_data: bool) -> Option<Duration> {
|
||||
get_metadata_slowtail_fault_request(bucket, object, read_data)?.delay_for_disk(disk_index)
|
||||
}
|
||||
|
||||
/// Check if multipart reads prefetch the next part's bitrot reader setup
|
||||
/// while the current part decodes (backlog#870).
|
||||
///
|
||||
|
||||
@@ -38,7 +38,6 @@ use std::collections::HashMap;
|
||||
/// - Account format is invalid
|
||||
/// - Credentials don't contain project_id
|
||||
/// - Account project_id doesn't match credentials project_id
|
||||
#[allow(dead_code)] // Used by Swift implementation
|
||||
pub fn validate_account_access(account: &str, credentials: &Credentials) -> SwiftResult<String> {
|
||||
// Extract project_id from account (strip "AUTH_" prefix)
|
||||
let account_project_id = account
|
||||
@@ -70,7 +69,6 @@ pub fn validate_account_access(account: &str, credentials: &Credentials) -> Swif
|
||||
///
|
||||
/// Admin users (with "admin" or "reseller_admin" roles) can perform
|
||||
/// cross-tenant operations and administrative tasks.
|
||||
#[allow(dead_code)] // Used by Swift implementation
|
||||
pub fn is_admin_user(credentials: &Credentials) -> bool {
|
||||
credentials
|
||||
.claims
|
||||
|
||||
@@ -144,7 +144,6 @@ impl ContainerMapper {
|
||||
/// - S3 bucket name compatible (only uses [a-z0-9-])
|
||||
/// - Deterministic mapping (same input always produces same bucket name)
|
||||
/// - Fixed-length prefix (16 hex chars = 8 bytes)
|
||||
#[allow(dead_code)] // Used in: create/delete container operations
|
||||
pub fn swift_to_s3_bucket(&self, container: &str, project_id: &str) -> String {
|
||||
if self.config.tenant_prefix_enabled {
|
||||
let hash = self.hash_project_id(project_id);
|
||||
@@ -216,7 +215,6 @@ pub fn bucket_info_to_container(info: &BucketInfo, mapper: &ContainerMapper, pro
|
||||
/// 2. Lists all S3 buckets
|
||||
/// 3. Filters to buckets belonging to this tenant (using tenant prefix)
|
||||
/// 4. Converts BucketInfo to Swift Container format
|
||||
#[allow(dead_code)] // Used by handler: list containers
|
||||
pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftResult<Vec<Container>> {
|
||||
// Validate account access and extract project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -279,7 +277,6 @@ pub async fn list_containers(account: &str, credentials: &Credentials) -> SwiftR
|
||||
/// - Returns 201 Created on success
|
||||
/// - Returns 202 Accepted if container already exists
|
||||
/// - Returns 400 Bad Request for invalid container names
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn create_container(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<bool> {
|
||||
// Validate account access and extract project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -348,7 +345,6 @@ fn validate_container_name(container: &str) -> SwiftResult<()> {
|
||||
}
|
||||
|
||||
/// Container metadata for HEAD response
|
||||
#[allow(dead_code)] // TODO: Remove once Swift API integration is complete
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ContainerMetadata {
|
||||
/// Number of objects in container
|
||||
@@ -411,7 +407,6 @@ pub(crate) async fn get_container_custom_metadata(
|
||||
/// - HEAD /v1/{account}/{container} returns container metadata
|
||||
/// - Returns 204 No Content on success with headers
|
||||
/// - Returns 404 Not Found if container doesn't exist
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn get_container_metadata(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<ContainerMetadata> {
|
||||
let (bucket_name, bucket_info, custom_metadata) = get_container_metadata_base(account, container, credentials).await?;
|
||||
|
||||
@@ -448,7 +443,6 @@ pub async fn get_container_metadata(account: &str, container: &str, credentials:
|
||||
/// - The update is additive: items the request does not name keep their stored
|
||||
/// value, and removal is explicit, via `X-Remove-Container-Meta-{name}` or an
|
||||
/// empty value
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn update_container_metadata(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -520,7 +514,6 @@ pub async fn update_container_metadata(
|
||||
/// - Returns 204 No Content on success
|
||||
/// - Returns 404 Not Found if container doesn't exist
|
||||
/// - Returns 409 Conflict if container is not empty
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn delete_container(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<()> {
|
||||
// Validate account access and extract project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -603,7 +596,6 @@ pub async fn delete_container(account: &str, container: &str, credentials: &Cred
|
||||
/// - Account validation fails
|
||||
/// - Container doesn't exist
|
||||
/// - Storage layer errors occur
|
||||
#[allow(dead_code)] // Handler integration: GET container
|
||||
pub async fn list_objects(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -706,7 +698,6 @@ pub async fn list_objects(
|
||||
/// Versioning configuration is stored as an S3 bucket tag:
|
||||
/// - Tag key: `swift-versions-location`
|
||||
/// - Tag value: archive container name
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn enable_versioning(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -795,7 +786,6 @@ pub async fn enable_versioning(
|
||||
/// * `account` - Account identifier
|
||||
/// * `container` - Container name to disable versioning on
|
||||
/// * `credentials` - Keystone credentials
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn disable_versioning(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<()> {
|
||||
// Validate account access
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -855,7 +845,6 @@ pub async fn disable_versioning(account: &str, container: &str, credentials: &Cr
|
||||
/// # Returns
|
||||
/// - Some(archive_container_name) if versioning is enabled
|
||||
/// - None if versioning is not enabled
|
||||
#[allow(dead_code)] // Used by handler and object.rs
|
||||
pub async fn get_versions_location(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<Option<String>> {
|
||||
// Validate account access
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -918,7 +907,6 @@ pub async fn get_versions_location(account: &str, container: &str, credentials:
|
||||
/// &credentials
|
||||
/// ).await?;
|
||||
/// ```
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn set_container_acl(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -1022,7 +1010,6 @@ pub async fn set_container_acl(
|
||||
/// println!("Container is publicly readable");
|
||||
/// }
|
||||
/// ```
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn get_container_acl(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -1083,7 +1070,6 @@ pub async fn get_container_acl(
|
||||
///
|
||||
/// # Returns
|
||||
/// Ok(()) if ACLs were deleted successfully
|
||||
#[allow(dead_code)] // Used by handler
|
||||
pub async fn delete_container_acl(account: &str, container: &str, credentials: &Credentials) -> SwiftResult<()> {
|
||||
// Setting both ACLs to None removes them
|
||||
set_container_acl(account, container, None, None, credentials).await
|
||||
|
||||
@@ -20,7 +20,6 @@ use std::fmt;
|
||||
|
||||
/// Swift-specific error type
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)] // Error variants used by Swift implementation
|
||||
pub enum SwiftError {
|
||||
/// 400 Bad Request
|
||||
BadRequest(String),
|
||||
|
||||
@@ -122,12 +122,10 @@ fn swift_user_metadata(headers: &HeaderMap) -> Option<HashMap<String, String>> {
|
||||
///
|
||||
/// Handles URL encoding/decoding and path normalization for Swift object keys.
|
||||
/// Swift object names can contain any UTF-8 characters except null bytes.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub struct ObjectKeyMapper;
|
||||
|
||||
impl ObjectKeyMapper {
|
||||
/// Create a new object key mapper
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
@@ -140,7 +138,6 @@ impl ObjectKeyMapper {
|
||||
/// - Not contain null bytes
|
||||
/// - Not contain '..' path segments (directory traversal)
|
||||
/// - Not start with '/' (leading slash handled by routing)
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn validate_object_name(object: &str) -> SwiftResult<()> {
|
||||
if object.is_empty() {
|
||||
return Err(SwiftError::BadRequest("Object name cannot be empty".to_string()));
|
||||
@@ -183,7 +180,6 @@ impl ObjectKeyMapper {
|
||||
/// Example:
|
||||
/// - Swift: "photos/vacation/beach photo.jpg"
|
||||
/// - S3: "photos/vacation/beach photo.jpg"
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn swift_to_s3_key(object: &str) -> SwiftResult<String> {
|
||||
Self::validate_object_name(object)?;
|
||||
Ok(object.to_string())
|
||||
@@ -193,7 +189,6 @@ impl ObjectKeyMapper {
|
||||
///
|
||||
/// This is essentially an identity transformation since we store
|
||||
/// Swift object names as-is in S3.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn s3_to_swift_name(key: &str) -> String {
|
||||
key.to_string()
|
||||
}
|
||||
@@ -208,7 +203,6 @@ impl ObjectKeyMapper {
|
||||
/// - Object: "vacation/beach.jpg"
|
||||
/// - Bucket: "abc123:photos"
|
||||
/// - Key: "vacation/beach.jpg"
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn build_s3_key(object: &str) -> SwiftResult<String> {
|
||||
Self::swift_to_s3_key(object)
|
||||
}
|
||||
@@ -220,7 +214,6 @@ impl ObjectKeyMapper {
|
||||
///
|
||||
/// Example URL: /v1/AUTH_abc/container/path%2Fto%2Ffile.txt
|
||||
/// Decoded: "path/to/file.txt"
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn decode_object_from_url(encoded: &str) -> SwiftResult<String> {
|
||||
// Decode percent-encoding
|
||||
let decoded = urlencoding::decode(encoded).map_err(|e| SwiftError::BadRequest(format!("Invalid URL encoding: {}", e)))?;
|
||||
@@ -233,7 +226,6 @@ impl ObjectKeyMapper {
|
||||
///
|
||||
/// When constructing URLs (e.g., for redirect responses), we need to
|
||||
/// percent-encode object names.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn encode_object_for_url(object: &str) -> String {
|
||||
urlencoding::encode(object).to_string()
|
||||
}
|
||||
@@ -241,7 +233,6 @@ impl ObjectKeyMapper {
|
||||
/// Check if object name represents a directory (pseudo-directory)
|
||||
///
|
||||
/// In Swift, objects ending with '/' are treated as directory markers.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn is_directory_marker(object: &str) -> bool {
|
||||
object.ends_with('/')
|
||||
}
|
||||
@@ -250,7 +241,6 @@ impl ObjectKeyMapper {
|
||||
///
|
||||
/// Removes redundant slashes and normalizes the path while preserving
|
||||
/// trailing slashes for directory markers.
|
||||
#[allow(dead_code)] // Used in: object operations
|
||||
pub fn normalize_path(object: &str) -> String {
|
||||
// Split by '/', filter out empty segments (except if it's the end)
|
||||
let has_trailing_slash = object.ends_with('/');
|
||||
@@ -324,7 +314,6 @@ fn sanitize_storage_error<E: std::fmt::Display>(operation: &str, error: E) -> Sw
|
||||
/// # Returns
|
||||
/// * `Ok(etag)` - Object ETag on success
|
||||
/// * `Err(SwiftError)` - Error if validation fails or upload fails
|
||||
#[allow(dead_code)] // Handler integration: PUT object
|
||||
pub async fn put_object<R>(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -445,7 +434,6 @@ where
|
||||
///
|
||||
/// Similar to put_object, but allows directly specifying metadata instead of extracting from headers.
|
||||
/// This is used internally for storing SLO manifests and marker objects.
|
||||
#[allow(dead_code)] // Used by SLO implementation
|
||||
pub async fn put_object_with_metadata<R>(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -549,7 +537,6 @@ where
|
||||
/// - `bytes=1000-1999` - Bytes 1000-1999
|
||||
/// - `bytes=1000-` - From byte 1000 to end
|
||||
/// - `bytes=-500` - Last 500 bytes
|
||||
#[allow(dead_code)] // Handler integration: GET object
|
||||
pub async fn get_object(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -608,7 +595,6 @@ pub async fn get_object(
|
||||
/// # Returns
|
||||
/// * `Ok(object_info)` - Object metadata (ObjectInfo)
|
||||
/// * `Err(SwiftError)` - Error if validation fails or object not found
|
||||
#[allow(dead_code)] // Handler integration: HEAD object
|
||||
pub async fn head_object(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -671,7 +657,6 @@ pub async fn head_object(
|
||||
/// # Returns
|
||||
/// * `Ok(())` - Object deleted successfully (or didn't exist)
|
||||
/// * `Err(SwiftError)` - Error if validation fails or deletion fails
|
||||
#[allow(dead_code)] // Handler integration: DELETE object
|
||||
pub async fn delete_object(account: &str, container: &str, object: &str, credentials: &Credentials) -> SwiftResult<()> {
|
||||
// 1. Validate account access and get project_id
|
||||
let project_id = validate_account_access(account, credentials)?;
|
||||
@@ -732,7 +717,6 @@ pub async fn delete_object(account: &str, container: &str, object: &str, credent
|
||||
/// # Returns
|
||||
/// * `Ok(())` - Metadata updated successfully
|
||||
/// * `Err(SwiftError)` - Error if validation fails, object not found, or update fails
|
||||
#[allow(dead_code)] // Handler integration: POST object
|
||||
pub async fn update_object_metadata(
|
||||
account: &str,
|
||||
container: &str,
|
||||
@@ -846,7 +830,6 @@ pub async fn update_object_metadata(
|
||||
/// # Handler Integration Note
|
||||
/// The current handler architecture needs to be updated to pass headers through
|
||||
/// to support COPY method and X-Copy-From header detection. See handler.rs for details.
|
||||
#[allow(dead_code)] // Handler integration: COPY object
|
||||
#[allow(clippy::too_many_arguments)] // Necessary for full copy functionality
|
||||
pub async fn copy_object(
|
||||
src_account: &str,
|
||||
@@ -979,7 +962,6 @@ pub async fn copy_object(
|
||||
/// assert_eq!(container, "my-container");
|
||||
/// assert_eq!(object, "path/to/file.txt");
|
||||
/// ```
|
||||
#[allow(dead_code)] // Handler integration: COPY method
|
||||
pub fn parse_destination_header(destination: &str) -> SwiftResult<(String, String)> {
|
||||
let destination = destination.trim_start_matches('/');
|
||||
let parts: Vec<&str> = destination.splitn(2, '/').collect();
|
||||
@@ -1013,7 +995,6 @@ pub fn parse_destination_header(destination: &str) -> SwiftResult<(String, Strin
|
||||
/// # Returns
|
||||
/// * `Ok((container, object))` - Parsed container and object names
|
||||
/// * `Err(SwiftError)` - Error if format is invalid
|
||||
#[allow(dead_code)] // Handler integration: X-Copy-From
|
||||
pub fn parse_copy_from_header(copy_from: &str) -> SwiftResult<(String, String)> {
|
||||
// Same parsing logic as Destination header
|
||||
parse_destination_header(copy_from)
|
||||
@@ -1042,7 +1023,6 @@ pub fn parse_copy_from_header(copy_from: &str) -> SwiftResult<(String, String)>
|
||||
/// assert_eq!(range.start, 0);
|
||||
/// assert_eq!(range.end, 1023);
|
||||
/// ```
|
||||
#[allow(dead_code)] // Handler integration: Range header
|
||||
pub fn parse_range_header(range_str: &str) -> SwiftResult<HTTPRangeSpec> {
|
||||
if !range_str.starts_with("bytes=") {
|
||||
return Err(SwiftError::BadRequest("Range header must start with 'bytes='".to_string()));
|
||||
@@ -1124,7 +1104,6 @@ pub fn parse_range_header(range_str: &str) -> SwiftResult<HTTPRangeSpec> {
|
||||
/// let header = format_content_range(0, 1023, 5000);
|
||||
/// assert_eq!(header, "bytes 0-1023/5000");
|
||||
/// ```
|
||||
#[allow(dead_code)] // Handler integration: Range header
|
||||
pub fn format_content_range(start: i64, end: i64, total: i64) -> String {
|
||||
format!("bytes {}-{}/{}", start, end, total)
|
||||
}
|
||||
|
||||
@@ -50,7 +50,6 @@ pub enum SwiftRoute {
|
||||
|
||||
impl SwiftRoute {
|
||||
/// Get the account identifier from the route
|
||||
#[allow(dead_code)] // Public API for future use
|
||||
pub fn account(&self) -> &str {
|
||||
match self {
|
||||
SwiftRoute::Account { account, .. } => account,
|
||||
@@ -60,7 +59,6 @@ impl SwiftRoute {
|
||||
}
|
||||
|
||||
/// Extract project_id from account string (removes AUTH_ prefix)
|
||||
#[allow(dead_code)] // Public API for future use
|
||||
pub fn project_id(&self) -> Option<&str> {
|
||||
let account = self.account();
|
||||
ACCOUNT_PATTERN
|
||||
|
||||
@@ -19,7 +19,6 @@ use std::collections::HashMap;
|
||||
|
||||
/// Swift container metadata
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[allow(dead_code)] // Used in container listing operations
|
||||
pub struct Container {
|
||||
/// Container name
|
||||
pub name: String,
|
||||
@@ -34,7 +33,6 @@ pub struct Container {
|
||||
|
||||
/// Swift object metadata
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[allow(dead_code)] // Used in object listing operations
|
||||
pub struct Object {
|
||||
/// Object name (key)
|
||||
pub name: String,
|
||||
@@ -50,7 +48,6 @@ pub struct Object {
|
||||
|
||||
/// Swift metadata extracted from headers
|
||||
#[derive(Debug, Clone, Default)]
|
||||
#[allow(dead_code)] // Used by Swift implementation
|
||||
pub struct SwiftMetadata {
|
||||
/// Custom metadata key-value pairs (from X-Container-Meta-* or X-Object-Meta-*)
|
||||
pub metadata: HashMap<String, String>,
|
||||
|
||||
Reference in New Issue
Block a user