mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 18:46:17 +00:00
Merge branch 'main' into cxymds/fix-1854-copyobject-error-cause
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -880,12 +880,44 @@ mod prepared_get_object_metadata_tests {
|
||||
use super::*;
|
||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||
use crate::object_api::{BLOCK_SIZE_V2, PutObjReader};
|
||||
use crate::set_disk::core::io_primitives::disk_call_counters;
|
||||
use crate::set_disk::core::io_primitives::{bounded_metadata_fanout_order, disk_call_counters, rename_fanout_barrier};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::test_metrics::CapturingRecorder;
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
|
||||
(0..1000)
|
||||
.map(|index| format!("{prefix}-{index}.bin"))
|
||||
.find(|name| {
|
||||
let order = bounded_metadata_fanout_order(bucket, name, 4, 2);
|
||||
let distribution = FileInfo::new(&[bucket, name].join("/"), 2, 2).erasure.distribution;
|
||||
let mut seen = [false; 2];
|
||||
for disk_index in order.into_iter().take(3) {
|
||||
if let Some(block_index @ 1..=2) = distribution.get(disk_index).copied() {
|
||||
seen[block_index - 1] = true;
|
||||
}
|
||||
}
|
||||
seen.into_iter().all(|seen| seen)
|
||||
})
|
||||
.expect("test should find an object whose initial fanout covers both data shards")
|
||||
}
|
||||
|
||||
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
|
||||
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
||||
.get(3)
|
||||
.expect("4-disk test geometry should leave one bounded spare disk")
|
||||
}
|
||||
|
||||
fn bounded_slow_initial_disk_index(bucket: &str, object: &str) -> usize {
|
||||
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
||||
.get(2)
|
||||
.expect("4-disk test geometry should include a third initial metadata disk")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_metadata_is_consumed_exactly_once() {
|
||||
let snapshot = GetObjectFileInfo::owned(FileInfo::default(), Vec::new(), Vec::new());
|
||||
@@ -1002,6 +1034,307 @@ mod prepared_get_object_metadata_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
fn inline_data_read_early_stop_reader_returns_exact_body() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("current-thread runtime should build");
|
||||
let bucket = "inline-data-read-early-stop-reader";
|
||||
let object = object_with_initial_data_shards(bucket, "inline-data-read-early-stop-reader-object");
|
||||
let payload = b"inline early-stop reader payload".repeat(256);
|
||||
let recorder = CapturingRecorder::default();
|
||||
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
let (restored, object_size, calls_total) = metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("inline object should be written");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let slow_initial_disk = bounded_slow_initial_disk_index(bucket, &object);
|
||||
let barrier =
|
||||
rename_fanout_barrier::arm(&object, slow_initial_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let set_disks_for_read = Arc::clone(&set_disks);
|
||||
let opts_for_read = opts.clone();
|
||||
let object_for_read = object.clone();
|
||||
let mut open_reader = tokio::spawn(async move {
|
||||
set_disks_for_read
|
||||
.get_object_reader(bucket, &object_for_read, None, HeaderMap::new(), &opts_for_read)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("bounded inline GET should pause a slow initial metadata read");
|
||||
let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut open_reader)
|
||||
.await
|
||||
.expect("production inline GET should return before the paused metadata response")
|
||||
.expect("inline GET reader task should not panic")
|
||||
.expect("inline GET reader should open");
|
||||
let object_size = reader.object_info.size;
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("inline GET body should stream");
|
||||
|
||||
(restored, object_size, calls.total(disk_call_counters::KIND_READ_VERSION))
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
});
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
|
||||
|
||||
assert_eq!(object_size, payload.len() as i64);
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(calls_total, 4, "bounded production GET should schedule the initial quorum plus one spare");
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_scheduled",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||
),
|
||||
vec![4.0],
|
||||
"bounded production GET should record all scheduled metadata tasks"
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_completed",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||
),
|
||||
vec![3.0],
|
||||
"bounded production GET should record only observed metadata responses as completed"
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_cancelled",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||
),
|
||||
vec![1.0],
|
||||
"bounded production GET should record the aborted slow metadata task"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
fn prepared_metadata_uses_full_fanout_even_when_data_read_early_stop_is_enabled() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("current-thread runtime should build");
|
||||
let bucket = "prepared-metadata-early-stop-enabled";
|
||||
let object = object_with_initial_data_shards(bucket, "prepared-metadata-early-stop-enabled-object");
|
||||
let payload = b"prepared metadata early-stop enabled payload".repeat(16);
|
||||
let recorder = CapturingRecorder::default();
|
||||
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
let (restored, calls_total) = metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let metadata = set_disks
|
||||
.prepare_get_object_metadata(bucket, &object, &opts)
|
||||
.await
|
||||
.expect("prepared metadata should resolve");
|
||||
let calls_total = calls.total(disk_call_counters::KIND_READ_VERSION);
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader_with_prepared_metadata(bucket, &object, None, HeaderMap::new(), &opts, metadata)
|
||||
.await
|
||||
.expect("prepared body reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("prepared body should stream");
|
||||
(restored, calls_total)
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
});
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
|
||||
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(
|
||||
calls_total, 4,
|
||||
"prepared metadata must opt out of data-read early-stop until the read shape is known"
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_scheduled",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||
),
|
||||
vec![4.0],
|
||||
"prepared metadata should schedule the full metadata fanout"
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_completed",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||
),
|
||||
vec![4.0],
|
||||
"prepared metadata must wait for every scheduled metadata response"
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_cancelled",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||
),
|
||||
vec![0.0],
|
||||
"prepared metadata must not cancel metadata responses"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
fn data_read_early_stop_request_shapes_full_wait_in_production_reader() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("current-thread runtime should build");
|
||||
let bucket = "data-read-early-stop-shape-reader";
|
||||
let payload = b"shape-gated inline reader payload".repeat(256);
|
||||
|
||||
for (object_prefix, range, configure_opts, expected_body) in [
|
||||
(
|
||||
"data-read-early-stop-range-reader-object",
|
||||
Some(HTTPRangeSpec {
|
||||
start: 0,
|
||||
end: 3,
|
||||
is_suffix_length: false,
|
||||
}),
|
||||
None,
|
||||
payload[..4].to_vec(),
|
||||
),
|
||||
("data-read-early-stop-part-reader-object", None, Some(1), payload.clone()),
|
||||
] {
|
||||
let recorder = CapturingRecorder::default();
|
||||
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
let (restored, calls_total) = metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let object = object_with_initial_data_shards(bucket, object_prefix);
|
||||
let mut opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
opts.part_number = configure_opts;
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("inline object should be written");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, &object, range, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("shape-gated GET reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("shape-gated GET body should stream");
|
||||
(restored, calls.total(disk_call_counters::KIND_READ_VERSION))
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
});
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_gate);
|
||||
|
||||
assert_eq!(restored, expected_body);
|
||||
assert_eq!(calls_total, 4, "shape-gated production GET should keep full metadata fanout");
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_scheduled",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||
),
|
||||
vec![4.0],
|
||||
"shape-gated production GET should schedule the full metadata fanout"
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_completed",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||
),
|
||||
vec![4.0],
|
||||
"shape-gated production GET must wait for every scheduled metadata response"
|
||||
);
|
||||
assert_eq!(
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_cancelled",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
|
||||
),
|
||||
vec![0.0],
|
||||
"shape-gated production GET must not cancel metadata responses"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn prepared_reader_rebuilds_object_info_when_precomputed_value_is_absent() {
|
||||
@@ -1105,7 +1438,7 @@ impl SetDisks {
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PreparedGetObjectMetadata> {
|
||||
let snapshot = self.get_object_fileinfo(bucket, object, opts, true, true).await?;
|
||||
let snapshot = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
|
||||
let object_info = build_get_object_info(snapshot.fi(), bucket, object, opts.versioned || opts.version_suspended);
|
||||
Ok(PreparedGetObjectMetadata {
|
||||
snapshot,
|
||||
@@ -2408,12 +2741,12 @@ mod write_layout_tests {
|
||||
|
||||
let held_layout = resolve_write_layout(&held, 0, 4, 2, None, false).expect("held snapshot should remain valid");
|
||||
assert_eq!(held_layout.parity_drives, 2);
|
||||
assert!(held.should_inline(512, false));
|
||||
assert!(held.should_inline(512, held_layout.data_drives, false));
|
||||
|
||||
let current = published.load_full();
|
||||
let current_layout = resolve_write_layout(¤t, 0, 4, 2, None, false).expect("new snapshot should resolve");
|
||||
assert_eq!(current_layout.parity_drives, 1);
|
||||
assert!(!current.should_inline(512, false));
|
||||
assert!(!current.should_inline(512, current_layout.data_drives, false));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2459,6 +2792,9 @@ pub struct SetDisks {
|
||||
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
||||
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
|
||||
get_object_metadata_cache_generations: Arc<[AtomicU64]>,
|
||||
/// GET codecs keyed by every persisted layout dimension that affects
|
||||
/// decoding. Clones of a set share the memoized shells.
|
||||
erasure_cache: Arc<ErasureCache>,
|
||||
pub lockers: Vec<Arc<dyn LockClient>>,
|
||||
shared_lockers: Arc<[Arc<dyn LockClient>]>,
|
||||
local_lock_manager: Arc<rustfs_lock::GlobalLockManager>,
|
||||
@@ -2481,6 +2817,137 @@ pub struct SetDisks {
|
||||
storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
|
||||
}
|
||||
|
||||
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
struct ErasureCacheKey {
|
||||
data_shards: usize,
|
||||
parity_shards: usize,
|
||||
block_size: usize,
|
||||
uses_legacy: bool,
|
||||
}
|
||||
|
||||
struct ErasureCache {
|
||||
entries: parking_lot::RwLock<HashMap<ErasureCacheKey, Arc<coding::Erasure>>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ErasureCache {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ErasureCache")
|
||||
.field("entries", &self.entries.read().len())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ErasureCache {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
entries: parking_lot::RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_or_try_insert(
|
||||
&self,
|
||||
key: ErasureCacheKey,
|
||||
) -> std::result::Result<Arc<coding::Erasure>, coding::ErasureConstructionError> {
|
||||
if let Some(erasure) = self.entries.read().get(&key) {
|
||||
return Ok(Arc::clone(erasure));
|
||||
}
|
||||
|
||||
// Serialize first construction for a key so concurrent cold GETs still
|
||||
// create exactly one shell. Codec construction never awaits.
|
||||
let mut entries = self.entries.write();
|
||||
if let Some(erasure) = entries.get(&key) {
|
||||
return Ok(Arc::clone(erasure));
|
||||
}
|
||||
let erasure = Arc::new(coding::Erasure::try_new_with_options(
|
||||
key.data_shards,
|
||||
key.parity_shards,
|
||||
key.block_size,
|
||||
key.uses_legacy,
|
||||
)?);
|
||||
if entries.len() < ERASURE_CACHE_MAX_ENTRIES {
|
||||
entries.insert(key, Arc::clone(&erasure));
|
||||
}
|
||||
Ok(erasure)
|
||||
}
|
||||
|
||||
fn get_for_file_info(&self, fi: &FileInfo) -> Result<Arc<coding::Erasure>> {
|
||||
self.get_or_try_insert(ErasureCacheKey {
|
||||
data_shards: fi.erasure.data_blocks,
|
||||
parity_shards: fi.erasure.parity_blocks,
|
||||
block_size: fi.erasure.block_size,
|
||||
uses_legacy: fi.uses_legacy_checksum,
|
||||
})
|
||||
.map_err(Error::from)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod erasure_cache_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reuses_shells_and_keeps_every_layout_dimension_in_the_key() {
|
||||
let cache = ErasureCache::new();
|
||||
let base = ErasureCacheKey {
|
||||
data_shards: 4,
|
||||
parity_shards: 2,
|
||||
block_size: 1_048_576,
|
||||
uses_legacy: false,
|
||||
};
|
||||
let first = cache.get_or_try_insert(base).expect("modern shell should construct");
|
||||
let reused = cache.get_or_try_insert(base).expect("same modern shell should be cached");
|
||||
assert!(Arc::ptr_eq(&first, &reused));
|
||||
|
||||
for distinct in [
|
||||
ErasureCacheKey { data_shards: 3, ..base },
|
||||
ErasureCacheKey {
|
||||
parity_shards: 1,
|
||||
..base
|
||||
},
|
||||
ErasureCacheKey {
|
||||
block_size: 524_288,
|
||||
..base
|
||||
},
|
||||
ErasureCacheKey {
|
||||
uses_legacy: true,
|
||||
..base
|
||||
},
|
||||
] {
|
||||
let shell = cache.get_or_try_insert(distinct).expect("distinct shell should construct");
|
||||
assert!(!Arc::ptr_eq(&first, &shell));
|
||||
}
|
||||
assert_eq!(cache.entries.read().len(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_cache_invalid_layouts_or_grow_past_the_bound() {
|
||||
let cache = ErasureCache::new();
|
||||
let invalid = ErasureCacheKey {
|
||||
data_shards: 4,
|
||||
parity_shards: 2,
|
||||
block_size: 0,
|
||||
uses_legacy: false,
|
||||
};
|
||||
assert!(cache.get_or_try_insert(invalid).is_err());
|
||||
assert!(cache.entries.read().is_empty());
|
||||
|
||||
for block_size in 1..=(ERASURE_CACHE_MAX_ENTRIES + 1) {
|
||||
cache
|
||||
.get_or_try_insert(ErasureCacheKey {
|
||||
data_shards: 4,
|
||||
parity_shards: 2,
|
||||
block_size,
|
||||
uses_legacy: false,
|
||||
})
|
||||
.expect("bounded cache fixture should construct");
|
||||
}
|
||||
assert_eq!(cache.entries.read().len(), ERASURE_CACHE_MAX_ENTRIES);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
struct GetObjectMetadataCacheKey {
|
||||
bucket: Arc<str>,
|
||||
@@ -2879,6 +3346,7 @@ impl SetDisks {
|
||||
.map(|_| AtomicU64::new(0))
|
||||
.collect::<Vec<_>>(),
|
||||
),
|
||||
erasure_cache: Arc::new(ErasureCache::new()),
|
||||
lockers,
|
||||
shared_lockers,
|
||||
// Sourced from the instance context so each instance owns its lock
|
||||
@@ -3411,9 +3879,15 @@ fn collect_inline_data_shard_fileinfos_by_index<'a>(
|
||||
if block_index == 0 || block_index > data_shards {
|
||||
continue;
|
||||
}
|
||||
if file_info.erasure.index != block_index {
|
||||
continue;
|
||||
}
|
||||
if !file_info.has_valid_erasure_geometry() {
|
||||
continue;
|
||||
}
|
||||
if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) {
|
||||
continue;
|
||||
}
|
||||
if file_info.data.as_ref().is_none_or(|data| data.is_empty()) {
|
||||
continue;
|
||||
}
|
||||
@@ -9221,6 +9695,9 @@ mod tests {
|
||||
HashAlgorithm::HighwayHash256S
|
||||
};
|
||||
let shards = erasure.encode_data(payload).expect("payload should encode");
|
||||
let version_id = Some(Uuid::new_v4());
|
||||
let data_dir = Some(Uuid::new_v4());
|
||||
let mod_time = Some(OffsetDateTime::now_utc());
|
||||
let mut files = Vec::with_capacity(shards.len());
|
||||
|
||||
for shard in shards {
|
||||
@@ -9233,6 +9710,16 @@ mod tests {
|
||||
writer.shutdown().await.expect("inline writer should shutdown");
|
||||
let data = writer.into_inline_data().expect("inline data should be retained");
|
||||
let mut file = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards);
|
||||
file.volume = "bucket".to_string();
|
||||
file.name = "object".to_string();
|
||||
file.size = i64::try_from(payload.len()).expect("test payload should fit i64");
|
||||
file.is_latest = true;
|
||||
file.version_id = version_id;
|
||||
file.data_dir = data_dir;
|
||||
file.mod_time = mod_time;
|
||||
file.metadata.insert("etag".to_string(), "etag-inline".to_string());
|
||||
file.add_object_part(1, "part-etag-inline".to_string(), payload.len(), file.mod_time, file.size, None, None);
|
||||
file.set_inline_data();
|
||||
file.erasure.index = files.len() + 1;
|
||||
file.data = Some(Bytes::from(data));
|
||||
files.push(file);
|
||||
@@ -9245,16 +9732,40 @@ mod tests {
|
||||
inline_bitrot_files_for_payload_with_mode(payload, false).await
|
||||
}
|
||||
|
||||
fn disk_ordered_fileinfos(files: &[FileInfo]) -> Vec<FileInfo> {
|
||||
let distribution = &files
|
||||
.first()
|
||||
.expect("inline data shard fixture should include metadata")
|
||||
.erasure
|
||||
.distribution;
|
||||
distribution
|
||||
.iter()
|
||||
.map(|block_index| {
|
||||
files
|
||||
.get(block_index.checked_sub(1).expect("erasure block indexes are one-based"))
|
||||
.expect("inline data shard fixture should include every distributed shard")
|
||||
.clone()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn inline_data_shard_fileinfo(
|
||||
name: &str,
|
||||
data_blocks: usize,
|
||||
parity_blocks: usize,
|
||||
erasure_index: usize,
|
||||
distribution: &[usize],
|
||||
data: Option<&'static [u8]>,
|
||||
) -> FileInfo {
|
||||
let mut fi = FileInfo::new(name, data_blocks, parity_blocks);
|
||||
fi.name = name.to_string();
|
||||
let mut fi = FileInfo::new("object", data_blocks, parity_blocks);
|
||||
fi.name = "object".to_string();
|
||||
fi.volume = "bucket".to_string();
|
||||
fi.size = 4;
|
||||
fi.is_latest = true;
|
||||
fi.data_dir = Some(Uuid::nil());
|
||||
fi.mod_time = Some(OffsetDateTime::UNIX_EPOCH);
|
||||
fi.metadata.insert("etag".to_string(), "etag-inline".to_string());
|
||||
fi.add_object_part(1, "part-etag-inline".to_string(), 4, fi.mod_time, 4, None, None);
|
||||
fi.set_inline_data();
|
||||
fi.erasure.index = erasure_index;
|
||||
fi.erasure.distribution = distribution.to_vec();
|
||||
fi.data = data.map(Bytes::from_static);
|
||||
@@ -9264,36 +9775,41 @@ mod tests {
|
||||
#[test]
|
||||
fn collect_inline_data_shards_by_index_uses_distribution_order() {
|
||||
let distribution = vec![3, 1, 5, 2, 4, 6];
|
||||
let mut fi = FileInfo::new("object", 4, 2);
|
||||
let mut fi = inline_data_shard_fileinfo(4, 2, 1, &distribution, Some(b"x"));
|
||||
fi.erasure.index = 1;
|
||||
fi.erasure.distribution = distribution.clone();
|
||||
let files = vec![
|
||||
inline_data_shard_fileinfo("block-3", 4, 2, 3, &distribution, Some(b"c")),
|
||||
inline_data_shard_fileinfo("block-1", 4, 2, 1, &distribution, Some(b"a")),
|
||||
inline_data_shard_fileinfo("parity-5", 4, 2, 5, &distribution, Some(b"p")),
|
||||
inline_data_shard_fileinfo("block-2", 4, 2, 2, &distribution, Some(b"b")),
|
||||
inline_data_shard_fileinfo("block-4", 4, 2, 4, &distribution, Some(b"d")),
|
||||
inline_data_shard_fileinfo("parity-6", 4, 2, 6, &distribution, Some(b"q")),
|
||||
inline_data_shard_fileinfo(4, 2, 3, &distribution, Some(b"c")),
|
||||
inline_data_shard_fileinfo(4, 2, 1, &distribution, Some(b"a")),
|
||||
inline_data_shard_fileinfo(4, 2, 5, &distribution, Some(b"p")),
|
||||
inline_data_shard_fileinfo(4, 2, 2, &distribution, Some(b"b")),
|
||||
inline_data_shard_fileinfo(4, 2, 4, &distribution, Some(b"d")),
|
||||
inline_data_shard_fileinfo(4, 2, 6, &distribution, Some(b"q")),
|
||||
];
|
||||
|
||||
let data_files =
|
||||
collect_inline_data_shard_fileinfos_by_index(&files, &fi, 4, |_| true).expect("all data shards should be collected");
|
||||
|
||||
assert_eq!(
|
||||
data_files.iter().map(|file| file.name.as_str()).collect::<Vec<_>>(),
|
||||
["block-1", "block-2", "block-3", "block-4"]
|
||||
data_files
|
||||
.iter()
|
||||
.map(|file| file.data.as_deref().expect("fixture carries inline bytes"))
|
||||
.collect::<Vec<_>>(),
|
||||
[b"a".as_slice(), b"b".as_slice(), b"c".as_slice(), b"d".as_slice()]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_inline_data_shards_by_index_rejects_missing_data_shard() {
|
||||
let distribution = vec![1, 2, 3, 4];
|
||||
let mut fi = FileInfo::new("object", 2, 2);
|
||||
let mut fi = inline_data_shard_fileinfo(2, 2, 1, &distribution, Some(b"x"));
|
||||
fi.erasure.index = 1;
|
||||
fi.erasure.distribution = distribution.clone();
|
||||
let files = vec![
|
||||
inline_data_shard_fileinfo("block-1", 2, 2, 1, &distribution, Some(b"a")),
|
||||
inline_data_shard_fileinfo("block-2", 2, 2, 2, &distribution, None),
|
||||
inline_data_shard_fileinfo("parity-3", 2, 2, 3, &distribution, Some(b"p")),
|
||||
inline_data_shard_fileinfo("parity-4", 2, 2, 4, &distribution, Some(b"q")),
|
||||
inline_data_shard_fileinfo(2, 2, 1, &distribution, Some(b"a")),
|
||||
inline_data_shard_fileinfo(2, 2, 2, &distribution, None),
|
||||
inline_data_shard_fileinfo(2, 2, 3, &distribution, Some(b"p")),
|
||||
inline_data_shard_fileinfo(2, 2, 4, &distribution, Some(b"q")),
|
||||
];
|
||||
|
||||
assert!(collect_inline_data_shard_fileinfos_by_index(&files, &fi, 2, |_| true).is_none());
|
||||
@@ -9426,10 +9942,8 @@ mod tests {
|
||||
|
||||
let payload = vec![b'i'; 192 * 1024];
|
||||
let (erasure, files, _read_length, _checksum_algo) = inline_bitrot_files_for_payload(&payload).await;
|
||||
let mut fi = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards);
|
||||
fi.size = payload.len() as i64;
|
||||
fi.data = files[0].data.clone();
|
||||
fi.add_object_part(1, String::new(), payload.len(), None, payload.len() as i64, None, None);
|
||||
let fi = files[0].clone();
|
||||
let disk_files = disk_ordered_fileinfos(&files);
|
||||
|
||||
let disks = vec![Some(disk); erasure.total_shard_count()];
|
||||
let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(fi.size);
|
||||
@@ -9437,8 +9951,9 @@ mod tests {
|
||||
let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo(
|
||||
"bucket",
|
||||
"object",
|
||||
Arc::new(ErasureCache::new()),
|
||||
&fi,
|
||||
&files,
|
||||
&disk_files,
|
||||
&disks,
|
||||
true,
|
||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
|
||||
@@ -9469,10 +9984,8 @@ mod tests {
|
||||
let payload = vec![b'v'; 64 * 1024];
|
||||
let payload_size = i64::try_from(payload.len()).expect("test payload size should fit i64");
|
||||
let (erasure, files, _read_length, _checksum_algo) = inline_bitrot_files_for_payload(&payload).await;
|
||||
let mut fi = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards);
|
||||
fi.size = payload_size;
|
||||
fi.data = files[0].data.clone();
|
||||
fi.add_object_part(1, String::new(), payload.len(), None, payload_size, None, None);
|
||||
let fi = files[0].clone();
|
||||
let disk_files = disk_ordered_fileinfos(&files);
|
||||
|
||||
let mut object_info = ObjectInfo {
|
||||
size: payload_size,
|
||||
@@ -9502,8 +10015,9 @@ mod tests {
|
||||
let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo(
|
||||
"bucket",
|
||||
"object",
|
||||
Arc::new(ErasureCache::new()),
|
||||
&fi,
|
||||
&files,
|
||||
&disk_files,
|
||||
&vec![Some(disk); erasure.total_shard_count()],
|
||||
true,
|
||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
|
||||
@@ -9582,6 +10096,7 @@ mod tests {
|
||||
let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::new(ErasureCache::new()),
|
||||
&fi,
|
||||
&files,
|
||||
&disks,
|
||||
@@ -9667,6 +10182,7 @@ mod tests {
|
||||
SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::new(ErasureCache::new()),
|
||||
range_offset,
|
||||
range_length as i64,
|
||||
&mut writer,
|
||||
@@ -9778,6 +10294,7 @@ mod tests {
|
||||
SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::new(ErasureCache::new()),
|
||||
0,
|
||||
total_size as i64,
|
||||
&mut writer,
|
||||
|
||||
@@ -1425,6 +1425,33 @@ impl SetDisks {
|
||||
/// post-heal tail — reclaim identically. Never fails the heal: delete errors
|
||||
/// are logged and swallowed. Callers must gate this on `!opts.dry_run`.
|
||||
async fn reclaim_orphan_data_dirs_best_effort(&self, bucket: &str, object: &str) {
|
||||
match self.reconcile_old_data_cleanup_receipts(bucket, object).await {
|
||||
Ok(removed) if removed > 0 => {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
removed,
|
||||
state = "old_data_cleanup_receipt_reconciled",
|
||||
"Set disk old-data cleanup receipts reconciled"
|
||||
);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_HEAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
error = %e,
|
||||
state = "old_data_cleanup_receipt_reconcile_failed",
|
||||
"Set disk old-data cleanup receipt reconcile failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
match self.reclaim_orphan_data_dirs(bucket, object).await {
|
||||
Ok(removed) if removed > 0 => {
|
||||
debug!(
|
||||
|
||||
@@ -22,6 +22,11 @@
|
||||
|
||||
use super::super::*;
|
||||
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
||||
use super::object::{
|
||||
assign_object_transaction_epoch, object_transaction_fencing_fleet_proof, object_transaction_fencing_fleet_proof_matches,
|
||||
object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence,
|
||||
verify_object_transaction_epoch_fence,
|
||||
};
|
||||
use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
use futures::{StreamExt, stream};
|
||||
@@ -63,6 +68,9 @@ pub(crate) enum MultipartCommitPause {
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterRename,
|
||||
BeforeLockLost,
|
||||
BeforeTransactionEpochVerify,
|
||||
BeforeObjectPublication,
|
||||
AfterObjectPublication,
|
||||
AfterRename,
|
||||
}
|
||||
|
||||
@@ -153,13 +161,24 @@ impl Drop for MultipartCommitBarrier {
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
|
||||
let barrier = MULTIPART_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("multipart commit barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
|
||||
.cloned();
|
||||
let barrier = {
|
||||
let mut slot = MULTIPART_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("multipart commit barrier mutex should not poison");
|
||||
if slot
|
||||
.as_ref()
|
||||
.is_some_and(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
|
||||
{
|
||||
if pause == MultipartCommitPause::BeforeTransactionEpochVerify {
|
||||
slot.take()
|
||||
} else {
|
||||
slot.clone()
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(barrier) = barrier
|
||||
&& let Ok(previous) = barrier.arrivals.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
|
||||
(current < barrier.expected_arrivals).then_some(current + 1)
|
||||
@@ -2296,140 +2315,196 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
// Crash-consistency injection: hard power loss after the upload is fully
|
||||
// staged and locked but before the authoritative rename_data commit. No
|
||||
// disk has moved the staged data, so a crash here must leave any prior
|
||||
// committed version byte-for-byte intact (rustfs/backlog#864) and the
|
||||
// upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`.
|
||||
if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, object) {
|
||||
return Err(StorageError::Unexpected);
|
||||
let transaction_fencing_proof = object_transaction_fencing_fleet_proof();
|
||||
if object_transaction_fencing_requested() && transaction_fencing_proof.is_none() {
|
||||
return Err(Error::other("object transaction fencing requires a live fleet capability proof"));
|
||||
}
|
||||
let transaction_epoch_fence = if transaction_fencing_proof.is_some() {
|
||||
Some(read_object_transaction_epoch_fence(self.as_ref(), bucket, object).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let transaction_epoch =
|
||||
transaction_epoch_fence.map(|_| assign_object_transaction_epoch(&shuffle_disks, &mut parts_metadatas));
|
||||
|
||||
// The trailing `_` drops the rename_data old-size backfill
|
||||
// (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
|
||||
// `get_object_info` lookup, so the backfill has no consumer here yet.
|
||||
let (online_disks, convergence, op_old_dir, cleanup_disks, _) = Self::rename_data(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&upload_id_path,
|
||||
&parts_metadatas,
|
||||
bucket,
|
||||
object,
|
||||
write_quorum,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Detach admission before any post-commit await: client cancellation
|
||||
// must not couple durable convergence repair to cleanup work.
|
||||
if convergence.needs_heal() {
|
||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(self.pool_index),
|
||||
Some(self.set_index),
|
||||
);
|
||||
request.object_version_id = fi
|
||||
.version_id
|
||||
.or_else(|| opts.version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
tokio::spawn(async move {
|
||||
let _ = rustfs_common::heal_channel::send_heal_request(request).await;
|
||||
});
|
||||
}
|
||||
|
||||
// Crash-consistency injection: hard power loss after the authoritative
|
||||
// rename_data commit succeeded but before the stale part.N.meta cleanup.
|
||||
// The new version is durably committed and visible, so a crash here must
|
||||
// leave the object readable as the new version; the un-reclaimed staging
|
||||
// parts are swept by a retried completion or upload GC (rustfs/backlog#946).
|
||||
// Compiles to a no-op outside `#[cfg(test)]`.
|
||||
if crash_inject::should_crash_at(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object) {
|
||||
return Err(StorageError::Unexpected);
|
||||
}
|
||||
|
||||
// backlog#946: reclaim the stale per-part metadata (and any superfluous
|
||||
// part.N data files no longer in the completed set) only *after* the
|
||||
// authoritative rename_data commit above has succeeded. If rename_data
|
||||
// fails write quorum and returns via `?`, the upload directory must keep
|
||||
// its part.N.meta so a retried CompleteMultipartUpload can still read the
|
||||
// parts; deleting them before the commit would strand the upload
|
||||
// permanently. This mirrors the "clean up only after commit" pattern
|
||||
// already used for the old data-dir GC and the upload-dir delete_all below.
|
||||
self.cleanup_multipart_path(&parts).await;
|
||||
|
||||
if let Some(old_dir) = op_old_dir {
|
||||
let committed_dir = fi.data_dir.unwrap_or_default().to_string();
|
||||
// backlog#898: best-effort reclaim of the dereferenced old data dir.
|
||||
// Returns a receipt (never `Err`); a failed GC must not turn an
|
||||
// already-committed multipart completion into a 503.
|
||||
let cleanup = self
|
||||
.commit_rename_data_dir(&cleanup_disks, bucket, object, &old_dir.to_string(), &committed_dir, write_quorum)
|
||||
.await;
|
||||
self.report_old_data_dir_cleanup(bucket, object, &old_dir.to_string(), &cleanup)
|
||||
.await;
|
||||
}
|
||||
|
||||
if let Some(stage_start) = complete_tail_stage_start {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
"multipart_complete_tail",
|
||||
stage_start.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::AfterRename).await;
|
||||
|
||||
let cleanup_store = self.clone();
|
||||
let cleanup_upload_id_path = upload_id_path.clone();
|
||||
let cleanup_bucket = bucket.to_owned();
|
||||
let cleanup_object = object.to_owned();
|
||||
let cleanup_upload_id = upload_id.to_owned();
|
||||
let cleanup_handle = tokio::spawn(async move {
|
||||
let commit_set = self.clone();
|
||||
let commit_bucket = bucket.to_owned();
|
||||
let commit_object = object.to_owned();
|
||||
let commit_upload_id = upload_id.to_owned();
|
||||
let commit_upload_id_path = upload_id_path.clone();
|
||||
let commit_version_suspended = opts.version_suspended;
|
||||
let commit_is_versioned = opts.versioned || opts.version_suspended;
|
||||
let commit_capacity_scope_token = opts.capacity_scope_token;
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some();
|
||||
let commit = async move {
|
||||
let _object_lock_guard = commit_object_lock_guard;
|
||||
let _upload_guard = upload_guard;
|
||||
if let Err(err) = cleanup_store
|
||||
.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &cleanup_upload_id_path, write_quorum)
|
||||
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
// Crash-consistency injection: hard power loss after the upload is fully
|
||||
// staged and locked but before the authoritative rename_data commit. No
|
||||
// disk has moved the staged data, so a crash here must leave any prior
|
||||
// committed version byte-for-byte intact (rustfs/backlog#864) and the
|
||||
// upload fully retryable. Compiles to a no-op outside `#[cfg(test)]`.
|
||||
if crash_inject::should_crash_at(CrashPoint::MultipartBeforeCommitRename, &commit_object) {
|
||||
return Err(StorageError::Unexpected);
|
||||
}
|
||||
|
||||
// The trailing `_` drops the rename_data old-size backfill
|
||||
// (rustfs/backlog#1009): CompleteMultipartUpload keeps its pre-commit
|
||||
// `get_object_info` lookup, so the backfill has no consumer here yet.
|
||||
if let Some(proof) = transaction_fencing_proof.as_ref()
|
||||
&& !object_transaction_fencing_fleet_proof_matches(proof)
|
||||
{
|
||||
return Err(Error::other(
|
||||
"object transaction fencing fleet capability changed during complete_multipart_upload",
|
||||
));
|
||||
}
|
||||
if let Some(expected) = transaction_epoch_fence {
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeTransactionEpochVerify).await;
|
||||
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
|
||||
}
|
||||
let (online_disks, convergence, op_old_dir, cleanup_disks, _) = SetDisks::rename_data(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&commit_upload_id_path,
|
||||
&parts_metadatas,
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
write_quorum,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Detach admission before any post-commit await: client cancellation
|
||||
// must not couple durable convergence repair to cleanup work.
|
||||
if convergence.needs_heal() {
|
||||
let mut request = rustfs_common::heal_channel::create_heal_request_with_options(
|
||||
commit_bucket.clone(),
|
||||
Some(commit_object.clone()),
|
||||
false,
|
||||
Some(HealChannelPriority::Normal),
|
||||
Some(commit_set.pool_index),
|
||||
Some(commit_set.set_index),
|
||||
);
|
||||
request.object_version_id = fi
|
||||
.version_id
|
||||
.or_else(|| commit_version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
tokio::spawn(async move {
|
||||
let _ = rustfs_common::heal_channel::send_heal_request(request).await;
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(old_dir) = op_old_dir {
|
||||
commit_set
|
||||
.persist_old_data_cleanup_receipts(
|
||||
&cleanup_disks,
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
old_dir,
|
||||
fi.data_dir,
|
||||
transaction_epoch,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Crash-consistency injection: hard power loss after the authoritative
|
||||
// rename_data commit succeeded but before the stale part.N.meta cleanup.
|
||||
// The new version is durably committed and visible, so a crash here must
|
||||
// leave the object readable as the new version; the un-reclaimed staging
|
||||
// parts are swept by a retried completion or upload GC (rustfs/backlog#946).
|
||||
// Compiles to a no-op outside `#[cfg(test)]`.
|
||||
if crash_inject::should_crash_at(CrashPoint::MultipartAfterCommitBeforePartsCleanup, &commit_object) {
|
||||
return Err(StorageError::Unexpected);
|
||||
}
|
||||
|
||||
if let Some(committed_slot) = online_disks.iter().position(Option::is_some) {
|
||||
fi = parts_metadatas[committed_slot].clone();
|
||||
}
|
||||
let committed_dir = fi.data_dir.unwrap_or_default().to_string();
|
||||
|
||||
commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &online_disks);
|
||||
|
||||
fi.is_latest = true;
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeObjectPublication).await;
|
||||
|
||||
commit_set
|
||||
.invalidate_get_object_metadata_cache(&commit_bucket, &commit_object)
|
||||
.await;
|
||||
|
||||
drop(_object_lock_guard); // release the object lock before multipart cleanup tail IO.
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterObjectPublication).await;
|
||||
|
||||
// backlog#946: reclaim the stale per-part metadata (and any superfluous
|
||||
// part.N data files no longer in the completed set) only *after* the
|
||||
// authoritative rename_data commit above has succeeded. If rename_data
|
||||
// fails write quorum and returns via `?`, the upload directory must keep
|
||||
// its part.N.meta so a retried CompleteMultipartUpload can still read the
|
||||
// parts; deleting them before the commit would strand the upload
|
||||
// permanently. This mirrors the "clean up only after commit" pattern
|
||||
// already used for the old data-dir GC and the upload-dir delete_all below.
|
||||
commit_set.cleanup_multipart_path(&parts).await;
|
||||
|
||||
if let Some(old_dir) = op_old_dir {
|
||||
// backlog#898: best-effort reclaim of the dereferenced old data dir.
|
||||
// Returns a receipt (never `Err`); a failed GC must not turn an
|
||||
// already-committed multipart completion into a 503.
|
||||
let cleanup = commit_set
|
||||
.commit_rename_data_dir(
|
||||
&cleanup_disks,
|
||||
&commit_bucket,
|
||||
&commit_object,
|
||||
&old_dir.to_string(),
|
||||
&committed_dir,
|
||||
write_quorum,
|
||||
)
|
||||
.await;
|
||||
commit_set
|
||||
.report_old_data_dir_cleanup(&commit_bucket, &commit_object, &old_dir.to_string(), &cleanup)
|
||||
.await;
|
||||
}
|
||||
|
||||
if let Some(stage_start) = complete_tail_stage_start {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
"multipart_complete_tail",
|
||||
stage_start.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterRename).await;
|
||||
|
||||
if let Err(err) = commit_set
|
||||
.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &commit_upload_id_path, write_quorum)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
bucket = %cleanup_bucket,
|
||||
object = %cleanup_object,
|
||||
upload_id = %cleanup_upload_id,
|
||||
bucket = %commit_bucket,
|
||||
object = %commit_object,
|
||||
upload_id = %commit_upload_id,
|
||||
error = ?err,
|
||||
"completed multipart upload staging cleanup did not reach write quorum"
|
||||
);
|
||||
}
|
||||
});
|
||||
if let Err(err) = cleanup_handle.await {
|
||||
warn!(
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
upload_id = %upload_id,
|
||||
error = ?err,
|
||||
"completed multipart upload staging cleanup task failed"
|
||||
);
|
||||
|
||||
drop(_upload_guard);
|
||||
|
||||
Ok(ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned))
|
||||
};
|
||||
|
||||
if detach_commit_owner {
|
||||
tokio::spawn(commit)
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("complete_multipart_upload commit task failed: {err}")))?
|
||||
} else {
|
||||
commit.await
|
||||
}
|
||||
drop(object_lock_guard); // drop object lock guard to release the lock
|
||||
|
||||
for (i, op_disk) in online_disks.iter().enumerate() {
|
||||
if let Some(disk) = op_disk
|
||||
&& disk.is_online().await
|
||||
{
|
||||
fi = parts_metadatas[i].clone();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &online_disks);
|
||||
|
||||
fi.is_latest = true;
|
||||
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
|
||||
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2452,9 +2527,10 @@ fn resolve_complete_etag(opts: &ObjectOptions, uploaded_parts: &[CompletePart])
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::storageclass::lookup_config_for_pools_without_env;
|
||||
use crate::disk::DiskAPI as _;
|
||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
||||
use crate::disk::{endpoint::Endpoint, format::FormatV3};
|
||||
use crate::layout::endpoints::SetupType;
|
||||
use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test;
|
||||
// No-locker helpers resolve to the isolated-context variants (see
|
||||
// `hermetic_set_disks_isolated`); the guard-based tests build through
|
||||
// `hermetic_set_disks_with_lockers`, which stays on the bootstrap context
|
||||
@@ -2463,6 +2539,7 @@ mod tests {
|
||||
hermetic_set_disks_for_pool_with_default_parity_isolated as hermetic_set_disks_for_pool_with_default_parity,
|
||||
hermetic_set_disks_isolated as hermetic_set_disks, hermetic_set_disks_with_lockers,
|
||||
};
|
||||
use crate::set_disk::ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_config::server_config::KVS;
|
||||
@@ -2865,6 +2942,208 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn object_transaction_epochs(disks: &[DiskStore], bucket: &str, object: &str) -> Vec<Option<Uuid>> {
|
||||
let mut epochs = Vec::with_capacity(disks.len());
|
||||
for (disk_index, disk) in disks.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist object metadata: {err}"));
|
||||
epochs.push(
|
||||
file_info
|
||||
.object_transaction_epoch()
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} transaction epoch should decode: {err}")),
|
||||
);
|
||||
}
|
||||
epochs
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn object_transaction_fencing_requires_live_fleet_proof_before_multipart_commit() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-transaction-fencing-no-proof";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (upload_id, parts) = stage_upload_with_create_opts(
|
||||
&set_disks,
|
||||
bucket,
|
||||
object,
|
||||
b"must-not-complete-without-proof",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let err = temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
async {
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
|
||||
.await
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("multipart completion must fail closed without a live fleet proof");
|
||||
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("object transaction fencing requires a live fleet capability proof"),
|
||||
"unexpected error: {err:?}"
|
||||
);
|
||||
set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("failed fenced completion must not publish object metadata");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn object_transaction_fencing_persists_epoch_on_multipart_commit() {
|
||||
let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-object-transaction-epoch";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, b"multipart fenced epoch", &ObjectOptions::default()).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
async {
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("fenced multipart completion should commit with a live proof");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let epochs = object_transaction_epochs(&disk_stores, bucket, object).await;
|
||||
let first = epochs[0].expect("fenced multipart completion should persist an epoch");
|
||||
assert!(!first.is_nil());
|
||||
assert!(epochs.into_iter().all(|epoch| epoch == Some(first)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn object_transaction_fencing_rejects_stale_multipart_epoch() {
|
||||
let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-object-transaction-stale-epoch";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
async {
|
||||
let mut initial_reader = PutObjReader::from_vec(b"initial fenced object".to_vec());
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut initial_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("initial fenced PUT should commit");
|
||||
let initial_epoch = object_transaction_epochs(&disk_stores, bucket, object)
|
||||
.await
|
||||
.into_iter()
|
||||
.next()
|
||||
.flatten()
|
||||
.expect("initial fenced PUT should persist an epoch");
|
||||
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, b"stale multipart body", &ObjectOptions::default())
|
||||
.await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeTransactionEpochVerify);
|
||||
let stale_set = Arc::clone(&set_disks);
|
||||
let stale = tokio::spawn(async move {
|
||||
stale_set
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
parts,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let mut winner_reader = PutObjReader::from_vec(b"winning put body".to_vec());
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut winner_reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("concurrent fenced PUT should advance the epoch");
|
||||
let winning_epoch = object_transaction_epochs(&disk_stores, bucket, object)
|
||||
.await
|
||||
.into_iter()
|
||||
.next()
|
||||
.flatten()
|
||||
.expect("winning fenced PUT should persist an epoch");
|
||||
assert_ne!(winning_epoch, initial_epoch);
|
||||
|
||||
barrier.release();
|
||||
let err = stale
|
||||
.await
|
||||
.expect("stale multipart task should not panic")
|
||||
.expect_err("stale epoch multipart completion must be rejected");
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
|
||||
let final_epochs = object_transaction_epochs(&disk_stores, bucket, object).await;
|
||||
assert!(final_epochs.into_iter().all(|epoch| epoch == Some(winning_epoch)));
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("winning object should remain readable");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("winning body should stream");
|
||||
assert_eq!(restored, b"winning put body");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_quota_rejection_preserves_destination_and_upload() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -4883,6 +5162,250 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn cancelled_complete_keeps_upload_lock_through_tail_cleanup() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
|
||||
],
|
||||
async {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-cancelled-tail-lock-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x53; 4096], &ObjectOptions::default()).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "abort must wait while completion tail owns the upload lock");
|
||||
|
||||
complete.abort();
|
||||
assert!(
|
||||
complete
|
||||
.await
|
||||
.expect_err("the completion request should be cancellable while the tail is paused")
|
||||
.is_cancelled()
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "cancelling the completion waiter must not release the upload lock");
|
||||
|
||||
barrier.release();
|
||||
let abort_err = abort
|
||||
.await
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("the committed upload should no longer exist when abort acquires the lock");
|
||||
assert!(
|
||||
matches!(abort_err, StorageError::InvalidUploadID(..)),
|
||||
"abort should return InvalidUploadID after the detached completion tail, got {abort_err:?}"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
|
||||
],
|
||||
async {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-object-lock-short-tail-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let completed_body = vec![0x63; 4096];
|
||||
let replacement_body = vec![0x64; 4096];
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &completed_body, &ObjectOptions::default()).await;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path));
|
||||
signaling.clear_observed();
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let completion_barrier =
|
||||
MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
completion_barrier.wait_until_paused().await;
|
||||
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("GET should not wait for multipart cleanup after object publication")
|
||||
.expect("completed object should be readable while the upload tail is paused");
|
||||
let mut observed_body = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut observed_body))
|
||||
.await
|
||||
.expect("completed object body should stream while the upload tail is paused")
|
||||
.expect("completed object body should read successfully");
|
||||
assert_eq!(observed_body, completed_body);
|
||||
|
||||
let put_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
|
||||
let put_store = set_disks.clone();
|
||||
let put_payload = replacement_body.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(put_payload);
|
||||
put_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
put_barrier.wait_until_paused().await;
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
!abort.is_finished(),
|
||||
"abort must still wait while the completion tail owns the upload lock"
|
||||
);
|
||||
|
||||
complete.abort();
|
||||
assert!(
|
||||
complete
|
||||
.await
|
||||
.expect_err("the completion waiter should remain cancellable after object publication")
|
||||
.is_cancelled()
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!abort.is_finished(), "cancelling the waiter must not release the upload lock");
|
||||
|
||||
completion_barrier.release();
|
||||
let abort_err = abort
|
||||
.await
|
||||
.expect("abort task should not panic")
|
||||
.expect_err("the committed upload should no longer exist after the detached tail drains");
|
||||
assert!(
|
||||
matches!(abort_err, StorageError::InvalidUploadID(..)),
|
||||
"abort should return InvalidUploadID after the completion tail, got {abort_err:?}"
|
||||
);
|
||||
|
||||
put_barrier.release();
|
||||
put.await
|
||||
.expect("same-key PUT task should not panic")
|
||||
.expect("same-key PUT should commit after the object lock is released early");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("final object should be readable");
|
||||
let mut final_body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut final_body)
|
||||
.await
|
||||
.expect("final object should stream fully");
|
||||
assert_eq!(final_body, replacement_body);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_keeps_object_lock_until_publication_fence() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
|
||||
],
|
||||
async {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-publication-fence-lock-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x65; 4096], &ObjectOptions::default()).await;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let completion_barrier =
|
||||
MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeObjectPublication);
|
||||
|
||||
let complete_store = set_disks.clone();
|
||||
let complete_upload_id = upload_id.clone();
|
||||
let complete = tokio::spawn(async move {
|
||||
complete_store
|
||||
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
completion_barrier.wait_until_paused().await;
|
||||
|
||||
let before_namespace_barrier =
|
||||
PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
|
||||
let after_namespace_barrier =
|
||||
PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
|
||||
let put_store = set_disks.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x66; 4096]);
|
||||
put_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
before_namespace_barrier.wait_until_paused().await;
|
||||
before_namespace_barrier.release_and_wait_until_namespace_pending().await;
|
||||
|
||||
completion_barrier.release();
|
||||
after_namespace_barrier.wait_until_paused().await;
|
||||
after_namespace_barrier.release();
|
||||
complete
|
||||
.await
|
||||
.expect("completion task should not panic")
|
||||
.expect("completion should commit after publication fence");
|
||||
put.await
|
||||
.expect("same-key PUT task should not panic")
|
||||
.expect("same-key PUT should commit after completion publishes and releases the object lock");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn complete_validates_parts_after_an_inflight_upload_part_commit() {
|
||||
@@ -6108,6 +6631,24 @@ mod tests {
|
||||
(body, etag)
|
||||
}
|
||||
|
||||
async fn current_data_dir(disk: &DiskStore, bucket: &str, object: &str) -> Uuid {
|
||||
disk.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("current object metadata should read")
|
||||
.data_dir
|
||||
.expect("test object should be stored out-of-line")
|
||||
}
|
||||
|
||||
async fn data_dir_exists(disk: &DiskStore, bucket: &str, object: &str, data_dir: Uuid) -> bool {
|
||||
disk.read_all(bucket, &format!("{object}/{data_dir}/part.1")).await.is_ok()
|
||||
}
|
||||
|
||||
async fn cleanup_receipt_exists(disk: &DiskStore, bucket: &str, object: &str, data_dir: Uuid) -> bool {
|
||||
disk.read_all(bucket, &old_data_cleanup_receipt_path(object, data_dir))
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
async fn upload_is_listed(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, upload_id: &str) -> bool {
|
||||
let page = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, object, None, None, None, 1000, None)
|
||||
@@ -6228,6 +6769,106 @@ mod tests {
|
||||
let (body_after, _) = read_object(&set_disks, bucket, object).await;
|
||||
assert_eq!(body_after, new, "reclaiming the leftover upload must not disturb the committed object");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(storage_class_env)]
|
||||
async fn post_commit_crash_receipt_reclaims_old_data_after_restart() {
|
||||
let _proof = install_remote_version_state_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-crash-old-data-receipt";
|
||||
let object = "crash-old-data-object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
async {
|
||||
let old = payload(0x51);
|
||||
let (u_old, parts_old) = stage_upload(&set_disks, bucket, object, &old).await;
|
||||
complete(&set_disks, bucket, object, &u_old, parts_old)
|
||||
.await
|
||||
.expect("the old version should commit");
|
||||
let old_dir = current_data_dir(&disk_stores[0], bucket, object).await;
|
||||
|
||||
let new = payload(0x52);
|
||||
let (u_new, parts_new) = stage_upload(&set_disks, bucket, object, &new).await;
|
||||
crash_inject::arm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
|
||||
let crashed = complete(&set_disks, bucket, object, &u_new, parts_new).await;
|
||||
assert!(
|
||||
matches!(crashed, Err(StorageError::Unexpected)),
|
||||
"the post-commit crash point must surface as unexpected, got {crashed:?}"
|
||||
);
|
||||
crash_inject::disarm(CrashPoint::MultipartAfterCommitBeforePartsCleanup, object);
|
||||
|
||||
let (body, _) = read_object(&set_disks, bucket, object).await;
|
||||
assert_eq!(body, new, "the committed replacement must remain readable after the crash");
|
||||
for disk in &disk_stores {
|
||||
assert!(
|
||||
cleanup_receipt_exists(disk, bucket, object, old_dir).await,
|
||||
"post-commit crash must leave a durable old-data cleanup receipt"
|
||||
);
|
||||
assert!(
|
||||
data_dir_exists(disk, bucket, object, old_dir).await,
|
||||
"post-commit crash must leave old data for restart reconciliation"
|
||||
);
|
||||
}
|
||||
|
||||
let restarted_endpoints = temp_dirs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(disk_idx, dir)| {
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8"))
|
||||
.expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_idx);
|
||||
endpoint
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut reloaded = Vec::with_capacity(restarted_endpoints.len());
|
||||
for endpoint in &restarted_endpoints {
|
||||
reloaded.push(
|
||||
new_disk(
|
||||
endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("disk should restart"),
|
||||
);
|
||||
}
|
||||
let restarted_set = SetDisks::new_with_instance_ctx(
|
||||
"restart-cleanup-receipt-test-owner".to_string(),
|
||||
Arc::new(RwLock::new(reloaded.iter().cloned().map(Some).collect())),
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
restarted_endpoints,
|
||||
set_disks.format.clone(),
|
||||
Vec::new(),
|
||||
Arc::new(crate::runtime::instance::InstanceContext::new()),
|
||||
)
|
||||
.await;
|
||||
let removed = restarted_set
|
||||
.reconcile_old_data_cleanup_receipts(bucket, object)
|
||||
.await
|
||||
.expect("restart receipt reconciliation should succeed");
|
||||
assert_eq!(removed, 4, "restart reconciler should delete all receipt targets");
|
||||
for disk in &reloaded {
|
||||
assert!(
|
||||
!data_dir_exists(disk, bucket, object, old_dir).await,
|
||||
"restart reconciler must reclaim the old data dir"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -482,6 +482,7 @@ impl SetDisks {
|
||||
pub(super) async fn try_get_object_direct_data_shards_with_fileinfo(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
erasure_cache: Arc<ErasureCache>,
|
||||
fi: &FileInfo,
|
||||
files: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
@@ -502,13 +503,7 @@ impl SetDisks {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let erasure = coding::Erasure::try_new_with_options(
|
||||
fi.erasure.data_blocks,
|
||||
fi.erasure.parity_blocks,
|
||||
fi.erasure.block_size,
|
||||
fi.uses_legacy_checksum,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
let erasure = erasure_cache.get_for_file_info(fi)?;
|
||||
|
||||
let checksum_info = fi.erasure.get_checksum_info(part.number);
|
||||
let checksum_algo = if fi.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
|
||||
@@ -636,6 +631,7 @@ impl SetDisks {
|
||||
// &self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
erasure_cache: Arc<ErasureCache>,
|
||||
offset: usize,
|
||||
length: i64,
|
||||
writer: &mut W,
|
||||
@@ -730,13 +726,7 @@ impl SetDisks {
|
||||
object, offset, length, end_offset, part_index, last_part_index, last_part_relative_offset, "Multipart read bounds"
|
||||
);
|
||||
|
||||
let erasure = coding::Erasure::try_new_with_options(
|
||||
fi.erasure.data_blocks,
|
||||
fi.erasure.parity_blocks,
|
||||
fi.erasure.block_size,
|
||||
fi.uses_legacy_checksum,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
let erasure = erasure_cache.get_for_file_info(&fi)?;
|
||||
|
||||
let part_indices: Vec<usize> = (part_index..=last_part_index).collect();
|
||||
debug!(bucket, object, ?part_indices, "Multipart part indices to stream");
|
||||
@@ -1170,6 +1160,7 @@ impl SetDisks {
|
||||
pub(super) async fn get_object_decode_reader_with_fileinfo(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
erasure_cache: Arc<ErasureCache>,
|
||||
fi: &FileInfo,
|
||||
files: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
@@ -1180,14 +1171,7 @@ impl SetDisks {
|
||||
metrics_size_bucket: &'static str,
|
||||
prefer_data_blocks_first_reader_setup: bool,
|
||||
) -> Result<GetCodecStreamingReaderBuildOutcome> {
|
||||
let erasure = coding::Erasure::try_new_with_options(
|
||||
fi.erasure.data_blocks,
|
||||
fi.erasure.parity_blocks,
|
||||
fi.erasure.block_size,
|
||||
fi.uses_legacy_checksum,
|
||||
)
|
||||
.map_err(Error::from)?;
|
||||
|
||||
let erasure = erasure_cache.get_for_file_info(fi)?;
|
||||
let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, files, fi);
|
||||
|
||||
if fi.parts.len() == 1 {
|
||||
@@ -1574,7 +1558,7 @@ struct LazyCodecPartContext {
|
||||
fi: FileInfo,
|
||||
files: Vec<FileInfo>,
|
||||
disks: Vec<Option<DiskStore>>,
|
||||
erasure: coding::Erasure,
|
||||
erasure: Arc<coding::Erasure>,
|
||||
skip_verify_bitrot: bool,
|
||||
metrics_object_class: &'static str,
|
||||
metrics_size_bucket: &'static str,
|
||||
@@ -2058,6 +2042,7 @@ mod metadata_cache_tests {
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
"bucket",
|
||||
"object",
|
||||
Arc::new(ErasureCache::new()),
|
||||
0,
|
||||
1,
|
||||
&mut output,
|
||||
@@ -2088,6 +2073,7 @@ mod metadata_cache_tests {
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::new(ErasureCache::new()),
|
||||
2,
|
||||
1,
|
||||
&mut output,
|
||||
@@ -2111,6 +2097,7 @@ mod metadata_cache_tests {
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::new(ErasureCache::new()),
|
||||
usize::MAX,
|
||||
1,
|
||||
&mut output,
|
||||
@@ -2132,6 +2119,7 @@ mod metadata_cache_tests {
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::new(ErasureCache::new()),
|
||||
1,
|
||||
1,
|
||||
&mut output,
|
||||
@@ -2155,6 +2143,7 @@ mod metadata_cache_tests {
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::new(ErasureCache::new()),
|
||||
0,
|
||||
1,
|
||||
&mut output,
|
||||
@@ -2192,6 +2181,7 @@ mod metadata_cache_tests {
|
||||
SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::new(ErasureCache::new()),
|
||||
0,
|
||||
0,
|
||||
&mut output,
|
||||
@@ -2224,6 +2214,7 @@ mod metadata_cache_tests {
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::new(ErasureCache::new()),
|
||||
0,
|
||||
1,
|
||||
&mut output,
|
||||
@@ -4128,6 +4119,7 @@ mod tests {
|
||||
let result = SetDisks::get_object_decode_reader_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
&fi,
|
||||
&[],
|
||||
&[],
|
||||
@@ -4150,6 +4142,7 @@ mod tests {
|
||||
let invalid_size = SetDisks::get_object_decode_reader_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
&single_part,
|
||||
&[],
|
||||
&[],
|
||||
@@ -4170,6 +4163,7 @@ mod tests {
|
||||
SetDisks::get_object_decode_reader_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
&multipart,
|
||||
&[],
|
||||
&[],
|
||||
@@ -4194,6 +4188,7 @@ mod tests {
|
||||
SetDisks::get_object_decode_reader_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
&multipart,
|
||||
&[],
|
||||
&[],
|
||||
@@ -4222,6 +4217,7 @@ mod tests {
|
||||
SetDisks::get_object_decode_reader_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
&multipart,
|
||||
&[],
|
||||
&[],
|
||||
@@ -4275,6 +4271,7 @@ mod tests {
|
||||
SetDisks::get_object_decode_reader_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
&fi,
|
||||
&files,
|
||||
&disks,
|
||||
@@ -4328,6 +4325,7 @@ mod tests {
|
||||
SetDisks::get_object_decode_reader_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
&fi,
|
||||
&files,
|
||||
&disks,
|
||||
@@ -4372,6 +4370,7 @@ mod tests {
|
||||
SetDisks::get_object_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
0,
|
||||
part_data.len() as i64,
|
||||
&mut output,
|
||||
|
||||
@@ -48,6 +48,23 @@ impl RestoreCleanupIdentity {
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_restore_metadata_lock_held(bucket: &str, object: &str, opts: &ObjectOptions, mode: &'static str) -> Result<()> {
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn finalize_restore_metadata(
|
||||
&self,
|
||||
@@ -88,6 +105,7 @@ impl SetDisks {
|
||||
if !expected.matches_file_info(&fi, &expected_etag) {
|
||||
return Err(Error::other("restored object changed before restore metadata finalization"));
|
||||
}
|
||||
ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
|
||||
let restore_expiry =
|
||||
lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
|
||||
fi.metadata.insert(
|
||||
@@ -159,6 +177,7 @@ impl SetDisks {
|
||||
if !expected.matches_file_info(&fi, &expected_etag) {
|
||||
return Ok(());
|
||||
}
|
||||
ensure_restore_metadata_lock_held(bucket, object, opts, "restore_cleanup_metadata")?;
|
||||
fi.metadata.remove(X_AMZ_RESTORE.as_str());
|
||||
fi.metadata.remove(AMZ_RESTORE_EXPIRY_DAYS);
|
||||
fi.metadata.remove(AMZ_RESTORE_REQUEST_DATE);
|
||||
|
||||
Reference in New Issue
Block a user