mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-05 12:57:42 +00:00
Merge remote-tracking branch 'origin/cxymds/fix-1358-fleet-version-gate' into cxymds/fix-1358-gcs-exact-generation
This commit is contained in:
@@ -303,9 +303,9 @@ pub mod disk {
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
|
||||
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, RUSTFS_META_BUCKET,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo,
|
||||
WalkDirOptions, new_disk, validate_batch_read_version_item_count,
|
||||
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
|
||||
@@ -1182,8 +1182,12 @@ impl TransitionState {
|
||||
}
|
||||
|
||||
fn new_with_capacity(capacity: usize) -> Arc<Self> {
|
||||
let capacity = capacity.max(1);
|
||||
let queue_send_timeout = resolve_transition_queue_send_timeout();
|
||||
Self::new_with_capacity_and_timeout(capacity, queue_send_timeout)
|
||||
}
|
||||
|
||||
fn new_with_capacity_and_timeout(capacity: usize, queue_send_timeout: StdDuration) -> Arc<Self> {
|
||||
let capacity = capacity.max(1);
|
||||
let (tx1, rx1) = bounded(capacity);
|
||||
Arc::new(Self {
|
||||
transition_tx: tx1,
|
||||
@@ -1250,13 +1254,12 @@ impl TransitionState {
|
||||
return false;
|
||||
}
|
||||
let bucket = bucket.to_string();
|
||||
let scheduled = Arc::clone(&self.compensation_buckets);
|
||||
let state = Arc::clone(self);
|
||||
tokio::spawn(async move {
|
||||
Self::inc_counter(&state.compensation_running_tasks);
|
||||
state.record_scanner_transition_state();
|
||||
let Some(api) = runtime_sources::object_store_handle() else {
|
||||
scheduled.lock().unwrap().remove(&bucket);
|
||||
state.finish_bucket_compensation(&bucket);
|
||||
Self::add_counter(&state.compensation_running_tasks, -1);
|
||||
state.record_scanner_transition_state();
|
||||
debug!(
|
||||
@@ -1292,13 +1295,25 @@ impl TransitionState {
|
||||
);
|
||||
}
|
||||
|
||||
scheduled.lock().unwrap().remove(&bucket);
|
||||
state.finish_bucket_compensation(&bucket);
|
||||
Self::add_counter(&state.compensation_running_tasks, -1);
|
||||
state.record_scanner_transition_state();
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
fn finish_bucket_compensation(&self, bucket: &str) {
|
||||
match self.compensation_buckets.lock() {
|
||||
Ok(mut scheduled) => {
|
||||
scheduled.remove(bucket);
|
||||
}
|
||||
Err(poisoned) => {
|
||||
poisoned.into_inner().remove(bucket);
|
||||
self.compensation_buckets.clear_poison();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn inc_counter(counter: &AtomicI64) {
|
||||
Self::add_counter(counter, 1);
|
||||
@@ -1535,17 +1550,35 @@ impl TransitionState {
|
||||
|
||||
let outcome = match self.transition_tx.try_send(Some(task)) {
|
||||
Ok(()) => TransitionEnqueueOutcome::Queued,
|
||||
Err(async_channel::TrySendError::Full(_)) => {
|
||||
Err(async_channel::TrySendError::Full(task)) => {
|
||||
Self::inc_counter(&self.queue_full_tasks);
|
||||
debug!(
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
source = ?src,
|
||||
"transition queue is full; deferring to scanner/backfill"
|
||||
);
|
||||
TransitionEnqueueOutcome::QueueFull
|
||||
let send_timeout = self.transition_queue_send_timeout;
|
||||
match tokio::time::timeout(send_timeout, self.transition_tx.send(task)).await {
|
||||
Ok(Ok(())) => TransitionEnqueueOutcome::Queued,
|
||||
Ok(Err(_)) => {
|
||||
self.schedule_bucket_compensation(&oi.bucket);
|
||||
TransitionEnqueueOutcome::QueueClosed
|
||||
}
|
||||
Err(_) => {
|
||||
Self::inc_counter(&self.queue_send_timeout_tasks);
|
||||
self.schedule_bucket_compensation(&oi.bucket);
|
||||
debug!(
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
source = ?src,
|
||||
timeout_ms = send_timeout.as_millis() as u64,
|
||||
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "queue_send_timed_out",
|
||||
"Scanner transition enqueue timed out; scheduled bucket compensation"
|
||||
);
|
||||
TransitionEnqueueOutcome::QueueFull
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(async_channel::TrySendError::Closed(_)) => {
|
||||
self.schedule_bucket_compensation(&oi.bucket);
|
||||
debug!(
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
@@ -1730,7 +1763,7 @@ impl TransitionState {
|
||||
}
|
||||
|
||||
pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
|
||||
let mut tier_stats = self.last_day_stats.lock().unwrap();
|
||||
let mut tier_stats = self.lock_last_day_stats();
|
||||
tier_stats
|
||||
.entry(tier.to_string())
|
||||
.and_modify(|e| e.add_stats(ts))
|
||||
@@ -1738,7 +1771,7 @@ impl TransitionState {
|
||||
}
|
||||
|
||||
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
|
||||
let tier_stats = self.last_day_stats.lock().unwrap();
|
||||
let tier_stats = self.lock_last_day_stats();
|
||||
let mut res = DailyAllTierStats::with_capacity(tier_stats.len());
|
||||
for (tier, st) in tier_stats.iter() {
|
||||
res.insert(tier.clone(), st.clone());
|
||||
@@ -1746,6 +1779,18 @@ impl TransitionState {
|
||||
res
|
||||
}
|
||||
|
||||
fn lock_last_day_stats(&self) -> std::sync::MutexGuard<'_, HashMap<String, LastDayTierStats>> {
|
||||
match self.last_day_stats.lock() {
|
||||
Ok(stats) => stats,
|
||||
Err(poisoned) => {
|
||||
let mut stats = poisoned.into_inner();
|
||||
stats.clear();
|
||||
self.last_day_stats.clear_poison();
|
||||
stats
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn update_workers(api: Arc<ECStore>, n: i64) {
|
||||
Self::update_workers_inner(api, n).await;
|
||||
}
|
||||
@@ -1767,7 +1812,27 @@ impl TransitionState {
|
||||
fn resize_workers_to(api: Arc<ECStore>, n: i64, requested: i64, absolute_max: i64) {
|
||||
let target = n as usize;
|
||||
let transition_state = runtime_sources::transition_state_handle();
|
||||
let mut workers = transition_state.workers.lock().unwrap();
|
||||
let runtime = match tokio::runtime::Handle::try_current() {
|
||||
Ok(runtime) => runtime,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
error = %err,
|
||||
state = "resize_failed",
|
||||
"Lifecycle worker pool requires a Tokio runtime"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
// Runtime lookup happens before locking, and the guard is dropped before
|
||||
// metrics/logging callbacks. Poison therefore means a Vec mutation may
|
||||
// have unwound and worker tracking cannot be reconstructed safely.
|
||||
let mut workers = transition_state
|
||||
.workers
|
||||
.lock()
|
||||
.expect("transition worker tracking mutex poisoned");
|
||||
let tracked_workers = workers.len();
|
||||
workers.retain(|worker| !worker.handle.is_finished());
|
||||
let pruned_finished_workers = tracked_workers.saturating_sub(workers.len());
|
||||
@@ -1777,7 +1842,7 @@ impl TransitionState {
|
||||
let clone_api = api.clone();
|
||||
let cancel = CancellationToken::new();
|
||||
let worker_cancel = cancel.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
let handle = runtime.spawn(async move {
|
||||
TransitionState::worker_with_cancel(clone_api, worker_cancel).await;
|
||||
});
|
||||
workers.push(TransitionWorker { cancel, handle });
|
||||
@@ -1791,6 +1856,7 @@ impl TransitionState {
|
||||
|
||||
let current_workers = workers.len() as i64;
|
||||
transition_state.num_workers.store(current_workers, Ordering::SeqCst);
|
||||
drop(workers);
|
||||
transition_state.record_scanner_transition_state();
|
||||
|
||||
debug!(
|
||||
@@ -4903,6 +4969,7 @@ mod tests {
|
||||
FreeVersionRecoveryStats, RecoveryWalkTestAction, list_tier_free_versions, recover_tier_free_versions_with_cancel,
|
||||
set_recovery_bucket_list_wait_hook, set_recovery_walk_test_hook,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
use crate::bucket::metadata_sys;
|
||||
@@ -4936,6 +5003,7 @@ mod tests {
|
||||
use http::HeaderMap;
|
||||
use rustfs_common::metrics::{IlmAction, global_metrics};
|
||||
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry, OutputLocation,
|
||||
@@ -5657,6 +5725,7 @@ mod tests {
|
||||
oi.transitioned_object.tier = "WARM".to_string();
|
||||
oi.transitioned_object.name = "remote/object".to_string();
|
||||
oi.transitioned_object.version_id = "remote-version".to_string();
|
||||
oi.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
|
||||
let local_delete_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
|
||||
let legacy_err = delete_free_version_remote_object_then(&oi, &manager, {
|
||||
@@ -5667,7 +5736,8 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect_err("legacy free-version without identity must be retained");
|
||||
assert!(legacy_err.to_string().contains("no durable backend identity"));
|
||||
assert_eq!(legacy_err.kind(), std::io::ErrorKind::Other);
|
||||
assert_eq!(old_backend.remove_count().await, 0);
|
||||
assert_eq!(local_delete_calls.load(Ordering::Relaxed), 0);
|
||||
|
||||
let mut invalid_metadata = HashMap::new();
|
||||
@@ -6546,6 +6616,96 @@ mod tests {
|
||||
assert_eq!(state.transition_rx.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_transition_enqueue_waits_for_saturated_queue_to_recover() {
|
||||
let state = TransitionState::new_with_capacity(1);
|
||||
let first_object = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "first".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let deferred_object = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "deferred".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::TransitionAction,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await,
|
||||
"first scanner transition should fill the queue"
|
||||
);
|
||||
|
||||
let deferred = state.queue_transition_task(&deferred_object, &event, &LcEventSrc::Scanner);
|
||||
tokio::pin!(deferred);
|
||||
assert!(
|
||||
(&mut deferred).now_or_never().is_none(),
|
||||
"a saturated scanner queue should apply bounded backpressure instead of dropping the task"
|
||||
);
|
||||
|
||||
let first_task = state
|
||||
.transition_rx
|
||||
.recv()
|
||||
.await
|
||||
.expect("queue should remain open")
|
||||
.expect("first queued transition task should be present");
|
||||
state.release_transition(&first_task.obj_info);
|
||||
|
||||
assert!(
|
||||
deferred.await,
|
||||
"the deferred scanner transition should enqueue as soon as capacity recovers"
|
||||
);
|
||||
let recovered_task = state
|
||||
.transition_rx
|
||||
.recv()
|
||||
.await
|
||||
.expect("queue should remain open")
|
||||
.expect("deferred transition task should be present");
|
||||
assert_eq!(recovered_task.obj_info.name, deferred_object.name);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_transition_sustained_saturation_schedules_compensation() {
|
||||
let state = TransitionState::new_with_capacity_and_timeout(1, StdDuration::ZERO);
|
||||
let first_object = ObjectInfo {
|
||||
bucket: "saturated-bucket".to_string(),
|
||||
name: "first".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let missed_object = ObjectInfo {
|
||||
bucket: "saturated-bucket".to_string(),
|
||||
name: "missed".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::TransitionAction,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await,
|
||||
"first scanner transition should fill the queue"
|
||||
);
|
||||
assert!(
|
||||
!state
|
||||
.queue_transition_task(&missed_object, &event, &LcEventSrc::Scanner)
|
||||
.await,
|
||||
"a continuously saturated queue should report that the object was not admitted"
|
||||
);
|
||||
assert_eq!(state.queue_full_tasks(), 1);
|
||||
assert_eq!(state.queue_send_timeout_tasks(), 1);
|
||||
assert_eq!(
|
||||
state.compensation_scheduled_tasks(),
|
||||
1,
|
||||
"timed-out scanner work must schedule a bounded bucket backfill"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_transition_enqueue_updates_transition_status() {
|
||||
@@ -7071,6 +7231,46 @@ mod tests {
|
||||
assert_eq!(state.compensation_pending_tasks(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poisoned_compensation_set_can_release_completed_bucket() {
|
||||
let state = TransitionState::new_with_capacity(1);
|
||||
state
|
||||
.compensation_buckets
|
||||
.lock()
|
||||
.expect("fresh mutex should lock")
|
||||
.insert("bucket-a".to_string());
|
||||
let poison_target = Arc::clone(&state.compensation_buckets);
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _guard = poison_target.lock().expect("fresh mutex should lock");
|
||||
panic!("poison compensation set");
|
||||
})
|
||||
.join();
|
||||
|
||||
state.finish_bucket_compensation("bucket-a");
|
||||
assert_eq!(state.compensation_pending_tasks(), 0);
|
||||
assert!(
|
||||
state.compensation_buckets.lock().is_ok(),
|
||||
"validated compensation state must clear poison"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poisoned_tier_stats_are_reset_before_reuse() {
|
||||
let state = TransitionState::new_with_capacity(1);
|
||||
let poison_target = Arc::clone(&state.last_day_stats);
|
||||
let _ = std::thread::spawn(move || {
|
||||
let mut stats = poison_target.lock().expect("fresh mutex should lock");
|
||||
stats.insert("stale".to_string(), LastDayTierStats::default());
|
||||
panic!("poison tier stats");
|
||||
})
|
||||
.join();
|
||||
|
||||
state.add_lastday_stats("fresh", TierStats::default());
|
||||
let stats = state.get_daily_all_tier_stats();
|
||||
assert!(!stats.contains_key("stale"), "possibly partial statistics must be discarded");
|
||||
assert!(stats.contains_key("fresh"), "statistics must accept new samples after recovery");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn scanner_transition_state_reports_compensation_pending_buckets() {
|
||||
let state = TransitionState::new_with_capacity(1);
|
||||
@@ -7361,6 +7561,23 @@ mod tests {
|
||||
TransitionState::resize_workers_to(ecstore, original_workers, original_workers, absolute_max);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn transition_worker_resize_without_runtime_does_not_poison_tracking() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let transition_state = runtime_sources::transition_state_handle();
|
||||
let resize = std::thread::spawn(move || {
|
||||
TransitionState::resize_workers_to(ecstore, 1, 1, resolve_transition_workers_absolute_max());
|
||||
})
|
||||
.join();
|
||||
|
||||
assert!(resize.is_ok(), "missing Tokio runtime must not panic while worker tracking is locked");
|
||||
assert!(
|
||||
transition_state.workers.lock().is_ok(),
|
||||
"failed resize must leave worker tracking unpoisoned"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
@@ -9979,6 +10196,61 @@ mod tests {
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn journal_replay_deletes_confirmed_exact_provider_token() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
backend
|
||||
.set_put_remote_version(Some("provider-version-token".to_string()))
|
||||
.await;
|
||||
lease
|
||||
.put(
|
||||
"remote/object",
|
||||
crate::client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")),
|
||||
9,
|
||||
)
|
||||
.await
|
||||
.expect("confirmed remote candidate should be seeded");
|
||||
backend.set_remove_failure(true);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
let je = Jentry {
|
||||
obj_name: "remote/object".to_string(),
|
||||
version_id: "provider-version-token".to_string(),
|
||||
tier_name: "WARM".to_string(),
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
};
|
||||
|
||||
crate::set_disk::cleanup_rejected_transition_upload_durably(
|
||||
&lease,
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
true,
|
||||
Some(ecstore.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("failed immediate cleanup should remain durable in the journal");
|
||||
assert!(backend.contains(&je.obj_name).await);
|
||||
|
||||
backend.set_remove_failure(false);
|
||||
crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
.await
|
||||
.expect("identity-bound exact journal must retry confirmed candidate cleanup");
|
||||
|
||||
assert!(!backend.contains(&je.obj_name).await);
|
||||
assert_eq!(backend.exact_remove_count(), 2);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), "provider-version-token".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed_recoverable_free_version(
|
||||
disk_paths: &[PathBuf],
|
||||
bucket: &str,
|
||||
|
||||
@@ -20,7 +20,10 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
@@ -270,15 +273,26 @@ pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -
|
||||
let backend_identity = je
|
||||
.backend_identity
|
||||
.ok_or_else(|| std::io::Error::other("legacy tier delete journal has no durable backend identity"))?;
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
je.version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
if je.version_id_exact {
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
|
||||
&je.obj_name,
|
||||
&je.version_id,
|
||||
&je.tier_name,
|
||||
backend_identity,
|
||||
&api.tier_config_mgr(),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
|
||||
@@ -252,7 +252,10 @@ impl ObjSweeper {
|
||||
version_id: self.transition_version_id.clone(),
|
||||
tier_name: self.transition_tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: false,
|
||||
version_id_exact: matches!(
|
||||
self.transition_version_state,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: self.transition_version_state,
|
||||
});
|
||||
}
|
||||
@@ -335,7 +338,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false).await
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
|
||||
}
|
||||
|
||||
async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
@@ -343,8 +346,11 @@ async fn delete_object_from_remote_tier_raw_with_lease(
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
validate_remote_version_id: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
lease.validate_remote_version_id(rv_id)?;
|
||||
if validate_remote_version_id {
|
||||
lease.validate_remote_version_id(rv_id)?;
|
||||
}
|
||||
|
||||
if remote_delete_breaker_is_open(Instant::now()).await {
|
||||
metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
|
||||
@@ -440,7 +446,53 @@ pub(crate) async fn delete_object_from_remote_tier_with_lease_idempotent(
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact).await {
|
||||
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, version_id_exact, true).await
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idempotent(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
if rv_id.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"confirmed versioned transition candidate requires a non-empty remote version",
|
||||
));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if obj_name == "remote/empty-guard-probe" {
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, true, false).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
tier_name: &str,
|
||||
backend_identity: TierDestinationId,
|
||||
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, tier_name, backend_identity)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent(obj_name, rv_id, &lease).await
|
||||
}
|
||||
|
||||
async fn delete_object_from_remote_tier_with_lease_idempotent_inner(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
lease: &TierOperationLease,
|
||||
version_id_exact: bool,
|
||||
validate_remote_version_id: bool,
|
||||
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
|
||||
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact, validate_remote_version_id)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(RemoteTierDeleteOutcome::Deleted),
|
||||
Err(err) if is_remote_tier_not_found_error(&err) => Ok(RemoteTierDeleteOutcome::AlreadyRemoved),
|
||||
Err(err) => {
|
||||
@@ -465,6 +517,7 @@ pub fn transitioned_delete_journal_entry(
|
||||
versioned: bool,
|
||||
suspended: bool,
|
||||
transitioned: &TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
let sweeper = ObjSweeper {
|
||||
version_id,
|
||||
@@ -473,6 +526,7 @@ pub fn transitioned_delete_journal_entry(
|
||||
transition_status: transitioned.status.clone(),
|
||||
transition_tier: transitioned.tier.clone(),
|
||||
transition_version_id: transitioned.version_id.clone(),
|
||||
transition_version_state,
|
||||
remote_object: transitioned.name.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -480,8 +534,13 @@ pub fn transitioned_delete_journal_entry(
|
||||
sweeper.should_remove_remote_object()
|
||||
}
|
||||
|
||||
pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject) -> Option<Jentry> {
|
||||
if transitioned.status != lifecycle::TRANSITION_COMPLETE {
|
||||
pub fn transitioned_force_delete_journal_entry(
|
||||
transitioned: &TransitionedObject,
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState,
|
||||
) -> Option<Jentry> {
|
||||
if transitioned.status != lifecycle::TRANSITION_COMPLETE
|
||||
|| transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -490,8 +549,11 @@ pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject
|
||||
version_id: transitioned.version_id.clone(),
|
||||
tier_name: transitioned.tier.clone(),
|
||||
backend_identity: None,
|
||||
version_id_exact: false,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
version_id_exact: matches!(
|
||||
transition_version_state,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: transition_version_state,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -500,11 +562,14 @@ mod test {
|
||||
use crate::client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, RemoteDeleteBreaker, RemoteTierDeleteOutcome,
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
use std::io::{Error, ErrorKind};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -548,6 +613,43 @@ mod test {
|
||||
assert!(should_record_remote_delete_failure(&Error::other("NoSuchVersion")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transitioned_delete_journal_preserves_remote_version_state() {
|
||||
let cases = [
|
||||
(TransitionVersionState::Unknown, "legacy-version", None),
|
||||
(TransitionVersionState::KnownDisabled, "", Some(false)),
|
||||
(TransitionVersionState::SuspendedNull, "null", Some(true)),
|
||||
(TransitionVersionState::Exact, "opaque-version", Some(true)),
|
||||
];
|
||||
|
||||
for (state, version_id, expected_exact) in cases {
|
||||
let transitioned = TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: version_id.to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
status: lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let regular = transitioned_delete_journal_entry(None, false, false, &transitioned, state);
|
||||
let forced = transitioned_force_delete_journal_entry(&transitioned, state);
|
||||
|
||||
match expected_exact {
|
||||
Some(expected_exact) => {
|
||||
let regular = regular.expect("known version state should produce a regular delete journal entry");
|
||||
assert_eq!(regular.version_state, state);
|
||||
assert_eq!(regular.version_id_exact, expected_exact);
|
||||
let forced = forced.expect("known version state should produce a forced delete journal entry");
|
||||
assert_eq!(forced.version_state, state);
|
||||
assert_eq!(forced.version_id_exact, expected_exact);
|
||||
}
|
||||
None => {
|
||||
assert!(regular.is_none());
|
||||
assert!(forced.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn idempotent_remote_delete_treats_hooked_nosuchversion_as_already_removed() {
|
||||
@@ -670,6 +772,55 @@ mod test {
|
||||
assert_eq!(backend.remove_versions().await, vec![("remote/object".to_string(), String::new())]);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn confirmed_transition_cleanup_deletes_exact_provider_token() {
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
let manager = crate::services::tier::tier::TierConfigMgr::new();
|
||||
let backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await;
|
||||
let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM")
|
||||
.await
|
||||
.expect("test tier lease should be available");
|
||||
let identity = lease.backend_identity();
|
||||
drop(lease);
|
||||
backend.set_reject_non_empty_remote_versions(true);
|
||||
|
||||
let outcome = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
"remote/object",
|
||||
"provider-version-token",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
.expect("confirmed upload compensation should delete the exact provider token");
|
||||
|
||||
assert_eq!(outcome, RemoteTierDeleteOutcome::Deleted);
|
||||
assert_eq!(backend.exact_remove_count(), 1);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![("remote/object".to_string(), "provider-version-token".to_string())]
|
||||
);
|
||||
|
||||
let err = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
"remote/empty-guard-probe",
|
||||
"",
|
||||
"WARM",
|
||||
identity,
|
||||
&manager,
|
||||
)
|
||||
.await
|
||||
.expect_err("confirmed versioned cleanup must reject an empty token");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
|
||||
assert_eq!(backend.remove_count().await, 1);
|
||||
assert_eq!(
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.load(std::sync::atomic::Ordering::Relaxed),
|
||||
0,
|
||||
"empty remote versions must be rejected before exact cleanup dispatch"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breaker_opens_at_threshold_and_recovers_after_window() {
|
||||
let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
|
||||
|
||||
@@ -22,7 +22,10 @@ use uuid::Uuid;
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent_with_manager_and_identity;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -708,6 +711,21 @@ async fn recover_unknown_upload_outcome(
|
||||
TransitionCandidateProbe::UnversionedPresent => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
|
||||
{
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
&transaction.remote_object,
|
||||
&version_id,
|
||||
&transaction.tier_name,
|
||||
transaction.backend_fingerprint,
|
||||
&api.tier_config_mgr(),
|
||||
)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
|
||||
}
|
||||
|
||||
@@ -741,6 +741,8 @@ impl BucketMetadataSys {
|
||||
|
||||
if let Some(config) = &bm.policy_config {
|
||||
Ok((config.clone(), bm.policy_config_updated_at))
|
||||
} else if !bm.policy_config_json.is_empty() {
|
||||
Ok((serde_json::from_slice(&bm.policy_config_json)?, bm.policy_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
}
|
||||
@@ -1051,6 +1053,22 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_bucket_policy_rejects_malformed_cached_policy() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let mut metadata = BucketMetadata::new("malformed-policy");
|
||||
metadata.policy_config_json = b"{".to_vec();
|
||||
sys.set("malformed-policy".to_string(), Arc::new(metadata)).await;
|
||||
|
||||
let err = sys
|
||||
.get_bucket_policy("malformed-policy")
|
||||
.await
|
||||
.expect_err("malformed persisted policy must not be treated as missing");
|
||||
|
||||
assert!(matches!(err, Error::Io(_)), "malformed persisted policy must surface its parse failure");
|
||||
}
|
||||
|
||||
fn target(bucket: &str, id: &str) -> BucketTarget {
|
||||
BucketTarget {
|
||||
source_bucket: bucket.to_string(),
|
||||
|
||||
@@ -15,23 +15,26 @@
|
||||
use super::metadata_sys::get_bucket_metadata_sys;
|
||||
use crate::error::{Result, StorageError};
|
||||
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
|
||||
use tracing::info;
|
||||
|
||||
pub struct PolicySys {}
|
||||
|
||||
impl PolicySys {
|
||||
pub async fn is_allowed(args: &BucketPolicyArgs<'_>) -> bool {
|
||||
match Self::get(args.bucket).await {
|
||||
Ok(cfg) => return cfg.is_allowed(args).await,
|
||||
Err(err) => {
|
||||
if err != StorageError::ConfigNotFound {
|
||||
info!("config get err {:?}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
args.is_owner
|
||||
matches!(Self::try_is_allowed(args).await, Ok(true))
|
||||
}
|
||||
|
||||
pub async fn try_is_allowed(args: &BucketPolicyArgs<'_>) -> Result<bool> {
|
||||
Self::is_allowed_with_policy(args, Self::get(args.bucket).await).await
|
||||
}
|
||||
|
||||
async fn is_allowed_with_policy(args: &BucketPolicyArgs<'_>, policy: Result<BucketPolicy>) -> Result<bool> {
|
||||
match policy {
|
||||
Ok(policy) => Ok(policy.is_allowed(args).await),
|
||||
Err(StorageError::ConfigNotFound) => Ok(args.is_owner),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get(bucket: &str) -> Result<BucketPolicy> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -41,3 +44,91 @@ impl PolicySys {
|
||||
Ok(cfg)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{PolicySys, StorageError};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn args<'a>(
|
||||
is_owner: bool,
|
||||
groups: &'a Option<Vec<String>>,
|
||||
conditions: &'a HashMap<String, Vec<String>>,
|
||||
) -> BucketPolicyArgs<'a> {
|
||||
BucketPolicyArgs {
|
||||
bucket: "bucket",
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
is_owner,
|
||||
account: "account",
|
||||
groups,
|
||||
conditions,
|
||||
object: "object",
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_policy_preserves_owner_and_iam_fallback_semantics() {
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
assert!(
|
||||
PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(StorageError::ConfigNotFound),)
|
||||
.await
|
||||
.expect("missing policy should preserve owner access")
|
||||
);
|
||||
assert!(
|
||||
!PolicySys::is_allowed_with_policy(&args(false, &groups, &conditions), Err(StorageError::ConfigNotFound),)
|
||||
.await
|
||||
.expect("missing policy should defer non-owner access to IAM")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn policy_load_failures_propagate() {
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
for (failure, expected_message) in [
|
||||
(StorageError::Io(std::io::Error::other("policy read failed")), "policy read failed"),
|
||||
(
|
||||
StorageError::other("bucket metadata sys not initialized for this instance"),
|
||||
"bucket metadata sys not initialized for this instance",
|
||||
),
|
||||
] {
|
||||
let result = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(failure)).await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(StorageError::Io(ref err)) if err.to_string().contains(expected_message)),
|
||||
"policy I/O and uninitialized metadata failures must propagate instead of granting owner access"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_bucket_deny_precedes_iam_allow() {
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
let policy: BucketPolicy = serde_json::from_str(
|
||||
r#"{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{
|
||||
"Effect":"Deny",
|
||||
"Principal":{"AWS":"*"},
|
||||
"Action":["s3:GetObject"],
|
||||
"Resource":["arn:aws:s3:::bucket/*"]
|
||||
}]
|
||||
}"#,
|
||||
)
|
||||
.expect("deny policy should parse");
|
||||
|
||||
let bucket_allowed = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Ok(policy))
|
||||
.await
|
||||
.expect("loaded bucket policy should evaluate");
|
||||
let iam_allowed = true;
|
||||
let request_allowed = bucket_allowed && iam_allowed;
|
||||
|
||||
assert!(iam_allowed, "test precondition: IAM grants the action");
|
||||
assert!(!bucket_allowed, "test precondition: bucket policy explicitly denies the action");
|
||||
assert!(!request_allowed, "explicit bucket Deny must reject before IAM Allow fallback");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ use bytes::Bytes;
|
||||
use futures::future::join_all;
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
||||
use std::io::Error;
|
||||
use std::sync::RwLock;
|
||||
use std::sync::{Mutex, MutexGuard, RwLock};
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::{OffsetDateTime, format_description};
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -46,6 +46,14 @@ use crate::client::utils::base64_encode;
|
||||
use rustfs_utils::path::trim_etag;
|
||||
use s3s::header::X_AMZ_EXPIRATION;
|
||||
|
||||
fn lock_md5_hasher(
|
||||
md5_hasher: &Mutex<Option<rustfs_utils::hash::HashAlgorithm>>,
|
||||
) -> Result<MutexGuard<'_, Option<rustfs_utils::hash::HashAlgorithm>>, std::io::Error> {
|
||||
md5_hasher
|
||||
.lock()
|
||||
.map_err(|_| std::io::Error::other("MD5 hasher state is unavailable"))
|
||||
}
|
||||
|
||||
/// Read exactly `want` bytes for a single multipart part, or fewer if the reader
|
||||
/// reaches EOF first. Advances the reader so the next call returns the following
|
||||
/// part. Replaces the previous per-part `read_all()`/`to_vec()`, which drained
|
||||
@@ -177,7 +185,7 @@ impl TransitionClient {
|
||||
let length = buf.len();
|
||||
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = self.md5_hasher.lock().unwrap();
|
||||
let mut md5_hasher = lock_md5_hasher(&self.md5_hasher)?;
|
||||
let md5_hash = match md5_hasher.as_mut() {
|
||||
Some(hasher) => hasher,
|
||||
None => return Err(std::io::Error::other("MD5 hasher not initialized")),
|
||||
@@ -370,7 +378,7 @@ impl TransitionClient {
|
||||
let mut md5_base64: String = "".to_string();
|
||||
|
||||
if opts.send_content_md5 {
|
||||
let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
|
||||
let mut md5_hasher = lock_md5_hasher(&clone_self.md5_hasher)?;
|
||||
let md5_hash = match md5_hasher.as_mut() {
|
||||
Some(hasher) => hasher,
|
||||
None => {
|
||||
@@ -418,6 +426,9 @@ impl TransitionClient {
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
result?;
|
||||
}
|
||||
|
||||
select! {
|
||||
err = err_rx.recv() => {
|
||||
@@ -620,10 +631,12 @@ fn collect_complete_parts(parts_info: &HashMap<i64, ObjectPart>, total_parts_cou
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ObjectPart, ReaderImpl, collect_complete_parts, read_multipart_part};
|
||||
use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part};
|
||||
use crate::object_api::GetObjectReader;
|
||||
use bytes::Bytes;
|
||||
use rustfs_utils::hash::HashAlgorithm;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
// Drive a reader through the same per-part loop the multipart stream uses and
|
||||
// collect the size of every part. Regression for rustfs/rustfs#4811: the old
|
||||
@@ -733,4 +746,18 @@ mod tests {
|
||||
"a gap in the parts map must be an error, not a panic"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poisoned_md5_state_fails_closed() {
|
||||
let hasher = Arc::new(Mutex::new(Some(HashAlgorithm::Md5)));
|
||||
let poison_target = Arc::clone(&hasher);
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _guard = poison_target.lock().expect("fresh mutex should lock");
|
||||
panic!("poison MD5 state");
|
||||
})
|
||||
.join();
|
||||
|
||||
let error = lock_md5_hasher(&hasher).expect_err("poisoned hash state must not be reused");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::Other);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,8 +24,8 @@ use crate::cluster::rpc::internode_data_transport::{
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
disk_store::{
|
||||
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
|
||||
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
|
||||
@@ -48,9 +48,10 @@ use rustfs_protos::proto_gen::node_service::RenamePartRequest;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
|
||||
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, StatVolumeRequest,
|
||||
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
|
||||
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
|
||||
RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest,
|
||||
WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -2481,6 +2482,71 @@ impl DiskAPI for RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: Bytes,
|
||||
) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(PreparePartTransactionRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_string(),
|
||||
src_path: src_path.to_string(),
|
||||
dst_volume: dst_volume.to_string(),
|
||||
dst_path: dst_path.to_string(),
|
||||
meta,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_prepare_part_transaction_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "prepare_part_transaction")?;
|
||||
|
||||
let response = client.prepare_part_transaction(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let mut request = Request::new(SettlePartTransactionRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_string(),
|
||||
path: path.to_string(),
|
||||
rollback: action == PartTransactionAction::Rollback,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_settle_part_transaction_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "settle_part_transaction")?;
|
||||
|
||||
let response = client.settle_part_transaction(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
trace!(
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
use crate::disk::error_reduce::count_errs;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets};
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
|
||||
@@ -49,7 +50,10 @@ use rustfs_filemeta::FileInfo;
|
||||
use rustfs_lock::NamespaceLockWrapper;
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
sync::Arc,
|
||||
};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::sync::broadcast::{Receiver, Sender};
|
||||
use tokio::time::Duration;
|
||||
@@ -63,6 +67,8 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Sets {
|
||||
pub id: Uuid,
|
||||
@@ -799,9 +805,52 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
|
||||
delimiter: Option<String>,
|
||||
max_uploads: usize,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
self.get_disks_by_key(prefix)
|
||||
.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads)
|
||||
.await
|
||||
let per_set_limit = max_uploads.saturating_add(1);
|
||||
let results = futures::stream::iter(self.disk_set.iter().cloned())
|
||||
.map(|set| {
|
||||
let key_marker = key_marker.clone();
|
||||
let upload_id_marker = upload_id_marker.clone();
|
||||
let delimiter = delimiter.clone();
|
||||
async move {
|
||||
set.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, per_set_limit)
|
||||
.await
|
||||
}
|
||||
})
|
||||
.buffer_unordered(LIST_MULTIPART_SETS_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let mut uploads = Vec::new();
|
||||
let mut common_prefixes = HashSet::new();
|
||||
let mut source_truncated = false;
|
||||
for result in results {
|
||||
let page = result?;
|
||||
uploads.extend(page.uploads);
|
||||
common_prefixes.extend(page.common_prefixes);
|
||||
source_truncated |= page.is_truncated;
|
||||
}
|
||||
|
||||
let page = paginate_multipart_listing(
|
||||
uploads,
|
||||
common_prefixes.into_iter().collect(),
|
||||
key_marker.as_deref(),
|
||||
key_marker.as_ref().and(upload_id_marker.as_deref()),
|
||||
max_uploads,
|
||||
source_truncated,
|
||||
);
|
||||
|
||||
Ok(ListMultipartsInfo {
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
next_key_marker: page.next_key_marker,
|
||||
next_upload_id_marker: page.next_upload_id_marker,
|
||||
max_uploads,
|
||||
is_truncated: page.is_truncated,
|
||||
uploads: page.uploads,
|
||||
common_prefixes: page.common_prefixes,
|
||||
prefix: prefix.to_owned(),
|
||||
delimiter,
|
||||
})
|
||||
}
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
|
||||
@@ -1111,6 +1160,7 @@ mod tests {
|
||||
use crate::layout::endpoints::SetupType;
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::list::ListOperations as _;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
use serial_test::serial;
|
||||
|
||||
@@ -1248,6 +1298,194 @@ mod tests {
|
||||
assert_eq!(result, (Some(3), Some(1), Some(0)));
|
||||
}
|
||||
|
||||
async fn multipart_listing_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
let format = FormatV3::new(2, 2);
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut all_endpoints = Vec::new();
|
||||
let mut disk_sets = Vec::new();
|
||||
|
||||
for set_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_index in 0..2 {
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
|
||||
.expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(set_index);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("disk should be created");
|
||||
let mut disk_format = format.clone();
|
||||
disk_format.erasure.this = format.erasure.sets[set_index][disk_index];
|
||||
save_format_file(&Some(disk.clone()), &Some(disk_format))
|
||||
.await
|
||||
.expect("format should be saved");
|
||||
temp_dirs.push(temp_dir);
|
||||
all_endpoints.push(endpoint.clone());
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
disk_sets.push(
|
||||
SetDisks::new(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
set_index,
|
||||
endpoints,
|
||||
format.clone(),
|
||||
vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())],
|
||||
)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
|
||||
let sets = Arc::new(Sets {
|
||||
id: format.id,
|
||||
disk_set: disk_sets,
|
||||
pool_idx: 0,
|
||||
endpoints: PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 2,
|
||||
drives_per_set: 2,
|
||||
endpoints: Endpoints::from(all_endpoints),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
},
|
||||
format,
|
||||
parity_count: 1,
|
||||
set_count: 2,
|
||||
set_drive_count: 2,
|
||||
default_parity_count: 1,
|
||||
distribution_algo: DistributionAlgoVersion::V1,
|
||||
exit_signal: None,
|
||||
ctx: bootstrap_ctx(),
|
||||
});
|
||||
(temp_dirs, sets)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
|
||||
let (_temp_dirs, sets) = multipart_listing_test_sets().await;
|
||||
let bucket = format!("multipart-list-{}", Uuid::new_v4().simple());
|
||||
sets.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
|
||||
let mut keys_by_set = [Vec::new(), Vec::new()];
|
||||
for index in 0..100 {
|
||||
let key = format!("logs/{index:03}.bin");
|
||||
let set_index = sets.get_hashed_set_index(&key);
|
||||
if keys_by_set[set_index].len() < 2 {
|
||||
keys_by_set[set_index].push(key);
|
||||
}
|
||||
if keys_by_set.iter().all(|keys| keys.len() == 2) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(keys_by_set.iter().all(|keys| keys.len() == 2), "test keys must span both sets");
|
||||
|
||||
let repeated_key = keys_by_set[0][0].clone();
|
||||
let mut expected = Vec::new();
|
||||
for key in keys_by_set.iter().flatten() {
|
||||
let upload = sets
|
||||
.new_multipart_upload(&bucket, key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
expected.push((key.clone(), upload.upload_id));
|
||||
}
|
||||
let second = sets
|
||||
.new_multipart_upload(&bucket, &repeated_key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("second upload for the same key should be created");
|
||||
expected.push((repeated_key, second.upload_id));
|
||||
expected.sort();
|
||||
|
||||
let mut actual = Vec::new();
|
||||
let mut key_marker = None;
|
||||
let mut upload_id_marker = None;
|
||||
for _ in 0..expected.len() + 1 {
|
||||
let page = sets
|
||||
.list_multipart_uploads(&bucket, "logs/", key_marker.clone(), upload_id_marker.clone(), None, 2)
|
||||
.await
|
||||
.expect("multipart page should list across every set");
|
||||
assert!(page.uploads.len() <= 2);
|
||||
actual.extend(
|
||||
page.uploads
|
||||
.iter()
|
||||
.map(|upload| (upload.object.clone(), upload.upload_id.clone())),
|
||||
);
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
key_marker = page.next_key_marker;
|
||||
upload_id_marker = page.next_upload_id_marker;
|
||||
}
|
||||
|
||||
assert_eq!(actual, expected, "set-level merge must return every upload exactly once");
|
||||
let mut deduped = actual.clone();
|
||||
deduped.dedup();
|
||||
assert_eq!(deduped.len(), actual.len(), "set-level pagination must not duplicate uploads");
|
||||
|
||||
let mut nested_by_set = [None, None];
|
||||
for index in 0..100 {
|
||||
let key = format!("nested/group-{index:03}/file.bin");
|
||||
let set_index = sets.get_hashed_set_index(&key);
|
||||
nested_by_set[set_index].get_or_insert(key);
|
||||
if nested_by_set.iter().all(Option::is_some) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for key in nested_by_set.iter().flatten() {
|
||||
sets.new_multipart_upload(&bucket, key, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("nested multipart upload should be created");
|
||||
}
|
||||
let mut expected_prefixes = nested_by_set
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|key| {
|
||||
key.rsplit_once('/')
|
||||
.expect("nested key should contain a delimiter")
|
||||
.0
|
||||
.to_string()
|
||||
+ "/"
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
expected_prefixes.sort();
|
||||
|
||||
let first = sets
|
||||
.list_multipart_uploads(&bucket, "nested/", None, None, Some("/".to_string()), 1)
|
||||
.await
|
||||
.expect("first delimiter page should list across every set");
|
||||
assert!(first.is_truncated);
|
||||
assert_eq!(first.common_prefixes, expected_prefixes[..1]);
|
||||
let second = sets
|
||||
.list_multipart_uploads(
|
||||
&bucket,
|
||||
"nested/",
|
||||
first.next_key_marker,
|
||||
first.next_upload_id_marker,
|
||||
Some("/".to_string()),
|
||||
1,
|
||||
)
|
||||
.await
|
||||
.expect("second delimiter page should list across every set");
|
||||
assert!(!second.is_truncated);
|
||||
assert_eq!(second.common_prefixes, expected_prefixes[1..]);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn sets_list_objects_v2_lists_objects_within_the_pool() {
|
||||
|
||||
@@ -35,7 +35,7 @@ use std::{
|
||||
Arc,
|
||||
atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
use tokio::{sync::RwLock, time};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -339,21 +339,19 @@ impl Drop for DiskHealthWaitingGuard<'_> {
|
||||
impl DiskHealthTracker {
|
||||
/// Create a new disk health tracker
|
||||
pub fn new() -> Self {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
let now = current_unix_time();
|
||||
let now_nanos = unix_nanos(now);
|
||||
|
||||
Self {
|
||||
last_success: AtomicI64::new(now),
|
||||
last_started: AtomicI64::new(now),
|
||||
last_success: AtomicI64::new(now_nanos),
|
||||
last_started: AtomicI64::new(now_nanos),
|
||||
status: AtomicU32::new(DISK_HEALTH_OK),
|
||||
waiting: AtomicU32::new(0),
|
||||
runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
|
||||
consecutive_failures: AtomicU32::new(0),
|
||||
consecutive_successes: AtomicU32::new(0),
|
||||
offline_since_unix_secs: AtomicI64::new(0),
|
||||
last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
|
||||
last_transition_unix_secs: AtomicI64::new(unix_secs_i64(now)),
|
||||
last_capacity_total: AtomicU64::new(0),
|
||||
last_capacity_used: AtomicU64::new(0),
|
||||
last_capacity_free: AtomicU64::new(0),
|
||||
@@ -363,11 +361,7 @@ impl DiskHealthTracker {
|
||||
|
||||
/// Log a successful operation
|
||||
pub fn log_success(&self) {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
self.last_success.store(now, Ordering::Relaxed);
|
||||
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
|
||||
@@ -429,11 +423,14 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
pub fn offline_duration(&self) -> Option<Duration> {
|
||||
self.offline_duration_at(current_unix_secs())
|
||||
}
|
||||
|
||||
fn offline_duration_at(&self, now: u64) -> Option<Duration> {
|
||||
let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
|
||||
if offline_since <= 0 {
|
||||
return None;
|
||||
}
|
||||
let now = current_unix_secs();
|
||||
Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
|
||||
}
|
||||
|
||||
@@ -492,6 +489,12 @@ impl DiskHealthTracker {
|
||||
/// Remote disks are marked faulty on timeout/network errors; the init loop retries with the
|
||||
/// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`.
|
||||
pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) {
|
||||
self.reset_for_store_init_retry_at(endpoint, current_unix_time());
|
||||
}
|
||||
|
||||
fn reset_for_store_init_retry_at(&self, endpoint: &Endpoint, now: Duration) {
|
||||
let now_nanos = unix_nanos(now);
|
||||
let now_secs = unix_secs_i64(now);
|
||||
self.status.store(DISK_HEALTH_OK, Ordering::Release);
|
||||
self.runtime_state
|
||||
.store(RuntimeDriveHealthState::Online as u32, Ordering::Release);
|
||||
@@ -499,11 +502,9 @@ impl DiskHealthTracker {
|
||||
self.consecutive_successes.store(0, Ordering::Release);
|
||||
self.offline_since_unix_secs.store(0, Ordering::Release);
|
||||
self.waiting.store(0, Ordering::Release);
|
||||
let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap();
|
||||
let now_nanos = now.as_nanos() as i64;
|
||||
self.last_success.store(now_nanos, Ordering::Relaxed);
|
||||
self.last_started.store(now_nanos, Ordering::Relaxed);
|
||||
self.last_transition_unix_secs.store(now.as_secs() as i64, Ordering::Release);
|
||||
self.last_transition_unix_secs.store(now_secs, Ordering::Release);
|
||||
record_drive_runtime_state(endpoint, RuntimeDriveHealthState::Online);
|
||||
}
|
||||
|
||||
@@ -612,10 +613,33 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
fn current_unix_secs() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
// Zero is reserved as "not recorded" by health timestamp atomics.
|
||||
current_unix_time().as_secs().max(1)
|
||||
}
|
||||
|
||||
fn current_unix_nanos() -> i64 {
|
||||
unix_nanos(current_unix_time())
|
||||
}
|
||||
|
||||
fn current_unix_time() -> Duration {
|
||||
unix_time_since_epoch(SystemTime::now())
|
||||
}
|
||||
|
||||
fn unix_time_since_epoch(time: SystemTime) -> Duration {
|
||||
time.duration_since(UNIX_EPOCH).unwrap_or(Duration::ZERO)
|
||||
}
|
||||
|
||||
fn unix_nanos(time: Duration) -> i64 {
|
||||
i64::try_from(time.as_nanos()).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
fn unix_secs_i64(time: Duration) -> i64 {
|
||||
i64::try_from(time.as_secs()).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
fn elapsed_since(last_nanos: i64, now_nanos: i64) -> Duration {
|
||||
let elapsed_nanos = now_nanos.saturating_sub(last_nanos).max(0);
|
||||
Duration::from_nanos(u64::try_from(elapsed_nanos).unwrap_or(u64::MAX))
|
||||
}
|
||||
|
||||
impl Default for DiskHealthTracker {
|
||||
@@ -635,11 +659,7 @@ struct HealthDiskCtxValue {
|
||||
|
||||
impl HealthDiskCtxValue {
|
||||
fn log_success(&self) {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
self.last_success.store(now, Ordering::Relaxed);
|
||||
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -774,12 +794,7 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
let last_success_nanos = health.last_success.load(Ordering::Relaxed);
|
||||
let elapsed = Duration::from_nanos(
|
||||
(std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64 - last_success_nanos) as u64
|
||||
);
|
||||
let elapsed = elapsed_since(last_success_nanos, current_unix_nanos());
|
||||
|
||||
if elapsed < SKIP_IF_SUCCESS_BEFORE {
|
||||
continue;
|
||||
@@ -1091,11 +1106,7 @@ impl LocalDiskWrapper {
|
||||
self.check_disk_stale().await?;
|
||||
|
||||
// Record operation start
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
self.health.last_started.store(now, Ordering::Relaxed);
|
||||
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
let _waiting_guard = self.health.waiting_guard();
|
||||
|
||||
if timeout_duration == Duration::ZERO {
|
||||
@@ -1317,11 +1328,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
// Record operation start
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos() as i64;
|
||||
self.health.last_started.store(now, Ordering::Relaxed);
|
||||
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
self.health.increment_waiting();
|
||||
|
||||
// Execute the operation
|
||||
@@ -1473,6 +1480,33 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: Bytes,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health(
|
||||
|| async {
|
||||
self.disk
|
||||
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: crate::disk::PartTransactionAction) -> Result<()> {
|
||||
self.track_disk_health(
|
||||
|| async { self.disk.settle_part_transaction(volume, path, action).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
self.track_disk_health(|| async { self.disk.delete(volume, path, opt).await }, get_max_timeout_duration())
|
||||
.await
|
||||
@@ -2243,4 +2277,53 @@ mod tests {
|
||||
assert!(health.mark_offline(&endpoint, "again"));
|
||||
assert!(health.is_faulty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unix_time_clamps_epoch_and_pre_epoch_to_zero() {
|
||||
let before_epoch = UNIX_EPOCH
|
||||
.checked_sub(Duration::from_nanos(1))
|
||||
.expect("one nanosecond before the Unix epoch should be representable");
|
||||
|
||||
assert_eq!(unix_time_since_epoch(UNIX_EPOCH), Duration::ZERO);
|
||||
assert_eq!(unix_time_since_epoch(before_epoch), Duration::ZERO);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elapsed_and_offline_duration_saturate_on_clock_rollback() {
|
||||
let health = DiskHealthTracker::new();
|
||||
health.offline_since_unix_secs.store(10, Ordering::Release);
|
||||
|
||||
assert_eq!(elapsed_since(10, 12), Duration::from_nanos(2));
|
||||
assert_eq!(elapsed_since(10, 9), Duration::ZERO);
|
||||
assert_eq!(health.offline_duration_at(9), Some(Duration::ZERO));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_epoch_retry_reset_updates_the_complete_health_state() {
|
||||
let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry-pre-epoch").expect("endpoint should parse");
|
||||
let health = DiskHealthTracker::new();
|
||||
health.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
|
||||
health
|
||||
.runtime_state
|
||||
.store(RuntimeDriveHealthState::Offline as u32, Ordering::Release);
|
||||
health.consecutive_failures.store(3, Ordering::Release);
|
||||
health.consecutive_successes.store(2, Ordering::Release);
|
||||
health.offline_since_unix_secs.store(11, Ordering::Release);
|
||||
health.waiting.store(4, Ordering::Release);
|
||||
health.last_success.store(12, Ordering::Release);
|
||||
health.last_started.store(13, Ordering::Release);
|
||||
health.last_transition_unix_secs.store(14, Ordering::Release);
|
||||
|
||||
health.reset_for_store_init_retry_at(&endpoint, unix_time_since_epoch(UNIX_EPOCH - Duration::from_nanos(1)));
|
||||
|
||||
assert_eq!(health.status.load(Ordering::Acquire), DISK_HEALTH_OK);
|
||||
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert_eq!(health.consecutive_failures.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.consecutive_successes.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.offline_since_unix_secs.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.waiting.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.last_success.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.last_started.load(Ordering::Acquire), 0);
|
||||
assert_eq!(health.last_transition_unix_secs.load(Ordering::Acquire), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,8 @@ use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_r
|
||||
use crate::disk::{
|
||||
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
|
||||
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
|
||||
FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, PART_TRANSACTION_NEW_META,
|
||||
PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE,
|
||||
STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int,
|
||||
endpoint::Endpoint,
|
||||
@@ -80,6 +81,10 @@ const ENV_BITROT_SIZE_MISMATCH_RETRY_COUNT: &str = "RUSTFS_BITROT_SIZE_MISMATCH_
|
||||
const ENV_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: &str = "RUSTFS_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS";
|
||||
const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_COUNT: u64 = 2;
|
||||
const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: u64 = 100;
|
||||
const PART_TRANSACTION_OLD_DATA: &str = "old.data";
|
||||
const PART_TRANSACTION_OLD_DATA_ABSENT: &str = "old.data.absent";
|
||||
const PART_TRANSACTION_OLD_META_ABSENT: &str = "old.meta.absent";
|
||||
const PART_TRANSACTION_PUBLISH_META: &str = "publish.meta";
|
||||
enum ReadAllError {
|
||||
Open(std::io::Error),
|
||||
Disk(DiskError),
|
||||
@@ -141,6 +146,28 @@ fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot_part_transaction_file(src: &Path, backup: &Path, absent: &Path) -> std::io::Result<()> {
|
||||
match std::fs::symlink_metadata(src) {
|
||||
Ok(metadata) if metadata.is_file() => std::fs::hard_link(src, backup),
|
||||
Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction source is not a file")),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => std::fs::write(absent, []),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, restore: &Path) -> std::io::Result<()> {
|
||||
match std::fs::symlink_metadata(backup) {
|
||||
Ok(metadata) if metadata.is_file() => {
|
||||
remove_file_if_exists(restore)?;
|
||||
std::fs::hard_link(backup, restore)?;
|
||||
std::fs::rename(restore, current)
|
||||
}
|
||||
Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction backup is not a file")),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound && absent.is_file() => remove_file_if_exists(current),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn rollback_committed_rename_std(
|
||||
dst_file_path: &Path,
|
||||
new_data_path: Option<&Path>,
|
||||
@@ -6447,6 +6474,151 @@ impl DiskAPI for LocalDisk {
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: Bytes,
|
||||
) -> Result<()> {
|
||||
let src_volume_dir = self.get_bucket_path(src_volume)?;
|
||||
let dst_volume_dir = self.get_bucket_path(dst_volume)?;
|
||||
if !skip_access_checks(src_volume) {
|
||||
super::fs::access_std(&src_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
if !skip_access_checks(dst_volume) {
|
||||
super::fs::access_std(&dst_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
|
||||
let src_file_path = self.get_object_path(src_volume, src_path)?;
|
||||
let dst_file_path = self.get_object_path(dst_volume, dst_path)?;
|
||||
let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
|
||||
let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
|
||||
for path in [&src_file_path, &dst_file_path, &dst_meta_path, &transaction_path] {
|
||||
check_path_length(path.to_string_lossy().as_ref())?;
|
||||
}
|
||||
|
||||
let durability = effective_durability(dst_volume);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let source = std::fs::symlink_metadata(&src_file_path).map_err(to_file_error)?;
|
||||
if !source.is_file() {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
if transaction_path.exists() {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
|
||||
let Some(transaction_parent) = transaction_path.parent() else {
|
||||
return Err(DiskError::InvalidPath);
|
||||
};
|
||||
std::fs::create_dir_all(transaction_parent).map_err(to_file_error)?;
|
||||
let staging_path = transaction_parent.join(format!(".part-txn-{}", Uuid::new_v4()));
|
||||
std::fs::create_dir(&staging_path).map_err(to_file_error)?;
|
||||
|
||||
let prepare_result = (|| -> std::io::Result<()> {
|
||||
snapshot_part_transaction_file(
|
||||
&dst_file_path,
|
||||
&staging_path.join(PART_TRANSACTION_OLD_DATA),
|
||||
&staging_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
|
||||
)?;
|
||||
snapshot_part_transaction_file(
|
||||
&dst_meta_path,
|
||||
&staging_path.join(PART_TRANSACTION_OLD_META),
|
||||
&staging_path.join(PART_TRANSACTION_OLD_META_ABSENT),
|
||||
)?;
|
||||
|
||||
let mut new_meta = std::fs::OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(staging_path.join(PART_TRANSACTION_NEW_META))?;
|
||||
std::io::Write::write_all(&mut new_meta, &meta)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
new_meta.sync_data()?;
|
||||
os::fsync_dir_std(&staging_path)?;
|
||||
}
|
||||
std::fs::rename(&staging_path, &transaction_path)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir_std(transaction_parent)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
|
||||
if let Err(err) = prepare_result {
|
||||
let _ = remove_dir_all_if_exists(&staging_path);
|
||||
return Err(to_file_error(err).into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
|
||||
self.get_bucket_path(volume)?;
|
||||
let current_data_path = self.get_object_path(volume, path)?;
|
||||
let current_meta_path = self.get_object_path(volume, &format!("{path}.meta"))?;
|
||||
let transaction_path = self.get_object_path(volume, &crate::disk::part_transaction_path(path))?;
|
||||
for candidate in [¤t_data_path, ¤t_meta_path, &transaction_path] {
|
||||
check_path_length(candidate.to_string_lossy().as_ref())?;
|
||||
}
|
||||
let durability = effective_durability(volume);
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
match std::fs::symlink_metadata(&transaction_path) {
|
||||
Ok(metadata) if metadata.is_dir() => {}
|
||||
Ok(_) => return Err(DiskError::FileCorrupt),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
|
||||
if action == PartTransactionAction::Rollback {
|
||||
std::fs::write(transaction_path.join(PART_TRANSACTION_ROLLBACK), []).map_err(to_file_error)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir_std(&transaction_path).map_err(to_file_error)?;
|
||||
}
|
||||
restore_part_transaction_file(
|
||||
¤t_data_path,
|
||||
&transaction_path.join(PART_TRANSACTION_OLD_DATA),
|
||||
&transaction_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
|
||||
&transaction_path.join("restore.data"),
|
||||
)
|
||||
.map_err(to_file_error)?;
|
||||
restore_part_transaction_file(
|
||||
¤t_meta_path,
|
||||
&transaction_path.join(PART_TRANSACTION_OLD_META),
|
||||
&transaction_path.join(PART_TRANSACTION_OLD_META_ABSENT),
|
||||
&transaction_path.join("restore.meta"),
|
||||
)
|
||||
.map_err(to_file_error)?;
|
||||
if durability.syncs_commit_metadata()
|
||||
&& let Some(parent) = current_data_path.parent()
|
||||
{
|
||||
os::fsync_dir_std(parent).map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
|
||||
let Some(parent) = transaction_path.parent() else {
|
||||
return Err(DiskError::InvalidPath);
|
||||
};
|
||||
let cleanup_path = parent.join(format!(".part-txn-settled-{}", Uuid::new_v4()));
|
||||
std::fs::rename(&transaction_path, &cleanup_path).map_err(to_file_error)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir_std(parent).map_err(to_file_error)?;
|
||||
}
|
||||
remove_dir_all_if_exists(&cleanup_path).map_err(to_file_error)?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)??;
|
||||
|
||||
self.io_backend.invalidate_cached_fd(volume, path).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
|
||||
let src_volume_dir = self.get_bucket_path(src_volume)?;
|
||||
@@ -6528,6 +6700,42 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
let transaction_publish_meta = if src_is_dir {
|
||||
None
|
||||
} else {
|
||||
let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
|
||||
let transaction_meta_path = transaction_path.join(PART_TRANSACTION_NEW_META);
|
||||
match fs::read(&transaction_meta_path).await {
|
||||
Ok(expected_meta) => {
|
||||
if expected_meta.as_slice() != meta.as_ref() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
let publish_meta_path = transaction_path.join(PART_TRANSACTION_PUBLISH_META);
|
||||
let source_meta_path = transaction_meta_path.clone();
|
||||
let publish_path = publish_meta_path.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
remove_file_if_exists(&publish_path)?;
|
||||
std::fs::hard_link(source_meta_path, &publish_path)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?;
|
||||
Some(publish_meta_path)
|
||||
}
|
||||
// Old peers know only RenamePart. The new coordinator never
|
||||
// reaches rename unless prepare succeeded, so an absent
|
||||
// transaction directory identifies the rolling-upgrade legacy
|
||||
// path. A present directory without new.meta is corruption.
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => match fs::metadata(&transaction_path).await {
|
||||
Ok(_) => return Err(DiskError::FileCorrupt),
|
||||
Err(meta_err) if meta_err.kind() == ErrorKind::NotFound => None,
|
||||
Err(meta_err) => return Err(to_file_error(meta_err).into()),
|
||||
},
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
};
|
||||
|
||||
// UploadPart is acknowledged once this rename lands, so the part data and
|
||||
// its directory entry must be durable before we return. Relaxed keeps the
|
||||
// part payload fdatasync but leaves the directory entry to the page cache.
|
||||
@@ -6561,7 +6769,17 @@ impl DiskAPI for LocalDisk {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
|
||||
self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?;
|
||||
if let Some(transaction_publish_meta) = transaction_publish_meta {
|
||||
let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
|
||||
rename_all(&transaction_publish_meta, &dst_meta_path, &dst_volume_dir).await?;
|
||||
if durability.syncs_commit_metadata()
|
||||
&& let Some(parent) = dst_meta_path.parent()
|
||||
{
|
||||
os::fsync_dir(parent).await.map_err(to_file_error)?;
|
||||
}
|
||||
} else {
|
||||
self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?;
|
||||
}
|
||||
|
||||
if let Some(parent) = src_file_path.parent() {
|
||||
self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await?;
|
||||
@@ -8282,6 +8500,12 @@ mod test {
|
||||
file_info.data_dir = data_dir;
|
||||
file_info.data = data;
|
||||
file_info.size = size;
|
||||
file_info.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: usize::try_from(size).expect("test object size should fit usize"),
|
||||
actual_size: size,
|
||||
..Default::default()
|
||||
}];
|
||||
file_info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
file_info
|
||||
}
|
||||
@@ -9370,9 +9594,15 @@ mod test {
|
||||
.await
|
||||
.expect("source part should be written");
|
||||
|
||||
disk.prepare_part_transaction(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone())
|
||||
.await
|
||||
.expect("part transaction should be prepared");
|
||||
disk.rename_part(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone())
|
||||
.await
|
||||
.expect("rename_part should commit part");
|
||||
disk.settle_part_transaction(bucket, "object/part.1", PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("part transaction should be committed");
|
||||
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, "object/part.1")
|
||||
@@ -9390,6 +9620,71 @@ mod test {
|
||||
matches!(disk.read_all(tmp_volume, "upload/part.1").await, Err(DiskError::FileNotFound)),
|
||||
"source part must be removed after a successful commit"
|
||||
);
|
||||
|
||||
let legacy_payload = Bytes::from_static(b"legacy peer payload");
|
||||
let legacy_meta = Bytes::from_static(b"legacy peer metadata");
|
||||
disk.write_all(tmp_volume, "legacy/part.1", legacy_payload.clone())
|
||||
.await
|
||||
.expect("legacy source part should be written");
|
||||
disk.rename_part(tmp_volume, "legacy/part.1", bucket, "object/part.1", legacy_meta.clone())
|
||||
.await
|
||||
.expect("pre-transaction peer RenamePart should remain supported");
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, "object/part.1")
|
||||
.await
|
||||
.expect("legacy destination part should be readable"),
|
||||
legacy_payload
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, "object/part.1.meta")
|
||||
.await
|
||||
.expect("legacy destination metadata should be readable"),
|
||||
legacy_meta
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_part_transaction_rolls_back_data_published_before_metadata() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
ensure_test_volume(&disk, "tmp").await;
|
||||
ensure_test_volume(&disk, "bucket").await;
|
||||
|
||||
disk.write_all("tmp", "upload/part.1", Bytes::from_static(b"new data"))
|
||||
.await
|
||||
.expect("new part should be staged");
|
||||
disk.write_all("bucket", "object/part.1", Bytes::from_static(b"old data"))
|
||||
.await
|
||||
.expect("old part data should be staged");
|
||||
disk.write_all("bucket", "object/part.1.meta", Bytes::from_static(b"old metadata"))
|
||||
.await
|
||||
.expect("old part metadata should be staged");
|
||||
|
||||
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", Bytes::from_static(b"new metadata"))
|
||||
.await
|
||||
.expect("part transaction should be prepared");
|
||||
disk.rename_file("tmp", "upload/part.1", "bucket", "object/part.1")
|
||||
.await
|
||||
.expect("data publication should succeed");
|
||||
disk.settle_part_transaction("bucket", "object/part.1", PartTransactionAction::Rollback)
|
||||
.await
|
||||
.expect("part transaction should roll back");
|
||||
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", "object/part.1")
|
||||
.await
|
||||
.expect("old part data should be restored"),
|
||||
Bytes::from_static(b"old data")
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", "object/part.1.meta")
|
||||
.await
|
||||
.expect("old part metadata should be restored"),
|
||||
Bytes::from_static(b"old metadata")
|
||||
);
|
||||
}
|
||||
|
||||
struct BlockingScanWriter {
|
||||
|
||||
@@ -38,6 +38,16 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
|
||||
pub const HEALING_MARKER_PATH: &str = "healing.bin";
|
||||
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
|
||||
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
|
||||
pub const PART_TRANSACTION_NEW_META: &str = "new.meta";
|
||||
pub const PART_TRANSACTION_OLD_META: &str = "old.meta";
|
||||
pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
|
||||
|
||||
pub fn part_transaction_path(part_path: &str) -> String {
|
||||
match part_path.rsplit_once('/') {
|
||||
Some((parent, name)) => format!("{parent}/.{name}.rustfs-txn"),
|
||||
None => format!(".{part_path}.rustfs-txn"),
|
||||
}
|
||||
}
|
||||
|
||||
use crate::cluster::rpc::RemoteDisk;
|
||||
use crate::cluster::rpc::build_internode_data_transport_from_env;
|
||||
@@ -62,6 +72,12 @@ pub type DiskStore = Arc<Disk>;
|
||||
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
|
||||
pub type FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PartTransactionAction {
|
||||
Commit,
|
||||
Rollback,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct MmapCopyStageMetrics {
|
||||
pub(crate) path: &'static str,
|
||||
@@ -381,6 +397,35 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
meta: Bytes,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.settle_part_transaction(volume, path, action).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.settle_part_transaction(volume, path, action).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
match self {
|
||||
@@ -659,6 +704,19 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
// ReadFileStream
|
||||
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
|
||||
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()>;
|
||||
async fn prepare_part_transaction(
|
||||
&self,
|
||||
_src_volume: &str,
|
||||
_src_path: &str,
|
||||
_dst_volume: &str,
|
||||
_dst_path: &str,
|
||||
_meta: Bytes,
|
||||
) -> Result<()> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn settle_part_transaction(&self, _volume: &str, _path: &str, _action: PartTransactionAction) -> Result<()> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
|
||||
// VerifyFile
|
||||
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp>;
|
||||
|
||||
@@ -443,7 +443,9 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorith
|
||||
size.div_ceil(shard_size) * algo.size() + size
|
||||
}
|
||||
|
||||
/// Verify an interleaved per-block bitrot shard file.
|
||||
/// Verify an interleaved per-block bitrot shard file and consume the reader
|
||||
/// through EOF. Bytes beyond the encoded length are corruption, even when every
|
||||
/// expected block has a valid hash.
|
||||
///
|
||||
/// The read loop below assumes every block on disk is `[hash][data]` (streaming
|
||||
/// bitrot). It is therefore only valid for the streaming Highway variants, whose
|
||||
@@ -487,6 +489,11 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
|
||||
left -= read;
|
||||
}
|
||||
|
||||
let mut trailing = [0u8; 1];
|
||||
if r.read(&mut trailing).await? != 0 {
|
||||
return Err(std::io::Error::other("bitrot shard file has trailing data"));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -860,12 +867,19 @@ mod tests {
|
||||
.await
|
||||
.expect("valid bitrot shard file should verify");
|
||||
|
||||
let mut truncated = written.clone();
|
||||
truncated.pop();
|
||||
let err = bitrot_verify(Cursor::new(truncated), written.len(), data.len(), algo.clone(), shard_size)
|
||||
.await
|
||||
.expect_err("one-byte-short shard file must be rejected while reading");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
|
||||
|
||||
let err = bitrot_verify(Cursor::new(written.clone()), written.len() - 1, data.len(), algo.clone(), shard_size)
|
||||
.await
|
||||
.expect_err("wrong file size must be rejected before reading data");
|
||||
assert!(err.to_string().contains("size mismatch"));
|
||||
|
||||
let mut corrupt = written;
|
||||
let mut corrupt = written.clone();
|
||||
let last = corrupt.len() - 1;
|
||||
corrupt[last] ^= 0x80;
|
||||
let err = bitrot_verify(
|
||||
@@ -878,6 +892,21 @@ mod tests {
|
||||
.await
|
||||
.expect_err("hash mismatch must reject corrupted data");
|
||||
assert!(err.to_string().contains("hash mismatch"));
|
||||
|
||||
for trailing in [vec![0xa5], vec![0xa5; 17]] {
|
||||
let mut oversized = written.clone();
|
||||
oversized.extend_from_slice(&trailing);
|
||||
let err = bitrot_verify(
|
||||
Cursor::new(oversized),
|
||||
written.len(),
|
||||
data.len(),
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
shard_size,
|
||||
)
|
||||
.await
|
||||
.expect_err("trailing bytes after a valid encoded shard must be rejected");
|
||||
assert!(err.to_string().contains("trailing data"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -909,6 +938,22 @@ mod tests {
|
||||
assert!(err.to_string().contains("hash mismatch"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_verify_accepts_exact_legacy_streaming_layout() {
|
||||
let data = b"legacy streaming bitrot";
|
||||
let shard_size = 8;
|
||||
let algo = HashAlgorithm::HighwayHash256SLegacy;
|
||||
let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), shard_size, algo.clone());
|
||||
for chunk in data.chunks(shard_size) {
|
||||
writer.write(chunk).await.expect("legacy streaming shard should encode");
|
||||
}
|
||||
let written = writer.into_inner().into_inner();
|
||||
|
||||
bitrot_verify(Cursor::new(written.clone()), written.len(), data.len(), algo, shard_size)
|
||||
.await
|
||||
.expect("exact legacy streaming shard should remain valid");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_all_vectored_retries_partial_hash_and_data_writes_and_rejects_zero_write() {
|
||||
let mut writer = LimitedVectoredWriter {
|
||||
|
||||
@@ -28,7 +28,6 @@ use md5::{Digest, Md5};
|
||||
use rustfs_kms::{KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext};
|
||||
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use serde::Deserialize;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use sha2::Sha256;
|
||||
@@ -44,6 +43,7 @@ const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
|
||||
const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size";
|
||||
const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size";
|
||||
const DEFAULT_SSE_ALGORITHM: &str = "AES256";
|
||||
const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
const DARE_PAYLOAD_SIZE: i64 = 64 * 1024;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
@@ -1716,16 +1716,39 @@ fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context
|
||||
|
||||
fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
|
||||
let encrypted_dek = std::str::from_utf8(encrypted_dek).map_err(|_| Error::other("managed DEK is not valid UTF-8"))?;
|
||||
let parts: Vec<&str> = encrypted_dek.split(':').collect();
|
||||
if parts.len() != 2 {
|
||||
return Err(Error::other("invalid managed DEK format"));
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct LocalSseDekEnvelope<'a> {
|
||||
version: u8,
|
||||
nonce: &'a str,
|
||||
ciphertext: &'a str,
|
||||
}
|
||||
|
||||
let (nonce, ciphertext) = match serde_json::from_str::<LocalSseDekEnvelope<'_>>(encrypted_dek) {
|
||||
Ok(envelope) => {
|
||||
if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION {
|
||||
return Err(Error::other(format!("unsupported managed DEK format version: {}", envelope.version)));
|
||||
}
|
||||
(envelope.nonce, envelope.ciphertext)
|
||||
}
|
||||
Err(_) => {
|
||||
// DEPRECATED: read-only compatibility for persisted colon-delimited DEKs.
|
||||
// RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs.
|
||||
let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else {
|
||||
return Err(Error::other("invalid managed DEK format"));
|
||||
};
|
||||
if ciphertext.contains(':') {
|
||||
return Err(Error::other("invalid managed DEK format"));
|
||||
}
|
||||
(nonce, ciphertext)
|
||||
}
|
||||
};
|
||||
|
||||
let nonce_vec = BASE64_STANDARD
|
||||
.decode(parts[0])
|
||||
.decode(nonce)
|
||||
.map_err(|_| Error::other("invalid managed DEK nonce"))?;
|
||||
let ciphertext = BASE64_STANDARD
|
||||
.decode(parts[1])
|
||||
.decode(ciphertext)
|
||||
.map_err(|_| Error::other("invalid managed DEK ciphertext"))?;
|
||||
|
||||
let nonce_array: [u8; 12] = nonce_vec
|
||||
@@ -2324,9 +2347,36 @@ mod tests {
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
let nonce = Nonce::from([0u8; 12]);
|
||||
let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt managed dek");
|
||||
serde_json::json!({
|
||||
"version": LOCAL_SSE_DEK_FORMAT_VERSION,
|
||||
"nonce": BASE64_STANDARD.encode(nonce),
|
||||
"ciphertext": BASE64_STANDARD.encode(ciphertext),
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn encrypt_legacy_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String {
|
||||
let key = Key::<Aes256Gcm>::from(master_key);
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
let nonce = Nonce::from([0u8; 12]);
|
||||
let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt legacy managed dek");
|
||||
format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decrypt_rustfs_local_sse_dek_rejects_unknown_json_version() {
|
||||
let envelope = serde_json::json!({
|
||||
"version": LOCAL_SSE_DEK_FORMAT_VERSION + 1,
|
||||
"nonce": BASE64_STANDARD.encode([0u8; 12]),
|
||||
"ciphertext": BASE64_STANDARD.encode([0u8; 48]),
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let error =
|
||||
decrypt_rustfs_local_sse_dek(envelope.as_bytes()).expect_err("unknown local SSE DEK versions must fail closed");
|
||||
assert!(error.to_string().contains("unsupported managed DEK format version"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
fn seal_managed_s3_object_key_for_test(
|
||||
bucket: &str,
|
||||
@@ -2433,7 +2483,7 @@ mod tests {
|
||||
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([7u8; 32])))], async {
|
||||
let data_key = [0x24; 32];
|
||||
let base_nonce = [0x14; 12];
|
||||
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [7u8; 32]);
|
||||
let encrypted_dek = encrypt_legacy_managed_dek_for_test(data_key, [7u8; 32]);
|
||||
let metadata = HashMap::from([
|
||||
(
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
|
||||
|
||||
@@ -46,7 +46,10 @@ use crate::diagnostics::get::{
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::disk::OldCurrentSize;
|
||||
use crate::disk::{
|
||||
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
||||
part_transaction_path,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
use crate::io_support::bitrot::{
|
||||
@@ -3176,6 +3179,155 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
async fn recover_part_transaction(&self, dst_object: &str, write_quorum: usize) -> disk::error::Result<bool> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
|
||||
let current_meta_path = format!("{dst_object}.meta");
|
||||
|
||||
let reads = disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let transaction_meta_path = transaction_meta_path.clone();
|
||||
let rollback_path = rollback_path.clone();
|
||||
let current_meta_path = current_meta_path.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Ok((None, None, false));
|
||||
};
|
||||
let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
|
||||
Ok(_) => true,
|
||||
Err(DiskError::FileNotFound) => false,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, ¤t_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None,
|
||||
Err(_) => None,
|
||||
};
|
||||
Ok((transaction_meta, current_meta, rollback))
|
||||
}
|
||||
});
|
||||
let observations = join_all(reads).await.into_iter().collect::<disk::error::Result<Vec<_>>>()?;
|
||||
if observations.iter().all(|(transaction, _, _)| transaction.is_none()) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut current_counts: HashMap<Bytes, usize> = HashMap::new();
|
||||
for (_, current, _) in &observations {
|
||||
if let Some(current) = current {
|
||||
*current_counts.entry(current.clone()).or_default() += 1;
|
||||
}
|
||||
}
|
||||
let current_quorum = current_counts
|
||||
.into_iter()
|
||||
.find_map(|(meta, count)| (count >= write_quorum).then_some(meta));
|
||||
|
||||
let old_meta_path = format!("{transaction_path}/{PART_TRANSACTION_OLD_META}");
|
||||
let old_meta_absent_path = format!("{transaction_path}/old.meta.absent");
|
||||
let decisions = observations
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, (transaction_meta, _, rollback))| {
|
||||
transaction_meta.as_ref().map(|meta| (index, meta.clone(), *rollback))
|
||||
})
|
||||
.map(|(index, transaction_meta, rollback)| {
|
||||
let disk = disks[index].clone();
|
||||
let old_meta_path = old_meta_path.clone();
|
||||
let old_meta_absent_path = old_meta_absent_path.clone();
|
||||
let current_quorum = current_quorum.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let action = if rollback {
|
||||
PartTransactionAction::Rollback
|
||||
} else if current_quorum.as_ref() == Some(&transaction_meta) {
|
||||
PartTransactionAction::Commit
|
||||
} else if let Some(current_quorum) = current_quorum {
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_path).await {
|
||||
Ok(old_meta) if old_meta == current_quorum => PartTransactionAction::Rollback,
|
||||
Ok(_) => PartTransactionAction::Commit,
|
||||
Err(DiskError::FileNotFound) => {
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_absent_path).await {
|
||||
Ok(_) => PartTransactionAction::Commit,
|
||||
Err(_) => return Err(DiskError::FileCorrupt),
|
||||
}
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
} else {
|
||||
PartTransactionAction::Rollback
|
||||
};
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_object, action)
|
||||
.await?;
|
||||
Ok(action == PartTransactionAction::Commit)
|
||||
}
|
||||
});
|
||||
|
||||
let results = join_all(decisions).await;
|
||||
if let Some(err) = results.iter().find_map(|result| result.as_ref().err()) {
|
||||
return Err(err.clone());
|
||||
}
|
||||
Ok(results.iter().any(|result| matches!(result, Ok(true))))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn recover_part_transactions(
|
||||
&self,
|
||||
part_path: &str,
|
||||
read_quorum: usize,
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let listings = join_all(disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
|
||||
.await
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
|
||||
let mut transaction_parts = HashSet::new();
|
||||
let mut errs = Vec::with_capacity(listings.len());
|
||||
for listing in listings {
|
||||
match listing {
|
||||
Ok(entries) => {
|
||||
errs.push(None);
|
||||
for entry in entries {
|
||||
let name = entry.trim_end_matches('/');
|
||||
let Some(part_number) = name
|
||||
.strip_prefix(".part.")
|
||||
.and_then(|name| name.strip_suffix(".rustfs-txn"))
|
||||
.and_then(|number| number.parse::<usize>().ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
transaction_parts.insert(part_number);
|
||||
}
|
||||
}
|
||||
Err(err) => errs.push(Some(err)),
|
||||
}
|
||||
}
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
for part_number in transaction_parts {
|
||||
self.recover_part_transaction(&format!("{part_path}part.{part_number}"), write_quorum)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(disks, meta))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(in crate::set_disk) async fn rename_part(
|
||||
@@ -3189,23 +3341,46 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
|
||||
) -> disk::error::Result<Vec<Option<DiskStore>>> {
|
||||
self.recover_part_transaction(dst_object, write_quorum).await?;
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
let dst_object = Arc::new(dst_object.to_string());
|
||||
|
||||
// Do NOT pre-delete the destination part before renaming: the per-disk
|
||||
// `rename_part` replaces `part.N` atomically (std::fs::rename) and rewrites
|
||||
// `part.N.meta`, so the pre-delete is redundant — and destructive. It
|
||||
// opened a window where an already-committed (ACKed) part was removed on
|
||||
// every disk before the new rename landed, so a re-upload that then failed
|
||||
// quorum destroyed the committed part outright (backlog#853 / #799 B4).
|
||||
// The atomic rename overwrites in place; on quorum failure below we roll
|
||||
// the destination back.
|
||||
let prepare_tasks = disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
let meta = meta.clone();
|
||||
async move {
|
||||
let disk = disk?;
|
||||
Some(
|
||||
disk.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
});
|
||||
let prepare_results = join_all(prepare_tasks).await;
|
||||
let prepare_errs = prepare_results
|
||||
.into_iter()
|
||||
.map(|result| match result {
|
||||
Some(Ok(())) => None,
|
||||
Some(Err(err)) => Some(err),
|
||||
None => Some(DiskError::DiskNotFound),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let prepared_disks = Self::eval_disks(disks, &prepare_errs);
|
||||
if reduce_write_quorum_errs(&prepare_errs, OBJECT_OP_IGNORED_ERRS, write_quorum).is_some() {
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?;
|
||||
return Err(DiskError::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
let futures = disks.iter().map(|disk| {
|
||||
let futures = prepared_disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let meta = meta.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
@@ -3255,19 +3430,42 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
let reduced_err = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
|
||||
if let Some(err) = reduced_err {
|
||||
let rollbacks = prepared_disks.iter().filter_map(|disk| {
|
||||
disk.clone().map(|disk| {
|
||||
let dst_object = dst_object.clone();
|
||||
async move {
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Rollback)
|
||||
.await
|
||||
}
|
||||
})
|
||||
});
|
||||
let rollback_results = join_all(rollbacks).await;
|
||||
self.recover_part_transaction(&dst_object, write_quorum).await?;
|
||||
if let Some(rollback_err) = rollback_results.iter().find_map(|result| result.as_ref().err()) {
|
||||
warn!(error = %rollback_err, "rename_part rollback did not settle on every prepared disk");
|
||||
}
|
||||
if let Some(context) = quorum_context {
|
||||
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
|
||||
} else {
|
||||
warn!("rename_part errs {:?}", &errs);
|
||||
}
|
||||
self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
|
||||
.await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let disks = Self::eval_disks(disks, &errs);
|
||||
Ok(disks)
|
||||
let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
|
||||
if !committed {
|
||||
let err = DiskError::ErasureWriteQuorum;
|
||||
if let Some(context) = quorum_context {
|
||||
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
|
||||
} else {
|
||||
warn!("rename_part errs {:?}", &errs);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(Self::eval_disks(&prepared_disks, &errs))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn eval_disks(disks: &[Option<DiskStore>], errs: &[Option<DiskError>]) -> Vec<Option<DiskStore>> {
|
||||
|
||||
@@ -690,6 +690,8 @@ mod ops;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||
mod read;
|
||||
mod replication;
|
||||
pub(crate) mod shard_source;
|
||||
|
||||
@@ -199,6 +199,32 @@ mod tests {
|
||||
.expect("healthy temp shard should verify");
|
||||
assert_eq!(verified, 1);
|
||||
|
||||
for trailing in [vec![0xa5], vec![0xa5; 17]] {
|
||||
let mut oversized = encoded.to_vec();
|
||||
oversized.extend_from_slice(&trailing);
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, path, Bytes::from(oversized))
|
||||
.await
|
||||
.expect("oversized temp shard should be staged");
|
||||
let err = verify_written_bitrot_shards(
|
||||
&[Some(disk.clone())],
|
||||
None,
|
||||
BitrotSelfVerifyTarget {
|
||||
operation: "put_object",
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
part_number: None,
|
||||
volume: RUSTFS_META_TMP_BUCKET,
|
||||
path,
|
||||
logical_shard_size,
|
||||
shard_size,
|
||||
write_quorum: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("disk shard with trailing bytes must not be committed");
|
||||
assert!(err.to_string().contains("trailing data"));
|
||||
}
|
||||
|
||||
let mut corrupt = encoded.to_vec();
|
||||
let last = corrupt.len() - 1;
|
||||
corrupt[last] ^= 0x01;
|
||||
@@ -225,4 +251,40 @@ mod tests {
|
||||
.expect_err("corrupt no-parity temp shard must not be committed");
|
||||
assert!(err.to_string().contains("bitrot self-verify failed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_parity_inline_self_verify_rejects_trailing_bytes() {
|
||||
let (_temp_dirs, disks, _set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
|
||||
let shard_size = 16usize;
|
||||
let payload = b"inline bitrot payload";
|
||||
let encoded = encode_streaming_shard(payload, shard_size).await;
|
||||
let disks = disks.into_iter().map(Some).collect::<Vec<_>>();
|
||||
|
||||
for trailing in [vec![0xa5], vec![0xa5; 17]] {
|
||||
let mut oversized = encoded.to_vec();
|
||||
oversized.extend_from_slice(&trailing);
|
||||
let parts = [FileInfo {
|
||||
data: Some(Bytes::from(oversized)),
|
||||
..Default::default()
|
||||
}];
|
||||
let err = verify_written_bitrot_shards(
|
||||
&disks,
|
||||
Some(&parts),
|
||||
BitrotSelfVerifyTarget {
|
||||
operation: "put_object",
|
||||
bucket: "bucket",
|
||||
object: "inline-object",
|
||||
part_number: None,
|
||||
volume: RUSTFS_META_TMP_BUCKET,
|
||||
path: "unused-for-inline",
|
||||
logical_shard_size: payload.len(),
|
||||
shard_size,
|
||||
write_quorum: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("inline shard with trailing bytes must not be committed");
|
||||
assert!(err.to_string().contains("trailing data"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,71 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
const EVENT_HEAL_OBJECT_RENAME: &str = "heal_object_rename";
|
||||
const HEAL_RENAME_INCOMPLETE: &str = "heal rename incomplete";
|
||||
|
||||
#[cfg(test)]
|
||||
static HEAL_RENAME_FAILURES: std::sync::Mutex<Vec<(String, String, usize)>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
#[cfg(test)]
|
||||
struct HealRenameFailureScope {
|
||||
bucket: String,
|
||||
object: String,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl HealRenameFailureScope {
|
||||
fn install(bucket: &str, object: &str, disk_indexes: &[usize]) -> Self {
|
||||
let mut failures = HEAL_RENAME_FAILURES
|
||||
.lock()
|
||||
.expect("heal rename failure registry should not poison");
|
||||
assert!(
|
||||
!failures
|
||||
.iter()
|
||||
.any(|(registered_bucket, registered_object, _)| { registered_bucket == bucket && registered_object == object }),
|
||||
"heal rename failures must be installed once per object"
|
||||
);
|
||||
failures.extend(
|
||||
disk_indexes
|
||||
.iter()
|
||||
.map(|index| (bucket.to_string(), object.to_string(), *index)),
|
||||
);
|
||||
Self {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for HealRenameFailureScope {
|
||||
fn drop(&mut self) {
|
||||
HEAL_RENAME_FAILURES
|
||||
.lock()
|
||||
.expect("heal rename failure registry should not poison")
|
||||
.retain(|(bucket, object, _)| bucket != &self.bucket || object != &self.object);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn should_fail_heal_rename(bucket: &str, object: &str, disk_index: usize) -> bool {
|
||||
let mut failures = HEAL_RENAME_FAILURES
|
||||
.lock()
|
||||
.expect("heal rename failure registry should not poison");
|
||||
if let Some(position) = failures
|
||||
.iter()
|
||||
.position(|entry| entry == &(bucket.to_string(), object.to_string(), disk_index))
|
||||
{
|
||||
failures.swap_remove(position);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn should_fail_heal_rename(_bucket: &str, _object: &str, _disk_index: usize) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct PartFailureSummary {
|
||||
@@ -659,8 +724,12 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
// Rename from tmp location to the actual location.
|
||||
// MinIO stops on the first RenameData error. RustFS intentionally
|
||||
// continues per target, but reports any residue after all attempts
|
||||
// so successful repairs survive and failed targets remain retryable.
|
||||
let mut rename_attempts = 0usize;
|
||||
let mut rename_successes = 0usize;
|
||||
let mut healed_disks = vec![None; out_dated_disks.len()];
|
||||
for (index, outdated_disk) in out_dated_disks.iter().enumerate() {
|
||||
if let Some(disk) = outdated_disk {
|
||||
rename_attempts += 1;
|
||||
@@ -669,9 +738,18 @@ impl SetDisks {
|
||||
// Attempt a rename now from healed data to final location.
|
||||
parts_metadata[index].set_healing();
|
||||
|
||||
let rename_result = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object)
|
||||
.await;
|
||||
let rename_result = if should_fail_heal_rename(bucket, object, index) {
|
||||
Err(DiskError::Unexpected)
|
||||
} else {
|
||||
disk.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_id,
|
||||
parts_metadata[index].clone(),
|
||||
bucket,
|
||||
object,
|
||||
)
|
||||
.await
|
||||
};
|
||||
|
||||
if let Err(err) = &rename_result {
|
||||
warn!(
|
||||
@@ -690,6 +768,7 @@ impl SetDisks {
|
||||
);
|
||||
} else {
|
||||
rename_successes += 1;
|
||||
healed_disks[index] = Some(disk.clone());
|
||||
if parts_metadata[index].is_remote() {
|
||||
let rm_data_dir =
|
||||
parts_metadata[index].data_dir.expect("operation should succeed").to_string();
|
||||
@@ -734,15 +813,18 @@ impl SetDisks {
|
||||
.await
|
||||
.map_err(DiskError::other)?;
|
||||
|
||||
if rename_attempts > 0 && rename_successes == 0 {
|
||||
self.record_healed_capacity_scope(&healed_disks);
|
||||
|
||||
if rename_successes < rename_attempts {
|
||||
return Ok((
|
||||
result,
|
||||
Some(DiskError::other(format!("all healed data rename attempts failed for {bucket}/{object}"))),
|
||||
Some(DiskError::other(format!(
|
||||
"{HEAL_RENAME_INCOMPLETE}: {rename_successes} of {rename_attempts} targets committed for \
|
||||
{bucket}/{object}"
|
||||
))),
|
||||
));
|
||||
}
|
||||
|
||||
self.record_healed_capacity_scope(&out_dated_disks);
|
||||
|
||||
// The object is healthy here; sweep any data dirs left behind
|
||||
// by pre-#3510 unversioned overwrites, which the dangling paths
|
||||
// above never touch (issues #3231, #3191). Best effort — a
|
||||
@@ -1312,11 +1394,13 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
|
||||
#[cfg(test)]
|
||||
mod heal_result_report_tests {
|
||||
use super::SetDisks;
|
||||
use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::format::FormatV3;
|
||||
use crate::disk::{DiskOption, DiskStore, new_disk};
|
||||
use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk};
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::{config::storageclass, store::init_format::save_format_file};
|
||||
@@ -1406,6 +1490,171 @@ mod heal_result_report_tests {
|
||||
(dir, set)
|
||||
}
|
||||
|
||||
async fn non_trash_tmp_entries(temp_dirs: &[TempDir]) -> Vec<String> {
|
||||
let mut entries = Vec::new();
|
||||
for dir in temp_dirs {
|
||||
let tmp = dir.path().join(RUSTFS_META_TMP_BUCKET);
|
||||
let mut read_dir = match tokio::fs::read_dir(&tmp).await {
|
||||
Ok(read_dir) => read_dir,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(err) => panic!("tmp directory should be readable: {err}"),
|
||||
};
|
||||
while let Some(entry) = read_dir.next_entry().await.expect("tmp entry should be readable") {
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
if name != ".trash" {
|
||||
entries.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
entries
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn heal_rename_outcome_matrix_reports_partial_and_retries_failed_targets() {
|
||||
for (case, failed_attempts, expect_error) in [
|
||||
("ok-ok", Vec::new(), false),
|
||||
("ok-err", vec![1], true),
|
||||
("err-ok", vec![0], true),
|
||||
("err-err", vec![0, 1], true),
|
||||
] {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = format!("heal-rename-{case}");
|
||||
let object = "object.bin";
|
||||
for disk in &disks {
|
||||
disk.make_volume(&bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let payload = vec![0x5a; 1024 * 1024];
|
||||
let mut reader = PutObjReader::from_vec(payload);
|
||||
set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let source = disks[2]
|
||||
.read_version("", &bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("source metadata should be readable");
|
||||
let data_dir = source.data_dir.expect("non-inline source should have a data directory");
|
||||
let tmp_entries_before_heal = non_trash_tmp_entries(&temp_dirs).await;
|
||||
let target_slots = {
|
||||
let mut slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1];
|
||||
slots.sort_unstable();
|
||||
slots
|
||||
};
|
||||
let failed_slots = failed_attempts
|
||||
.iter()
|
||||
.map(|attempt| target_slots[*attempt])
|
||||
.collect::<Vec<_>>();
|
||||
let failed_physical_indexes = [0, 1]
|
||||
.into_iter()
|
||||
.filter(|index| failed_slots.contains(&(source.erasure.distribution[*index] - 1)))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
for index in [0, 1] {
|
||||
tokio::fs::remove_file(
|
||||
temp_dirs[index]
|
||||
.path()
|
||||
.join(&bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("target shard should be removed before heal");
|
||||
}
|
||||
|
||||
let failure_scope = HealRenameFailureScope::install(&bucket, object, &failed_slots);
|
||||
let (first_result, first_error) = set
|
||||
.heal_object(
|
||||
&bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("heal should report its per-target rename outcome");
|
||||
drop(failure_scope);
|
||||
|
||||
assert_eq!(first_error.is_some(), expect_error, "{case}: aggregate status must match target outcomes");
|
||||
if let Some(error) = first_error {
|
||||
let error = error.to_string();
|
||||
assert!(
|
||||
error.contains(HEAL_RENAME_INCOMPLETE),
|
||||
"{case}: partial/all failure must have an explicit retryable status: {error}"
|
||||
);
|
||||
assert!(
|
||||
error.contains(&format!("{} of 2 targets committed", 2 - failed_slots.len())),
|
||||
"{case}: aggregate status must distinguish partial from all-target failure: {error}"
|
||||
);
|
||||
}
|
||||
for index in [0, 1] {
|
||||
let expected = if failed_physical_indexes.contains(&index) {
|
||||
DriveState::Missing
|
||||
} else {
|
||||
DriveState::Ok
|
||||
};
|
||||
assert_eq!(
|
||||
first_result.after.drives[index].state,
|
||||
expected.to_string(),
|
||||
"{case}: after.drives must reflect the actual rename outcome at index {index}"
|
||||
);
|
||||
assert_eq!(
|
||||
temp_dirs[index]
|
||||
.path()
|
||||
.join(&bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1")
|
||||
.exists(),
|
||||
!failed_physical_indexes.contains(&index),
|
||||
"{case}: tmp cleanup must neither delete committed shards nor expose failed targets"
|
||||
);
|
||||
}
|
||||
let tmp_entries_after_heal = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert!(
|
||||
tmp_entries_after_heal
|
||||
.iter()
|
||||
.all(|entry| tmp_entries_before_heal.contains(entry)),
|
||||
"{case}: first heal must not leave a new temporary shard: {tmp_entries_after_heal:?}"
|
||||
);
|
||||
|
||||
if !failed_slots.is_empty() {
|
||||
let (retry_result, retry_error) = set
|
||||
.heal_object(
|
||||
&bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
no_lock: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("second heal should retry failed targets");
|
||||
assert!(retry_error.is_none(), "{case}: second heal should complete remaining targets");
|
||||
for index in [0, 1] {
|
||||
assert_eq!(
|
||||
retry_result.after.drives[index].state,
|
||||
DriveState::Ok.to_string(),
|
||||
"{case}: second heal must converge target {index}"
|
||||
);
|
||||
}
|
||||
let tmp_entries_after_retry = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert!(
|
||||
tmp_entries_after_retry
|
||||
.iter()
|
||||
.all(|entry| tmp_entries_before_heal.contains(entry)),
|
||||
"{case}: retry must not leave a new temporary shard: {tmp_entries_after_retry:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Regression for #955: an offline disk must contribute exactly one drive
|
||||
// record. Before the fix the offline branch fell through and pushed a second
|
||||
// (Corrupt) record for the same disk, so `before/after.drives` grew to
|
||||
|
||||
@@ -69,6 +69,13 @@ struct HealWalkCollector {
|
||||
}
|
||||
|
||||
impl HealWalkCollector {
|
||||
fn lock_objects(&self) -> disk::error::Result<std::sync::MutexGuard<'_, Vec<HealWalkObject>>> {
|
||||
self.objects.lock().map_err(|_| {
|
||||
self.cancel.cancel();
|
||||
DiskError::FileCorrupt
|
||||
})
|
||||
}
|
||||
|
||||
/// Expand one resolved entry into its versions and record it. Cancels the
|
||||
/// walk once EITHER page bound (distinct object names OR expanded versions)
|
||||
/// is met — always at a sorted object-key boundary so a heavily-versioned
|
||||
@@ -105,7 +112,9 @@ impl HealWalkCollector {
|
||||
|
||||
let added = versions.len();
|
||||
let (objs_len, ver_total) = {
|
||||
let mut objects = self.objects.lock().unwrap();
|
||||
let Ok(mut objects) = self.lock_objects() else {
|
||||
return;
|
||||
};
|
||||
objects.push(HealWalkObject {
|
||||
name: entry.name,
|
||||
versions,
|
||||
@@ -239,7 +248,10 @@ impl SetDisks {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let objects = std::mem::take(&mut *collector.objects.lock().unwrap());
|
||||
let objects = {
|
||||
let mut objects = collector.lock_objects()?;
|
||||
std::mem::take(&mut *objects)
|
||||
};
|
||||
let truncated = collector.truncated.load(Ordering::SeqCst);
|
||||
Ok(finalize_heal_walk_page(objects, forward_to, truncated))
|
||||
}
|
||||
@@ -310,4 +322,28 @@ mod tests {
|
||||
assert_eq!(next_forward, None, "a complete final page must not carry a resume cursor");
|
||||
assert!(!truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poisoned_collector_state_cancels_the_walk() {
|
||||
let collector = Arc::new(HealWalkCollector {
|
||||
bucket: "bucket".to_string(),
|
||||
batch_objects: 2,
|
||||
version_budget: 2,
|
||||
objects: Mutex::new(Vec::new()),
|
||||
version_total: AtomicUsize::new(0),
|
||||
truncated: AtomicBool::new(false),
|
||||
cancel: CancellationToken::new(),
|
||||
});
|
||||
let poison_target = Arc::clone(&collector);
|
||||
let _ = std::thread::spawn(move || {
|
||||
let _guard = poison_target.objects.lock().expect("fresh mutex should lock");
|
||||
panic!("poison heal collector");
|
||||
})
|
||||
.join();
|
||||
|
||||
let error = collector.lock_objects().expect_err("poisoned collection must fail closed");
|
||||
assert_eq!(error, DiskError::FileCorrupt);
|
||||
assert!(collector.cancel.is_cancelled(), "a poisoned page collector must cancel its walk");
|
||||
assert!(collector.objects.lock().is_err(), "poisoned state must remain fail-closed");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1289,8 +1289,18 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.no_lock && object_lock_guard.is_none() {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
let upload_guard = self
|
||||
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
|
||||
.await?;
|
||||
|
||||
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
|
||||
let (mut fi, mut files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
||||
let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
||||
let has_layout_candidate = range_seek_rollout_enabled
|
||||
&& fi
|
||||
.data_dir
|
||||
@@ -1303,22 +1313,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
&token,
|
||||
)
|
||||
});
|
||||
let upload_guard = if has_layout_candidate {
|
||||
if object_lock_guard.is_none() {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
let guard = self
|
||||
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
|
||||
.await?;
|
||||
(fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
|
||||
guard
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let quorum_validated_layout_token = upload_guard.as_ref().and_then(|_| {
|
||||
let quorum_validated_layout_token = if has_layout_candidate {
|
||||
fi.data_dir
|
||||
.filter(|data_dir| !data_dir.is_nil())
|
||||
.map(|data_dir| data_dir.to_string())
|
||||
@@ -1329,7 +1324,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
token,
|
||||
)
|
||||
})
|
||||
});
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut fi.metadata,
|
||||
crate::object_api::ENCRYPTED_PART_LAYOUT_CANDIDATE_SUFFIX,
|
||||
@@ -1356,6 +1353,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
|
||||
|
||||
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
|
||||
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
|
||||
let part_meta_paths = uploaded_parts
|
||||
.iter()
|
||||
@@ -1713,12 +1713,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.no_lock && object_lock_guard.is_none() {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
// Phase 2 (backlog#899): fence the commit on lock loss before any destructive
|
||||
// step. If the refresh heartbeat has observed a refresh-quorum loss, another
|
||||
// writer may have re-acquired this object's lock; proceeding would race a
|
||||
@@ -2183,6 +2177,201 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
async fn assert_quorum_minus_one_retry_preserves_completable_part(
|
||||
disk_count: usize,
|
||||
parity: usize,
|
||||
success_indices: &[usize],
|
||||
) {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(disk_count, 0, parity).await;
|
||||
let bucket = format!("multipart-retry-{disk_count}-{}", success_indices[0]);
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, &bucket).await;
|
||||
|
||||
let payload = vec![0x41; 1 << 20];
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, &bucket, object, &payload, &ObjectOptions::default()).await;
|
||||
let (upload_meta, _) = set_disks
|
||||
.check_upload_id_exists(&bucket, object, &upload_id, false)
|
||||
.await
|
||||
.expect("staged upload metadata should be readable");
|
||||
let upload_path = SetDisks::get_upload_id_dir(&bucket, object, &upload_id);
|
||||
let write_quorum = upload_meta.write_quorum(set_disks.default_write_quorum());
|
||||
assert_eq!(success_indices.len() + 1, write_quorum);
|
||||
let part_path = format!(
|
||||
"{}/{}/part.1",
|
||||
upload_path,
|
||||
upload_meta.data_dir.expect("multipart upload should have a data directory")
|
||||
);
|
||||
let acknowledged_meta = disk_stores[0]
|
||||
.read_all(RUSTFS_META_MULTIPART_BUCKET, &format!("{part_path}.meta"))
|
||||
.await
|
||||
.expect("acknowledged part metadata should be readable");
|
||||
let retry_path = format!("{}/part.1", Uuid::new_v4());
|
||||
|
||||
let mut retry_disks = vec![None; disk_stores.len()];
|
||||
for &index in success_indices {
|
||||
disk_stores[index]
|
||||
.write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"retry shard"))
|
||||
.await
|
||||
.expect("retry shard should be staged");
|
||||
retry_disks[index] = Some(disk_stores[index].clone());
|
||||
}
|
||||
|
||||
let err = set_disks
|
||||
.rename_part(
|
||||
&retry_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&retry_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
acknowledged_meta,
|
||||
write_quorum,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("quorum-minus-one renamed shards must remain below write quorum");
|
||||
assert_eq!(err, DiskError::ErasureWriteQuorum);
|
||||
|
||||
for (index, disk) in disk_stores.iter().enumerate() {
|
||||
assert!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path).await.is_ok(),
|
||||
"old acknowledged shard on disk {index} must survive"
|
||||
);
|
||||
}
|
||||
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(&bucket, object, &upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the old acknowledged part should remain completable");
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object should be readable");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("completed object should stream fully");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upload_part_retry_quorum_failure_preserves_old_part_across_ec_geometries() {
|
||||
assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[0, 1]).await;
|
||||
assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[2, 3]).await;
|
||||
assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[0, 1, 2]).await;
|
||||
assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[3, 4, 5]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_upload_recovers_interrupted_part_retry() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(6, 0, 2).await;
|
||||
let bucket = "multipart-retry-recovery";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let payload = vec![0x42; 1 << 20];
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
|
||||
let (upload_meta, _) = set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload_id, false)
|
||||
.await
|
||||
.expect("staged upload metadata should be readable");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let part_path = format!(
|
||||
"{}/{}/part.1",
|
||||
upload_path,
|
||||
upload_meta.data_dir.expect("multipart upload should have a data directory")
|
||||
);
|
||||
let retry_meta = Bytes::from_static(b"interrupted retry metadata");
|
||||
|
||||
for (index, disk) in disk_stores.iter().enumerate().take(3) {
|
||||
let retry_path = format!("{}/part.1", Uuid::new_v4());
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"interrupted retry shard"))
|
||||
.await
|
||||
.expect("retry shard should be staged");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&retry_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
retry_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("part transaction should be prepared");
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&retry_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
retry_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("retry shard {index} should be published: {err}"));
|
||||
}
|
||||
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completion should roll back the interrupted quorum-minus-one retry");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object should be readable");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("completed object should stream fully");
|
||||
assert_eq!(restored, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_part_quorum_failure_without_old_part_removes_new_shards() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
|
||||
let src_path = format!("{}/part.1", Uuid::new_v4());
|
||||
let dst_path = format!("{}/part.1", Uuid::new_v4());
|
||||
|
||||
let mut retry_disks = vec![None; disk_stores.len()];
|
||||
for index in [0, 1] {
|
||||
disk_stores[index]
|
||||
.write_all(RUSTFS_META_TMP_BUCKET, &src_path, Bytes::from_static(b"new shard"))
|
||||
.await
|
||||
.expect("new shard should be staged");
|
||||
retry_disks[index] = Some(disk_stores[index].clone());
|
||||
}
|
||||
|
||||
let err = set_disks
|
||||
.rename_part(
|
||||
&retry_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&src_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&dst_path,
|
||||
Bytes::from_static(b"retry metadata"),
|
||||
3,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("two renamed shards must remain below write quorum");
|
||||
assert_eq!(err, DiskError::ErasureWriteQuorum);
|
||||
for (index, disk) in disk_stores.iter().enumerate() {
|
||||
assert!(
|
||||
matches!(disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &dst_path).await, Err(DiskError::FileNotFound)),
|
||||
"failed first upload must not leave a destination on disk {index}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn make_multipart_lock_test_set_disks() -> Arc<SetDisks> {
|
||||
let endpoints = vec![
|
||||
Endpoint::try_from("http://127.0.0.1:9000/data").expect("first endpoint should parse"),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -207,11 +207,9 @@ impl SetDisks {
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<Vec<FileInfo>> {
|
||||
// Use existing disk selection logic
|
||||
let disks = self.disks.read().await;
|
||||
let required_reads = self.format.erasure.sets.len();
|
||||
let disks = self.disks.read().await.clone();
|
||||
let required_reads = self.default_read_quorum();
|
||||
|
||||
// Clone parameters outside the closure to avoid lifetime issues
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.to_string();
|
||||
let version_id = version_id.to_string();
|
||||
@@ -220,7 +218,6 @@ impl SetDisks {
|
||||
let processor = runtime_sources::batch_processors().read_processor();
|
||||
let tasks: Vec<_> = disks
|
||||
.iter()
|
||||
.take(required_reads + 2) // Read a few extra for reliability
|
||||
.filter_map(|disk| {
|
||||
disk.as_ref().map(|d| {
|
||||
let disk = d.clone();
|
||||
@@ -234,10 +231,10 @@ impl SetDisks {
|
||||
})
|
||||
.collect();
|
||||
|
||||
match processor.execute_batch_with_quorum(tasks, required_reads).await {
|
||||
Ok(results) => Ok(results),
|
||||
Err(_) => Err(DiskError::FileNotFound.into()), // Use existing error type
|
||||
}
|
||||
processor
|
||||
.execute_batch_with_quorum(tasks, required_reads)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
@@ -1960,6 +1957,42 @@ mod metadata_cache_tests {
|
||||
(dir, disk)
|
||||
}
|
||||
|
||||
async fn read_version_quorum_test_set(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
pool_set_count: usize,
|
||||
readable_disks: usize,
|
||||
) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
|
||||
let mut dirs = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let (dir, disk) = new_read_version_test_disk(bucket).await;
|
||||
if disk_index < readable_disks {
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.erasure.index = fi.erasure.distribution[disk_index];
|
||||
disk.write_metadata(bucket, bucket, object, fi)
|
||||
.await
|
||||
.expect("metadata should be written before quorum read");
|
||||
}
|
||||
dirs.push(dir);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let set = SetDisks::new(
|
||||
"read-version-quorum-test".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
4,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
Vec::new(),
|
||||
FormatV3::new(pool_set_count, 4),
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
(dirs, set)
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn get_object_metadata_cache_capacity_uses_default_and_env_override() {
|
||||
@@ -1986,6 +2019,33 @@ mod metadata_cache_tests {
|
||||
fi
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_with_fileinfo_rejects_positive_size_without_parts() {
|
||||
let mut output = Vec::new();
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
"bucket",
|
||||
"object",
|
||||
0,
|
||||
1,
|
||||
&mut output,
|
||||
valid_test_fileinfo("object"),
|
||||
Vec::new(),
|
||||
&[],
|
||||
0,
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"small",
|
||||
)
|
||||
.await
|
||||
.expect_err("positive-size metadata without parts must fail without panicking");
|
||||
|
||||
assert_eq!(err, Error::FileCorrupt);
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_with_fileinfo_rejects_invalid_ranges_before_reader_setup() {
|
||||
let bucket = "bucket";
|
||||
@@ -2059,6 +2119,12 @@ mod metadata_cache_tests {
|
||||
|
||||
let mut invalid_erasure = valid_test_fileinfo(object);
|
||||
invalid_erasure.erasure.block_size = 0;
|
||||
invalid_erasure.parts.push(ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
});
|
||||
let err = SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
@@ -2087,6 +2153,37 @@ mod metadata_cache_tests {
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_with_fileinfo_accepts_zero_size_without_parts() {
|
||||
let bucket = "bucket";
|
||||
let object = "empty";
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.size = 0;
|
||||
|
||||
let mut output = Vec::new();
|
||||
SetDisks::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
0,
|
||||
0,
|
||||
&mut output,
|
||||
fi,
|
||||
Vec::new(),
|
||||
&[],
|
||||
0,
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"empty",
|
||||
)
|
||||
.await
|
||||
.expect("zero-byte object without parts must remain readable");
|
||||
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_with_fileinfo_fails_closed_without_read_quorum() {
|
||||
let bucket = "bucket";
|
||||
@@ -2133,6 +2230,7 @@ mod metadata_cache_tests {
|
||||
let object = "object";
|
||||
let (_dir, disk) = new_read_version_test_disk(bucket).await;
|
||||
let mut fi = valid_test_fileinfo(object);
|
||||
fi.size = 0;
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
disk.write_metadata(bucket, bucket, object, fi.clone())
|
||||
.await
|
||||
@@ -2165,11 +2263,70 @@ mod metadata_cache_tests {
|
||||
.await
|
||||
.expect_err("empty disk set must fail closed");
|
||||
assert!(
|
||||
missing_quorum.to_string().to_ascii_lowercase().contains("file"),
|
||||
"optimized read failure should map to file-not-found style error: {missing_quorum}"
|
||||
missing_quorum.to_string().contains("Insufficient successful results"),
|
||||
"optimized read failure should preserve the batch quorum diagnostic: {missing_quorum}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_version_optimized_uses_current_set_drive_quorum_not_pool_set_count() {
|
||||
let bucket = "read-version-layout-quorum-bucket";
|
||||
let object = "object";
|
||||
|
||||
let (_single_set_dirs, single_set) = read_version_quorum_test_set(bucket, object, 1, 1).await;
|
||||
single_set
|
||||
.read_version_optimized(bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect_err("one metadata copy must not satisfy a four-drive 2+2 set");
|
||||
|
||||
let (_multi_set_dirs, multi_set) = read_version_quorum_test_set(bucket, object, 3, 2).await;
|
||||
let versions = multi_set
|
||||
.read_version_optimized(bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("two metadata copies should satisfy the current set's read quorum");
|
||||
assert_eq!(versions.len(), 2);
|
||||
assert!(versions.iter().all(|version| version.name == object));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_version_optimized_counts_only_valid_metadata_toward_quorum() {
|
||||
let bucket = "read-version-corrupt-quorum-bucket";
|
||||
let object = "object";
|
||||
|
||||
let (quorum_minus_one_dirs, quorum_minus_one) = read_version_quorum_test_set(bucket, object, 1, 2).await;
|
||||
tokio::fs::write(
|
||||
quorum_minus_one_dirs[1]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE),
|
||||
b"corrupt metadata",
|
||||
)
|
||||
.await
|
||||
.expect("metadata should be corrupted for the test");
|
||||
quorum_minus_one
|
||||
.read_version_optimized(bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect_err("one valid and one corrupt metadata copy must not satisfy read quorum");
|
||||
|
||||
let (quorum_dirs, quorum) = read_version_quorum_test_set(bucket, object, 1, 3).await;
|
||||
tokio::fs::write(
|
||||
quorum_dirs[2]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE),
|
||||
b"corrupt metadata",
|
||||
)
|
||||
.await
|
||||
.expect("metadata should be corrupted for the test");
|
||||
let versions = quorum
|
||||
.read_version_optimized(bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("two valid metadata copies must satisfy read quorum despite one corrupt copy");
|
||||
assert_eq!(versions.len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_info_and_quorum_maps_delete_marker_and_purge_states() {
|
||||
let bucket = "get-object-info-marker-bucket";
|
||||
|
||||
@@ -2722,10 +2722,12 @@ mod tests {
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn transition_transaction_recovery_deletes_provider_recovered_unknown_upload() {
|
||||
let versioned_remote = uuid::Uuid::new_v4().to_string();
|
||||
let nil_remote = uuid::Uuid::nil().to_string();
|
||||
for (case, tier_name, remote_version) in [
|
||||
("missing", "TXPROBEMISSING", None),
|
||||
("unversioned", "TXPROBEUNVERSIONED", Some(String::new())),
|
||||
("versioned", "TXPROBEVERSIONED", Some(versioned_remote)),
|
||||
("nil-version", "TXPROBENILVERSION", Some(nil_remote)),
|
||||
] {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
|
||||
|
||||
@@ -6696,7 +6696,7 @@ mod test {
|
||||
use crate::object_api::ObjectInfo;
|
||||
use rustfs_filemeta::{
|
||||
FileInfo, FileMeta, FileMetaVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetaDeleteMarker,
|
||||
VersionType,
|
||||
ObjectPartInfo, VersionType,
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
@@ -6899,6 +6899,12 @@ mod test {
|
||||
fi.volume = "bucket".to_owned();
|
||||
fi.name = "object".to_owned();
|
||||
fi.size = 1;
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.fresh = true;
|
||||
fi.erasure.index = 1;
|
||||
fi.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"));
|
||||
@@ -6954,6 +6960,12 @@ mod test {
|
||||
fi.version_id = Some(Uuid::from_u128(version_idx));
|
||||
fi.versioned = true;
|
||||
fi.size = 1;
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.mod_time = Some(*mod_time);
|
||||
fi.metadata = metadata;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user