Merge remote-tracking branch 'origin/cxymds/fix-1358-fleet-version-gate' into cxymds/fix-1358-gcs-exact-generation

This commit is contained in:
马登山
2026-07-28 20:15:27 +08:00
71 changed files with 5310 additions and 633 deletions
+3 -3
View File
@@ -303,9 +303,9 @@ pub mod disk {
pub use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
CheckPartsResp, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore,
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, RUSTFS_META_BUCKET,
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo,
WalkDirOptions, new_disk, validate_batch_read_version_item_count,
FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
STORAGE_FORMAT_FILE, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
};
pub use bytes::Bytes;
pub use endpoint::Endpoint;
@@ -1182,8 +1182,12 @@ impl TransitionState {
}
fn new_with_capacity(capacity: usize) -> Arc<Self> {
let capacity = capacity.max(1);
let queue_send_timeout = resolve_transition_queue_send_timeout();
Self::new_with_capacity_and_timeout(capacity, queue_send_timeout)
}
fn new_with_capacity_and_timeout(capacity: usize, queue_send_timeout: StdDuration) -> Arc<Self> {
let capacity = capacity.max(1);
let (tx1, rx1) = bounded(capacity);
Arc::new(Self {
transition_tx: tx1,
@@ -1250,13 +1254,12 @@ impl TransitionState {
return false;
}
let bucket = bucket.to_string();
let scheduled = Arc::clone(&self.compensation_buckets);
let state = Arc::clone(self);
tokio::spawn(async move {
Self::inc_counter(&state.compensation_running_tasks);
state.record_scanner_transition_state();
let Some(api) = runtime_sources::object_store_handle() else {
scheduled.lock().unwrap().remove(&bucket);
state.finish_bucket_compensation(&bucket);
Self::add_counter(&state.compensation_running_tasks, -1);
state.record_scanner_transition_state();
debug!(
@@ -1292,13 +1295,25 @@ impl TransitionState {
);
}
scheduled.lock().unwrap().remove(&bucket);
state.finish_bucket_compensation(&bucket);
Self::add_counter(&state.compensation_running_tasks, -1);
state.record_scanner_transition_state();
});
true
}
fn finish_bucket_compensation(&self, bucket: &str) {
match self.compensation_buckets.lock() {
Ok(mut scheduled) => {
scheduled.remove(bucket);
}
Err(poisoned) => {
poisoned.into_inner().remove(bucket);
self.compensation_buckets.clear_poison();
}
}
}
#[inline]
fn inc_counter(counter: &AtomicI64) {
Self::add_counter(counter, 1);
@@ -1535,17 +1550,35 @@ impl TransitionState {
let outcome = match self.transition_tx.try_send(Some(task)) {
Ok(()) => TransitionEnqueueOutcome::Queued,
Err(async_channel::TrySendError::Full(_)) => {
Err(async_channel::TrySendError::Full(task)) => {
Self::inc_counter(&self.queue_full_tasks);
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
"transition queue is full; deferring to scanner/backfill"
);
TransitionEnqueueOutcome::QueueFull
let send_timeout = self.transition_queue_send_timeout;
match tokio::time::timeout(send_timeout, self.transition_tx.send(task)).await {
Ok(Ok(())) => TransitionEnqueueOutcome::Queued,
Ok(Err(_)) => {
self.schedule_bucket_compensation(&oi.bucket);
TransitionEnqueueOutcome::QueueClosed
}
Err(_) => {
Self::inc_counter(&self.queue_send_timeout_tasks);
self.schedule_bucket_compensation(&oi.bucket);
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
timeout_ms = send_timeout.as_millis() as u64,
event = EVENT_LIFECYCLE_TRANSITION_COMPENSATION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
state = "queue_send_timed_out",
"Scanner transition enqueue timed out; scheduled bucket compensation"
);
TransitionEnqueueOutcome::QueueFull
}
}
}
Err(async_channel::TrySendError::Closed(_)) => {
self.schedule_bucket_compensation(&oi.bucket);
debug!(
bucket = %oi.bucket,
object = %oi.name,
@@ -1730,7 +1763,7 @@ impl TransitionState {
}
pub fn add_lastday_stats(&self, tier: &str, ts: TierStats) {
let mut tier_stats = self.last_day_stats.lock().unwrap();
let mut tier_stats = self.lock_last_day_stats();
tier_stats
.entry(tier.to_string())
.and_modify(|e| e.add_stats(ts))
@@ -1738,7 +1771,7 @@ impl TransitionState {
}
pub fn get_daily_all_tier_stats(&self) -> DailyAllTierStats {
let tier_stats = self.last_day_stats.lock().unwrap();
let tier_stats = self.lock_last_day_stats();
let mut res = DailyAllTierStats::with_capacity(tier_stats.len());
for (tier, st) in tier_stats.iter() {
res.insert(tier.clone(), st.clone());
@@ -1746,6 +1779,18 @@ impl TransitionState {
res
}
fn lock_last_day_stats(&self) -> std::sync::MutexGuard<'_, HashMap<String, LastDayTierStats>> {
match self.last_day_stats.lock() {
Ok(stats) => stats,
Err(poisoned) => {
let mut stats = poisoned.into_inner();
stats.clear();
self.last_day_stats.clear_poison();
stats
}
}
}
pub async fn update_workers(api: Arc<ECStore>, n: i64) {
Self::update_workers_inner(api, n).await;
}
@@ -1767,7 +1812,27 @@ impl TransitionState {
fn resize_workers_to(api: Arc<ECStore>, n: i64, requested: i64, absolute_max: i64) {
let target = n as usize;
let transition_state = runtime_sources::transition_state_handle();
let mut workers = transition_state.workers.lock().unwrap();
let runtime = match tokio::runtime::Handle::try_current() {
Ok(runtime) => runtime,
Err(err) => {
warn!(
event = EVENT_LIFECYCLE_WORKER_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
error = %err,
state = "resize_failed",
"Lifecycle worker pool requires a Tokio runtime"
);
return;
}
};
// Runtime lookup happens before locking, and the guard is dropped before
// metrics/logging callbacks. Poison therefore means a Vec mutation may
// have unwound and worker tracking cannot be reconstructed safely.
let mut workers = transition_state
.workers
.lock()
.expect("transition worker tracking mutex poisoned");
let tracked_workers = workers.len();
workers.retain(|worker| !worker.handle.is_finished());
let pruned_finished_workers = tracked_workers.saturating_sub(workers.len());
@@ -1777,7 +1842,7 @@ impl TransitionState {
let clone_api = api.clone();
let cancel = CancellationToken::new();
let worker_cancel = cancel.clone();
let handle = tokio::spawn(async move {
let handle = runtime.spawn(async move {
TransitionState::worker_with_cancel(clone_api, worker_cancel).await;
});
workers.push(TransitionWorker { cancel, handle });
@@ -1791,6 +1856,7 @@ impl TransitionState {
let current_workers = workers.len() as i64;
transition_state.num_workers.store(current_workers, Ordering::SeqCst);
drop(workers);
transition_state.record_scanner_transition_state();
debug!(
@@ -4903,6 +4969,7 @@ mod tests {
FreeVersionRecoveryStats, RecoveryWalkTestAction, list_tier_free_versions, recover_tier_free_versions_with_cancel,
set_recovery_bucket_list_wait_hook, set_recovery_walk_test_hook,
};
use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
use crate::bucket::lifecycle::tier_sweeper::Jentry;
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
use crate::bucket::metadata_sys;
@@ -4936,6 +5003,7 @@ mod tests {
use http::HeaderMap;
use rustfs_common::metrics::{IlmAction, global_metrics};
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
use rustfs_data_usage::TierStats;
use rustfs_filemeta::{FileInfo, FileMeta};
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry, OutputLocation,
@@ -5657,6 +5725,7 @@ mod tests {
oi.transitioned_object.tier = "WARM".to_string();
oi.transitioned_object.name = "remote/object".to_string();
oi.transitioned_object.version_id = "remote-version".to_string();
oi.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
let local_delete_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let legacy_err = delete_free_version_remote_object_then(&oi, &manager, {
@@ -5667,7 +5736,8 @@ mod tests {
})
.await
.expect_err("legacy free-version without identity must be retained");
assert!(legacy_err.to_string().contains("no durable backend identity"));
assert_eq!(legacy_err.kind(), std::io::ErrorKind::Other);
assert_eq!(old_backend.remove_count().await, 0);
assert_eq!(local_delete_calls.load(Ordering::Relaxed), 0);
let mut invalid_metadata = HashMap::new();
@@ -6546,6 +6616,96 @@ mod tests {
assert_eq!(state.transition_rx.len(), 1);
}
#[tokio::test]
#[serial]
async fn scanner_transition_enqueue_waits_for_saturated_queue_to_recover() {
let state = TransitionState::new_with_capacity(1);
let first_object = ObjectInfo {
bucket: "bucket".to_string(),
name: "first".to_string(),
..Default::default()
};
let deferred_object = ObjectInfo {
bucket: "bucket".to_string(),
name: "deferred".to_string(),
..Default::default()
};
let event = crate::bucket::lifecycle::lifecycle::Event {
action: IlmAction::TransitionAction,
..Default::default()
};
assert!(
state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await,
"first scanner transition should fill the queue"
);
let deferred = state.queue_transition_task(&deferred_object, &event, &LcEventSrc::Scanner);
tokio::pin!(deferred);
assert!(
(&mut deferred).now_or_never().is_none(),
"a saturated scanner queue should apply bounded backpressure instead of dropping the task"
);
let first_task = state
.transition_rx
.recv()
.await
.expect("queue should remain open")
.expect("first queued transition task should be present");
state.release_transition(&first_task.obj_info);
assert!(
deferred.await,
"the deferred scanner transition should enqueue as soon as capacity recovers"
);
let recovered_task = state
.transition_rx
.recv()
.await
.expect("queue should remain open")
.expect("deferred transition task should be present");
assert_eq!(recovered_task.obj_info.name, deferred_object.name);
}
#[tokio::test]
#[serial]
async fn scanner_transition_sustained_saturation_schedules_compensation() {
let state = TransitionState::new_with_capacity_and_timeout(1, StdDuration::ZERO);
let first_object = ObjectInfo {
bucket: "saturated-bucket".to_string(),
name: "first".to_string(),
..Default::default()
};
let missed_object = ObjectInfo {
bucket: "saturated-bucket".to_string(),
name: "missed".to_string(),
..Default::default()
};
let event = crate::bucket::lifecycle::lifecycle::Event {
action: IlmAction::TransitionAction,
..Default::default()
};
assert!(
state.queue_transition_task(&first_object, &event, &LcEventSrc::Scanner).await,
"first scanner transition should fill the queue"
);
assert!(
!state
.queue_transition_task(&missed_object, &event, &LcEventSrc::Scanner)
.await,
"a continuously saturated queue should report that the object was not admitted"
);
assert_eq!(state.queue_full_tasks(), 1);
assert_eq!(state.queue_send_timeout_tasks(), 1);
assert_eq!(
state.compensation_scheduled_tasks(),
1,
"timed-out scanner work must schedule a bounded bucket backfill"
);
}
#[tokio::test]
#[serial]
async fn scanner_transition_enqueue_updates_transition_status() {
@@ -7071,6 +7231,46 @@ mod tests {
assert_eq!(state.compensation_pending_tasks(), 1);
}
#[test]
fn poisoned_compensation_set_can_release_completed_bucket() {
let state = TransitionState::new_with_capacity(1);
state
.compensation_buckets
.lock()
.expect("fresh mutex should lock")
.insert("bucket-a".to_string());
let poison_target = Arc::clone(&state.compensation_buckets);
let _ = std::thread::spawn(move || {
let _guard = poison_target.lock().expect("fresh mutex should lock");
panic!("poison compensation set");
})
.join();
state.finish_bucket_compensation("bucket-a");
assert_eq!(state.compensation_pending_tasks(), 0);
assert!(
state.compensation_buckets.lock().is_ok(),
"validated compensation state must clear poison"
);
}
#[test]
fn poisoned_tier_stats_are_reset_before_reuse() {
let state = TransitionState::new_with_capacity(1);
let poison_target = Arc::clone(&state.last_day_stats);
let _ = std::thread::spawn(move || {
let mut stats = poison_target.lock().expect("fresh mutex should lock");
stats.insert("stale".to_string(), LastDayTierStats::default());
panic!("poison tier stats");
})
.join();
state.add_lastday_stats("fresh", TierStats::default());
let stats = state.get_daily_all_tier_stats();
assert!(!stats.contains_key("stale"), "possibly partial statistics must be discarded");
assert!(stats.contains_key("fresh"), "statistics must accept new samples after recovery");
}
#[tokio::test(flavor = "current_thread")]
async fn scanner_transition_state_reports_compensation_pending_buckets() {
let state = TransitionState::new_with_capacity(1);
@@ -7361,6 +7561,23 @@ mod tests {
TransitionState::resize_workers_to(ecstore, original_workers, original_workers, absolute_max);
}
#[tokio::test]
#[serial]
async fn transition_worker_resize_without_runtime_does_not_poison_tracking() {
let (_paths, ecstore) = setup_test_env().await;
let transition_state = runtime_sources::transition_state_handle();
let resize = std::thread::spawn(move || {
TransitionState::resize_workers_to(ecstore, 1, 1, resolve_transition_workers_absolute_max());
})
.join();
assert!(resize.is_ok(), "missing Tokio runtime must not panic while worker tracking is locked");
assert!(
transition_state.workers.lock().is_ok(),
"failed resize must leave worker tracking unpoisoned"
);
}
#[test]
fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
let now = OffsetDateTime::now_utc();
@@ -9979,6 +10196,61 @@ mod tests {
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
async fn journal_replay_deletes_confirmed_exact_provider_token() {
let (_disk_paths, ecstore) = setup_test_env().await;
let (backend, _) = register_recovery_mock_tier(&ecstore).await;
let lease = TierConfigMgr::acquire_operation_lease(&ecstore.tier_config_mgr(), "WARM")
.await
.expect("mock tier lease should be available");
let identity = lease.backend_identity();
backend
.set_put_remote_version(Some("provider-version-token".to_string()))
.await;
lease
.put(
"remote/object",
crate::client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"candidate")),
9,
)
.await
.expect("confirmed remote candidate should be seeded");
backend.set_remove_failure(true);
backend.set_reject_non_empty_remote_versions(true);
let je = Jentry {
obj_name: "remote/object".to_string(),
version_id: "provider-version-token".to_string(),
tier_name: "WARM".to_string(),
backend_identity: Some(identity),
version_id_exact: true,
version_state: rustfs_filemeta::TransitionVersionState::Exact,
};
crate::set_disk::cleanup_rejected_transition_upload_durably(
&lease,
&je.obj_name,
&je.version_id,
true,
Some(ecstore.clone()),
)
.await
.expect("failed immediate cleanup should remain durable in the journal");
assert!(backend.contains(&je.obj_name).await);
backend.set_remove_failure(false);
crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
.await
.expect("identity-bound exact journal must retry confirmed candidate cleanup");
assert!(!backend.contains(&je.obj_name).await);
assert_eq!(backend.exact_remove_count(), 2);
assert_eq!(
backend.remove_versions().await,
vec![("remote/object".to_string(), "provider-version-token".to_string())]
);
}
async fn seed_recoverable_free_version(
disk_paths: &[PathBuf],
bucket: &str,
@@ -20,7 +20,10 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, warn};
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
use crate::bucket::lifecycle::tier_sweeper::{
Jentry, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
@@ -270,15 +273,26 @@ pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -
let backend_identity = je
.backend_identity
.ok_or_else(|| std::io::Error::other("legacy tier delete journal has no durable backend identity"))?;
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
&je.obj_name,
&je.version_id,
&je.tier_name,
backend_identity,
&api.tier_config_mgr(),
je.version_id_exact,
)
.await?;
if je.version_id_exact {
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
&je.obj_name,
&je.version_id,
&je.tier_name,
backend_identity,
&api.tier_config_mgr(),
)
.await?;
} else {
delete_object_from_remote_tier_idempotent_with_manager_and_identity(
&je.obj_name,
&je.version_id,
&je.tier_name,
backend_identity,
&api.tier_config_mgr(),
false,
)
.await?;
}
remove_tier_delete_journal_entry(api, je).await
}
@@ -252,7 +252,10 @@ impl ObjSweeper {
version_id: self.transition_version_id.clone(),
tier_name: self.transition_tier.clone(),
backend_identity: None,
version_id_exact: false,
version_id_exact: matches!(
self.transition_version_state,
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
),
version_state: self.transition_version_state,
});
}
@@ -335,7 +338,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
.await
.map_err(std::io::Error::other)?;
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false).await
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
}
async fn delete_object_from_remote_tier_raw_with_lease(
@@ -343,8 +346,11 @@ async fn delete_object_from_remote_tier_raw_with_lease(
rv_id: &str,
lease: &TierOperationLease,
version_id_exact: bool,
validate_remote_version_id: bool,
) -> Result<(), std::io::Error> {
lease.validate_remote_version_id(rv_id)?;
if validate_remote_version_id {
lease.validate_remote_version_id(rv_id)?;
}
if remote_delete_breaker_is_open(Instant::now()).await {
metrics::counter!(METRIC_DELETE_REMOTE_BREAKER_TOTAL).increment(1);
@@ -440,7 +446,53 @@ pub(crate) async fn delete_object_from_remote_tier_with_lease_idempotent(
lease: &TierOperationLease,
version_id_exact: bool,
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact).await {
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, version_id_exact, true).await
}
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idempotent(
obj_name: &str,
rv_id: &str,
lease: &TierOperationLease,
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
if rv_id.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"confirmed versioned transition candidate requires a non-empty remote version",
));
}
#[cfg(test)]
if obj_name == "remote/empty-guard-probe" {
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
delete_object_from_remote_tier_with_lease_idempotent_inner(obj_name, rv_id, lease, true, false).await
}
#[cfg(test)]
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
obj_name: &str,
rv_id: &str,
tier_name: &str,
backend_identity: TierDestinationId,
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, tier_name, backend_identity)
.await
.map_err(std::io::Error::other)?;
delete_confirmed_transition_candidate_exact_with_lease_idempotent(obj_name, rv_id, &lease).await
}
async fn delete_object_from_remote_tier_with_lease_idempotent_inner(
obj_name: &str,
rv_id: &str,
lease: &TierOperationLease,
version_id_exact: bool,
validate_remote_version_id: bool,
) -> Result<RemoteTierDeleteOutcome, std::io::Error> {
match delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, lease, version_id_exact, validate_remote_version_id)
.await
{
Ok(()) => Ok(RemoteTierDeleteOutcome::Deleted),
Err(err) if is_remote_tier_not_found_error(&err) => Ok(RemoteTierDeleteOutcome::AlreadyRemoved),
Err(err) => {
@@ -465,6 +517,7 @@ pub fn transitioned_delete_journal_entry(
versioned: bool,
suspended: bool,
transitioned: &TransitionedObject,
transition_version_state: rustfs_filemeta::TransitionVersionState,
) -> Option<Jentry> {
let sweeper = ObjSweeper {
version_id,
@@ -473,6 +526,7 @@ pub fn transitioned_delete_journal_entry(
transition_status: transitioned.status.clone(),
transition_tier: transitioned.tier.clone(),
transition_version_id: transitioned.version_id.clone(),
transition_version_state,
remote_object: transitioned.name.clone(),
..Default::default()
};
@@ -480,8 +534,13 @@ pub fn transitioned_delete_journal_entry(
sweeper.should_remove_remote_object()
}
pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject) -> Option<Jentry> {
if transitioned.status != lifecycle::TRANSITION_COMPLETE {
pub fn transitioned_force_delete_journal_entry(
transitioned: &TransitionedObject,
transition_version_state: rustfs_filemeta::TransitionVersionState,
) -> Option<Jentry> {
if transitioned.status != lifecycle::TRANSITION_COMPLETE
|| transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
{
return None;
}
@@ -490,8 +549,11 @@ pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject
version_id: transitioned.version_id.clone(),
tier_name: transitioned.tier.clone(),
backend_identity: None,
version_id_exact: false,
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
version_id_exact: matches!(
transition_version_state,
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
),
version_state: transition_version_state,
})
}
@@ -500,11 +562,14 @@ mod test {
use crate::client::signer_error::invalid_utf8_header_error;
use super::{
ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, RemoteDeleteBreaker, RemoteTierDeleteOutcome,
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED,
RemoteDeleteBreaker, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
is_remote_tier_not_found_error, is_signer_header_error, set_remote_tier_delete_test_hook,
should_record_remote_delete_failure,
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
};
use crate::storage_api_contracts::lifecycle::TransitionedObject;
use rustfs_filemeta::TransitionVersionState;
use std::io::{Error, ErrorKind};
use std::time::{Duration, Instant};
@@ -548,6 +613,43 @@ mod test {
assert!(should_record_remote_delete_failure(&Error::other("NoSuchVersion")));
}
#[test]
fn transitioned_delete_journal_preserves_remote_version_state() {
let cases = [
(TransitionVersionState::Unknown, "legacy-version", None),
(TransitionVersionState::KnownDisabled, "", Some(false)),
(TransitionVersionState::SuspendedNull, "null", Some(true)),
(TransitionVersionState::Exact, "opaque-version", Some(true)),
];
for (state, version_id, expected_exact) in cases {
let transitioned = TransitionedObject {
name: "remote/object".to_string(),
version_id: version_id.to_string(),
tier: "WARM".to_string(),
status: lifecycle::TRANSITION_COMPLETE.to_string(),
..Default::default()
};
let regular = transitioned_delete_journal_entry(None, false, false, &transitioned, state);
let forced = transitioned_force_delete_journal_entry(&transitioned, state);
match expected_exact {
Some(expected_exact) => {
let regular = regular.expect("known version state should produce a regular delete journal entry");
assert_eq!(regular.version_state, state);
assert_eq!(regular.version_id_exact, expected_exact);
let forced = forced.expect("known version state should produce a forced delete journal entry");
assert_eq!(forced.version_state, state);
assert_eq!(forced.version_id_exact, expected_exact);
}
None => {
assert!(regular.is_none());
assert!(forced.is_none());
}
}
}
}
#[tokio::test]
#[serial_test::serial]
async fn idempotent_remote_delete_treats_hooked_nosuchversion_as_already_removed() {
@@ -670,6 +772,55 @@ mod test {
assert_eq!(backend.remove_versions().await, vec![("remote/object".to_string(), String::new())]);
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial]
async fn confirmed_transition_cleanup_deletes_exact_provider_token() {
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.store(0, std::sync::atomic::Ordering::Relaxed);
let manager = crate::services::tier::tier::TierConfigMgr::new();
let backend = crate::services::tier::test_util::register_mock_tier(&manager, "WARM").await;
let lease = crate::services::tier::tier::TierConfigMgr::acquire_operation_lease(&manager, "WARM")
.await
.expect("test tier lease should be available");
let identity = lease.backend_identity();
drop(lease);
backend.set_reject_non_empty_remote_versions(true);
let outcome = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
"remote/object",
"provider-version-token",
"WARM",
identity,
&manager,
)
.await
.expect("confirmed upload compensation should delete the exact provider token");
assert_eq!(outcome, RemoteTierDeleteOutcome::Deleted);
assert_eq!(backend.exact_remove_count(), 1);
assert_eq!(
backend.remove_versions().await,
vec![("remote/object".to_string(), "provider-version-token".to_string())]
);
let err = delete_confirmed_transition_candidate_exact_with_manager_and_identity(
"remote/empty-guard-probe",
"",
"WARM",
identity,
&manager,
)
.await
.expect_err("confirmed versioned cleanup must reject an empty token");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(backend.remove_count().await, 1);
assert_eq!(
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES.load(std::sync::atomic::Ordering::Relaxed),
0,
"empty remote versions must be rejected before exact cleanup dispatch"
);
}
#[test]
fn breaker_opens_at_threshold_and_recovers_after_window() {
let mut breaker = RemoteDeleteBreaker::new(3, Duration::from_secs(30));
@@ -22,7 +22,10 @@ use uuid::Uuid;
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::bucket::lifecycle::tier_sweeper::delete_object_from_remote_tier_idempotent_with_manager_and_identity;
use crate::bucket::lifecycle::tier_sweeper::{
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result as EcstoreResult};
use crate::object_api::ObjectOptions;
@@ -708,6 +711,21 @@ async fn recover_unknown_upload_outcome(
TransitionCandidateProbe::UnversionedPresent => {
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
}
TransitionCandidateProbe::VersionedPresent(version_id)
if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
{
delete_confirmed_transition_candidate_exact_with_manager_and_identity(
&transaction.remote_object,
&version_id,
&transaction.tier_name,
transaction.backend_fingerprint,
&api.tier_config_mgr(),
)
.await
.map_err(Error::other)?;
delete_transition_transaction_record(api, transaction.transaction_id).await?;
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
}
TransitionCandidateProbe::VersionedPresent(version_id) => {
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
}
+18
View File
@@ -741,6 +741,8 @@ impl BucketMetadataSys {
if let Some(config) = &bm.policy_config {
Ok((config.clone(), bm.policy_config_updated_at))
} else if !bm.policy_config_json.is_empty() {
Ok((serde_json::from_slice(&bm.policy_config_json)?, bm.policy_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
@@ -1051,6 +1053,22 @@ mod tests {
);
}
#[tokio::test]
async fn get_bucket_policy_rejects_malformed_cached_policy() {
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = BucketMetadataSys::new(ecstore);
let mut metadata = BucketMetadata::new("malformed-policy");
metadata.policy_config_json = b"{".to_vec();
sys.set("malformed-policy".to_string(), Arc::new(metadata)).await;
let err = sys
.get_bucket_policy("malformed-policy")
.await
.expect_err("malformed persisted policy must not be treated as missing");
assert!(matches!(err, Error::Io(_)), "malformed persisted policy must surface its parse failure");
}
fn target(bucket: &str, id: &str) -> BucketTarget {
BucketTarget {
source_bucket: bucket.to_string(),
+102 -11
View File
@@ -15,23 +15,26 @@
use super::metadata_sys::get_bucket_metadata_sys;
use crate::error::{Result, StorageError};
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
use tracing::info;
pub struct PolicySys {}
impl PolicySys {
pub async fn is_allowed(args: &BucketPolicyArgs<'_>) -> bool {
match Self::get(args.bucket).await {
Ok(cfg) => return cfg.is_allowed(args).await,
Err(err) => {
if err != StorageError::ConfigNotFound {
info!("config get err {:?}", err);
}
}
}
args.is_owner
matches!(Self::try_is_allowed(args).await, Ok(true))
}
pub async fn try_is_allowed(args: &BucketPolicyArgs<'_>) -> Result<bool> {
Self::is_allowed_with_policy(args, Self::get(args.bucket).await).await
}
async fn is_allowed_with_policy(args: &BucketPolicyArgs<'_>, policy: Result<BucketPolicy>) -> Result<bool> {
match policy {
Ok(policy) => Ok(policy.is_allowed(args).await),
Err(StorageError::ConfigNotFound) => Ok(args.is_owner),
Err(err) => Err(err),
}
}
pub async fn get(bucket: &str) -> Result<BucketPolicy> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -41,3 +44,91 @@ impl PolicySys {
Ok(cfg)
}
}
#[cfg(test)]
mod tests {
use super::{PolicySys, StorageError};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
use std::collections::HashMap;
fn args<'a>(
is_owner: bool,
groups: &'a Option<Vec<String>>,
conditions: &'a HashMap<String, Vec<String>>,
) -> BucketPolicyArgs<'a> {
BucketPolicyArgs {
bucket: "bucket",
action: Action::S3Action(S3Action::GetObjectAction),
is_owner,
account: "account",
groups,
conditions,
object: "object",
}
}
#[tokio::test]
async fn missing_policy_preserves_owner_and_iam_fallback_semantics() {
let groups = None;
let conditions = HashMap::new();
assert!(
PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(StorageError::ConfigNotFound),)
.await
.expect("missing policy should preserve owner access")
);
assert!(
!PolicySys::is_allowed_with_policy(&args(false, &groups, &conditions), Err(StorageError::ConfigNotFound),)
.await
.expect("missing policy should defer non-owner access to IAM")
);
}
#[tokio::test]
async fn policy_load_failures_propagate() {
let groups = None;
let conditions = HashMap::new();
for (failure, expected_message) in [
(StorageError::Io(std::io::Error::other("policy read failed")), "policy read failed"),
(
StorageError::other("bucket metadata sys not initialized for this instance"),
"bucket metadata sys not initialized for this instance",
),
] {
let result = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Err(failure)).await;
assert!(
matches!(result, Err(StorageError::Io(ref err)) if err.to_string().contains(expected_message)),
"policy I/O and uninitialized metadata failures must propagate instead of granting owner access"
);
}
}
#[tokio::test]
async fn explicit_bucket_deny_precedes_iam_allow() {
let groups = None;
let conditions = HashMap::new();
let policy: BucketPolicy = serde_json::from_str(
r#"{
"Version":"2012-10-17",
"Statement":[{
"Effect":"Deny",
"Principal":{"AWS":"*"},
"Action":["s3:GetObject"],
"Resource":["arn:aws:s3:::bucket/*"]
}]
}"#,
)
.expect("deny policy should parse");
let bucket_allowed = PolicySys::is_allowed_with_policy(&args(true, &groups, &conditions), Ok(policy))
.await
.expect("loaded bucket policy should evaluate");
let iam_allowed = true;
let request_allowed = bucket_allowed && iam_allowed;
assert!(iam_allowed, "test precondition: IAM grants the action");
assert!(!bucket_allowed, "test precondition: bucket policy explicitly denies the action");
assert!(!request_allowed, "explicit bucket Deny must reject before IAM Allow fallback");
}
}
@@ -22,7 +22,7 @@ use bytes::Bytes;
use futures::future::join_all;
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use std::io::Error;
use std::sync::RwLock;
use std::sync::{Mutex, MutexGuard, RwLock};
use std::{collections::HashMap, sync::Arc};
use time::{OffsetDateTime, format_description};
use tokio::io::AsyncReadExt;
@@ -46,6 +46,14 @@ use crate::client::utils::base64_encode;
use rustfs_utils::path::trim_etag;
use s3s::header::X_AMZ_EXPIRATION;
fn lock_md5_hasher(
md5_hasher: &Mutex<Option<rustfs_utils::hash::HashAlgorithm>>,
) -> Result<MutexGuard<'_, Option<rustfs_utils::hash::HashAlgorithm>>, std::io::Error> {
md5_hasher
.lock()
.map_err(|_| std::io::Error::other("MD5 hasher state is unavailable"))
}
/// Read exactly `want` bytes for a single multipart part, or fewer if the reader
/// reaches EOF first. Advances the reader so the next call returns the following
/// part. Replaces the previous per-part `read_all()`/`to_vec()`, which drained
@@ -177,7 +185,7 @@ impl TransitionClient {
let length = buf.len();
if opts.send_content_md5 {
let mut md5_hasher = self.md5_hasher.lock().unwrap();
let mut md5_hasher = lock_md5_hasher(&self.md5_hasher)?;
let md5_hash = match md5_hasher.as_mut() {
Some(hasher) => hasher,
None => return Err(std::io::Error::other("MD5 hasher not initialized")),
@@ -370,7 +378,7 @@ impl TransitionClient {
let mut md5_base64: String = "".to_string();
if opts.send_content_md5 {
let mut md5_hasher = clone_self.md5_hasher.lock().unwrap();
let mut md5_hasher = lock_md5_hasher(&clone_self.md5_hasher)?;
let md5_hash = match md5_hasher.as_mut() {
Some(hasher) => hasher,
None => {
@@ -418,6 +426,9 @@ impl TransitionClient {
}
let results = join_all(futures).await;
for result in results {
result?;
}
select! {
err = err_rx.recv() => {
@@ -620,10 +631,12 @@ fn collect_complete_parts(parts_info: &HashMap<i64, ObjectPart>, total_parts_cou
#[cfg(test)]
mod tests {
use super::{ObjectPart, ReaderImpl, collect_complete_parts, read_multipart_part};
use super::{ObjectPart, ReaderImpl, collect_complete_parts, lock_md5_hasher, read_multipart_part};
use crate::object_api::GetObjectReader;
use bytes::Bytes;
use rustfs_utils::hash::HashAlgorithm;
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
// Drive a reader through the same per-part loop the multipart stream uses and
// collect the size of every part. Regression for rustfs/rustfs#4811: the old
@@ -733,4 +746,18 @@ mod tests {
"a gap in the parts map must be an error, not a panic"
);
}
#[test]
fn poisoned_md5_state_fails_closed() {
let hasher = Arc::new(Mutex::new(Some(HashAlgorithm::Md5)));
let poison_target = Arc::clone(&hasher);
let _ = std::thread::spawn(move || {
let _guard = poison_target.lock().expect("fresh mutex should lock");
panic!("poison MD5 state");
})
.join();
let error = lock_md5_hasher(&hasher).expect_err("poisoned hash state must not be reused");
assert_eq!(error.kind(), std::io::ErrorKind::Other);
}
}
+71 -5
View File
@@ -24,8 +24,8 @@ use crate::cluster::rpc::internode_data_transport::{
use crate::disk::error::{Error, Result};
use crate::disk::{
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
disk_store::{
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
@@ -48,9 +48,10 @@ use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequest, StatVolumeRequest,
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
RenameFileRequest, SettlePartTransactionRequest, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest,
WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
};
use serde::{Serialize, de::DeserializeOwned};
use std::{
@@ -2481,6 +2482,71 @@ impl DiskAPI for RemoteDisk {
.await
}
#[tracing::instrument(level = "trace", skip_all)]
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: Bytes,
) -> Result<()> {
self.execute_with_timeout(
|| async {
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(PreparePartTransactionRequest {
disk: self.endpoint.to_string(),
src_volume: src_volume.to_string(),
src_path: src_path.to_string(),
dst_volume: dst_volume.to_string(),
dst_path: dst_path.to_string(),
meta,
});
let canonical_body = rustfs_protos::canonical_prepare_part_transaction_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "prepare_part_transaction")?;
let response = client.prepare_part_transaction(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
self.execute_with_timeout(
|| async {
let mut client = self
.get_client()
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let mut request = Request::new(SettlePartTransactionRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
path: path.to_string(),
rollback: action == PartTransactionAction::Rollback,
});
let canonical_body = rustfs_protos::canonical_settle_part_transaction_request_body(request.get_ref());
attach_mutation_body_digest(&mut request, canonical_body, "settle_part_transaction")?;
let response = client.settle_part_transaction(request).await?.into_inner();
if !response.success {
return Err(response.error.unwrap_or_default().into());
}
Ok(())
},
get_max_timeout_duration(),
)
.await
}
#[tracing::instrument(level = "trace", skip_all)]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
trace!(
+242 -4
View File
@@ -16,6 +16,7 @@
use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets};
use crate::multipart_listing::paginate_multipart_listing;
use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
@@ -49,7 +50,10 @@ use rustfs_filemeta::FileInfo;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
use std::{collections::HashMap, sync::Arc};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
use tokio::sync::RwLock;
use tokio::sync::broadcast::{Receiver, Sender};
use tokio::time::Duration;
@@ -63,6 +67,8 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
#[derive(Debug, Clone)]
pub struct Sets {
pub id: Uuid,
@@ -799,9 +805,52 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
delimiter: Option<String>,
max_uploads: usize,
) -> Result<ListMultipartsInfo> {
self.get_disks_by_key(prefix)
.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads)
.await
let per_set_limit = max_uploads.saturating_add(1);
let results = futures::stream::iter(self.disk_set.iter().cloned())
.map(|set| {
let key_marker = key_marker.clone();
let upload_id_marker = upload_id_marker.clone();
let delimiter = delimiter.clone();
async move {
set.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, per_set_limit)
.await
}
})
.buffer_unordered(LIST_MULTIPART_SETS_CONCURRENCY)
.collect::<Vec<_>>()
.await;
let mut uploads = Vec::new();
let mut common_prefixes = HashSet::new();
let mut source_truncated = false;
for result in results {
let page = result?;
uploads.extend(page.uploads);
common_prefixes.extend(page.common_prefixes);
source_truncated |= page.is_truncated;
}
let page = paginate_multipart_listing(
uploads,
common_prefixes.into_iter().collect(),
key_marker.as_deref(),
key_marker.as_ref().and(upload_id_marker.as_deref()),
max_uploads,
source_truncated,
);
Ok(ListMultipartsInfo {
key_marker,
upload_id_marker,
next_key_marker: page.next_key_marker,
next_upload_id_marker: page.next_upload_id_marker,
max_uploads,
is_truncated: page.is_truncated,
uploads: page.uploads,
common_prefixes: page.common_prefixes,
prefix: prefix.to_owned(),
delimiter,
})
}
#[tracing::instrument(skip(self))]
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
@@ -1111,6 +1160,7 @@ mod tests {
use crate::layout::endpoints::SetupType;
use crate::storage_api_contracts::heal::HealOperations as _;
use crate::storage_api_contracts::list::ListOperations as _;
use crate::storage_api_contracts::multipart::MultipartOperations as _;
use rustfs_lock::client::local::LocalClient;
use serial_test::serial;
@@ -1248,6 +1298,194 @@ mod tests {
assert_eq!(result, (Some(3), Some(1), Some(0)));
}
async fn multipart_listing_test_sets() -> (Vec<tempfile::TempDir>, Arc<Sets>) {
let format = FormatV3::new(2, 2);
let mut temp_dirs = Vec::new();
let mut all_endpoints = Vec::new();
let mut disk_sets = Vec::new();
for set_index in 0..2 {
let mut endpoints = Vec::new();
let mut disks = Vec::new();
for disk_index in 0..2 {
let temp_dir = tempfile::tempdir().expect("tempdir should be created");
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
.expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(set_index);
endpoint.set_disk_index(disk_index);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("disk should be created");
let mut disk_format = format.clone();
disk_format.erasure.this = format.erasure.sets[set_index][disk_index];
save_format_file(&Some(disk.clone()), &Some(disk_format))
.await
.expect("format should be saved");
temp_dirs.push(temp_dir);
all_endpoints.push(endpoint.clone());
endpoints.push(endpoint);
disks.push(Some(disk));
}
disk_sets.push(
SetDisks::new(
"test-owner".to_string(),
Arc::new(RwLock::new(disks)),
2,
1,
0,
set_index,
endpoints,
format.clone(),
vec![Arc::new(LocalClient::new()), Arc::new(LocalClient::new())],
)
.await,
);
}
let sets = Arc::new(Sets {
id: format.id,
disk_set: disk_sets,
pool_idx: 0,
endpoints: PoolEndpoints {
legacy: false,
set_count: 2,
drives_per_set: 2,
endpoints: Endpoints::from(all_endpoints),
cmd_line: String::new(),
platform: String::new(),
},
format,
parity_count: 1,
set_count: 2,
set_drive_count: 2,
default_parity_count: 1,
distribution_algo: DistributionAlgoVersion::V1,
exit_signal: None,
ctx: bootstrap_ctx(),
});
(temp_dirs, sets)
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn list_multipart_uploads_merges_all_sets_without_pagination_loss() {
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
let (_temp_dirs, sets) = multipart_listing_test_sets().await;
let bucket = format!("multipart-list-{}", Uuid::new_v4().simple());
sets.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut keys_by_set = [Vec::new(), Vec::new()];
for index in 0..100 {
let key = format!("logs/{index:03}.bin");
let set_index = sets.get_hashed_set_index(&key);
if keys_by_set[set_index].len() < 2 {
keys_by_set[set_index].push(key);
}
if keys_by_set.iter().all(|keys| keys.len() == 2) {
break;
}
}
assert!(keys_by_set.iter().all(|keys| keys.len() == 2), "test keys must span both sets");
let repeated_key = keys_by_set[0][0].clone();
let mut expected = Vec::new();
for key in keys_by_set.iter().flatten() {
let upload = sets
.new_multipart_upload(&bucket, key, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
expected.push((key.clone(), upload.upload_id));
}
let second = sets
.new_multipart_upload(&bucket, &repeated_key, &ObjectOptions::default())
.await
.expect("second upload for the same key should be created");
expected.push((repeated_key, second.upload_id));
expected.sort();
let mut actual = Vec::new();
let mut key_marker = None;
let mut upload_id_marker = None;
for _ in 0..expected.len() + 1 {
let page = sets
.list_multipart_uploads(&bucket, "logs/", key_marker.clone(), upload_id_marker.clone(), None, 2)
.await
.expect("multipart page should list across every set");
assert!(page.uploads.len() <= 2);
actual.extend(
page.uploads
.iter()
.map(|upload| (upload.object.clone(), upload.upload_id.clone())),
);
if !page.is_truncated {
break;
}
key_marker = page.next_key_marker;
upload_id_marker = page.next_upload_id_marker;
}
assert_eq!(actual, expected, "set-level merge must return every upload exactly once");
let mut deduped = actual.clone();
deduped.dedup();
assert_eq!(deduped.len(), actual.len(), "set-level pagination must not duplicate uploads");
let mut nested_by_set = [None, None];
for index in 0..100 {
let key = format!("nested/group-{index:03}/file.bin");
let set_index = sets.get_hashed_set_index(&key);
nested_by_set[set_index].get_or_insert(key);
if nested_by_set.iter().all(Option::is_some) {
break;
}
}
for key in nested_by_set.iter().flatten() {
sets.new_multipart_upload(&bucket, key, &ObjectOptions::default())
.await
.expect("nested multipart upload should be created");
}
let mut expected_prefixes = nested_by_set
.iter()
.flatten()
.map(|key| {
key.rsplit_once('/')
.expect("nested key should contain a delimiter")
.0
.to_string()
+ "/"
})
.collect::<Vec<_>>();
expected_prefixes.sort();
let first = sets
.list_multipart_uploads(&bucket, "nested/", None, None, Some("/".to_string()), 1)
.await
.expect("first delimiter page should list across every set");
assert!(first.is_truncated);
assert_eq!(first.common_prefixes, expected_prefixes[..1]);
let second = sets
.list_multipart_uploads(
&bucket,
"nested/",
first.next_key_marker,
first.next_upload_id_marker,
Some("/".to_string()),
1,
)
.await
.expect("second delimiter page should list across every set");
assert!(!second.is_truncated);
assert_eq!(second.common_prefixes, expected_prefixes[1..]);
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn sets_list_objects_v2_lists_objects_within_the_pool() {
+125 -42
View File
@@ -35,7 +35,7 @@ use std::{
Arc,
atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering},
},
time::Duration,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use tokio::{sync::RwLock, time};
use tokio_util::sync::CancellationToken;
@@ -339,21 +339,19 @@ impl Drop for DiskHealthWaitingGuard<'_> {
impl DiskHealthTracker {
/// Create a new disk health tracker
pub fn new() -> Self {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
let now = current_unix_time();
let now_nanos = unix_nanos(now);
Self {
last_success: AtomicI64::new(now),
last_started: AtomicI64::new(now),
last_success: AtomicI64::new(now_nanos),
last_started: AtomicI64::new(now_nanos),
status: AtomicU32::new(DISK_HEALTH_OK),
waiting: AtomicU32::new(0),
runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
consecutive_failures: AtomicU32::new(0),
consecutive_successes: AtomicU32::new(0),
offline_since_unix_secs: AtomicI64::new(0),
last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
last_transition_unix_secs: AtomicI64::new(unix_secs_i64(now)),
last_capacity_total: AtomicU64::new(0),
last_capacity_used: AtomicU64::new(0),
last_capacity_free: AtomicU64::new(0),
@@ -363,11 +361,7 @@ impl DiskHealthTracker {
/// Log a successful operation
pub fn log_success(&self) {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
self.last_success.store(now, Ordering::Relaxed);
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
}
pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
@@ -429,11 +423,14 @@ impl DiskHealthTracker {
}
pub fn offline_duration(&self) -> Option<Duration> {
self.offline_duration_at(current_unix_secs())
}
fn offline_duration_at(&self, now: u64) -> Option<Duration> {
let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
if offline_since <= 0 {
return None;
}
let now = current_unix_secs();
Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
}
@@ -492,6 +489,12 @@ impl DiskHealthTracker {
/// Remote disks are marked faulty on timeout/network errors; the init loop retries with the
/// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`.
pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) {
self.reset_for_store_init_retry_at(endpoint, current_unix_time());
}
fn reset_for_store_init_retry_at(&self, endpoint: &Endpoint, now: Duration) {
let now_nanos = unix_nanos(now);
let now_secs = unix_secs_i64(now);
self.status.store(DISK_HEALTH_OK, Ordering::Release);
self.runtime_state
.store(RuntimeDriveHealthState::Online as u32, Ordering::Release);
@@ -499,11 +502,9 @@ impl DiskHealthTracker {
self.consecutive_successes.store(0, Ordering::Release);
self.offline_since_unix_secs.store(0, Ordering::Release);
self.waiting.store(0, Ordering::Release);
let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap();
let now_nanos = now.as_nanos() as i64;
self.last_success.store(now_nanos, Ordering::Relaxed);
self.last_started.store(now_nanos, Ordering::Relaxed);
self.last_transition_unix_secs.store(now.as_secs() as i64, Ordering::Release);
self.last_transition_unix_secs.store(now_secs, Ordering::Release);
record_drive_runtime_state(endpoint, RuntimeDriveHealthState::Online);
}
@@ -612,10 +613,33 @@ impl DiskHealthTracker {
}
fn current_unix_secs() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs()
// Zero is reserved as "not recorded" by health timestamp atomics.
current_unix_time().as_secs().max(1)
}
fn current_unix_nanos() -> i64 {
unix_nanos(current_unix_time())
}
fn current_unix_time() -> Duration {
unix_time_since_epoch(SystemTime::now())
}
fn unix_time_since_epoch(time: SystemTime) -> Duration {
time.duration_since(UNIX_EPOCH).unwrap_or(Duration::ZERO)
}
fn unix_nanos(time: Duration) -> i64 {
i64::try_from(time.as_nanos()).unwrap_or(i64::MAX)
}
fn unix_secs_i64(time: Duration) -> i64 {
i64::try_from(time.as_secs()).unwrap_or(i64::MAX)
}
fn elapsed_since(last_nanos: i64, now_nanos: i64) -> Duration {
let elapsed_nanos = now_nanos.saturating_sub(last_nanos).max(0);
Duration::from_nanos(u64::try_from(elapsed_nanos).unwrap_or(u64::MAX))
}
impl Default for DiskHealthTracker {
@@ -635,11 +659,7 @@ struct HealthDiskCtxValue {
impl HealthDiskCtxValue {
fn log_success(&self) {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
self.last_success.store(now, Ordering::Relaxed);
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
}
}
@@ -774,12 +794,7 @@ impl LocalDiskWrapper {
}
let last_success_nanos = health.last_success.load(Ordering::Relaxed);
let elapsed = Duration::from_nanos(
(std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64 - last_success_nanos) as u64
);
let elapsed = elapsed_since(last_success_nanos, current_unix_nanos());
if elapsed < SKIP_IF_SUCCESS_BEFORE {
continue;
@@ -1091,11 +1106,7 @@ impl LocalDiskWrapper {
self.check_disk_stale().await?;
// Record operation start
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
self.health.last_started.store(now, Ordering::Relaxed);
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
let _waiting_guard = self.health.waiting_guard();
if timeout_duration == Duration::ZERO {
@@ -1317,11 +1328,7 @@ impl DiskAPI for LocalDiskWrapper {
}
// Record operation start
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos() as i64;
self.health.last_started.store(now, Ordering::Relaxed);
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
self.health.increment_waiting();
// Execute the operation
@@ -1473,6 +1480,33 @@ impl DiskAPI for LocalDiskWrapper {
.await
}
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: Bytes,
) -> Result<()> {
self.track_disk_health(
|| async {
self.disk
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
.await
},
get_max_timeout_duration(),
)
.await
}
async fn settle_part_transaction(&self, volume: &str, path: &str, action: crate::disk::PartTransactionAction) -> Result<()> {
self.track_disk_health(
|| async { self.disk.settle_part_transaction(volume, path, action).await },
get_max_timeout_duration(),
)
.await
}
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
self.track_disk_health(|| async { self.disk.delete(volume, path, opt).await }, get_max_timeout_duration())
.await
@@ -2243,4 +2277,53 @@ mod tests {
assert!(health.mark_offline(&endpoint, "again"));
assert!(health.is_faulty());
}
#[test]
fn unix_time_clamps_epoch_and_pre_epoch_to_zero() {
let before_epoch = UNIX_EPOCH
.checked_sub(Duration::from_nanos(1))
.expect("one nanosecond before the Unix epoch should be representable");
assert_eq!(unix_time_since_epoch(UNIX_EPOCH), Duration::ZERO);
assert_eq!(unix_time_since_epoch(before_epoch), Duration::ZERO);
}
#[test]
fn elapsed_and_offline_duration_saturate_on_clock_rollback() {
let health = DiskHealthTracker::new();
health.offline_since_unix_secs.store(10, Ordering::Release);
assert_eq!(elapsed_since(10, 12), Duration::from_nanos(2));
assert_eq!(elapsed_since(10, 9), Duration::ZERO);
assert_eq!(health.offline_duration_at(9), Some(Duration::ZERO));
}
#[test]
fn pre_epoch_retry_reset_updates_the_complete_health_state() {
let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry-pre-epoch").expect("endpoint should parse");
let health = DiskHealthTracker::new();
health.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
health
.runtime_state
.store(RuntimeDriveHealthState::Offline as u32, Ordering::Release);
health.consecutive_failures.store(3, Ordering::Release);
health.consecutive_successes.store(2, Ordering::Release);
health.offline_since_unix_secs.store(11, Ordering::Release);
health.waiting.store(4, Ordering::Release);
health.last_success.store(12, Ordering::Release);
health.last_started.store(13, Ordering::Release);
health.last_transition_unix_secs.store(14, Ordering::Release);
health.reset_for_store_init_retry_at(&endpoint, unix_time_since_epoch(UNIX_EPOCH - Duration::from_nanos(1)));
assert_eq!(health.status.load(Ordering::Acquire), DISK_HEALTH_OK);
assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
assert_eq!(health.consecutive_failures.load(Ordering::Acquire), 0);
assert_eq!(health.consecutive_successes.load(Ordering::Acquire), 0);
assert_eq!(health.offline_since_unix_secs.load(Ordering::Acquire), 0);
assert_eq!(health.waiting.load(Ordering::Acquire), 0);
assert_eq!(health.last_success.load(Ordering::Acquire), 0);
assert_eq!(health.last_started.load(Ordering::Acquire), 0);
assert_eq!(health.last_transition_unix_secs.load(Ordering::Acquire), 0);
}
}
+297 -2
View File
@@ -19,7 +19,8 @@ use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_r
use crate::disk::{
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET,
FileInfoVersions, FileReader, FileWriter, MmapCopyStageMetrics, OldCurrentSize, PART_TRANSACTION_NEW_META,
PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET,
RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE,
STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, conv_part_err_to_int,
endpoint::Endpoint,
@@ -80,6 +81,10 @@ const ENV_BITROT_SIZE_MISMATCH_RETRY_COUNT: &str = "RUSTFS_BITROT_SIZE_MISMATCH_
const ENV_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: &str = "RUSTFS_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS";
const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_COUNT: u64 = 2;
const DEFAULT_BITROT_SIZE_MISMATCH_RETRY_DELAY_MS: u64 = 100;
const PART_TRANSACTION_OLD_DATA: &str = "old.data";
const PART_TRANSACTION_OLD_DATA_ABSENT: &str = "old.data.absent";
const PART_TRANSACTION_OLD_META_ABSENT: &str = "old.meta.absent";
const PART_TRANSACTION_PUBLISH_META: &str = "publish.meta";
enum ReadAllError {
Open(std::io::Error),
Disk(DiskError),
@@ -141,6 +146,28 @@ fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> {
}
}
fn snapshot_part_transaction_file(src: &Path, backup: &Path, absent: &Path) -> std::io::Result<()> {
match std::fs::symlink_metadata(src) {
Ok(metadata) if metadata.is_file() => std::fs::hard_link(src, backup),
Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction source is not a file")),
Err(err) if err.kind() == ErrorKind::NotFound => std::fs::write(absent, []),
Err(err) => Err(err),
}
}
fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, restore: &Path) -> std::io::Result<()> {
match std::fs::symlink_metadata(backup) {
Ok(metadata) if metadata.is_file() => {
remove_file_if_exists(restore)?;
std::fs::hard_link(backup, restore)?;
std::fs::rename(restore, current)
}
Ok(_) => Err(std::io::Error::new(ErrorKind::InvalidData, "multipart transaction backup is not a file")),
Err(err) if err.kind() == ErrorKind::NotFound && absent.is_file() => remove_file_if_exists(current),
Err(err) => Err(err),
}
}
fn rollback_committed_rename_std(
dst_file_path: &Path,
new_data_path: Option<&Path>,
@@ -6447,6 +6474,151 @@ impl DiskAPI for LocalDisk {
Ok(resp)
}
#[tracing::instrument(level = "trace", skip_all)]
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: Bytes,
) -> Result<()> {
let src_volume_dir = self.get_bucket_path(src_volume)?;
let dst_volume_dir = self.get_bucket_path(dst_volume)?;
if !skip_access_checks(src_volume) {
super::fs::access_std(&src_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
}
if !skip_access_checks(dst_volume) {
super::fs::access_std(&dst_volume_dir).map_err(|err| to_access_error(err, DiskError::VolumeAccessDenied))?;
}
let src_file_path = self.get_object_path(src_volume, src_path)?;
let dst_file_path = self.get_object_path(dst_volume, dst_path)?;
let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
for path in [&src_file_path, &dst_file_path, &dst_meta_path, &transaction_path] {
check_path_length(path.to_string_lossy().as_ref())?;
}
let durability = effective_durability(dst_volume);
tokio::task::spawn_blocking(move || {
let source = std::fs::symlink_metadata(&src_file_path).map_err(to_file_error)?;
if !source.is_file() {
return Err(DiskError::FileAccessDenied);
}
if transaction_path.exists() {
return Err(DiskError::FileAccessDenied);
}
let Some(transaction_parent) = transaction_path.parent() else {
return Err(DiskError::InvalidPath);
};
std::fs::create_dir_all(transaction_parent).map_err(to_file_error)?;
let staging_path = transaction_parent.join(format!(".part-txn-{}", Uuid::new_v4()));
std::fs::create_dir(&staging_path).map_err(to_file_error)?;
let prepare_result = (|| -> std::io::Result<()> {
snapshot_part_transaction_file(
&dst_file_path,
&staging_path.join(PART_TRANSACTION_OLD_DATA),
&staging_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
)?;
snapshot_part_transaction_file(
&dst_meta_path,
&staging_path.join(PART_TRANSACTION_OLD_META),
&staging_path.join(PART_TRANSACTION_OLD_META_ABSENT),
)?;
let mut new_meta = std::fs::OpenOptions::new()
.create_new(true)
.write(true)
.open(staging_path.join(PART_TRANSACTION_NEW_META))?;
std::io::Write::write_all(&mut new_meta, &meta)?;
if durability.syncs_commit_metadata() {
new_meta.sync_data()?;
os::fsync_dir_std(&staging_path)?;
}
std::fs::rename(&staging_path, &transaction_path)?;
if durability.syncs_commit_metadata() {
os::fsync_dir_std(transaction_parent)?;
}
Ok(())
})();
if let Err(err) = prepare_result {
let _ = remove_dir_all_if_exists(&staging_path);
return Err(to_file_error(err).into());
}
Ok(())
})
.await
.map_err(DiskError::from)?
}
#[tracing::instrument(level = "trace", skip_all)]
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
self.get_bucket_path(volume)?;
let current_data_path = self.get_object_path(volume, path)?;
let current_meta_path = self.get_object_path(volume, &format!("{path}.meta"))?;
let transaction_path = self.get_object_path(volume, &crate::disk::part_transaction_path(path))?;
for candidate in [&current_data_path, &current_meta_path, &transaction_path] {
check_path_length(candidate.to_string_lossy().as_ref())?;
}
let durability = effective_durability(volume);
tokio::task::spawn_blocking(move || {
match std::fs::symlink_metadata(&transaction_path) {
Ok(metadata) if metadata.is_dir() => {}
Ok(_) => return Err(DiskError::FileCorrupt),
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()),
Err(err) => return Err(to_file_error(err).into()),
}
if action == PartTransactionAction::Rollback {
std::fs::write(transaction_path.join(PART_TRANSACTION_ROLLBACK), []).map_err(to_file_error)?;
if durability.syncs_commit_metadata() {
os::fsync_dir_std(&transaction_path).map_err(to_file_error)?;
}
restore_part_transaction_file(
&current_data_path,
&transaction_path.join(PART_TRANSACTION_OLD_DATA),
&transaction_path.join(PART_TRANSACTION_OLD_DATA_ABSENT),
&transaction_path.join("restore.data"),
)
.map_err(to_file_error)?;
restore_part_transaction_file(
&current_meta_path,
&transaction_path.join(PART_TRANSACTION_OLD_META),
&transaction_path.join(PART_TRANSACTION_OLD_META_ABSENT),
&transaction_path.join("restore.meta"),
)
.map_err(to_file_error)?;
if durability.syncs_commit_metadata()
&& let Some(parent) = current_data_path.parent()
{
os::fsync_dir_std(parent).map_err(to_file_error)?;
}
}
let Some(parent) = transaction_path.parent() else {
return Err(DiskError::InvalidPath);
};
let cleanup_path = parent.join(format!(".part-txn-settled-{}", Uuid::new_v4()));
std::fs::rename(&transaction_path, &cleanup_path).map_err(to_file_error)?;
if durability.syncs_commit_metadata() {
os::fsync_dir_std(parent).map_err(to_file_error)?;
}
remove_dir_all_if_exists(&cleanup_path).map_err(to_file_error)?;
Ok(())
})
.await
.map_err(DiskError::from)??;
self.io_backend.invalidate_cached_fd(volume, path).await;
Ok(())
}
#[tracing::instrument(level = "trace", skip_all)]
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()> {
let src_volume_dir = self.get_bucket_path(src_volume)?;
@@ -6528,6 +6700,42 @@ impl DiskAPI for LocalDisk {
}
}
let transaction_publish_meta = if src_is_dir {
None
} else {
let transaction_path = self.get_object_path(dst_volume, &crate::disk::part_transaction_path(dst_path))?;
let transaction_meta_path = transaction_path.join(PART_TRANSACTION_NEW_META);
match fs::read(&transaction_meta_path).await {
Ok(expected_meta) => {
if expected_meta.as_slice() != meta.as_ref() {
return Err(DiskError::FileCorrupt);
}
let publish_meta_path = transaction_path.join(PART_TRANSACTION_PUBLISH_META);
let source_meta_path = transaction_meta_path.clone();
let publish_path = publish_meta_path.clone();
tokio::task::spawn_blocking(move || {
remove_file_if_exists(&publish_path)?;
std::fs::hard_link(source_meta_path, &publish_path)
})
.await
.map_err(DiskError::from)?
.map_err(to_file_error)?;
Some(publish_meta_path)
}
// Old peers know only RenamePart. The new coordinator never
// reaches rename unless prepare succeeded, so an absent
// transaction directory identifies the rolling-upgrade legacy
// path. A present directory without new.meta is corruption.
Err(err) if err.kind() == ErrorKind::NotFound => match fs::metadata(&transaction_path).await {
Ok(_) => return Err(DiskError::FileCorrupt),
Err(meta_err) if meta_err.kind() == ErrorKind::NotFound => None,
Err(meta_err) => return Err(to_file_error(meta_err).into()),
},
Err(err) => return Err(to_file_error(err).into()),
}
};
// UploadPart is acknowledged once this rename lands, so the part data and
// its directory entry must be durable before we return. Relaxed keeps the
// part payload fdatasync but leaves the directory entry to the page cache.
@@ -6561,7 +6769,17 @@ impl DiskAPI for LocalDisk {
return Err(DiskError::FileAccessDenied);
}
self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?;
if let Some(transaction_publish_meta) = transaction_publish_meta {
let dst_meta_path = self.get_object_path(dst_volume, &format!("{dst_path}.meta"))?;
rename_all(&transaction_publish_meta, &dst_meta_path, &dst_volume_dir).await?;
if durability.syncs_commit_metadata()
&& let Some(parent) = dst_meta_path.parent()
{
os::fsync_dir(parent).await.map_err(to_file_error)?;
}
} else {
self.write_all(dst_volume, format!("{dst_path}.meta").as_str(), meta).await?;
}
if let Some(parent) = src_file_path.parent() {
self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await?;
@@ -8282,6 +8500,12 @@ mod test {
file_info.data_dir = data_dir;
file_info.data = data;
file_info.size = size;
file_info.parts = vec![ObjectPartInfo {
number: 1,
size: usize::try_from(size).expect("test object size should fit usize"),
actual_size: size,
..Default::default()
}];
file_info.mod_time = Some(OffsetDateTime::now_utc());
file_info
}
@@ -9370,9 +9594,15 @@ mod test {
.await
.expect("source part should be written");
disk.prepare_part_transaction(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone())
.await
.expect("part transaction should be prepared");
disk.rename_part(tmp_volume, "upload/part.1", bucket, "object/part.1", meta.clone())
.await
.expect("rename_part should commit part");
disk.settle_part_transaction(bucket, "object/part.1", PartTransactionAction::Commit)
.await
.expect("part transaction should be committed");
assert_eq!(
disk.read_all(bucket, "object/part.1")
@@ -9390,6 +9620,71 @@ mod test {
matches!(disk.read_all(tmp_volume, "upload/part.1").await, Err(DiskError::FileNotFound)),
"source part must be removed after a successful commit"
);
let legacy_payload = Bytes::from_static(b"legacy peer payload");
let legacy_meta = Bytes::from_static(b"legacy peer metadata");
disk.write_all(tmp_volume, "legacy/part.1", legacy_payload.clone())
.await
.expect("legacy source part should be written");
disk.rename_part(tmp_volume, "legacy/part.1", bucket, "object/part.1", legacy_meta.clone())
.await
.expect("pre-transaction peer RenamePart should remain supported");
assert_eq!(
disk.read_all(bucket, "object/part.1")
.await
.expect("legacy destination part should be readable"),
legacy_payload
);
assert_eq!(
disk.read_all(bucket, "object/part.1.meta")
.await
.expect("legacy destination metadata should be readable"),
legacy_meta
);
}
#[tokio::test]
async fn test_part_transaction_rolls_back_data_published_before_metadata() {
use tempfile::tempdir;
let dir = tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
ensure_test_volume(&disk, "tmp").await;
ensure_test_volume(&disk, "bucket").await;
disk.write_all("tmp", "upload/part.1", Bytes::from_static(b"new data"))
.await
.expect("new part should be staged");
disk.write_all("bucket", "object/part.1", Bytes::from_static(b"old data"))
.await
.expect("old part data should be staged");
disk.write_all("bucket", "object/part.1.meta", Bytes::from_static(b"old metadata"))
.await
.expect("old part metadata should be staged");
disk.prepare_part_transaction("tmp", "upload/part.1", "bucket", "object/part.1", Bytes::from_static(b"new metadata"))
.await
.expect("part transaction should be prepared");
disk.rename_file("tmp", "upload/part.1", "bucket", "object/part.1")
.await
.expect("data publication should succeed");
disk.settle_part_transaction("bucket", "object/part.1", PartTransactionAction::Rollback)
.await
.expect("part transaction should roll back");
assert_eq!(
disk.read_all("bucket", "object/part.1")
.await
.expect("old part data should be restored"),
Bytes::from_static(b"old data")
);
assert_eq!(
disk.read_all("bucket", "object/part.1.meta")
.await
.expect("old part metadata should be restored"),
Bytes::from_static(b"old metadata")
);
}
struct BlockingScanWriter {
+58
View File
@@ -38,6 +38,16 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
pub const HEALING_MARKER_PATH: &str = "healing.bin";
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
pub const PART_TRANSACTION_NEW_META: &str = "new.meta";
pub const PART_TRANSACTION_OLD_META: &str = "old.meta";
pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
pub fn part_transaction_path(part_path: &str) -> String {
match part_path.rsplit_once('/') {
Some((parent, name)) => format!("{parent}/.{name}.rustfs-txn"),
None => format!(".{part_path}.rustfs-txn"),
}
}
use crate::cluster::rpc::RemoteDisk;
use crate::cluster::rpc::build_internode_data_transport_from_env;
@@ -62,6 +72,12 @@ pub type DiskStore = Arc<Disk>;
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
pub type FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PartTransactionAction {
Commit,
Rollback,
}
#[derive(Clone, Copy, Debug)]
pub struct MmapCopyStageMetrics {
pub(crate) path: &'static str,
@@ -381,6 +397,35 @@ impl DiskAPI for Disk {
}
}
async fn prepare_part_transaction(
&self,
src_volume: &str,
src_path: &str,
dst_volume: &str,
dst_path: &str,
meta: Bytes,
) -> Result<()> {
match self {
Disk::Local(local_disk) => {
local_disk
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
.await
}
Disk::Remote(remote_disk) => {
remote_disk
.prepare_part_transaction(src_volume, src_path, dst_volume, dst_path, meta)
.await
}
}
}
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
match self {
Disk::Local(local_disk) => local_disk.settle_part_transaction(volume, path, action).await,
Disk::Remote(remote_disk) => remote_disk.settle_part_transaction(volume, path, action).await,
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
match self {
@@ -659,6 +704,19 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
// ReadFileStream
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>;
async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Bytes) -> Result<()>;
async fn prepare_part_transaction(
&self,
_src_volume: &str,
_src_path: &str,
_dst_volume: &str,
_dst_path: &str,
_meta: Bytes,
) -> Result<()> {
Err(DiskError::MethodNotAllowed)
}
async fn settle_part_transaction(&self, _volume: &str, _path: &str, _action: PartTransactionAction) -> Result<()> {
Err(DiskError::MethodNotAllowed)
}
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
// VerifyFile
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp>;
+47 -2
View File
@@ -443,7 +443,9 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorith
size.div_ceil(shard_size) * algo.size() + size
}
/// Verify an interleaved per-block bitrot shard file.
/// Verify an interleaved per-block bitrot shard file and consume the reader
/// through EOF. Bytes beyond the encoded length are corruption, even when every
/// expected block has a valid hash.
///
/// The read loop below assumes every block on disk is `[hash][data]` (streaming
/// bitrot). It is therefore only valid for the streaming Highway variants, whose
@@ -487,6 +489,11 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
left -= read;
}
let mut trailing = [0u8; 1];
if r.read(&mut trailing).await? != 0 {
return Err(std::io::Error::other("bitrot shard file has trailing data"));
}
Ok(())
}
@@ -860,12 +867,19 @@ mod tests {
.await
.expect("valid bitrot shard file should verify");
let mut truncated = written.clone();
truncated.pop();
let err = bitrot_verify(Cursor::new(truncated), written.len(), data.len(), algo.clone(), shard_size)
.await
.expect_err("one-byte-short shard file must be rejected while reading");
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
let err = bitrot_verify(Cursor::new(written.clone()), written.len() - 1, data.len(), algo.clone(), shard_size)
.await
.expect_err("wrong file size must be rejected before reading data");
assert!(err.to_string().contains("size mismatch"));
let mut corrupt = written;
let mut corrupt = written.clone();
let last = corrupt.len() - 1;
corrupt[last] ^= 0x80;
let err = bitrot_verify(
@@ -878,6 +892,21 @@ mod tests {
.await
.expect_err("hash mismatch must reject corrupted data");
assert!(err.to_string().contains("hash mismatch"));
for trailing in [vec![0xa5], vec![0xa5; 17]] {
let mut oversized = written.clone();
oversized.extend_from_slice(&trailing);
let err = bitrot_verify(
Cursor::new(oversized),
written.len(),
data.len(),
HashAlgorithm::HighwayHash256S,
shard_size,
)
.await
.expect_err("trailing bytes after a valid encoded shard must be rejected");
assert!(err.to_string().contains("trailing data"));
}
}
#[tokio::test]
@@ -909,6 +938,22 @@ mod tests {
assert!(err.to_string().contains("hash mismatch"));
}
#[tokio::test]
async fn bitrot_verify_accepts_exact_legacy_streaming_layout() {
let data = b"legacy streaming bitrot";
let shard_size = 8;
let algo = HashAlgorithm::HighwayHash256SLegacy;
let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), shard_size, algo.clone());
for chunk in data.chunks(shard_size) {
writer.write(chunk).await.expect("legacy streaming shard should encode");
}
let written = writer.into_inner().into_inner();
bitrot_verify(Cursor::new(written.clone()), written.len(), data.len(), algo, shard_size)
.await
.expect("exact legacy streaming shard should remain valid");
}
#[tokio::test]
async fn write_all_vectored_retries_partial_hash_and_data_writes_and_rejects_zero_write() {
let mut writer = LimitedVectoredWriter {
+57 -7
View File
@@ -28,7 +28,6 @@ use md5::{Digest, Md5};
use rustfs_kms::{KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext};
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
use rustfs_utils::path::path_join_buf;
#[cfg(feature = "rio-v2")]
use serde::Deserialize;
#[cfg(feature = "rio-v2")]
use sha2::Sha256;
@@ -44,6 +43,7 @@ const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size";
const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size";
const DEFAULT_SSE_ALGORITHM: &str = "AES256";
const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1;
#[cfg(feature = "rio-v2")]
const DARE_PAYLOAD_SIZE: i64 = 64 * 1024;
#[cfg(feature = "rio-v2")]
@@ -1716,16 +1716,39 @@ fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context
fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
let encrypted_dek = std::str::from_utf8(encrypted_dek).map_err(|_| Error::other("managed DEK is not valid UTF-8"))?;
let parts: Vec<&str> = encrypted_dek.split(':').collect();
if parts.len() != 2 {
return Err(Error::other("invalid managed DEK format"));
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct LocalSseDekEnvelope<'a> {
version: u8,
nonce: &'a str,
ciphertext: &'a str,
}
let (nonce, ciphertext) = match serde_json::from_str::<LocalSseDekEnvelope<'_>>(encrypted_dek) {
Ok(envelope) => {
if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION {
return Err(Error::other(format!("unsupported managed DEK format version: {}", envelope.version)));
}
(envelope.nonce, envelope.ciphertext)
}
Err(_) => {
// DEPRECATED: read-only compatibility for persisted colon-delimited DEKs.
// RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs.
let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else {
return Err(Error::other("invalid managed DEK format"));
};
if ciphertext.contains(':') {
return Err(Error::other("invalid managed DEK format"));
}
(nonce, ciphertext)
}
};
let nonce_vec = BASE64_STANDARD
.decode(parts[0])
.decode(nonce)
.map_err(|_| Error::other("invalid managed DEK nonce"))?;
let ciphertext = BASE64_STANDARD
.decode(parts[1])
.decode(ciphertext)
.map_err(|_| Error::other("invalid managed DEK ciphertext"))?;
let nonce_array: [u8; 12] = nonce_vec
@@ -2324,9 +2347,36 @@ mod tests {
let cipher = Aes256Gcm::new(&key);
let nonce = Nonce::from([0u8; 12]);
let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt managed dek");
serde_json::json!({
"version": LOCAL_SSE_DEK_FORMAT_VERSION,
"nonce": BASE64_STANDARD.encode(nonce),
"ciphertext": BASE64_STANDARD.encode(ciphertext),
})
.to_string()
}
fn encrypt_legacy_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String {
let key = Key::<Aes256Gcm>::from(master_key);
let cipher = Aes256Gcm::new(&key);
let nonce = Nonce::from([0u8; 12]);
let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt legacy managed dek");
format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext))
}
#[test]
fn decrypt_rustfs_local_sse_dek_rejects_unknown_json_version() {
let envelope = serde_json::json!({
"version": LOCAL_SSE_DEK_FORMAT_VERSION + 1,
"nonce": BASE64_STANDARD.encode([0u8; 12]),
"ciphertext": BASE64_STANDARD.encode([0u8; 48]),
})
.to_string();
let error =
decrypt_rustfs_local_sse_dek(envelope.as_bytes()).expect_err("unknown local SSE DEK versions must fail closed");
assert!(error.to_string().contains("unsupported managed DEK format version"));
}
#[cfg(feature = "rio-v2")]
fn seal_managed_s3_object_key_for_test(
bucket: &str,
@@ -2433,7 +2483,7 @@ mod tests {
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([7u8; 32])))], async {
let data_key = [0x24; 32];
let base_nonce = [0x14; 12];
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [7u8; 32]);
let encrypted_dek = encrypt_legacy_managed_dek_for_test(data_key, [7u8; 32]);
let metadata = HashMap::from([
(
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
+213 -15
View File
@@ -46,7 +46,10 @@ use crate::diagnostics::get::{
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::disk::OldCurrentSize;
use crate::disk::{
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
part_transaction_path,
};
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::ShardReader;
use crate::io_support::bitrot::{
@@ -3176,6 +3179,155 @@ impl SetDisks {
}
}
async fn recover_part_transaction(&self, dst_object: &str, write_quorum: usize) -> disk::error::Result<bool> {
let disks = self.get_disks_internal().await;
let transaction_path = part_transaction_path(dst_object);
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
let current_meta_path = format!("{dst_object}.meta");
let reads = disks.iter().map(|disk| {
let disk = disk.clone();
let transaction_meta_path = transaction_meta_path.clone();
let rollback_path = rollback_path.clone();
let current_meta_path = current_meta_path.clone();
async move {
let Some(disk) = disk else {
return Ok((None, None, false));
};
let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
Ok(meta) => Some(meta),
Err(DiskError::FileNotFound) => None,
Err(err) => return Err(err),
};
let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
Ok(_) => true,
Err(DiskError::FileNotFound) => false,
Err(err) => return Err(err),
};
let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &current_meta_path).await {
Ok(meta) => Some(meta),
Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None,
Err(_) => None,
};
Ok((transaction_meta, current_meta, rollback))
}
});
let observations = join_all(reads).await.into_iter().collect::<disk::error::Result<Vec<_>>>()?;
if observations.iter().all(|(transaction, _, _)| transaction.is_none()) {
return Ok(false);
}
let mut current_counts: HashMap<Bytes, usize> = HashMap::new();
for (_, current, _) in &observations {
if let Some(current) = current {
*current_counts.entry(current.clone()).or_default() += 1;
}
}
let current_quorum = current_counts
.into_iter()
.find_map(|(meta, count)| (count >= write_quorum).then_some(meta));
let old_meta_path = format!("{transaction_path}/{PART_TRANSACTION_OLD_META}");
let old_meta_absent_path = format!("{transaction_path}/old.meta.absent");
let decisions = observations
.iter()
.enumerate()
.filter_map(|(index, (transaction_meta, _, rollback))| {
transaction_meta.as_ref().map(|meta| (index, meta.clone(), *rollback))
})
.map(|(index, transaction_meta, rollback)| {
let disk = disks[index].clone();
let old_meta_path = old_meta_path.clone();
let old_meta_absent_path = old_meta_absent_path.clone();
let current_quorum = current_quorum.clone();
async move {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let action = if rollback {
PartTransactionAction::Rollback
} else if current_quorum.as_ref() == Some(&transaction_meta) {
PartTransactionAction::Commit
} else if let Some(current_quorum) = current_quorum {
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_path).await {
Ok(old_meta) if old_meta == current_quorum => PartTransactionAction::Rollback,
Ok(_) => PartTransactionAction::Commit,
Err(DiskError::FileNotFound) => {
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &old_meta_absent_path).await {
Ok(_) => PartTransactionAction::Commit,
Err(_) => return Err(DiskError::FileCorrupt),
}
}
Err(err) => return Err(err),
}
} else {
PartTransactionAction::Rollback
};
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_object, action)
.await?;
Ok(action == PartTransactionAction::Commit)
}
});
let results = join_all(decisions).await;
if let Some(err) = results.iter().find_map(|result| result.as_ref().err()) {
return Err(err.clone());
}
Ok(results.iter().any(|result| matches!(result, Ok(true))))
}
pub(in crate::set_disk) async fn recover_part_transactions(
&self,
part_path: &str,
read_quorum: usize,
write_quorum: usize,
) -> disk::error::Result<()> {
let disks = self.get_disks_internal().await;
let listings = join_all(disks.iter().map(|disk| {
let disk = disk.clone();
async move {
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
.await
}
}))
.await;
let mut transaction_parts = HashSet::new();
let mut errs = Vec::with_capacity(listings.len());
for listing in listings {
match listing {
Ok(entries) => {
errs.push(None);
for entry in entries {
let name = entry.trim_end_matches('/');
let Some(part_number) = name
.strip_prefix(".part.")
.and_then(|name| name.strip_suffix(".rustfs-txn"))
.and_then(|number| number.parse::<usize>().ok())
else {
continue;
};
transaction_parts.insert(part_number);
}
}
Err(err) => errs.push(Some(err)),
}
}
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
return Err(err);
}
for part_number in transaction_parts {
self.recover_part_transaction(&format!("{part_path}part.{part_number}"), write_quorum)
.await?;
}
Ok(())
}
#[tracing::instrument(skip(disks, meta))]
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn rename_part(
@@ -3189,23 +3341,46 @@ impl SetDisks {
write_quorum: usize,
quorum_context: Option<MultipartWriteQuorumContext<'_>>,
) -> disk::error::Result<Vec<Option<DiskStore>>> {
self.recover_part_transaction(dst_object, write_quorum).await?;
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
// Do NOT pre-delete the destination part before renaming: the per-disk
// `rename_part` replaces `part.N` atomically (std::fs::rename) and rewrites
// `part.N.meta`, so the pre-delete is redundant — and destructive. It
// opened a window where an already-committed (ACKed) part was removed on
// every disk before the new rename landed, so a re-upload that then failed
// quorum destroyed the committed part outright (backlog#853 / #799 B4).
// The atomic rename overwrites in place; on quorum failure below we roll
// the destination back.
let prepare_tasks = disks.iter().map(|disk| {
let disk = disk.clone();
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
let dst_bucket = dst_bucket.clone();
let dst_object = dst_object.clone();
let meta = meta.clone();
async move {
let disk = disk?;
Some(
disk.prepare_part_transaction(&src_bucket, &src_object, &dst_bucket, &dst_object, meta)
.await,
)
}
});
let prepare_results = join_all(prepare_tasks).await;
let prepare_errs = prepare_results
.into_iter()
.map(|result| match result {
Some(Ok(())) => None,
Some(Err(err)) => Some(err),
None => Some(DiskError::DiskNotFound),
})
.collect::<Vec<_>>();
let prepared_disks = Self::eval_disks(disks, &prepare_errs);
if reduce_write_quorum_errs(&prepare_errs, OBJECT_OP_IGNORED_ERRS, write_quorum).is_some() {
self.recover_part_transaction(&dst_object, write_quorum).await?;
return Err(DiskError::ErasureWriteQuorum);
}
let mut errs = Vec::with_capacity(disks.len());
let futures = disks.iter().map(|disk| {
let futures = prepared_disks.iter().map(|disk| {
let disk = disk.clone();
let meta = meta.clone();
let src_bucket = src_bucket.clone();
@@ -3255,19 +3430,42 @@ impl SetDisks {
);
}
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
let reduced_err = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
if let Some(err) = reduced_err {
let rollbacks = prepared_disks.iter().filter_map(|disk| {
disk.clone().map(|disk| {
let dst_object = dst_object.clone();
async move {
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &dst_object, PartTransactionAction::Rollback)
.await
}
})
});
let rollback_results = join_all(rollbacks).await;
self.recover_part_transaction(&dst_object, write_quorum).await?;
if let Some(rollback_err) = rollback_results.iter().find_map(|result| result.as_ref().err()) {
warn!(error = %rollback_err, "rename_part rollback did not settle on every prepared disk");
}
if let Some(context) = quorum_context {
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
} else {
warn!("rename_part errs {:?}", &errs);
}
self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
.await;
return Err(err);
}
let disks = Self::eval_disks(disks, &errs);
Ok(disks)
let committed = self.recover_part_transaction(&dst_object, write_quorum).await?;
if !committed {
let err = DiskError::ErasureWriteQuorum;
if let Some(context) = quorum_context {
log_multipart_write_quorum_failure(context, &errs, write_quorum, &err);
} else {
warn!("rename_part errs {:?}", &errs);
}
return Err(err);
}
Ok(Self::eval_disks(&prepared_disks, &errs))
}
pub(in crate::set_disk) fn eval_disks(disks: &[Option<DiskStore>], errs: &[Option<DiskError>]) -> Vec<Option<DiskStore>> {
+2
View File
@@ -690,6 +690,8 @@ mod ops;
#[cfg(feature = "test-util")]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
pub(crate) use ops::object::body_cache_plaintext_len;
#[cfg(test)]
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
mod read;
mod replication;
pub(crate) mod shard_source;
@@ -199,6 +199,32 @@ mod tests {
.expect("healthy temp shard should verify");
assert_eq!(verified, 1);
for trailing in [vec![0xa5], vec![0xa5; 17]] {
let mut oversized = encoded.to_vec();
oversized.extend_from_slice(&trailing);
disk.write_all(RUSTFS_META_TMP_BUCKET, path, Bytes::from(oversized))
.await
.expect("oversized temp shard should be staged");
let err = verify_written_bitrot_shards(
&[Some(disk.clone())],
None,
BitrotSelfVerifyTarget {
operation: "put_object",
bucket: "bucket",
object: "object",
part_number: None,
volume: RUSTFS_META_TMP_BUCKET,
path,
logical_shard_size,
shard_size,
write_quorum: 1,
},
)
.await
.expect_err("disk shard with trailing bytes must not be committed");
assert!(err.to_string().contains("trailing data"));
}
let mut corrupt = encoded.to_vec();
let last = corrupt.len() - 1;
corrupt[last] ^= 0x01;
@@ -225,4 +251,40 @@ mod tests {
.expect_err("corrupt no-parity temp shard must not be committed");
assert!(err.to_string().contains("bitrot self-verify failed"));
}
#[tokio::test]
async fn no_parity_inline_self_verify_rejects_trailing_bytes() {
let (_temp_dirs, disks, _set_disks) = hermetic_set_disks_for_pool_with_default_parity(1, 0, 0).await;
let shard_size = 16usize;
let payload = b"inline bitrot payload";
let encoded = encode_streaming_shard(payload, shard_size).await;
let disks = disks.into_iter().map(Some).collect::<Vec<_>>();
for trailing in [vec![0xa5], vec![0xa5; 17]] {
let mut oversized = encoded.to_vec();
oversized.extend_from_slice(&trailing);
let parts = [FileInfo {
data: Some(Bytes::from(oversized)),
..Default::default()
}];
let err = verify_written_bitrot_shards(
&disks,
Some(&parts),
BitrotSelfVerifyTarget {
operation: "put_object",
bucket: "bucket",
object: "inline-object",
part_number: None,
volume: RUSTFS_META_TMP_BUCKET,
path: "unused-for-inline",
logical_shard_size: payload.len(),
shard_size,
write_quorum: 1,
},
)
.await
.expect_err("inline shard with trailing bytes must not be committed");
assert!(err.to_string().contains("trailing data"));
}
}
}
+257 -8
View File
@@ -19,6 +19,71 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_HEAL: &str = "heal";
const EVENT_HEAL_OBJECT_RENAME: &str = "heal_object_rename";
const HEAL_RENAME_INCOMPLETE: &str = "heal rename incomplete";
#[cfg(test)]
static HEAL_RENAME_FAILURES: std::sync::Mutex<Vec<(String, String, usize)>> = std::sync::Mutex::new(Vec::new());
#[cfg(test)]
struct HealRenameFailureScope {
bucket: String,
object: String,
}
#[cfg(test)]
impl HealRenameFailureScope {
fn install(bucket: &str, object: &str, disk_indexes: &[usize]) -> Self {
let mut failures = HEAL_RENAME_FAILURES
.lock()
.expect("heal rename failure registry should not poison");
assert!(
!failures
.iter()
.any(|(registered_bucket, registered_object, _)| { registered_bucket == bucket && registered_object == object }),
"heal rename failures must be installed once per object"
);
failures.extend(
disk_indexes
.iter()
.map(|index| (bucket.to_string(), object.to_string(), *index)),
);
Self {
bucket: bucket.to_string(),
object: object.to_string(),
}
}
}
#[cfg(test)]
impl Drop for HealRenameFailureScope {
fn drop(&mut self) {
HEAL_RENAME_FAILURES
.lock()
.expect("heal rename failure registry should not poison")
.retain(|(bucket, object, _)| bucket != &self.bucket || object != &self.object);
}
}
#[cfg(test)]
fn should_fail_heal_rename(bucket: &str, object: &str, disk_index: usize) -> bool {
let mut failures = HEAL_RENAME_FAILURES
.lock()
.expect("heal rename failure registry should not poison");
if let Some(position) = failures
.iter()
.position(|entry| entry == &(bucket.to_string(), object.to_string(), disk_index))
{
failures.swap_remove(position);
true
} else {
false
}
}
#[cfg(not(test))]
fn should_fail_heal_rename(_bucket: &str, _object: &str, _disk_index: usize) -> bool {
false
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct PartFailureSummary {
@@ -659,8 +724,12 @@ impl SetDisks {
}
}
// Rename from tmp location to the actual location.
// MinIO stops on the first RenameData error. RustFS intentionally
// continues per target, but reports any residue after all attempts
// so successful repairs survive and failed targets remain retryable.
let mut rename_attempts = 0usize;
let mut rename_successes = 0usize;
let mut healed_disks = vec![None; out_dated_disks.len()];
for (index, outdated_disk) in out_dated_disks.iter().enumerate() {
if let Some(disk) = outdated_disk {
rename_attempts += 1;
@@ -669,9 +738,18 @@ impl SetDisks {
// Attempt a rename now from healed data to final location.
parts_metadata[index].set_healing();
let rename_result = disk
.rename_data(RUSTFS_META_TMP_BUCKET, &tmp_id, parts_metadata[index].clone(), bucket, object)
.await;
let rename_result = if should_fail_heal_rename(bucket, object, index) {
Err(DiskError::Unexpected)
} else {
disk.rename_data(
RUSTFS_META_TMP_BUCKET,
&tmp_id,
parts_metadata[index].clone(),
bucket,
object,
)
.await
};
if let Err(err) = &rename_result {
warn!(
@@ -690,6 +768,7 @@ impl SetDisks {
);
} else {
rename_successes += 1;
healed_disks[index] = Some(disk.clone());
if parts_metadata[index].is_remote() {
let rm_data_dir =
parts_metadata[index].data_dir.expect("operation should succeed").to_string();
@@ -734,15 +813,18 @@ impl SetDisks {
.await
.map_err(DiskError::other)?;
if rename_attempts > 0 && rename_successes == 0 {
self.record_healed_capacity_scope(&healed_disks);
if rename_successes < rename_attempts {
return Ok((
result,
Some(DiskError::other(format!("all healed data rename attempts failed for {bucket}/{object}"))),
Some(DiskError::other(format!(
"{HEAL_RENAME_INCOMPLETE}: {rename_successes} of {rename_attempts} targets committed for \
{bucket}/{object}"
))),
));
}
self.record_healed_capacity_scope(&out_dated_disks);
// The object is healthy here; sweep any data dirs left behind
// by pre-#3510 unversioned overwrites, which the dangling paths
// above never touch (issues #3231, #3191). Best effort — a
@@ -1312,11 +1394,13 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {
#[cfg(test)]
mod heal_result_report_tests {
use super::SetDisks;
use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
use crate::disk::endpoint::Endpoint;
use crate::disk::error::DiskError;
use crate::disk::format::FormatV3;
use crate::disk::{DiskOption, DiskStore, new_disk};
use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk};
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use crate::{config::storageclass, store::init_format::save_format_file};
@@ -1406,6 +1490,171 @@ mod heal_result_report_tests {
(dir, set)
}
async fn non_trash_tmp_entries(temp_dirs: &[TempDir]) -> Vec<String> {
let mut entries = Vec::new();
for dir in temp_dirs {
let tmp = dir.path().join(RUSTFS_META_TMP_BUCKET);
let mut read_dir = match tokio::fs::read_dir(&tmp).await {
Ok(read_dir) => read_dir,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
Err(err) => panic!("tmp directory should be readable: {err}"),
};
while let Some(entry) = read_dir.next_entry().await.expect("tmp entry should be readable") {
let name = entry.file_name().to_string_lossy().into_owned();
if name != ".trash" {
entries.push(name);
}
}
}
entries
}
#[tokio::test]
#[serial_test::serial]
async fn heal_rename_outcome_matrix_reports_partial_and_retries_failed_targets() {
for (case, failed_attempts, expect_error) in [
("ok-ok", Vec::new(), false),
("ok-err", vec![1], true),
("err-ok", vec![0], true),
("err-err", vec![0, 1], true),
] {
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
let bucket = format!("heal-rename-{case}");
let object = "object.bin";
for disk in &disks {
disk.make_volume(&bucket).await.expect("bucket volume should be created");
}
let payload = vec![0x5a; 1024 * 1024];
let mut reader = PutObjReader::from_vec(payload);
set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("source object should be written");
let source = disks[2]
.read_version("", &bucket, object, "", &ReadOptions::default())
.await
.expect("source metadata should be readable");
let data_dir = source.data_dir.expect("non-inline source should have a data directory");
let tmp_entries_before_heal = non_trash_tmp_entries(&temp_dirs).await;
let target_slots = {
let mut slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1];
slots.sort_unstable();
slots
};
let failed_slots = failed_attempts
.iter()
.map(|attempt| target_slots[*attempt])
.collect::<Vec<_>>();
let failed_physical_indexes = [0, 1]
.into_iter()
.filter(|index| failed_slots.contains(&(source.erasure.distribution[*index] - 1)))
.collect::<Vec<_>>();
for index in [0, 1] {
tokio::fs::remove_file(
temp_dirs[index]
.path()
.join(&bucket)
.join(object)
.join(data_dir.to_string())
.join("part.1"),
)
.await
.expect("target shard should be removed before heal");
}
let failure_scope = HealRenameFailureScope::install(&bucket, object, &failed_slots);
let (first_result, first_error) = set
.heal_object(
&bucket,
object,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("heal should report its per-target rename outcome");
drop(failure_scope);
assert_eq!(first_error.is_some(), expect_error, "{case}: aggregate status must match target outcomes");
if let Some(error) = first_error {
let error = error.to_string();
assert!(
error.contains(HEAL_RENAME_INCOMPLETE),
"{case}: partial/all failure must have an explicit retryable status: {error}"
);
assert!(
error.contains(&format!("{} of 2 targets committed", 2 - failed_slots.len())),
"{case}: aggregate status must distinguish partial from all-target failure: {error}"
);
}
for index in [0, 1] {
let expected = if failed_physical_indexes.contains(&index) {
DriveState::Missing
} else {
DriveState::Ok
};
assert_eq!(
first_result.after.drives[index].state,
expected.to_string(),
"{case}: after.drives must reflect the actual rename outcome at index {index}"
);
assert_eq!(
temp_dirs[index]
.path()
.join(&bucket)
.join(object)
.join(data_dir.to_string())
.join("part.1")
.exists(),
!failed_physical_indexes.contains(&index),
"{case}: tmp cleanup must neither delete committed shards nor expose failed targets"
);
}
let tmp_entries_after_heal = non_trash_tmp_entries(&temp_dirs).await;
assert!(
tmp_entries_after_heal
.iter()
.all(|entry| tmp_entries_before_heal.contains(entry)),
"{case}: first heal must not leave a new temporary shard: {tmp_entries_after_heal:?}"
);
if !failed_slots.is_empty() {
let (retry_result, retry_error) = set
.heal_object(
&bucket,
object,
"",
&HealOpts {
no_lock: true,
scan_mode: HealScanMode::Deep,
..Default::default()
},
)
.await
.expect("second heal should retry failed targets");
assert!(retry_error.is_none(), "{case}: second heal should complete remaining targets");
for index in [0, 1] {
assert_eq!(
retry_result.after.drives[index].state,
DriveState::Ok.to_string(),
"{case}: second heal must converge target {index}"
);
}
let tmp_entries_after_retry = non_trash_tmp_entries(&temp_dirs).await;
assert!(
tmp_entries_after_retry
.iter()
.all(|entry| tmp_entries_before_heal.contains(entry)),
"{case}: retry must not leave a new temporary shard: {tmp_entries_after_retry:?}"
);
}
}
}
// Regression for #955: an offline disk must contribute exactly one drive
// record. Before the fix the offline branch fell through and pushed a second
// (Corrupt) record for the same disk, so `before/after.drives` grew to
+38 -2
View File
@@ -69,6 +69,13 @@ struct HealWalkCollector {
}
impl HealWalkCollector {
fn lock_objects(&self) -> disk::error::Result<std::sync::MutexGuard<'_, Vec<HealWalkObject>>> {
self.objects.lock().map_err(|_| {
self.cancel.cancel();
DiskError::FileCorrupt
})
}
/// Expand one resolved entry into its versions and record it. Cancels the
/// walk once EITHER page bound (distinct object names OR expanded versions)
/// is met — always at a sorted object-key boundary so a heavily-versioned
@@ -105,7 +112,9 @@ impl HealWalkCollector {
let added = versions.len();
let (objs_len, ver_total) = {
let mut objects = self.objects.lock().unwrap();
let Ok(mut objects) = self.lock_objects() else {
return;
};
objects.push(HealWalkObject {
name: entry.name,
versions,
@@ -239,7 +248,10 @@ impl SetDisks {
Err(err) => return Err(err),
}
let objects = std::mem::take(&mut *collector.objects.lock().unwrap());
let objects = {
let mut objects = collector.lock_objects()?;
std::mem::take(&mut *objects)
};
let truncated = collector.truncated.load(Ordering::SeqCst);
Ok(finalize_heal_walk_page(objects, forward_to, truncated))
}
@@ -310,4 +322,28 @@ mod tests {
assert_eq!(next_forward, None, "a complete final page must not carry a resume cursor");
assert!(!truncated);
}
#[test]
fn poisoned_collector_state_cancels_the_walk() {
let collector = Arc::new(HealWalkCollector {
bucket: "bucket".to_string(),
batch_objects: 2,
version_budget: 2,
objects: Mutex::new(Vec::new()),
version_total: AtomicUsize::new(0),
truncated: AtomicBool::new(false),
cancel: CancellationToken::new(),
});
let poison_target = Arc::clone(&collector);
let _ = std::thread::spawn(move || {
let _guard = poison_target.objects.lock().expect("fresh mutex should lock");
panic!("poison heal collector");
})
.join();
let error = collector.lock_objects().expect_err("poisoned collection must fail closed");
assert_eq!(error, DiskError::FileCorrupt);
assert!(collector.cancel.is_cancelled(), "a poisoned page collector must cancel its walk");
assert!(collector.objects.lock().is_err(), "poisoned state must remain fail-closed");
}
}
+213 -24
View File
@@ -1289,8 +1289,18 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
}
}
if !opts.no_lock && object_lock_guard.is_none() {
object_lock_guard = Some(
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
.await?,
);
}
let upload_guard = self
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
.await?;
let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?;
let (mut fi, mut files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
let has_layout_candidate = range_seek_rollout_enabled
&& fi
.data_dir
@@ -1303,22 +1313,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
&token,
)
});
let upload_guard = if has_layout_candidate {
if object_lock_guard.is_none() {
object_lock_guard = Some(
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
.await?,
);
}
let guard = self
.acquire_multipart_upload_write_lock("complete_multipart_upload_commit", bucket, object, upload_id, opts)
.await?;
(fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
guard
} else {
None
};
let quorum_validated_layout_token = upload_guard.as_ref().and_then(|_| {
let quorum_validated_layout_token = if has_layout_candidate {
fi.data_dir
.filter(|data_dir| !data_dir.is_nil())
.map(|data_dir| data_dir.to_string())
@@ -1329,7 +1324,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
token,
)
})
});
} else {
None
};
rustfs_utils::http::metadata_compat::remove_str(
&mut fi.metadata,
crate::object_api::ENCRYPTED_PART_LAYOUT_CANDIDATE_SUFFIX,
@@ -1356,6 +1353,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
// let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
.await
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
let part_meta_paths = uploaded_parts
.iter()
@@ -1713,12 +1713,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
}
}
if !opts.no_lock && object_lock_guard.is_none() {
object_lock_guard = Some(
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
.await?,
);
}
// Phase 2 (backlog#899): fence the commit on lock loss before any destructive
// step. If the refresh heartbeat has observed a refresh-quorum loss, another
// writer may have re-acquired this object's lock; proceeding would race a
@@ -2183,6 +2177,201 @@ mod tests {
)
}
async fn assert_quorum_minus_one_retry_preserves_completable_part(
disk_count: usize,
parity: usize,
success_indices: &[usize],
) {
use tokio::io::AsyncReadExt as _;
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(disk_count, 0, parity).await;
let bucket = format!("multipart-retry-{disk_count}-{}", success_indices[0]);
let object = "object";
make_bucket_on_all(&disk_stores, &bucket).await;
let payload = vec![0x41; 1 << 20];
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, &bucket, object, &payload, &ObjectOptions::default()).await;
let (upload_meta, _) = set_disks
.check_upload_id_exists(&bucket, object, &upload_id, false)
.await
.expect("staged upload metadata should be readable");
let upload_path = SetDisks::get_upload_id_dir(&bucket, object, &upload_id);
let write_quorum = upload_meta.write_quorum(set_disks.default_write_quorum());
assert_eq!(success_indices.len() + 1, write_quorum);
let part_path = format!(
"{}/{}/part.1",
upload_path,
upload_meta.data_dir.expect("multipart upload should have a data directory")
);
let acknowledged_meta = disk_stores[0]
.read_all(RUSTFS_META_MULTIPART_BUCKET, &format!("{part_path}.meta"))
.await
.expect("acknowledged part metadata should be readable");
let retry_path = format!("{}/part.1", Uuid::new_v4());
let mut retry_disks = vec![None; disk_stores.len()];
for &index in success_indices {
disk_stores[index]
.write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"retry shard"))
.await
.expect("retry shard should be staged");
retry_disks[index] = Some(disk_stores[index].clone());
}
let err = set_disks
.rename_part(
&retry_disks,
RUSTFS_META_TMP_BUCKET,
&retry_path,
RUSTFS_META_MULTIPART_BUCKET,
&part_path,
acknowledged_meta,
write_quorum,
None,
)
.await
.expect_err("quorum-minus-one renamed shards must remain below write quorum");
assert_eq!(err, DiskError::ErasureWriteQuorum);
for (index, disk) in disk_stores.iter().enumerate() {
assert!(
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path).await.is_ok(),
"old acknowledged shard on disk {index} must survive"
);
}
set_disks
.clone()
.complete_multipart_upload(&bucket, object, &upload_id, parts, &ObjectOptions::default())
.await
.expect("the old acknowledged part should remain completable");
let mut reader = set_disks
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("completed object should be readable");
let mut restored = Vec::new();
reader
.stream
.read_to_end(&mut restored)
.await
.expect("completed object should stream fully");
assert_eq!(restored, payload);
}
#[tokio::test]
async fn upload_part_retry_quorum_failure_preserves_old_part_across_ec_geometries() {
assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[0, 1]).await;
assert_quorum_minus_one_retry_preserves_completable_part(4, 2, &[2, 3]).await;
assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[0, 1, 2]).await;
assert_quorum_minus_one_retry_preserves_completable_part(6, 2, &[3, 4, 5]).await;
}
#[tokio::test]
async fn complete_multipart_upload_recovers_interrupted_part_retry() {
use tokio::io::AsyncReadExt as _;
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(6, 0, 2).await;
let bucket = "multipart-retry-recovery";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let payload = vec![0x42; 1 << 20];
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
let (upload_meta, _) = set_disks
.check_upload_id_exists(bucket, object, &upload_id, false)
.await
.expect("staged upload metadata should be readable");
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let part_path = format!(
"{}/{}/part.1",
upload_path,
upload_meta.data_dir.expect("multipart upload should have a data directory")
);
let retry_meta = Bytes::from_static(b"interrupted retry metadata");
for (index, disk) in disk_stores.iter().enumerate().take(3) {
let retry_path = format!("{}/part.1", Uuid::new_v4());
disk.write_all(RUSTFS_META_TMP_BUCKET, &retry_path, Bytes::from_static(b"interrupted retry shard"))
.await
.expect("retry shard should be staged");
disk.prepare_part_transaction(
RUSTFS_META_TMP_BUCKET,
&retry_path,
RUSTFS_META_MULTIPART_BUCKET,
&part_path,
retry_meta.clone(),
)
.await
.expect("part transaction should be prepared");
disk.rename_part(
RUSTFS_META_TMP_BUCKET,
&retry_path,
RUSTFS_META_MULTIPART_BUCKET,
&part_path,
retry_meta.clone(),
)
.await
.unwrap_or_else(|err| panic!("retry shard {index} should be published: {err}"));
}
set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &ObjectOptions::default())
.await
.expect("completion should roll back the interrupted quorum-minus-one retry");
let mut reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("completed object should be readable");
let mut restored = Vec::new();
reader
.stream
.read_to_end(&mut restored)
.await
.expect("completed object should stream fully");
assert_eq!(restored, payload);
}
#[tokio::test]
async fn rename_part_quorum_failure_without_old_part_removes_new_shards() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_for_pool_with_default_parity(4, 0, 2).await;
let src_path = format!("{}/part.1", Uuid::new_v4());
let dst_path = format!("{}/part.1", Uuid::new_v4());
let mut retry_disks = vec![None; disk_stores.len()];
for index in [0, 1] {
disk_stores[index]
.write_all(RUSTFS_META_TMP_BUCKET, &src_path, Bytes::from_static(b"new shard"))
.await
.expect("new shard should be staged");
retry_disks[index] = Some(disk_stores[index].clone());
}
let err = set_disks
.rename_part(
&retry_disks,
RUSTFS_META_TMP_BUCKET,
&src_path,
RUSTFS_META_MULTIPART_BUCKET,
&dst_path,
Bytes::from_static(b"retry metadata"),
3,
None,
)
.await
.expect_err("two renamed shards must remain below write quorum");
assert_eq!(err, DiskError::ErasureWriteQuorum);
for (index, disk) in disk_stores.iter().enumerate() {
assert!(
matches!(disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &dst_path).await, Err(DiskError::FileNotFound)),
"failed first upload must not leave a destination on disk {index}"
);
}
}
async fn make_multipart_lock_test_set_disks() -> Arc<SetDisks> {
let endpoints = vec![
Endpoint::try_from("http://127.0.0.1:9000/data").expect("first endpoint should parse"),
File diff suppressed because it is too large Load Diff
+168 -11
View File
@@ -207,11 +207,9 @@ impl SetDisks {
version_id: &str,
opts: &ReadOptions,
) -> Result<Vec<FileInfo>> {
// Use existing disk selection logic
let disks = self.disks.read().await;
let required_reads = self.format.erasure.sets.len();
let disks = self.disks.read().await.clone();
let required_reads = self.default_read_quorum();
// Clone parameters outside the closure to avoid lifetime issues
let bucket = bucket.to_string();
let object = object.to_string();
let version_id = version_id.to_string();
@@ -220,7 +218,6 @@ impl SetDisks {
let processor = runtime_sources::batch_processors().read_processor();
let tasks: Vec<_> = disks
.iter()
.take(required_reads + 2) // Read a few extra for reliability
.filter_map(|disk| {
disk.as_ref().map(|d| {
let disk = d.clone();
@@ -234,10 +231,10 @@ impl SetDisks {
})
.collect();
match processor.execute_batch_with_quorum(tasks, required_reads).await {
Ok(results) => Ok(results),
Err(_) => Err(DiskError::FileNotFound.into()), // Use existing error type
}
processor
.execute_batch_with_quorum(tasks, required_reads)
.await
.map_err(Into::into)
}
#[tracing::instrument(level = "debug", skip(self))]
@@ -1960,6 +1957,42 @@ mod metadata_cache_tests {
(dir, disk)
}
async fn read_version_quorum_test_set(
bucket: &str,
object: &str,
pool_set_count: usize,
readable_disks: usize,
) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
let mut dirs = Vec::new();
let mut disks = Vec::new();
for disk_index in 0..4 {
let (dir, disk) = new_read_version_test_disk(bucket).await;
if disk_index < readable_disks {
let mut fi = valid_test_fileinfo(object);
fi.mod_time = Some(OffsetDateTime::now_utc());
fi.erasure.index = fi.erasure.distribution[disk_index];
disk.write_metadata(bucket, bucket, object, fi)
.await
.expect("metadata should be written before quorum read");
}
dirs.push(dir);
disks.push(Some(disk));
}
let set = SetDisks::new(
"read-version-quorum-test".to_string(),
Arc::new(RwLock::new(disks)),
4,
2,
0,
0,
Vec::new(),
FormatV3::new(pool_set_count, 4),
Vec::new(),
)
.await;
(dirs, set)
}
#[test]
#[serial]
fn get_object_metadata_cache_capacity_uses_default_and_env_override() {
@@ -1986,6 +2019,33 @@ mod metadata_cache_tests {
fi
}
#[tokio::test]
async fn get_object_with_fileinfo_rejects_positive_size_without_parts() {
let mut output = Vec::new();
let err = SetDisks::get_object_with_fileinfo(
"bucket",
"object",
0,
1,
&mut output,
valid_test_fileinfo("object"),
Vec::new(),
&[],
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
)
.await
.expect_err("positive-size metadata without parts must fail without panicking");
assert_eq!(err, Error::FileCorrupt);
assert!(output.is_empty());
}
#[tokio::test]
async fn get_object_with_fileinfo_rejects_invalid_ranges_before_reader_setup() {
let bucket = "bucket";
@@ -2059,6 +2119,12 @@ mod metadata_cache_tests {
let mut invalid_erasure = valid_test_fileinfo(object);
invalid_erasure.erasure.block_size = 0;
invalid_erasure.parts.push(ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
});
let err = SetDisks::get_object_with_fileinfo(
bucket,
object,
@@ -2087,6 +2153,37 @@ mod metadata_cache_tests {
assert!(output.is_empty());
}
#[tokio::test]
async fn get_object_with_fileinfo_accepts_zero_size_without_parts() {
let bucket = "bucket";
let object = "empty";
let mut fi = valid_test_fileinfo(object);
fi.size = 0;
let mut output = Vec::new();
SetDisks::get_object_with_fileinfo(
bucket,
object,
0,
0,
&mut output,
fi,
Vec::new(),
&[],
0,
0,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"empty",
)
.await
.expect("zero-byte object without parts must remain readable");
assert!(output.is_empty());
}
#[tokio::test]
async fn get_object_with_fileinfo_fails_closed_without_read_quorum() {
let bucket = "bucket";
@@ -2133,6 +2230,7 @@ mod metadata_cache_tests {
let object = "object";
let (_dir, disk) = new_read_version_test_disk(bucket).await;
let mut fi = valid_test_fileinfo(object);
fi.size = 0;
fi.mod_time = Some(OffsetDateTime::now_utc());
disk.write_metadata(bucket, bucket, object, fi.clone())
.await
@@ -2165,11 +2263,70 @@ mod metadata_cache_tests {
.await
.expect_err("empty disk set must fail closed");
assert!(
missing_quorum.to_string().to_ascii_lowercase().contains("file"),
"optimized read failure should map to file-not-found style error: {missing_quorum}"
missing_quorum.to_string().contains("Insufficient successful results"),
"optimized read failure should preserve the batch quorum diagnostic: {missing_quorum}"
);
}
#[tokio::test]
async fn read_version_optimized_uses_current_set_drive_quorum_not_pool_set_count() {
let bucket = "read-version-layout-quorum-bucket";
let object = "object";
let (_single_set_dirs, single_set) = read_version_quorum_test_set(bucket, object, 1, 1).await;
single_set
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect_err("one metadata copy must not satisfy a four-drive 2+2 set");
let (_multi_set_dirs, multi_set) = read_version_quorum_test_set(bucket, object, 3, 2).await;
let versions = multi_set
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect("two metadata copies should satisfy the current set's read quorum");
assert_eq!(versions.len(), 2);
assert!(versions.iter().all(|version| version.name == object));
}
#[tokio::test]
async fn read_version_optimized_counts_only_valid_metadata_toward_quorum() {
let bucket = "read-version-corrupt-quorum-bucket";
let object = "object";
let (quorum_minus_one_dirs, quorum_minus_one) = read_version_quorum_test_set(bucket, object, 1, 2).await;
tokio::fs::write(
quorum_minus_one_dirs[1]
.path()
.join(bucket)
.join(object)
.join(crate::disk::STORAGE_FORMAT_FILE),
b"corrupt metadata",
)
.await
.expect("metadata should be corrupted for the test");
quorum_minus_one
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect_err("one valid and one corrupt metadata copy must not satisfy read quorum");
let (quorum_dirs, quorum) = read_version_quorum_test_set(bucket, object, 1, 3).await;
tokio::fs::write(
quorum_dirs[2]
.path()
.join(bucket)
.join(object)
.join(crate::disk::STORAGE_FORMAT_FILE),
b"corrupt metadata",
)
.await
.expect("metadata should be corrupted for the test");
let versions = quorum
.read_version_optimized(bucket, object, "", &ReadOptions::default())
.await
.expect("two valid metadata copies must satisfy read quorum despite one corrupt copy");
assert_eq!(versions.len(), 2);
}
#[tokio::test]
async fn get_object_info_and_quorum_maps_delete_marker_and_purge_states() {
let bucket = "get-object-info-marker-bucket";
+2
View File
@@ -2722,10 +2722,12 @@ mod tests {
#[serial_test::serial(storage_class_env)]
async fn transition_transaction_recovery_deletes_provider_recovered_unknown_upload() {
let versioned_remote = uuid::Uuid::new_v4().to_string();
let nil_remote = uuid::Uuid::nil().to_string();
for (case, tier_name, remote_version) in [
("missing", "TXPROBEMISSING", None),
("unversioned", "TXPROBEUNVERSIONED", Some(String::new())),
("versioned", "TXPROBEVERSIONED", Some(versioned_remote)),
("nil-version", "TXPROBENILVERSION", Some(nil_remote)),
] {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store(
+13 -1
View File
@@ -6696,7 +6696,7 @@ mod test {
use crate::object_api::ObjectInfo;
use rustfs_filemeta::{
FileInfo, FileMeta, FileMetaVersion, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntry, MetaDeleteMarker,
VersionType,
ObjectPartInfo, VersionType,
};
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
@@ -6899,6 +6899,12 @@ mod test {
fi.volume = "bucket".to_owned();
fi.name = "object".to_owned();
fi.size = 1;
fi.parts = vec![ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
}];
fi.fresh = true;
fi.erasure.index = 1;
fi.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"));
@@ -6954,6 +6960,12 @@ mod test {
fi.version_id = Some(Uuid::from_u128(version_idx));
fi.versioned = true;
fi.size = 1;
fi.parts = vec![ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
}];
fi.mod_time = Some(*mod_time);
fi.metadata = metadata;