Merge branch 'main' into fix/b5-t4-b2-remote-target-serde

This commit is contained in:
houseme
2026-08-23 12:28:53 +08:00
committed by GitHub
254 changed files with 22099 additions and 4287 deletions
-1
View File
@@ -49,4 +49,3 @@ Applies to `crates/ecstore/`.
## Suggested Validation
- `cargo test -p rustfs-ecstore`
- Full gate before commit: `make pre-commit`
+2 -2
View File
@@ -317,8 +317,6 @@ pub mod config {
}
pub mod data_usage {
#[cfg(feature = "test-util")]
pub use crate::data_usage::seed_bucket_usage_memory_for_test;
pub use crate::data_usage::{
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
@@ -330,6 +328,8 @@ pub mod data_usage {
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
store_data_usage_in_backend,
};
#[cfg(feature = "test-util")]
pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
}
pub mod disk {
+39 -1
View File
@@ -866,7 +866,7 @@ impl BucketTargetSys {
return Some(cli);
}
// TODO: spawn a task to reload the target
// TODO(backlog): spawn an async task to proactively reload the replication target
if self.is_reloading_target(bucket, arn).await {
return None;
}
@@ -3425,6 +3425,44 @@ mod tests {
assert!(mutexes.contains_key("second"));
}
#[tokio::test]
async fn update_all_targets_publishes_disable_proxy_on_target_client() {
// The read-proxy selector (replication_proxy::get_proxy_targets) skips
// targets whose TargetClient carries disable_proxy — the persisted
// per-target opt-out must survive client publication.
let sys = BucketTargetSys::default();
let target = |arn: &str, disable_proxy: bool| BucketTarget {
arn: arn.to_string(),
endpoint: "192.168.1.10:9000".to_string(),
target_bucket: "target-bucket".to_string(),
region: "us-east-1".to_string(),
disable_proxy,
credentials: Some(Credentials {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
session_token: None,
expiration: None,
}),
..Default::default()
};
let targets = BucketTargets {
targets: vec![target("arn:proxied", false), target("arn:opted-out", true)],
};
sys.update_all_targets("bucket", Some(&targets)).await;
let proxied = sys
.get_remote_target_client("bucket", "arn:proxied")
.await
.expect("client should be published");
assert!(!proxied.disable_proxy);
let opted_out = sys
.get_remote_target_client("bucket", "arn:opted-out")
.await
.expect("client should be published");
assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn target_updates_serialize_client_build_through_publication_per_bucket() {
let sys = Arc::new(BucketTargetSys::default());
@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
.map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
let mut rstate = oi.replication_state();
rstate.replicate_decision_str = dsc.to_string();
let asz = oi.get_actual_size().unwrap_or_default();
let asz = oi.get_actual_size_or_physical();
let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
let checksum = if ssec { oi.checksum.clone() } else { None };
@@ -1412,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
};
let mut replication_state = oi.replication_state();
replication_state.replicate_decision_str = dsc.to_string();
let actual_size = oi.get_actual_size().unwrap_or_default();
let actual_size = oi.get_actual_size_or_physical();
Ok(ReplicateObjectInfo {
name: oi.name.clone(),
@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
.map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
version_id: object_info.version_id.map(|version_id| version_id.to_string()),
etag: object_info.etag.as_deref(),
actual_size: object_info.get_actual_size().unwrap_or_default(),
actual_size: object_info.get_actual_size_or_physical(),
delete_marker: object_info.delete_marker,
content_type: object_info.content_type.as_deref(),
content_encoding: object_info.content_encoding.as_deref(),
@@ -542,6 +542,20 @@ mod tests {
assert!(replication_target_head_is_newer_null_version(&source, &target));
}
#[test]
fn replication_source_uses_physical_size_for_unknown_compressed_object() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
let source = ObjectInfo {
size: 128,
actual_size: -1,
user_defined: Arc::new(metadata),
..Default::default()
};
assert_eq!(replication_source_object(&source).actual_size, 128);
}
#[test]
fn replication_target_head_content_matches_compare_etag_only() {
let source = ObjectInfo {
@@ -454,7 +454,7 @@ impl S3PeerSys {
}
}
topology_complete &= bucket_map.values().all(|count| *count >= quorum);
// TODO: MRF
// TODO(backlog): integrate MRF backlog stats into scanner bucket listing
}
let mut buckets: Vec<BucketInfo> = result_map.into_values().collect();
+174 -24
View File
@@ -42,18 +42,19 @@ use futures::lock::Mutex;
use metrics::counter;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
};
use rustfs_protos::ChannelClass;
use rustfs_protos::evict_failed_connection;
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_protos::proto_gen::node_service::{
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest,
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest,
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, node_service_client::NodeServiceClient,
};
@@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const BATCH_METADATA_RPC_OFF: &str = "off";
const BATCH_METADATA_RPC_AUTO: &str = "auto";
const BATCH_METADATA_RPC_ON: &str = "on";
@@ -112,6 +114,28 @@ const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
fn decode_delete_versions_errors(response: DeleteVersionsResponse, expected_len: usize) -> Vec<Option<Error>> {
if !response.item_errors.is_empty() {
if response.item_errors.len() != expected_len {
return vec![Some(Error::other("malformed delete_versions item errors")); expected_len];
}
return response
.item_errors
.into_iter()
.map(|error| (error.code != 0).then(|| error.into()))
.collect();
}
if response.errors.len() != expected_len {
return vec![Some(Error::other("malformed delete_versions errors")); expected_len];
}
response
.errors
.into_iter()
.map(|error| (!error.is_empty()).then(|| Error::other(error)))
.collect()
}
fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
if !response.success {
return Err(response.error.unwrap_or_default().into());
@@ -180,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
}
fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
.or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
.as_deref()
.map(parse_batch_metadata_rpc_mode)
@@ -1804,6 +1829,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
}
}
fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
if let Some(started_at) = started_at {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
}
}
/// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
/// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
/// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
@@ -1914,6 +1945,27 @@ fn decode_batch_read_version_response_items(
Ok(batch_read_version_resps)
}
fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
req.items
.iter()
.fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
total
.saturating_add(item.org_volume.len())
.saturating_add(item.volume.len())
.saturating_add(item.path.len())
.saturating_add(item.version_id.len())
})
}
fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
response
.batch_read_version_resps
.iter()
.map(String::len)
.sum::<usize>()
.saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
}
fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
file_info.validate_for_metadata_read().map_err(Into::into)
}
@@ -2406,8 +2458,6 @@ impl DiskAPI for RemoteDisk {
return errors;
}
// TODO: use Error not string
let result = self
.execute_with_timeout(
|| async {
@@ -2439,17 +2489,7 @@ impl DiskAPI for RemoteDisk {
}
return errors;
}
response
.errors
.iter()
.map(|error| {
if error.is_empty() {
None
} else {
Some(Error::other(error.to_string()))
}
})
.collect()
decode_delete_versions_errors(response, versions.len())
}
#[tracing::instrument(level = "trace", skip_all)]
@@ -2827,14 +2867,19 @@ impl DiskAPI for RemoteDisk {
state = "started",
"Remote disk RPC started"
);
let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let batch_read_version_req = compat_json(&req)?;
let batch_read_version_req_bin = encode_msgpack(&req)?;
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
let request_payload_bytes = batch_read_version_attribution_enabled
.then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
let batch_result = self
.execute_with_timeout_for_op(
"batch_read_version",
move || async move {
let disk = self.disk_ref().await;
let disk_len = disk.len();
let mut client = self
.get_bulk_client()
.await
@@ -2845,9 +2890,20 @@ impl DiskAPI for RemoteDisk {
batch_read_version_req_bin: batch_read_version_req_bin.into(),
});
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
if let Some(request_payload_bytes) = request_payload_bytes {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
request_payload_bytes.saturating_add(disk_len),
);
}
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let response = match client.batch_read_version(request).await {
Ok(response) => response.into_inner(),
Ok(response) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
response.into_inner()
}
Err(status) if status.code() == Code::Unimplemented => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
if mode.should_fallback_on_unimplemented() {
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
warn!(
@@ -2864,6 +2920,7 @@ impl DiskAPI for RemoteDisk {
}
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
warn!(
event = EVENT_REMOTE_DISK_RPC,
component = LOG_COMPONENT_ECSTORE,
@@ -2876,14 +2933,33 @@ impl DiskAPI for RemoteDisk {
);
return Err(Error::from(status));
}
Err(status) => return Err(Error::from(status)),
Err(status) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(Error::from(status));
}
};
if !response.success {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
return Err(response.error.unwrap_or_default().into());
}
decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
batch_read_version_response_payload_len(&response),
);
let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
match decode_batch_read_version_response_items(response, &self.endpoint) {
Ok(batch_read_version_resps) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
Ok(Some(batch_read_version_resps))
}
Err(err) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
Err(err)
}
}
},
get_max_timeout_duration(),
)
@@ -3760,6 +3836,63 @@ mod tests {
static INIT: Once = Once::new();
#[test]
fn delete_versions_response_preserves_typed_item_errors() {
let errors = decode_delete_versions_errors(
DeleteVersionsResponse {
success: true,
errors: vec!["file not found".to_string(), String::new()],
error: None,
item_errors: vec![
rustfs_protos::proto_gen::node_service::Error {
code: DiskError::FileNotFound.to_u32(),
error_info: "file not found".to_string(),
},
rustfs_protos::proto_gen::node_service::Error::default(),
],
},
2,
);
assert!(matches!(errors.as_slice(), [Some(DiskError::FileNotFound), None]));
}
#[test]
fn delete_versions_response_accepts_legacy_string_errors() {
let errors = decode_delete_versions_errors(
DeleteVersionsResponse {
success: true,
errors: vec!["legacy error".to_string(), String::new()],
error: None,
item_errors: Vec::new(),
},
2,
);
assert_eq!(errors.len(), 2);
assert_eq!(errors[0].as_ref().map(ToString::to_string).as_deref(), Some("io error legacy error"));
assert!(errors[1].is_none());
}
#[test]
fn delete_versions_response_rejects_misaligned_item_errors() {
let errors = decode_delete_versions_errors(
DeleteVersionsResponse {
success: true,
errors: vec!["file not found".to_string()],
error: None,
item_errors: vec![rustfs_protos::proto_gen::node_service::Error {
code: DiskError::FileNotFound.to_u32(),
error_info: "file not found".to_string(),
}],
},
2,
);
assert_eq!(errors.len(), 2);
assert!(errors.iter().all(Option::is_some));
}
#[test]
fn disk_mutation_digest_marks_rolling_compatibility() {
let mut request = Request::new(());
@@ -4554,6 +4687,7 @@ mod tests {
} else {
"file version not found".to_string()
},
error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
}
}
@@ -4673,6 +4807,7 @@ mod tests {
fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
@@ -4682,10 +4817,25 @@ mod tests {
);
}
#[test]
fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
(ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
],
|| {
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
},
);
}
#[test]
fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
(ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
],
@@ -14,9 +14,10 @@
use rustfs_io_metrics::internode_metrics::{
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
global_internode_metrics,
};
use std::time::Duration;
@@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_stage_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
stage,
duration,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics()
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
bytes,
);
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
}
pub(crate) fn record_remote_disk_grpc_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
+4 -1
View File
@@ -248,10 +248,13 @@ impl Config {
let shard_size = shard_size as usize;
// Keep the historical two-data-shard object budget while preventing
// wider EC layouts from multiplying the maximum inline object size.
// Use div_ceil to match the shard_file_size calculation (which also uses
// div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline
// for objects right at the threshold.
let inline_block = if self.initialized && self.inline_block_explicit {
self.inline_block
} else {
(DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK)
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
};
if versioned {
File diff suppressed because it is too large Load Diff
+84 -69
View File
@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
range::HTTPRangeSpec,
};
use crate::{
@@ -249,7 +249,7 @@ impl Sets {
self.connect_disks().await;
// TODO: config interval
// TODO(backlog): make monitor_and_connect interval configurable instead of hardcoded 15s
let mut interval = tokio::time::interval(Duration::from_secs(15));
loop {
tokio::select! {
@@ -414,6 +414,66 @@ fn apply_delete_objects_results(
}
}
fn apply_delete_accounting_results(
accounting: &mut [Option<DeleteAccounting>],
set_objects: &[DelObj],
set_accounting: &[Option<DeleteAccounting>],
) {
for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
accounting[obj.orig_idx] = value.clone();
}
}
impl Sets {
pub(crate) async fn delete_objects_with_accounting(
&self,
bucket: &str,
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut del_errs = vec![None; objects.len()];
let mut accounting = vec![None; objects.len()];
let mut set_obj_map = HashMap::new();
for (i, obj) in objects.iter().enumerate() {
let idx = self.get_hashed_set_index(obj.object_name.as_str());
set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
orig_idx: i,
obj: obj.clone(),
});
}
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
let mut futures = FuturesUnordered::new();
let bucket = bucket.to_owned();
for (set_index, set_objects) in set_obj_map {
let disks = self.get_disks(set_index);
let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
let bucket = bucket.clone();
let opts = opts.clone();
let semaphore = semaphore.clone();
futures.push(async move {
let _permit = semaphore
.acquire_owned()
.await
.expect("delete_objects semaphore should remain open");
let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
(set_objects, deleted, errors, accounting)
});
}
while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
}
(del_objects, del_errs, accounting)
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::object::ObjectIO for Sets {
type Error = Error;
@@ -655,65 +715,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
// Default return value
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut del_errs = Vec::with_capacity(objects.len());
for _ in 0..objects.len() {
del_errs.push(None)
}
let mut set_obj_map = HashMap::new();
// hash key
for (i, obj) in objects.iter().enumerate() {
let idx = self.get_hashed_set_index(obj.object_name.as_str());
if !set_obj_map.contains_key(&idx) {
set_obj_map.insert(
idx,
vec![DelObj {
// set_idx: idx,
orig_idx: i,
obj: obj.clone(),
}],
);
} else if let Some(val) = set_obj_map.get_mut(&idx) {
val.push(DelObj {
// set_idx: idx,
orig_idx: i,
obj: obj.clone(),
});
}
}
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
let mut futures = FuturesUnordered::new();
let bucket = bucket.to_string();
for (k, v) in set_obj_map {
let disks = self.get_disks(k);
let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
let bucket = bucket.clone();
let opts = opts.clone();
let semaphore = semaphore.clone();
futures.push(async move {
let _permit = semaphore
.acquire_owned()
.await
.expect("delete_objects semaphore should remain open");
let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
(v, dobjects, errs)
});
}
while let Some((v, dobjects, errs)) = futures.next().await {
apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
}
(del_objects, del_errs)
let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
(deleted, errors)
}
#[tracing::instrument(skip(self))]
@@ -985,14 +988,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets {
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::heal::HealOperations for Sets {
type Error = Error;
type HealResultItem = HealResultItem;
type HealOptions = HealOpts;
#[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
impl Sets {
pub(crate) async fn heal_format_with_fence<F>(&self, dry_run: bool, fence_lost: F) -> Result<(HealResultItem, Option<Error>)>
where
F: Fn() -> bool + Send + Sync,
{
let (disks, init_errs) = init_storage_disks_with_errors(
&self.endpoints.endpoints,
&DiskOption {
@@ -1065,6 +1065,9 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
// Save new formats `format.json` on unformatted disks.
for (index, (fm, disk)) in tmp_new_formats.iter_mut().zip(disks.iter()).enumerate() {
if fm.is_some() && disk.is_some() {
if fence_lost() {
return Ok((res, Some(StorageError::SlowDown)));
}
if let Err(err) = save_format_file(disk, fm).await {
if let Some(disk) = disk.as_ref() {
let _ = disk.close().await;
@@ -1098,6 +1101,18 @@ impl crate::storage_api_contracts::heal::HealOperations for Sets {
}
Ok((res, None))
}
}
#[async_trait::async_trait]
impl crate::storage_api_contracts::heal::HealOperations for Sets {
type Error = Error;
type HealResultItem = HealResultItem;
type HealOptions = HealOpts;
#[tracing::instrument(skip(self))]
async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
self.heal_format_with_fence(dry_run, || false).await
}
#[tracing::instrument(skip(self))]
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
let mut result = HealResultItem {
+161 -42
View File
@@ -26,7 +26,7 @@ use crate::storage_api_contracts::{
namespace::NamespaceLocking as _,
object::{HTTPPreconditions, ObjectOperations as _},
};
use crate::store::ECStore;
use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
@@ -856,7 +856,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
is_unversioned_data_movement_object(source)
&& is_unversioned_data_movement_object(target)
&& !target.delete_marker
&& source
.mod_time
.zip(target.mod_time)
@@ -1028,6 +1027,7 @@ pub(crate) enum SourceCleanupError {
pub(crate) struct SourceCleanupBucketFence<'a> {
pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
}
fn ensure_source_cleanup_versions_match(
@@ -1065,7 +1065,9 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
struct SourceCleanupDeleteBarrierState {
bucket: String,
object: String,
fence_pending: tokio::sync::Notify,
arrived: tokio::sync::Notify,
is_paused: AtomicBool,
release: tokio::sync::Notify,
}
@@ -1079,7 +1081,7 @@ pub(crate) struct SourceCleanupDeleteBarrier {
}
#[cfg(test)]
static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<SourceCleanupDeleteBarrierState>>>> =
std::sync::OnceLock::new();
#[cfg(test)]
@@ -1092,15 +1094,22 @@ impl SourceCleanupDeleteBarrier {
let state = Arc::new(SourceCleanupDeleteBarrierState {
bucket: bucket.to_string(),
object: object.to_string(),
fence_pending: tokio::sync::Notify::new(),
arrived: tokio::sync::Notify::new(),
is_paused: AtomicBool::new(false),
release: tokio::sync::Notify::new(),
});
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison");
assert!(slot.is_none(), "source cleanup delete barrier must be unique");
*slot = Some(Arc::clone(&state));
assert!(
!barriers
.iter()
.any(|barrier| barrier.bucket == bucket && barrier.object == object),
"source cleanup delete barrier must be unique per object"
);
barriers.push(Arc::clone(&state));
Self { state }
}
@@ -1110,35 +1119,58 @@ impl SourceCleanupDeleteBarrier {
.expect("source cleanup should reach the pre-delete barrier");
}
pub(crate) async fn wait_until_fence_pending(&self) {
tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified())
.await
.expect("source cleanup should attempt the fixed mutation fence");
}
pub(crate) fn is_paused(&self) -> bool {
self.state.is_paused.load(Ordering::Acquire)
}
pub(crate) fn release(&self) {
self.state.release.notify_one();
}
}
#[cfg(test)]
pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) {
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison")
.iter()
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
.cloned();
if let Some(barrier) = barrier {
barrier.fence_pending.notify_one();
}
}
#[cfg(test)]
impl Drop for SourceCleanupDeleteBarrier {
fn drop(&mut self) {
self.state.release.notify_one();
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison");
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
*slot = None;
}
barriers.retain(|state| !Arc::ptr_eq(state, &self.state));
}
}
#[cfg(test)]
async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
let barrier = SOURCE_CLEANUP_DELETE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("source cleanup delete barrier mutex should not poison")
.as_ref()
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
.iter()
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
.cloned();
if let Some(barrier) = barrier {
barrier.is_paused.store(true, Ordering::Release);
barrier.arrived.notify_one();
barrier.release.notified().await;
}
@@ -1154,11 +1186,20 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
op_label: &str,
) -> std::result::Result<ObjectInfo, SourceCleanupError> {
let cleanup_key = encode_dir_object(object);
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
let _guard = ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(Error::from)?;
let source_guard = if bucket_fence
.object_mutation_fence
.is_some_and(SourceCleanupMutationFence::source_lock_covered)
{
None
} else {
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
Some(
ns_lock
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(Error::from)?,
)
};
if bucket_fence
.lifecycle_guard
@@ -1168,6 +1209,14 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
"{op_label}: bucket incarnation fence was lost before source cleanup"
))));
}
if bucket_fence
.object_mutation_fence
.is_some_and(SourceCleanupMutationFence::is_lock_lost)
{
return Err(SourceCleanupError::Storage(Error::other(format!(
"{op_label}: object mutation fence was lost before source cleanup"
))));
}
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
@@ -1182,7 +1231,12 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id,
..Default::default()
};
opts.add_namespace_lock_guard(&_guard);
if let Some(source_guard) = source_guard.as_ref() {
opts.add_namespace_lock_guard(source_guard);
}
if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
object_mutation_fence.add_namespace_lock_fence(&mut opts);
}
if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
}
@@ -1330,6 +1384,37 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str {
}
}
pub(crate) async fn migrate_decommission_object(
store: Arc<ECStore>,
pool_idx: usize,
bucket: String,
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
) -> Result<()> {
let source = rd.object_info.clone();
let _mutation_fence = store
.acquire_decommission_object_mutation_fence(&bucket, &source.name)
.await?;
let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
.await?
.ok_or(Error::FileNotFound)?;
if !is_equivalent_data_movement_object_identity(&source, &current, true, false) {
return Err(Error::FileNotFound);
}
migrate_object_inner(
store,
pool_idx,
bucket,
rd,
source_bucket_incarnation_id,
op_label,
Some(&_mutation_fence),
)
.await
}
pub(crate) async fn migrate_object(
store: Arc<ECStore>,
pool_idx: usize,
@@ -1337,6 +1422,18 @@ pub(crate) async fn migrate_object(
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
) -> Result<()> {
migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
}
async fn migrate_object_inner(
store: Arc<ECStore>,
pool_idx: usize,
bucket: String,
rd: GetObjectReader,
source_bucket_incarnation_id: Option<uuid::Uuid>,
op_label: &str,
mutation_fence: Option<&ObjectLockDiagGuard>,
) -> Result<()> {
let object_info = rd.object_info.clone();
let has_part_checksums = object_info
@@ -1350,7 +1447,7 @@ pub(crate) async fn migrate_object(
let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts)
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
.await
{
Ok(res) => res,
@@ -1448,7 +1545,7 @@ pub(crate) async fn migrate_object(
if let Err(err) = store
.clone()
.complete_multipart_upload_for_data_movement(
target_pool_idx,
(target_pool_idx, mutation_fence),
&bucket,
&object_info.name,
&res.upload_id,
@@ -1609,7 +1706,7 @@ pub(crate) async fn migrate_object(
let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
let (target_pool_idx, put_result) = store
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts)
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
.await
.map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
if let Err(err) = put_result {
@@ -3541,25 +3638,47 @@ mod tests {
}
#[test]
fn test_precondition_conflict_rejects_newer_delete_marker() {
let source = ObjectInfo {
size: 128,
etag: Some("etag-source".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let target = ObjectInfo {
delete_marker: true,
etag: None,
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
..source.clone()
};
fn test_precondition_conflict_accepts_only_newer_null_delete_marker() {
for version_id in [None, Some(Uuid::nil())] {
let source = ObjectInfo {
version_id,
size: 128,
etag: Some("etag-source".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let target = ObjectInfo {
delete_marker: true,
etag: None,
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
..source.clone()
};
let should_resume =
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
.expect("delete marker conflict should be evaluated");
assert!(
resolve_data_movement_overwrite_resume_result(
&Error::PreconditionFailed,
Ok(Some(target.clone())),
&source,
0,
1,
)
.expect("newer null delete marker should be evaluated")
);
assert!(!should_resume);
let mut same_time = target.clone();
same_time.mod_time = source.mod_time;
assert!(
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,)
.expect("same-generation null delete marker should be rejected")
);
let mut versioned = target;
versioned.version_id = Some(Uuid::new_v4());
assert!(
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,)
.expect("a UUID delete marker must not erase a null source version")
);
}
}
#[test]
+108 -8
View File
@@ -1391,7 +1391,37 @@ impl BucketUsageAccumulator {
}
pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
// A compressed object may carry -1 while the transformed size is unknown
// (legacy streaming sentinel). In that case the persisted physical size
// is still a valid accounting floor; every other negative value is corrupt.
// An explicit negative `actual-size` metadata value is corrupt, however:
// the sentinel is only valid in the in-memory/object-part field written by
// the legacy streaming path, not as a persisted declared size.
let compressed = object.is_compressed();
if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
return Err(Error::PartMissingOrCorrupt);
}
if object
.parts
.iter()
.any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
{
return Err(Error::PartMissingOrCorrupt);
}
let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
.filter(|value| !value.is_empty());
if declared_actual_size
.as_deref()
.and_then(|value| value.parse::<i64>().ok())
.is_some_and(|size| size < 0)
{
return Err(Error::PartMissingOrCorrupt);
}
let logical_size = match object.get_actual_size().map_err(Error::other)? {
size if size == -1 && compressed && declared_actual_size.is_none() => None,
size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
_ => return Err(Error::PartMissingOrCorrupt),
};
let persisted_part_size = if object.parts.is_empty() {
u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
} else {
@@ -1399,12 +1429,8 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
// Compressed streaming objects persist -1 when the transformed
// part size is unknown. The physical part size remains a valid
// quota floor; reject only non-negative values that overflow.
let actual_size = if part.actual_size < 0 {
if object.is_compressed() {
0
} else {
return Err(Error::PartMissingOrCorrupt);
}
let actual_size = if part.actual_size == -1 {
0
} else {
u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
};
@@ -1412,7 +1438,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
})?
};
Ok(logical_size.max(persisted_part_size))
Ok(logical_size.unwrap_or(0).max(persisted_part_size))
}
type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
@@ -3320,6 +3346,80 @@ mod tests {
);
}
#[test]
fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
let object = ObjectInfo {
size: 400,
actual_size: -1,
user_defined: Arc::new(metadata),
..Default::default()
};
assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
}
#[test]
fn quota_object_size_rejects_compressed_part_sum_overflow() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
let object = ObjectInfo {
size: 1,
user_defined: Arc::new(metadata),
parts: Arc::new(vec![
rustfs_filemeta::ObjectPartInfo {
actual_size: i64::MAX,
..Default::default()
},
rustfs_filemeta::ObjectPartInfo {
actual_size: 1,
..Default::default()
},
]),
..Default::default()
};
assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
}
#[test]
fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
let mut metadata = HashMap::new();
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
let corrupt_object = ObjectInfo {
size: 400,
actual_size: -2,
user_defined: Arc::new(metadata.clone()),
..Default::default()
};
assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
let corrupt_part = ObjectInfo {
size: 400,
user_defined: Arc::new(metadata),
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
size: 400,
actual_size: -2,
..Default::default()
}]),
..Default::default()
};
assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
}
#[tokio::test]
#[serial]
async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
+62 -31
View File
@@ -858,6 +858,7 @@ const EVENT_DISK_LOCAL_DIRECT_IO_FALLBACK: &str = "disk_local_direct_io_fallback
#[cfg(target_os = "linux")]
const EVENT_DISK_LOCAL_URING_LATCH_OFF: &str = "disk_local_uring_latch_off";
const EVENT_DISK_LOCAL_DELETE_FAILED: &str = "disk_local_delete_failed";
const EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED: &str = "disk_local_delete_rollback_failed";
const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
@@ -5215,8 +5216,8 @@ impl LocalDisk {
let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default());
// TODO: DIRECT support
// TODD: DiskInfo
// TODO(backlog): add O_DIRECT I/O support for performance-critical paths
// TODO(backlog): populate DiskInfo in constructor
let mut disk = Self {
root: root.clone(),
publication_root,
@@ -5751,7 +5752,7 @@ impl LocalDisk {
// return Ok(());
// TODO: async notifications for disk space checks and trash cleanup
// TODO(backlog): make disk space checks and trash cleanup event-driven instead of poll-based
let trash_path = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
// if let Some(parent) = trash_path.parent() {
@@ -5997,7 +5998,7 @@ impl LocalDisk {
#[hotpath::measure(impl_type = "LocalDisk")]
async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
// TODO: timeout support
// TODO(backlog): add configurable timeout for read_all_data operations
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
Ok(data)
}
@@ -6106,6 +6107,43 @@ impl LocalDisk {
Ok((bytes, modtime))
}
async fn write_missing_delete_marker(
&self,
volume: &str,
path: &str,
fi: FileInfo,
object_dir: &Path,
xl_path: &Path,
rollback_dir: Option<Uuid>,
) -> Result<()> {
if let Some(rollback_dir) = rollback_dir {
let rollback_path = object_dir.join(rollback_dir.to_string());
fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
.await
.map_err(to_file_error)?;
}
if let Err(err) = self.write_metadata("", volume, path, fi).await {
if let Some(rollback_dir) = rollback_dir
&& let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await
{
warn!(
event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
result = "failed",
volume,
path,
rollback_dir = %rollback_dir,
error = ?restore_err,
"Disk local delete rollback failed"
);
}
return Err(err);
}
Ok(())
}
async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
let volume_dir = self.io_get_bucket_path(volume)?;
let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
@@ -6123,7 +6161,20 @@ impl LocalDisk {
return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
}
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?;
let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
Ok(data) => data,
Err(DiskError::FileNotFound) => {
// `deleted` alone can be an explicit marker purge; only
// `mark_deleted` may create metadata that was not present.
let Some(delete_marker) = fis.iter().find(|fi| fi.deleted && fi.mark_deleted).cloned() else {
return Err(DiskError::FileNotFound);
};
return self
.write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir)
.await;
}
Err(err) => return Err(err),
};
if data.is_empty() {
return Err(DiskError::FileNotFound);
@@ -6674,7 +6725,7 @@ impl LocalDisk {
return Ok(());
}
// TODO: add lock
// TODO(backlog): add directory listing lock to prevent concurrent enumeration
let stall = opts.stall_timeout_duration();
@@ -8796,7 +8847,7 @@ impl DiskAPI for LocalDisk {
Ok(entries)
}
// FIXME: TODO: io.writer TODO cancel
// TODO(backlog): support io.writer cancellation and early termination in walk_dir
#[tracing::instrument(level = "trace", skip_all)]
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
self.wait_for_startup_cleanup().await;
@@ -9880,7 +9931,7 @@ impl DiskAPI for LocalDisk {
);
return Err(e);
}
// TODO: health check
// TODO(backlog): add post-setup disk health verification
}
Ok(())
}
@@ -10422,29 +10473,9 @@ impl DiskAPI for LocalDisk {
}
if fi.deleted && force_del_marker {
if let Some(rollback_dir) = rollback_dir {
let rollback_path = file_path.join(rollback_dir.to_string());
fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
.await
.map_err(to_file_error)?;
}
if let Err(err) = self.write_metadata("", volume, path, fi).await {
if let Some(rollback_dir) = rollback_dir
&& let Err(restore_err) =
restore_delete_rollback(file_path.as_path(), &xl_path, rollback_dir, &self.publication_root).await
{
warn!(
volume,
path,
rollback_dir = %rollback_dir,
error = ?restore_err,
"failed to restore metadata after delete marker commit error"
);
}
return Err(err);
}
return Ok(());
return self
.write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir)
.await;
}
return if fi.version_id.is_some() {
+69 -25
View File
@@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_DISK: &str = "disk";
const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;
pub fn part_transaction_path(part_path: &str) -> String {
match part_path.rsplit_once('/') {
@@ -62,6 +64,7 @@ use bytes::Bytes;
use endpoint::Endpoint;
use error::DiskError;
use error::{Error, Result};
use futures::stream::{self, StreamExt};
use local::LocalDisk;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_madmin::info_commands::DiskMetrics;
@@ -417,6 +420,14 @@ impl DiskAPI for Disk {
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
match self {
Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
match self {
@@ -1028,36 +1039,47 @@ where
D: DiskAPI + ?Sized,
{
validate_batch_read_version_item_count(req.items.len())?;
let parallelism = batch_read_version_server_parallelism();
let mut responses = Vec::with_capacity(req.items.len());
for (index, item) in req.items.iter().enumerate() {
let response = match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: true,
file_info,
error: String::new(),
},
Err(err) => BatchReadVersionResp {
index,
path: item.path.clone(),
version_id: item.version_id.clone(),
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
},
};
responses.push(response);
}
let mut responses = stream::iter(req.items.into_iter().enumerate())
.map(|(index, item)| async move {
match disk
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
.await
{
Ok(file_info) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: true,
file_info,
error: String::new(),
error_code: 0,
},
Err(err) => BatchReadVersionResp {
index,
path: item.path,
version_id: item.version_id,
success: false,
file_info: FileInfo::default(),
error: err.to_string(),
error_code: err.to_u32(),
},
}
})
.buffer_unordered(parallelism)
.collect::<Vec<_>>()
.await;
responses.sort_unstable_by_key(|response| response.index);
Ok(responses)
}
fn batch_read_version_server_parallelism() -> usize {
rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
.clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct CheckPartsResp {
pub results: Vec<usize>,
@@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp {
pub success: bool,
pub file_info: FileInfo,
pub error: String,
#[serde(default)]
pub error_code: u32,
}
pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
@@ -1417,6 +1441,26 @@ mod tests {
assert!(!partial_valid_location.valid());
}
#[test]
fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
assert_eq!(batch_read_version_server_parallelism(), 4);
});
}
#[test]
fn batch_read_version_server_parallelism_honors_env_with_bounds() {
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
assert_eq!(batch_read_version_server_parallelism(), 8);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
assert_eq!(batch_read_version_server_parallelism(), 1);
});
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
});
}
/// Test FileInfoVersions find_version_index
#[test]
fn test_file_info_versions_find_version_index() {
+42 -4
View File
@@ -315,7 +315,7 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
#[cfg(unix)]
{
let dir = dir.as_ref().to_path_buf();
tokio::task::spawn_blocking(move || fsync_dir_std(dir)).await?
fsync_spawn_blocking(move || fsync_dir_std(dir)).await?
}
#[cfg(not(unix))]
@@ -683,7 +683,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
#[cfg(test)]
let dir = group.dir.clone();
let dir_file = group.dir_file.clone();
tokio::task::spawn_blocking(move || {
fsync_spawn_blocking(move || {
#[cfg(test)]
{
if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -1080,6 +1080,44 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
/// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
/// configured with >1 threads, isolates device-bound fsync from the main
/// blocking pool so reads (pread/stat/open) are not starved. `None` means
/// fall back to the main runtime (zero behavior change).
static FSYNC_RUNTIME: LazyLock<Option<tokio::runtime::Runtime>> = LazyLock::new(|| {
let threads =
rustfs_utils::get_env_usize(rustfs_config::ENV_FSYNC_BLOCKING_THREADS, rustfs_config::DEFAULT_FSYNC_BLOCKING_THREADS);
if threads <= 1 {
return None;
}
let mut builder = tokio::runtime::Builder::new_multi_thread();
builder
.worker_threads(num_cpus::get().min(8))
.max_blocking_threads(threads)
.thread_name("rustfs-fsync")
.thread_stack_size(512 * 1024)
.enable_all();
match builder.build() {
Ok(rt) => {
tracing::info!(threads, "fsync dedicated blocking pool enabled");
Some(rt)
}
Err(err) => {
tracing::warn!(%err, "failed to build fsync runtime, falling back to main pool");
None
}
}
});
/// Spawn a blocking task on the fsync-dedicated runtime if configured,
/// otherwise fall back to the main tokio blocking pool.
fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle<T> {
match FSYNC_RUNTIME.as_ref() {
Some(rt) => rt.spawn_blocking(f),
None => tokio::task::spawn_blocking(f),
}
}
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
type NamespaceMutationLock = AsyncMutex<()>;
@@ -1217,7 +1255,7 @@ where
F: FnOnce() -> io::Result<T> + Send + 'static,
{
let (disk_permit, global_permit) = acquire_file_sync_permits(disk_permits).await?;
let result = tokio::task::spawn_blocking(move || {
let result = fsync_spawn_blocking(move || {
let _disk_permit = disk_permit;
work()
})
@@ -2146,7 +2184,7 @@ async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'stati
wait_started,
);
let disk_permit = admission.disk_permit.clone();
let result = tokio::task::spawn_blocking(move || {
let result = fsync_spawn_blocking(move || {
let _lease = lease;
let _disk_permit = disk_permit;
operation()
+2 -2
View File
@@ -249,7 +249,7 @@ impl PoolEndpointList {
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
// TODO Check for cross device mounts if any.
// TODO(backlog): check for cross-device mounts in single-drive setup
return Ok(Self {
inner: vec![Endpoints::from(vec![endpoint])],
@@ -264,7 +264,7 @@ impl PoolEndpointList {
// Convert args to endpoints
let mut eps = Endpoints::try_from(set_layout.as_slice())?;
// TODO Check for cross device mounts if any.
// TODO(backlog): check for cross-device mounts in multi-pool setup
for (disk_idx, ep) in eps.as_mut().iter_mut().enumerate() {
ep.set_pool_index(pool_idx);
+8
View File
@@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() {
cluster::rpc::shutdown_background_monitors();
}
/// Publish that the process is ready to serve user-object GET traffic.
///
/// Experimental metadata coalescing is allowed to run only after this point so
/// startup and internal metadata reads keep the original per-disk path.
pub fn mark_get_metadata_read_version_coalescing_service_ready() {
runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
}
#[cfg(test)]
mod rio_tests {
#[test]
+55 -15
View File
@@ -24,7 +24,7 @@ use crate::storage_api_contracts::{
pub struct NamespaceLockFence {
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
#[cfg(test)]
forced_lost: Arc<std::sync::atomic::AtomicBool>,
forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
}
impl Debug for NamespaceLockFence {
@@ -40,13 +40,17 @@ impl NamespaceLockFence {
Self {
signals: Arc::default(),
#[cfg(test)]
forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)),
forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
}
}
pub(crate) fn is_lock_lost(&self) -> bool {
#[cfg(test)]
if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
if self
.forced_lost
.iter()
.any(|lost| lost.load(std::sync::atomic::Ordering::Acquire))
{
return true;
}
self.signals.iter().any(|signal| signal.is_lost())
@@ -57,27 +61,26 @@ impl NamespaceLockFence {
}
fn extend(&mut self, other: &Self) {
if Arc::ptr_eq(&self.signals, &other.signals) {
return;
if !Arc::ptr_eq(&self.signals, &other.signals) {
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
}
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
#[cfg(test)]
if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
self.forced_lost.store(true, std::sync::atomic::Ordering::Release);
if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
}
}
#[cfg(test)]
pub(crate) fn lost_for_test() -> Self {
let fence = Self::new();
fence.forced_lost.store(true, std::sync::atomic::Ordering::Release);
fence.forced_lost[0].store(true, std::sync::atomic::Ordering::Release);
fence
}
#[cfg(test)]
pub(crate) fn loss_handle_for_test() -> (Self, Arc<std::sync::atomic::AtomicBool>) {
let fence = Self::new();
(fence.clone(), Arc::clone(&fence.forced_lost))
(fence.clone(), Arc::clone(&fence.forced_lost[0]))
}
}
@@ -411,6 +414,13 @@ impl ObjectOptions {
self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
}
#[cfg(test)]
pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
self.namespace_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
.extend(fence);
}
pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
self.lifecycle_delete_all_journal
.get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
@@ -689,6 +699,9 @@ impl ObjectInfo {
}
pub fn get_actual_size(&self) -> std::io::Result<i64> {
if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
return Err(std::io::Error::other("invalid negative actual size"));
}
if self.actual_size > 0 {
return Ok(self.actual_size);
}
@@ -700,10 +713,25 @@ impl ObjectInfo {
let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
return Ok(size);
}
let mut actual_size = 0;
self.parts.iter().for_each(|part| {
actual_size += part.actual_size;
});
if self.actual_size == -1 && self.parts.is_empty() {
return Ok(-1);
}
let mut actual_size = 0_i64;
let mut unknown = false;
for part in self.parts.iter() {
match part.actual_size {
-1 => unknown = true,
size if size >= 0 => {
actual_size = actual_size
.checked_add(size)
.ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
}
_ => return Err(std::io::Error::other("invalid negative compressed part size")),
}
}
if unknown {
return Ok(-1);
}
if actual_size == 0 && actual_size != self.size {
return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
}
@@ -718,6 +746,18 @@ impl ObjectInfo {
Ok(self.size)
}
/// Returns a non-negative size for client and replication boundaries.
///
/// Compressed legacy metadata can retain the internal `-1` unknown-size
/// sentinel. Those boundaries cannot emit a negative length, so they use
/// the persisted physical size while quota accounting keeps the sentinel
/// distinction in [`crate::data_usage::quota_object_size`].
pub fn get_actual_size_or_physical(&self) -> i64 {
self.get_actual_size()
.map(|size| if size >= 0 { size } else { self.size.max(0) })
.unwrap_or_else(|_| self.size.max(0))
}
pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
let mut version_id = fi.version_id;
@@ -1091,7 +1131,7 @@ impl ObjectInfo {
}
};
// TODO:VersionPurgeStatus
// TODO(backlog): handle VersionPurgeStatus in object listing
let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned));
+14 -1
View File
@@ -25,7 +25,10 @@ use lazy_static::lazy_static;
use rustfs_lock::client::LockClient;
use std::{
collections::HashMap,
sync::{Arc, OnceLock},
sync::{
Arc, OnceLock,
atomic::{AtomicBool, Ordering},
},
time::SystemTime,
};
use tokio::sync::{OnceCell, RwLock};
@@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000;
pub const DISK_FILL_FRACTION: f64 = 0.99;
pub const DISK_RESERVE_FRACTION: f64 = 0.15;
static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
}
pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
}
// Global singletons for backward compatibility with MinIO port.
// These should be migrated to AppContext over time.
// See issue #730 for migration plan.
+9
View File
@@ -160,6 +160,10 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Serializes decommission data-movement operations with cancellation and
/// a subsequent restart. Readers are held across one object side effect;
/// the transition path takes the writer after cancelling the routine.
decommission_operation_gate: Arc<RwLock<()>>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -200,6 +204,7 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
decommission_operation_gate: Arc::new(RwLock::new(())),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -218,6 +223,10 @@ impl InstanceContext {
self.lock_manager.clone()
}
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.decommission_operation_gate)
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
@@ -256,6 +256,10 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
cycle_max_objects: metrics.cycle_max_objects,
cycle_max_directories: metrics.cycle_max_directories,
cycle_timeout_total: metrics.cycle_timeout_total,
cycle_recovery_required_total: metrics.cycle_recovery_required_total,
cycle_last_progress_age: metrics.cycle_last_progress_age,
leader_lease_without_progress: metrics.leader_lease_without_progress,
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
@@ -611,6 +615,10 @@ mod test {
current_started: chrono_to_jiff_timestamp(current_started),
last_cycle_partial_source: "usage".to_string(),
last_cycle_partial_source_code: 1,
cycle_timeout_total: 3,
cycle_recovery_required_total: 2,
cycle_last_progress_age: 17,
leader_lease_without_progress: true,
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
source: "usage".to_string(),
cycles: 2,
@@ -622,6 +630,10 @@ mod test {
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
assert_eq!(scanner.last_cycle_partial_source, "usage");
assert_eq!(scanner.last_cycle_partial_source_code, 1);
assert_eq!(scanner.cycle_timeout_total, 3);
assert_eq!(scanner.cycle_recovery_required_total, 2);
assert_eq!(scanner.cycle_last_progress_age, 17);
assert!(scanner.leader_lease_without_progress);
let usage = scanner
.partial_cycles_by_source
.iter()
@@ -334,6 +334,7 @@ impl ECStore {
lifecycle_guard: bucket_incarnation_fence
.as_ref()
.and_then(|guard| guard.namespace_lock_guard()),
..Default::default()
},
"rebalance",
),
@@ -53,11 +53,12 @@ use crate::diagnostics::get::{
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::disk::disk_store::DiskStoreRenameDataExt;
use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout};
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
use crate::disk::{
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
};
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::ShardReader;
@@ -75,7 +76,7 @@ use std::{
future::Future,
pin::Pin,
sync::{
OnceLock,
Arc, OnceLock,
atomic::{AtomicUsize, Ordering},
},
task::{Context, Poll},
@@ -94,6 +95,242 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String {
[bucket, object].join("/")
}
fn read_version_coalescing_enabled() -> bool {
let enabled = || {
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
.is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on"))
};
#[cfg(test)]
{
enabled()
}
#[cfg(not(test))]
{
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(enabled)
}
}
fn read_version_coalescing_delay() -> Duration {
#[cfg(test)]
{
let micros = rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
);
Duration::from_micros(micros)
}
#[cfg(not(test))]
{
static DELAY: OnceLock<Duration> = OnceLock::new();
*DELAY.get_or_init(|| {
Duration::from_micros(rustfs_utils::get_env_u64(
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
))
})
}
}
struct CoalescedReadVersionRequest {
item: BatchReadVersionItem,
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
struct ReadVersionCoalescerKey {
disk: usize,
incl_free_versions: bool,
read_data: bool,
healing: bool,
}
impl ReadVersionCoalescerKey {
fn new(disk: &DiskStore, opts: &ReadOptions) -> Self {
Self {
disk: Arc::as_ptr(disk) as usize,
incl_free_versions: opts.incl_free_versions,
read_data: opts.read_data,
healing: opts.healing,
}
}
}
#[derive(Default)]
struct ReadVersionCoalescer {
lanes: HashMap<ReadVersionCoalescerKey, Vec<CoalescedReadVersionRequest>>,
}
fn read_version_coalescer() -> &'static Mutex<ReadVersionCoalescer> {
static COALESCER: OnceLock<Mutex<ReadVersionCoalescer>> = OnceLock::new();
COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default()))
}
fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
counter!(
METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL,
"event" => event,
"item_count" => item_count.to_string()
)
.increment(1);
}
async fn read_version_via_coalescer(
disk: DiskStore,
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
opts: &ReadOptions,
allow_coalescing: bool,
) -> disk::error::Result<FileInfo> {
if !allow_coalescing || !read_version_coalescing_enabled() {
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
if !matches!(disk.as_ref(), Disk::Remote(_)) {
record_read_version_coalescer_event("bypass_non_remote", 1);
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
}
let (tx, rx) = oneshot::channel();
let item = BatchReadVersionItem {
org_volume: org_bucket.to_string(),
volume: bucket.to_string(),
path: object.to_string(),
version_id: version_id.to_string(),
};
let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
let lane = coalescer.lanes.entry(lane_key).or_default();
let schedule_delayed_flush = lane.is_empty();
lane.push(CoalescedReadVersionRequest { item, tx });
if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
coalescer.lanes.remove(&lane_key)
} else if schedule_delayed_flush {
let disk = disk.clone();
let task_opts = *opts;
tokio::spawn(async move {
tokio::time::sleep(read_version_coalescing_delay()).await;
flush_read_version_coalescer_lane(lane_key, disk, task_opts).await;
});
None
} else {
None
}
};
if let Some(pending) = pending {
flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
}
rx.await
.unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
}
async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
let pending = {
let mut coalescer = read_version_coalescer().lock().await;
coalescer.lanes.remove(&lane_key).unwrap_or_default()
};
flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await;
}
async fn flush_read_version_coalescer_pending(
lane_key: ReadVersionCoalescerKey,
disk: DiskStore,
opts: ReadOptions,
pending: Vec<CoalescedReadVersionRequest>,
) {
if pending.is_empty() {
return;
}
#[cfg(test)]
{
let mut observed_paths = HashSet::new();
for request in &pending {
if observed_paths.insert(request.item.path.as_str()) {
disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk);
}
}
}
let mut senders = Vec::with_capacity(pending.len());
let mut items = Vec::with_capacity(pending.len());
for request in pending {
senders.push(request.tx);
items.push(request.item);
}
let expected_items = items.clone();
record_read_version_coalescer_event("attempted_batch", items.len());
let result =
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
.await
{
Ok(result) => result,
Err(_) => Err(DiskError::Timeout),
};
match result {
Ok(responses) => {
let results = map_batch_read_version_responses(&expected_items, responses);
for (tx, result) in senders.into_iter().zip(results) {
let _ = tx.send(result);
}
}
Err(err) => {
let message = err.to_string();
for tx in senders {
let _ = tx.send(Err(DiskError::other(message.clone())));
}
}
}
}
fn map_batch_read_version_responses(
expected_items: &[BatchReadVersionItem],
responses: Vec<BatchReadVersionResp>,
) -> Vec<crate::disk::error::Result<FileInfo>> {
let mut results = (0..expected_items.len())
.map(|_| Err(DiskError::other("coalesced read_version response missing")))
.collect::<Vec<_>>();
let mut seen = vec![false; expected_items.len()];
for response in responses {
let Some(expected) = expected_items.get(response.index) else {
continue;
};
let Some(slot) = results.get_mut(response.index) else {
continue;
};
if seen[response.index] {
*slot = Err(DiskError::other("coalesced read_version response duplicate index"));
continue;
}
seen[response.index] = true;
if response.path != expected.path || response.version_id != expected.version_id {
*slot = Err(DiskError::other("coalesced read_version response identity mismatch"));
} else {
*slot = if response.success {
Ok(response.file_info)
} else {
Err(batch_read_version_response_error(response.error_code, response.error))
};
}
}
results
}
fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError {
match DiskError::from_u32(error_code) {
Some(DiskError::Io(_)) | None => DiskError::other(error),
Some(error) => error,
}
}
pub(in crate::set_disk) fn bounded_metadata_fanout_order(
bucket: &str,
object: &str,
@@ -133,11 +370,15 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
order
}
use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::RwLock;
use tokio::sync::{Mutex, RwLock, oneshot};
use tokio::task::JoinSet;
pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
/// Default reader-setup strategy for the GET read path (rustfs/backlog#1215,
/// #1159, #923).
@@ -2356,6 +2597,7 @@ impl SetDisks {
false,
true,
0,
false,
)
.await?;
Ok((ress, errors))
@@ -2386,6 +2628,36 @@ impl SetDisks {
true,
caller_allows_early_stop,
default_parity_count,
false,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object(
disks: &[Option<DiskStore>],
org_bucket: &str,
bucket: &str,
object: &str,
version_id: &str,
read_data: bool,
incl_free_versions: bool,
caller_allows_early_stop: bool,
default_parity_count: usize,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
Self::read_all_fileinfo_inner(
disks,
org_bucket,
bucket,
object,
version_id,
read_data,
false,
incl_free_versions,
true,
caller_allows_early_stop,
default_parity_count,
true,
)
.await
}
@@ -2408,6 +2680,7 @@ impl SetDisks {
// subset would fail write quorum (backlog#872 regression).
caller_allows_early_stop: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let early_stop_enabled =
caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled());
@@ -2424,6 +2697,7 @@ impl SetDisks {
healing,
incl_free_versions,
default_parity_count,
allow_coalescing,
)
.await;
}
@@ -2446,6 +2720,7 @@ impl SetDisks {
healing,
incl_free_versions,
observe,
allow_coalescing,
)
.await
}
@@ -2461,6 +2736,7 @@ impl SetDisks {
healing: bool,
incl_free_versions: bool,
observe: bool,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = observe.then(Instant::now);
let mut ress = Vec::with_capacity(disks.len());
@@ -2492,7 +2768,7 @@ impl SetDisks {
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
@@ -2559,6 +2835,7 @@ impl SetDisks {
healing: bool,
incl_free_versions: bool,
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
let fanout_start = Instant::now();
let mut ress = vec![FileInfo::default(); disks.len()];
@@ -2607,7 +2884,7 @@ impl SetDisks {
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
.await
} else {
Err(DiskError::DiskNotFound)
@@ -5737,6 +6014,7 @@ pub(crate) mod disk_call_counters {
/// Kind label for the per-disk `read_version` metadata RPC.
pub const KIND_READ_VERSION: &str = "read_version";
pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version";
/// Registry key: (object, kind, disk_index).
type CountKey = (String, String, usize);
@@ -6460,6 +6738,286 @@ mod tests {
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn metadata_read_version_coalescer_bypasses_local_disks() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-local-bypass-bucket";
let object_a = "coalesced-local-object-a";
let object_b = "coalesced-local-object-b";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await;
install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await;
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")),
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")),
],
async {
let calls = disk_call_counters::observe(object_a);
let disks_a = disks.clone();
let disks_b = disks.clone();
let read_a = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_a, "", bucket, object_a, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
tokio::task::yield_now().await;
let read_b = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed_for_get_object(
&disks_b, "", bucket, object_b, "", false, false, false, 2,
)
.await
.map(|(file_infos, errors, _)| (file_infos, errors))
});
let (metadata_a, errs_a) = read_a
.await
.expect("first read task should not panic")
.expect("first coalesced read should resolve");
let (metadata_b, errs_b) = read_b
.await
.expect("second read task should not panic")
.expect("second coalesced read should resolve");
assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS);
assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS);
assert!(errs_a.iter().all(Option::is_none));
assert!(errs_b.iter().all(Option::is_none));
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64,
"local disks still execute the ordinary per-disk read_version path"
);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"GET coalescing targets internode RPC count only and must not batch local disk reads"
);
},
)
.await;
drop(dirs);
}
#[tokio::test]
async fn metadata_read_version_coalescer_requires_get_object_intent() {
const DISKS: usize = 4;
let bucket = "coalesced-read-version-default-bypass-bucket";
let object = "default-bypass-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async {
let calls = disk_call_counters::observe(object);
let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false)
.await
.expect("default metadata read should resolve");
assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64);
assert_eq!(
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
0,
"non-GET metadata paths must bypass coalescer even when the env gate is enabled"
);
})
.await;
drop(dirs);
}
#[test]
fn batch_read_version_response_mapping_preserves_index_and_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-c".to_string(),
version_id: "v-c".to_string(),
},
];
let ok_file_info = FileInfo {
name: "object-a".to_string(),
..Default::default()
};
let responses = vec![
BatchReadVersionResp {
index: 2,
path: "object-c".to_string(),
version_id: "v-c".to_string(),
success: false,
file_info: FileInfo::default(),
error: "disk read failed".to_string(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: ok_file_info,
error: String::new(),
error_code: 0,
},
];
let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
let first = results
.next()
.expect("slot 0 should exist")
.expect("slot 0 should map the success response by index");
assert_eq!(first.name, "object-a");
let missing = results
.next()
.expect("slot 1 should exist")
.expect_err("slot 1 should stay missing");
assert!(
missing.to_string().contains("response missing"),
"unexpected missing response error: {missing}"
);
let failed = results
.next()
.expect("slot 2 should exist")
.expect_err("slot 2 should map the response error");
assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}");
assert!(results.next().is_none());
}
#[test]
fn batch_read_version_response_mapping_preserves_typed_not_found_errors() {
let expected_items = vec![
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
},
BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-b".to_string(),
version_id: "v-b".to_string(),
},
];
let results = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileNotFound.to_string(),
error_code: DiskError::FileNotFound.to_u32(),
},
BatchReadVersionResp {
index: 1,
path: "object-b".to_string(),
version_id: "v-b".to_string(),
success: false,
file_info: FileInfo::default(),
error: DiskError::FileVersionNotFound.to_string(),
error_code: DiskError::FileVersionNotFound.to_u32(),
},
],
);
assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound)));
assert!(matches!(
results.get(1).expect("slot 1 should exist"),
Err(DiskError::FileVersionNotFound)
));
}
#[test]
fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() {
let expected_items = vec![BatchReadVersionItem {
org_volume: String::new(),
volume: "bucket".to_string(),
path: "object-a".to_string(),
version_id: "v-a".to_string(),
}];
let mismatched = map_batch_read_version_responses(
&expected_items,
vec![BatchReadVersionResp {
index: 0,
path: "object-b".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-b".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
}],
)
.pop()
.expect("slot 0 should exist")
.expect_err("identity mismatch should fail closed");
assert!(
mismatched.to_string().contains("identity mismatch"),
"unexpected mismatch error: {mismatched}"
);
let duplicate = map_batch_read_version_responses(
&expected_items,
vec![
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
BatchReadVersionResp {
index: 0,
path: "object-a".to_string(),
version_id: "v-a".to_string(),
success: true,
file_info: FileInfo {
name: "object-a".to_string(),
..Default::default()
},
error: String::new(),
error_code: 0,
},
],
)
.pop()
.expect("slot 0 should exist")
.expect_err("duplicate response index should fail closed");
assert!(
duplicate.to_string().contains("duplicate index"),
"unexpected duplicate error: {duplicate}"
);
}
/// Isolation guard: unobserved objects record nothing (so parallel tests do
/// not inflate one another), and a scope clears its own counts on drop.
#[tokio::test]
+30 -11
View File
@@ -97,7 +97,7 @@ use crate::storage_api_contracts::{
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
},
namespace::NamespaceLocking as _,
object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
range::HTTPRangeSpec,
};
use crate::store::utils::is_reserved_or_invalid_bucket;
@@ -735,8 +735,12 @@ pub(crate) use core::io_primitives::disk_call_counters;
mod ctx;
mod metadata;
mod ops;
#[cfg(test)]
pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
#[cfg(any(test, feature = "test-util"))]
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
#[cfg(test)]
pub(crate) use ops::object::DeleteObjectCommitBarrier;
#[cfg(feature = "test-util")]
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
pub(crate) use ops::object::body_cache_plaintext_len;
@@ -922,14 +926,10 @@ mod prepared_get_object_metadata_tests {
.expect("test should find an object whose initial fanout covers both data shards")
}
#[allow(
dead_code,
reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)"
)]
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
fn bounded_initial_parity_disk_index(bucket: &str, object: &str) -> usize {
*bounded_metadata_fanout_order(bucket, object, 4, 2)
.get(3)
.expect("4-disk test geometry should leave one bounded spare disk")
.get(2)
.expect("4-disk test geometry should schedule one parity disk initially")
}
#[tokio::test]
@@ -1087,7 +1087,7 @@ mod prepared_get_object_metadata_tests {
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
],
async {
let slow_parity_disk = bounded_spare_disk_index(bucket, &object);
let slow_parity_disk = bounded_initial_parity_disk_index(bucket, &object);
let barrier =
rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION);
let calls = disk_call_counters::observe(&object);
@@ -3029,6 +3029,16 @@ pub struct SetDisks {
storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
}
// DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones
// each endpoint's canonical Arc, so an exact Arc set identifies the lock domain.
pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
left.iter()
.all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client)))
&& right
.iter()
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
}
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
@@ -3604,6 +3614,15 @@ impl SetDisks {
&self.ctx
}
/// Whether both sets' namespace-lock implementations cover the same object key.
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
(true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers),
_ => false,
}
}
/// The lock manager this set actually uses (test-only; Phase 5 Slice 3).
#[cfg(test)]
pub(crate) fn local_lock_manager_for_test(&self) -> &Arc<rustfs_lock::GlobalLockManager> {
@@ -4588,11 +4607,11 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool {
}
fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool {
opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement)
opts.delete_marker || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.data_movement)
}
fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none());
let mut mark_delete = goi.version_id.is_some() || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none());
let mut delete_marker = opts.versioned;
if opts.version_id.is_some() {
+97 -2
View File
@@ -32,6 +32,8 @@ use crate::crash_inject::{self, CrashPoint};
use crate::multipart_listing::paginate_multipart_listing;
use futures::{StreamExt, stream};
use std::future::Future;
#[cfg(test)]
use std::sync::atomic::AtomicBool;
#[cfg(any(test, feature = "test-util"))]
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
@@ -65,6 +67,7 @@ impl StaleMultipartCleanupGuard {
#[cfg(any(test, feature = "test-util"))]
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum MultipartCommitPause {
NewUploadBeforeLockLost,
PutPartBeforeLockAcquire,
PutPartBeforeLockLost,
PutPartAfterRename,
@@ -156,6 +159,72 @@ impl Drop for MultipartCommitBarrier {
}
}
#[cfg(test)]
struct NewMultipartUploadCommitObservationState {
bucket: String,
object: String,
committed: AtomicBool,
}
#[cfg(test)]
pub(crate) struct NewMultipartUploadCommitObservation {
state: Arc<NewMultipartUploadCommitObservationState>,
}
#[cfg(test)]
static NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION: std::sync::OnceLock<
std::sync::Mutex<Option<Arc<NewMultipartUploadCommitObservationState>>>,
> = std::sync::OnceLock::new();
#[cfg(test)]
impl NewMultipartUploadCommitObservation {
pub(crate) fn install(bucket: &str, object: &str) -> Self {
let state = Arc::new(NewMultipartUploadCommitObservationState {
bucket: bucket.to_string(),
object: object.to_string(),
committed: AtomicBool::new(false),
});
let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("new multipart upload commit observation mutex should not poison");
assert!(slot.is_none(), "new multipart upload commit observation must be unique");
*slot = Some(Arc::clone(&state));
Self { state }
}
pub(crate) fn committed(&self) -> bool {
self.state.committed.load(Ordering::Acquire)
}
}
#[cfg(test)]
impl Drop for NewMultipartUploadCommitObservation {
fn drop(&mut self) {
let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("new multipart upload commit observation mutex should not poison");
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
*slot = None;
}
}
}
#[cfg(test)]
fn observe_new_multipart_upload_commit(bucket: &str, object: &str) {
let state = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("new multipart upload commit observation mutex should not poison")
.as_ref()
.filter(|state| state.bucket == bucket && state.object == object)
.cloned();
if let Some(state) = state {
state.committed.store(true, Ordering::Release);
}
}
#[cfg(any(test, feature = "test-util"))]
async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
let barrier = {
@@ -1575,7 +1644,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let parts_metadata = vec![fi.clone(); disks.len()];
if !user_defined.contains_key("content-type") {
// TODO: get content-type
// TODO(backlog): detect content-type from part data when header is missing
}
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
@@ -1615,6 +1684,30 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement);
#[cfg(any(test, feature = "test-util"))]
pause_multipart_commit(bucket, object, MultipartCommitPause::NewUploadBeforeLockLost).await;
if _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "new_multipart_upload_commit",
bucket: bucket.to_string(),
object: object.to_string(),
required: 1,
achieved: 0,
});
}
if opts
.namespace_lock_fence
.as_ref()
.is_some_and(NamespaceLockFence::is_lock_lost)
{
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "new_multipart_upload_outer_lock",
bucket: bucket.to_string(),
object: object.to_string(),
required: 1,
achieved: 0,
});
}
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
Self::write_unique_file_info(
&shuffle_disks,
@@ -1626,6 +1719,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
)
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
#[cfg(test)]
observe_new_multipart_upload_commit(bucket, object);
// evalDisks
@@ -1971,7 +2066,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
}
// TODO: crypto
// TODO(backlog): integrate encryption verification during complete multipart
if (i < uploaded_parts.len() - 1)
&& !(opts.data_movement && ext_part.actual_size < 0)
+195 -15
View File
@@ -45,6 +45,7 @@ use crate::bucket::replication::{
DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
};
use crate::data_usage::quota_object_size;
use crate::diagnostics::get::GetObjectFailureReason;
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
use crate::error::is_err_invalid_upload_id;
@@ -1293,7 +1294,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
(prepared.snapshot, prepared.object_info)
} else {
match self
.get_object_fileinfo(
.get_object_fileinfo_for_get_object_reader(
bucket,
object,
opts,
@@ -2122,11 +2123,11 @@ impl SetDisks {
let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
let put_object_size = known_put_object_storage_size(data.size());
let is_inline_buffer =
storage_class_config.should_inline(erasure.shard_file_size(put_object_size), erasure.data_shards, opts.versioned);
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
let is_inline_buffer = storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
let shard_file_size = erasure.shard_file_size(put_object_size);
let shard_file_size = shard_file_size_raw;
let shard_size = erasure.shard_size();
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
@@ -2483,6 +2484,7 @@ impl SetDisks {
})
.await?,
);
notify_put_object_commit_namespace_acquired(bucket, object);
}
#[cfg(not(any(test, feature = "test-util")))]
{
@@ -4630,6 +4632,7 @@ struct PutObjectCommitBarrierState {
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
namespace_pending: tokio::sync::Notify,
namespace_acquired: std::sync::atomic::AtomicBool,
}
#[cfg(any(test, feature = "test-util"))]
@@ -4651,6 +4654,7 @@ impl PutObjectCommitBarrier {
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Notify::new(),
namespace_pending: tokio::sync::Notify::new(),
namespace_acquired: std::sync::atomic::AtomicBool::new(false),
});
let mut slot = PUT_OBJECT_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
@@ -4685,6 +4689,10 @@ impl PutObjectCommitBarrier {
.await
.expect("put object should wait for the namespace lock after leaving the commit barrier");
}
pub fn namespace_acquired(&self) -> bool {
self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
}
}
#[cfg(any(test, feature = "test-util"))]
@@ -4741,6 +4749,22 @@ fn notify_put_object_commit_namespace_pending(bucket: &str, object: &str) {
}
}
#[cfg(any(test, feature = "test-util"))]
fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
let barrier = PUT_OBJECT_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
.lock()
.expect("put object commit barrier mutex should not poison")
.iter()
.find(|barrier| {
barrier.bucket == bucket && barrier.object == object && barrier.pause == PutObjectCommitPause::BeforeNamespace
})
.cloned();
if let Some(barrier) = barrier {
barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
}
}
#[cfg(test)]
struct DeleteObjectCommitBarrierState {
bucket: String,
@@ -4750,7 +4774,7 @@ struct DeleteObjectCommitBarrierState {
}
#[cfg(test)]
struct DeleteObjectCommitBarrier {
pub(crate) struct DeleteObjectCommitBarrier {
state: Arc<DeleteObjectCommitBarrierState>,
}
@@ -4760,7 +4784,7 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
#[cfg(test)]
impl DeleteObjectCommitBarrier {
fn install(bucket: &str, object: &str) -> Self {
pub(crate) fn install(bucket: &str, object: &str) -> Self {
let state = Arc::new(DeleteObjectCommitBarrierState {
bucket: bucket.to_string(),
object: object.to_string(),
@@ -4776,13 +4800,13 @@ impl DeleteObjectCommitBarrier {
Self { state }
}
async fn wait_until_paused(&self) {
pub(crate) async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await
.expect("delete object should reach the deterministic commit barrier");
}
fn release(&self) {
pub(crate) fn release(&self) {
self.state.release.notify_one();
}
}
@@ -5655,7 +5679,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
(deleted, errors)
}
async fn delete_objects_with_accounting(
&self,
bucket: &str,
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut accounting = vec![None; objects.len()];
let delete_config_snapshot = opts
.delete_replication_config_snapshot
.clone()
@@ -5745,7 +5780,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
*item = Some(Error::other(message.clone()));
}
}
return (del_objects, del_errs);
return (del_objects, del_errs, accounting);
}
},
}
@@ -5792,6 +5827,22 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let source_missing = gerr
.as_ref()
.is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
// Resolve accounting from the generation selected under this
// object's write lock. A request-layer pre-stat is only an
// optimization and cannot identify a concurrent overwrite.
let (accounting_size, accounting_version_id, removed_current_object) = if source_missing
|| dobj.synthetic_version_id
|| set_disk_delete_creates_delete_marker(&check_opts)
|| goi.delete_marker
{
(None, None, false)
} else {
(
quota_object_size(&goi).ok(),
goi.version_id.filter(|version_id| !version_id.is_nil()),
(dobj.version_id.is_none() || is_explicit_null_version(dobj.version_id)) && !dobj.synthetic_version_id,
)
};
// Normalize both sides before comparing. `goi.version_id` is the
// client-facing identity, where `from_file_info` synthesizes
// `Some(Uuid::nil())` for a null version on a versioned or
@@ -5877,6 +5928,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
if dobj.version_id.is_none() && (version_suspended || versioned) {
vr.mod_time = Some(OffsetDateTime::now_utc());
vr.deleted = true;
vr.mark_deleted = true;
if versioned {
vr.version_id = Some(Uuid::new_v4());
}
@@ -5920,7 +5972,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
},
replication_state: vr.replication_state_internal.clone(),
..Default::default()
}
};
accounting[i] = Some(DeleteAccounting {
size: accounting_size,
version_id: accounting_version_id,
removed_current_object,
});
}
// Only add to vers_map if we hold the lock
@@ -5966,7 +6023,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
});
}
}
return (del_objects, del_errs);
return (del_objects, del_errs, accounting);
}
let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len());
@@ -6161,7 +6218,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
join_all(rollback_futures).await;
// TODO: add_partial
// TODO(backlog): support partial object deletion for multi-part objects
if let Some(api) = opts.tier_delete_journal_api.as_ref() {
for (idx, je) in persisted_journal_entries {
@@ -6204,7 +6261,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
}
(del_objects, del_errs)
// An accounting identity is actionable only when the delete result is
// successful. Never let a failed commit (including a partial quorum
// failure) reach the request-layer fast delta path.
for (index, err) in del_errs.iter().enumerate() {
if err.is_some() {
accounting[index] = None;
}
}
(del_objects, del_errs, accounting)
}
#[tracing::instrument(skip(self))]
@@ -6371,7 +6437,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
}
// TODO: Lifecycle
// TODO(backlog): integrate lifecycle evaluation before object deletion
let mut version_found = true;
// delete_object_version below derives its own majority quorum from the
@@ -6465,7 +6531,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
mark_deleted: mark_delete,
mod_time: Some(mod_time),
replication_state_internal: opts.delete_replication.as_ref().map(replication_state_to_filemeta),
..Default::default() // TODO: Transition
..Default::default() // TODO(backlog): populate transition state on delete markers
};
fi.set_tier_free_version_id(&find_vid.to_string());
@@ -6533,6 +6599,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
obj_info.size = goi.size;
// Keep the committed source metadata on the internal delete result so
// the request layer can derive canonical accounting for this exact
// generation. Delete responses do not expose these fields.
obj_info.actual_size = goi.actual_size;
obj_info.user_defined = Arc::clone(&goi.user_defined);
obj_info.parts = Arc::clone(&goi.parts);
obj_info.user_tags = Arc::clone(&goi.user_tags);
self.invalidate_get_object_metadata_cache(bucket, object).await;
Ok(obj_info)
@@ -7824,6 +7896,113 @@ mod replication_quota_safety_tests {
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
}
#[tokio::test]
async fn delete_returns_canonical_compressed_accounting_size() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
let bucket = "compressed-delete-accounting";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut user_defined = HashMap::new();
insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
.expect("compressed fixture reader should be valid"),
);
set_disks
.put_object(
bucket,
"object",
&mut reader,
&ObjectOptions {
user_defined,
..Default::default()
},
)
.await
.expect("compressed object should be written");
let (deleted, errors, accounting) = set_disks
.delete_objects_with_accounting(
bucket,
vec![ObjectToDelete {
object_name: "object".to_string(),
..Default::default()
}],
ObjectOptions {
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
)
.await;
assert!(errors[0].is_none(), "compressed delete should succeed: {:?}", errors[0]);
assert!(deleted[0].found, "the committed object must be reported as found");
assert_eq!(accounting[0].as_ref().and_then(|value| value.size), Some(1000));
assert!(accounting[0].as_ref().is_some_and(|value| value.version_id.is_none()));
assert!(accounting[0].as_ref().is_some_and(|value| value.removed_current_object));
}
#[tokio::test]
async fn suspended_delete_marker_does_not_return_body_accounting() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
let bucket = "suspended-delete-accounting";
for disk in &disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut user_defined = HashMap::new();
insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_COMPRESSION,
"klauspost/compress/s2".to_string(),
);
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
let mut reader = PutObjReader::new(
HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
.expect("compressed fixture reader should be valid"),
);
let suspended_opts = ObjectOptions {
version_suspended: true,
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
s3s::dto::VersioningConfiguration {
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
..Default::default()
},
None,
))),
user_defined,
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
..Default::default()
};
set_disks
.put_object(bucket, "object", &mut reader, &suspended_opts)
.await
.expect("compressed object should be written");
let (deleted, errors, accounting) = set_disks
.delete_objects_with_accounting(
bucket,
vec![ObjectToDelete {
object_name: "object".to_string(),
..Default::default()
}],
suspended_opts,
)
.await;
assert!(errors[0].is_none(), "suspended delete should create a marker: {:?}", errors[0]);
assert!(deleted[0].delete_marker);
assert!(accounting[0].is_none(), "a delete marker must not carry body accounting");
}
#[tokio::test]
async fn direct_put_cannot_persist_a_tiny_logical_size() {
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
@@ -11636,6 +11815,7 @@ mod transition_upload_integrity_tests {
crate::data_movement::SourceCleanupBucketFence {
expected_incarnation_id: None,
lifecycle_guard: Some(&bucket_guard),
..Default::default()
},
"test_data_movement",
)
+66 -14
View File
@@ -259,10 +259,33 @@ impl SetDisks {
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false)
.await
}
#[tracing::instrument(level = "debug", skip(self))]
#[hotpath::measure(impl_type = "SetDisks")]
pub(super) async fn get_object_fileinfo_for_get_object_reader(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<GetObjectFileInfo> {
let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket)
&& crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
self.get_object_fileinfo_gated_inner(
bucket,
object,
opts,
read_data,
caller_allows_early_stop,
allow_read_version_coalescing,
)
.await
}
/// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
/// quorum fanout. Read-before-write callers (object tagging) must use this:
/// the returned online-disk set is the write target, and the early-stop
@@ -275,6 +298,20 @@ impl SetDisks {
opts: &ObjectOptions,
read_data: bool,
allow_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false)
.await
}
#[allow(clippy::too_many_arguments)]
async fn get_object_fileinfo_gated_inner(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
read_data: bool,
allow_early_stop: bool,
allow_read_version_coalescing: bool,
) -> Result<GetObjectFileInfo> {
let vid = opts.version_id.clone().unwrap_or_default();
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
@@ -337,19 +374,34 @@ impl SetDisks {
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
// core/io_primitives.rs); unsafe requests and callers that opt out
// (allow_early_stop=false) fall back to full-wait.
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
false,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?;
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing {
Self::read_all_fileinfo_observed_for_get_object(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?
} else {
Self::read_all_fileinfo_observed(
&disks,
"",
bucket,
object,
vid.as_str(),
read_data,
false,
opts.incl_free_versions,
allow_early_stop,
self.default_parity_count,
)
.await?
};
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
GET_OBJECT_PATH_INTERNAL_META
} else {
@@ -62,8 +62,8 @@ pub(crate) mod object {
use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::range::HTTPRangeSpec;
pub(crate) use rustfs_storage_api::{
DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, ObjectOperations,
ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions,
ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
};
pub(crate) trait EcstoreObjectIO:
+1 -1
View File
@@ -601,7 +601,7 @@ impl ECStore {
#[instrument(skip(self))]
pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
// TODO: opts.cached
// TODO(backlog): support cached bucket listing via opts.cached
let mut buckets = self.peer_sys.list_bucket(opts).await?;
+608 -5
View File
@@ -13,7 +13,12 @@
// limitations under the License.
use super::*;
use crate::core::pools::POOL_META_NAME;
use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
use crate::set_disk::get_lock_acquire_timeout;
use crate::storage_api_contracts::heal::HealOperations as _;
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use rustfs_lock::NamespaceLockGuard;
use tracing::trace;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
@@ -30,7 +35,119 @@ fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
)
}
#[derive(Debug, Clone, Copy)]
enum HealFormatPoolSkip {
Completed,
Retryable,
}
fn classify_heal_format_pool(
pool_idx: usize,
pool_cmd_line: &str,
pool_meta: &PoolMeta,
rebalance_meta: Option<&RebalanceMeta>,
) -> Option<HealFormatPoolSkip> {
let Some(pool) = pool_meta.pools.get(pool_idx) else {
return Some(HealFormatPoolSkip::Retryable);
};
if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line {
return Some(HealFormatPoolSkip::Retryable);
}
if let Some(decommission) = pool.decommission.as_ref() {
if decommission.complete {
return Some(HealFormatPoolSkip::Completed);
}
if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) {
return Some(HealFormatPoolSkip::Retryable);
}
}
if let Some(meta) = rebalance_meta {
let Some(pool_stats) = meta.pool_stats.get(pool_idx) else {
return Some(HealFormatPoolSkip::Retryable);
};
if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) {
return Some(HealFormatPoolSkip::Retryable);
}
}
None
}
fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error {
match skip {
HealFormatPoolSkip::Completed => StorageError::NoHealRequired,
HealFormatPoolSkip::Retryable => StorageError::SlowDown,
}
}
fn heal_format_fence_lost_error() -> Error {
StorageError::SlowDown
}
impl ECStore {
async fn acquire_heal_format_fence(
&self,
) -> Result<(NamespaceLockGuard, NamespaceLockGuard, PoolMeta, Option<RebalanceMeta>)> {
let metadata_pool = self
.pools
.first()
.cloned()
.ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
// Metadata fence order is part of the decommission/rebalance protocol:
// pool.bin must always be acquired before rebalance.bin.
let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
let pool_guard = pool_lock.get_write_lock(get_lock_acquire_timeout()).await?;
let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?;
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
return Err(heal_format_fence_lost_error());
}
let mut pool_meta = PoolMeta::default();
pool_meta.load_no_lock(metadata_pool.clone()).await?;
if pool_meta.pools.len() != self.pools.len()
|| pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line
})
{
return Err(heal_format_fence_lost_error());
}
let mut rebalance_meta = RebalanceMeta::new();
let rebalance_meta = match rebalance_meta
.load_with_opts(
metadata_pool,
ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
{
Ok(()) => Some(rebalance_meta),
Err(Error::ConfigNotFound) => None,
Err(err) => return Err(err),
};
if rebalance_meta
.as_ref()
.is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
{
return Err(heal_format_fence_lost_error());
}
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
return Err(heal_format_fence_lost_error());
}
Ok((pool_guard, rebalance_guard, pool_meta, rebalance_meta))
}
fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
match opts.pool {
Some(pool_idx) => Ok(vec![
@@ -52,9 +169,26 @@ impl ECStore {
};
let mut count_no_heal = 0;
let mut count_completed = 0;
let mut first_error = None;
for pool in self.pools.iter() {
let (mut result, err) = pool.heal_format(dry_run).await?;
for (pool_idx, pool) in self.pools.iter().enumerate() {
let (pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
first_error.get_or_insert(heal_format_fence_lost_error());
break;
}
if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
{
if matches!(skip, HealFormatPoolSkip::Completed) {
count_completed += 1;
} else {
first_error.get_or_insert(heal_format_pool_skip_error(skip));
}
continue;
}
let fence_lost = || pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost();
let (mut result, err) = pool.heal_format_with_fence(dry_run, fence_lost).await?;
if let Some(err) = err {
match err {
StorageError::NoHealRequired => {
@@ -69,11 +203,18 @@ impl ECStore {
r.set_count += result.set_count;
r.before.drives.append(&mut result.before.drives);
r.after.drives.append(&mut result.after.drives);
// A lease can be lost after the final write; fail closed before
// reporting the pool as successfully healed.
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
first_error.get_or_insert(heal_format_fence_lost_error());
break;
}
}
if let Some(err) = first_error {
return Ok((r, Some(err)));
}
if count_no_heal == self.pools.len() {
if count_no_heal + count_completed == self.pools.len() {
info!(
event = EVENT_HEAL_FORMAT_COMPLETED,
component = LOG_COMPONENT_ECSTORE,
@@ -297,10 +438,17 @@ impl ECStore {
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::metadata_sys;
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
use crate::disk::{DiskOption, format::FormatV3, new_disk};
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk};
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::runtime::instance::InstanceContext;
use crate::services::rebalance::{RebalanceInfo, RebalanceStats};
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
use crate::store::init_format::{load_format_erasure, save_format_file};
use crate::store::init_local_disks_with_instance_ctx;
use tokio_util::sync::CancellationToken;
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
let format = FormatV3::new(1, 1);
@@ -347,6 +495,209 @@ mod tests {
}
}
fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
PoolMeta {
pools: vec![PoolStatus {
id: 0,
cmd_line: "pool-0".to_string(),
last_update: OffsetDateTime::UNIX_EPOCH,
decommission: Some(info),
}],
..Default::default()
}
}
#[test]
fn heal_format_pool_state_barriers_are_classified() {
let active = pool_meta_with_decommission(PoolDecommissionInfo {
start_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
});
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &active, None),
Some(HealFormatPoolSkip::Retryable)
));
for info in [
PoolDecommissionInfo {
failed: true,
..Default::default()
},
PoolDecommissionInfo {
canceled: true,
..Default::default()
},
] {
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None),
Some(HealFormatPoolSkip::Retryable)
));
}
let completed = pool_meta_with_decommission(PoolDecommissionInfo {
complete: true,
..Default::default()
});
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &completed, None),
Some(HealFormatPoolSkip::Completed)
));
}
#[test]
fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() {
let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default());
let rebalance = RebalanceMeta {
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)),
Some(HealFormatPoolSkip::Retryable)
));
let stopping = RebalanceMeta {
pool_stats: vec![RebalanceStats {
info: RebalanceInfo {
stopping: true,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)),
Some(HealFormatPoolSkip::Retryable)
));
let identity = pool_meta_with_decommission(PoolDecommissionInfo::default());
assert!(matches!(
classify_heal_format_pool(0, "pool-new", &identity, None),
Some(HealFormatPoolSkip::Retryable)
));
let identity_without_decommission = PoolMeta {
pools: vec![PoolStatus {
id: 0,
cmd_line: "pool-0".to_string(),
last_update: OffsetDateTime::UNIX_EPOCH,
decommission: None,
}],
..Default::default()
};
assert!(matches!(
classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None),
Some(HealFormatPoolSkip::Retryable)
));
assert!(matches!(
classify_heal_format_pool(0, "", &identity_meta, None),
Some(HealFormatPoolSkip::Retryable)
));
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None),
Some(HealFormatPoolSkip::Retryable)
));
let stopped = RebalanceMeta {
stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Stopped,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none());
let stopping_after_stop = RebalanceMeta {
stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
pool_stats: vec![RebalanceStats {
participating: true,
info: RebalanceInfo {
status: RebalStatus::Started,
stopping: true,
..Default::default()
},
..Default::default()
}],
..Default::default()
};
assert!(matches!(
classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)),
Some(HealFormatPoolSkip::Retryable)
));
}
#[test]
fn skipped_heal_format_pool_is_never_reported_as_success() {
assert!(matches!(
heal_format_pool_skip_error(HealFormatPoolSkip::Retryable),
StorageError::SlowDown
));
assert!(matches!(
heal_format_pool_skip_error(HealFormatPoolSkip::Completed),
StorageError::NoHealRequired
));
}
async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
let mut pool_endpoints = Vec::new();
for pool_index in 0..2 {
let mut endpoints = Vec::new();
for disk_index in 0..4 {
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
tokio::fs::create_dir_all(&disk_path)
.await
.expect("multi-pool heal test disk should be created");
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
.expect("test endpoint should parse");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
}
pool_endpoints.push(PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: format!("heal-owner-pool-{pool_index}"),
platform: "test".to_string(),
});
}
let endpoint_pools = EndpointServerPools::from(pool_endpoints);
let instance_ctx = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("multi-pool local disks should initialize");
let shutdown = CancellationToken::new();
let store = ECStore::new_with_instance_ctx(
"127.0.0.1:0".parse().expect("test address should parse"),
endpoint_pools,
shutdown.clone(),
instance_ctx,
)
.await
.expect("multi-pool test store should initialize");
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
(temp_dir, store, shutdown)
}
#[tokio::test]
async fn heal_object_pool_scope_selects_only_requested_pool() {
let store = minimal_heal_store().await;
@@ -506,6 +857,229 @@ mod tests {
}
}
#[tokio::test]
#[serial_test::serial]
async fn unscoped_heal_object_suspended_owner_semantics() {
let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
let active_object = "active-owner";
let suspended_only_object = "suspended-only";
let duplicate_object = "duplicate-owner";
let marker_object = "marker-owner";
let quorum_object = "quorum-owner";
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created in all pools");
let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
store.pools[0]
.put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
.await
.expect("active owner object should be written");
let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
missing_active_disk
.delete(
&bucket,
active_object,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("active owner shard should be removed for repair");
assert!(
missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
"the active owner fixture must start with one missing metadata copy"
);
let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
store.pools[1]
.put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
.await
.expect("suspended owner object should be written");
for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
store.pools[pool_index]
.put_object(
&bucket,
duplicate_object,
&mut duplicate_reader,
&ObjectOptions {
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
..Default::default()
},
)
.await
.expect("duplicate owner object should be written");
}
let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
.clone()
.expect("duplicate active owner disk should be online");
duplicate_missing_disk
.delete(
&bucket,
duplicate_object,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("duplicate active owner shard should be removed for repair");
let history_version = Uuid::new_v4();
let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
store.pools[0]
.put_object(
&bucket,
marker_object,
&mut history_reader,
&ObjectOptions {
versioned: true,
version_id: Some(history_version.to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
..Default::default()
},
)
.await
.expect("versioned marker history should be written");
store.pools[0]
.delete_object(
&bucket,
marker_object,
ObjectOptions {
versioned: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
..Default::default()
},
)
.await
.expect("delete marker should be written");
let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
store.pools[0]
.put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
.await
.expect("quorum boundary object should be written");
{
let mut pool_meta = store.pool_meta.write().await;
let mut next = PoolMeta::new(&store.pools, &pool_meta);
next.pools[1].decommission = Some(PoolDecommissionInfo {
start_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
});
*pool_meta = next;
}
let (_, duplicate_owner) = store
.get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
.await
.expect("duplicate owner should resolve");
assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
let (_, active_duplicate_owner) = store
.get_latest_object_info_with_idx(
&bucket,
duplicate_object,
&ObjectOptions {
skip_decommissioned: true,
..Default::default()
},
)
.await
.expect("active duplicate owner should resolve");
assert_eq!(
active_duplicate_owner, 0,
"suspended duplicate must be excluded from active owner selection"
);
let (duplicate_result, duplicate_err) = store
.handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
.await
.expect("duplicate owner heal should complete through the production path");
assert_eq!(duplicate_result.object, duplicate_object);
assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
assert!(
duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
"production heal must repair the active duplicate owner rather than the suspended owner"
);
let (marker_info, marker_owner) = store
.get_latest_object_info_with_idx(
&bucket,
marker_object,
&ObjectOptions {
skip_decommissioned: true,
versioned: true,
..Default::default()
},
)
.await
.expect("latest delete marker should resolve");
assert_eq!(marker_owner, 0);
assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
let (active_result, active_err) = store
.handle_heal_object(&bucket, active_object, "", &HealOpts::default())
.await
.expect("unscoped active-owner heal should complete");
assert_eq!(active_result.object, active_object);
assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
assert!(
missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
"active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
);
assert!(
store.pools[1]
.get_object_info(&bucket, active_object, &ObjectOptions::default())
.await
.is_err(),
"the suspended pool must not be written for an active-owner object"
);
let (suspended_result, suspended_err) = store
.handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
.await
.expect("unscoped suspended-only heal should return a terminal result");
assert!(suspended_result.object.is_empty());
assert!(matches!(suspended_err, Some(Error::FileNotFound)));
assert!(
store.pools[1]
.get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
.await
.is_ok(),
"suspended-only data must remain untouched when unscoped heal reports absent"
);
let (_, explicit_err) = store
.handle_heal_object(
&bucket,
suspended_only_object,
"",
&HealOpts {
pool: Some(1),
..Default::default()
},
)
.await
.expect("explicit suspended-owner heal should return a mapped error");
assert!(matches!(explicit_err, Some(Error::SlowDown)));
let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
let surviving_quorum_disk = original_quorum_disks[3].clone();
*store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
let (_, quorum_err) = store
.handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
.await
.expect("quorum boundary heal should return a mapped result");
*store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
assert!(
matches!(quorum_err, Some(Error::ErasureReadQuorum)),
"quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
);
shutdown.cancel();
}
#[tokio::test]
async fn handle_heal_format_continues_after_a_pool_error() {
let canonical_format = FormatV3::new(1, 3);
@@ -615,6 +1189,18 @@ mod tests {
bucket_fence_registry: std::sync::Arc::default(),
};
let err = store
.handle_heal_format(false)
.await
.expect_err("missing pool metadata must fail closed before format writes");
assert!(matches!(err, StorageError::SlowDown));
let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
pool_meta
.save(store.pools.clone())
.await
.expect("pool metadata should be persisted before format heal");
let (result, err) = store
.handle_heal_format(false)
.await
@@ -628,5 +1214,22 @@ mod tests {
.await
.expect("the later pool should be healed despite the first pool error");
assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
for status in &mut completed_meta.pools {
status.decommission = Some(PoolDecommissionInfo {
complete: true,
..Default::default()
});
}
completed_meta
.save(store.pools.clone())
.await
.expect("completed pool metadata should be persisted");
let (_, err) = store
.handle_heal_format(false)
.await
.expect("completed pools should be reported as a no-op");
assert!(matches!(err, Some(StorageError::NoHealRequired)));
}
}
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -4673,7 +4673,7 @@ async fn gather_results(
entry.name = entry.name.replace("\\", "/");
}
// TODO: rx.recv()
// TODO(backlog): integrate rx.recv() for incremental listing results
if let Some(marker) = &opts.marker
&& ((!opts.include_marker && &entry.name <= marker) || (opts.include_marker && &entry.name < marker))
@@ -4703,7 +4703,7 @@ async fn gather_results(
continue;
}
// TODO: Lifecycle
// TODO(backlog): integrate lifecycle evaluation during object listing
entries.push(Some(entry));
candidate_entries += 1;
+3 -3
View File
@@ -33,7 +33,7 @@ use crate::bucket::utils::check_put_object_part_args;
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
use crate::cluster::rpc::{RemoteClient, S3PeerSys};
use crate::config::storageclass;
use crate::core::pools::PoolMeta;
use crate::core::pools::{DecommissionCanceler, PoolMeta};
use crate::disk::endpoint::{Endpoint, EndpointType};
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
use crate::error::{Error, Result};
@@ -151,7 +151,7 @@ pub(crate) mod init_format;
pub(crate) mod list_objects;
mod multipart;
mod object;
pub(crate) use object::ObjectLockDiagGuard;
pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence};
pub use object::{
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
SnapshotConsistencyError,
@@ -176,7 +176,7 @@ pub struct ECStore {
// pub local_disks: Vec<DiskStore>,
pub pool_meta: RwLock<PoolMeta>,
pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
pub decommission_cancelers: RwLock<Vec<Option<CancellationToken>>>,
pub decommission_cancelers: RwLock<Vec<Option<DecommissionCanceler>>>,
/// Serializes rebalance/decommission start transitions.
///
/// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`,
+22 -11
View File
@@ -332,7 +332,7 @@ impl ECStore {
let expected_incarnation_id = opts.expected_bucket_incarnation_id;
if request.prefix.is_empty() {
// TODO: return from cache
// TODO(backlog): return cached multipart listing when prefix is empty
}
if self.single_pool() {
@@ -400,7 +400,7 @@ impl ECStore {
object: &str,
opts: &ObjectOptions,
) -> Result<MultipartUploadResult> {
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts, None)
.await
.map(|(res, _, _)| res)
}
@@ -410,20 +410,22 @@ impl ECStore {
bucket: &str,
object: &str,
opts: &ObjectOptions,
mutation_fence: Option<&ObjectLockDiagGuard>,
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
check_new_multipart_args(bucket, object)?;
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let opts = &opts;
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
if self.single_pool() {
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
.await;
return self.pools[0]
.new_multipart_upload(bucket, object, opts)
.new_multipart_upload(bucket, object, &opts)
.await
.map(|res| (res, 0, opts.expected_bucket_incarnation_id));
}
if opts.data_movement && opts.version_id.is_some() {
let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?;
let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?;
if idx == opts.src_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
@@ -431,7 +433,9 @@ impl ECStore {
opts.version_id.clone().unwrap_or_default(),
));
}
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
return Ok((res, idx, opts.expected_bucket_incarnation_id));
}
@@ -454,7 +458,9 @@ impl ECStore {
.await?;
if !res.uploads.is_empty() {
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
return Ok((res, idx, opts.expected_bucket_incarnation_id));
}
}
@@ -467,7 +473,9 @@ impl ECStore {
));
}
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
.await;
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
Ok((res, idx, opts.expected_bucket_incarnation_id))
}
@@ -610,7 +618,7 @@ impl ECStore {
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
let opts = &opts;
// TODO: defer DeleteUploadID
// TODO(backlog): defer DeleteUploadID to background for faster abort response
if self.single_pool() {
return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;
@@ -704,13 +712,14 @@ impl ECStore {
pub(crate) async fn complete_multipart_upload_for_data_movement(
self: Arc<Self>,
target_pool_idx: usize,
target: (usize, Option<&ObjectLockDiagGuard>),
bucket: &str,
object: &str,
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let (target_pool_idx, mutation_fence) = target;
check_complete_multipart_args(bucket, object, upload_id)?;
if !opts.data_movement {
return Err(Error::other("targeted multipart completion requires data_movement options"));
@@ -739,6 +748,8 @@ impl ECStore {
snapshot.add_lock_fences(&mut opts);
opts.object_lock_config_snapshot = Some(snapshot);
}
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
.await;
#[cfg(test)]
pause_data_movement_multipart_before_selected_completion(bucket).await;
let pool = self
File diff suppressed because it is too large Load Diff
+387 -2
View File
@@ -385,7 +385,7 @@ impl ECStore {
}
pub(super) async fn is_suspended(&self, idx: usize) -> bool {
// TODO: LOCK
// TODO(backlog): acquire pool metadata lock for consistent suspension check
let pool_meta = self.pool_meta.read().await;
@@ -859,6 +859,7 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
use crate::disk::error::DiskError;
use crate::layout::endpoint::Endpoint;
@@ -1423,6 +1424,14 @@ mod tests {
}
}
fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
ObjectInfo {
version_id: Some(version_id),
etag,
..object_info_with_mod_time(unix_ts, delete_marker)
}
}
#[test]
fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
let candidates = vec![
@@ -1446,7 +1455,7 @@ mod tests {
}
#[test]
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() {
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_mod_time(10, false)),
@@ -1466,6 +1475,382 @@ mod tests {
assert_eq!(idx, 1);
}
#[test]
fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 2,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 7,
err: None,
},
];
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect("equivalent replicas must resolve deterministically");
assert_eq!(idx, 7);
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("divergent version ids must not silently resolve to the higher pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("divergent etags must not silently resolve to the higher pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 1,
err: None,
},
];
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect_err("a delete marker tied with a live version must not be masked by the pool index");
assert_eq!(err, Error::ErasureReadQuorum);
}
fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
let err = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(left),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(right),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect_err("equal-time identity divergence must fail closed");
assert_eq!(err, Error::ErasureReadQuorum);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut data_dir = base.clone();
data_dir.data_dir = Some(Uuid::from_u128(2));
assert_equal_time_identity_conflict(base.clone(), data_dir);
let mut size = base.clone();
size.size = 1;
assert_equal_time_identity_conflict(base.clone(), size);
let mut actual_size = base.clone();
actual_size.actual_size = 1;
assert_equal_time_identity_conflict(base.clone(), actual_size);
let mut checksum = base.clone();
checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
assert_equal_time_identity_conflict(base.clone(), checksum);
let mut parts = base.clone();
parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
etag: "part-etag".to_string(),
number: 1,
size: 1,
..Default::default()
}]);
assert_equal_time_identity_conflict(base.clone(), parts);
let mut transition = base;
transition.transitioned_object.tier = "tier-a".to_string();
assert_equal_time_identity_conflict(
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
transition,
);
}
#[test]
fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let mut minio_alias = base.clone();
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"X-MINIO-INTERNAL-COMPRESSION".to_string(),
"zstd".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(rustfs_alias),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(minio_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("same-value internal aliases should resolve");
assert_eq!(idx, 1);
let mut dual_alias = base.clone();
dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
("x-minio-internal-compression".to_string(), "zstd".to_string()),
]));
let mut single_alias = base;
single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(dual_alias),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(single_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("dual-key and single-key internal metadata should resolve");
assert_eq!(idx, 1);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-rustfs-internal-compression".to_string(),
"zstd".to_string(),
)]));
let mut minio_alias = base;
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
"x-minio-internal-compression".to_string(),
"snappy".to_string(),
)]));
assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
}
#[test]
fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut rustfs_alias = base.clone();
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!(
"X-RUSTFS-INTERNAL-{}{suffix}",
suffix_prefix.to_uppercase(),
suffix = "arn:aws:s3:::Bucket"
),
"value".to_string(),
)]));
let mut minio_alias = base.clone();
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
"value".to_string(),
)]));
let (_, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(rustfs_alias.clone()),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(minio_alias),
idx: 1,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("dynamic internal aliases with the same target should resolve");
assert_eq!(idx, 1);
let mut different_target_case = base;
different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
"value".to_string(),
)]));
assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
}
}
#[test]
fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut first = base.clone();
first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
("x-minio-internal-compression".to_string(), "snappy".to_string()),
]));
let mut second = base;
second.user_defined = first.user_defined.clone();
assert_equal_time_identity_conflict(first, second);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
let mut replication = base.clone();
replication.replication_status_internal = Some("PENDING".to_string());
replication.replication_status = ReplicationStatusType::Pending;
assert_equal_time_identity_conflict(base.clone(), replication);
let mut purge = base.clone();
purge.version_purge_status_internal = Some("PENDING".to_string());
purge.version_purge_status = VersionPurgeStatusType::Pending;
assert_equal_time_identity_conflict(base.clone(), purge);
let mut decision = base;
decision.replication_decision = "replicate".to_string();
assert_equal_time_identity_conflict(
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
decision,
);
}
#[test]
fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
without_mod_time.mod_time = None;
let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
}
#[test]
fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
older.data_dir = Some(Uuid::from_u128(2));
let (info, idx) = resolve_latest_object_info_candidates(
vec![
LatestObjectInfoCandidate {
info: Some(latest),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: Some(older),
idx: 9,
err: None,
},
],
"bucket",
"object",
&ObjectOptions::default(),
)
.expect("older identity divergence must not affect the latest candidate");
assert_eq!(idx, 0);
assert_eq!(
info.mod_time,
Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
);
}
#[test]
fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
let candidates = vec![
LatestObjectInfoCandidate {
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
idx: 0,
err: None,
},
LatestObjectInfoCandidate {
info: None,
idx: 1,
err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
];
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
.expect("not-found pools must not block resolution of found candidates");
assert_eq!(idx, 0);
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
}
#[test]
fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
let err = resolve_latest_object_info_candidates(
+175 -23
View File
@@ -12,10 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::cmp::Ordering;
use std::collections::HashMap;
use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{ObjectInfo, ObjectOptions};
use rustfs_utils::http::metadata_compat::{
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
strip_internal_prefix_preserving_case,
};
use rustfs_utils::path::decode_dir_object;
use time::OffsetDateTime;
@@ -73,7 +77,7 @@ pub(super) fn resolve_rebalance_delete_from_all_pools_result(
object: &str,
) -> Result<ObjectInfo> {
result.map_err(|err| {
if err == Error::PreconditionFailed {
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
err
} else {
Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}"))
@@ -86,7 +90,7 @@ fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
}
fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
if err == Error::PreconditionFailed {
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
err
} else {
Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
@@ -137,37 +141,158 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p
))
}
fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option<OffsetDateTime> {
candidate
.info
.as_ref()
.map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
}
fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
left.transition_version_state == right.transition_version_state
&& left.transitioned_object.name == right.transitioned_object.name
&& left.transitioned_object.version_id == right.transitioned_object.version_id
&& left.transitioned_object.tier == right.transitioned_object.tier
&& left.transitioned_object.free_version == right.transitioned_object.free_version
&& left.transitioned_object.status == right.transitioned_object.status
}
#[derive(PartialEq, Eq)]
struct LatestUserDefinedIdentity {
internal: HashMap<String, String>,
other: HashMap<String, String>,
}
fn normalize_internal_identity_suffix(key: &str) -> Option<String> {
let suffix = strip_internal_prefix_preserving_case(key)?;
for dynamic_prefix in [
SUFFIX_REPLICATION_RESET_ARN_PREFIX,
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
] {
let prefix_len = dynamic_prefix.len();
if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..))
&& prefix.eq_ignore_ascii_case(dynamic_prefix)
{
return Some(format!("{dynamic_prefix}{remainder}"));
}
}
Some(suffix.to_lowercase())
}
fn normalize_user_defined_identity(user_defined: &HashMap<String, String>) -> Option<LatestUserDefinedIdentity> {
let mut identity = LatestUserDefinedIdentity {
internal: HashMap::with_capacity(user_defined.len()),
other: HashMap::with_capacity(user_defined.len()),
};
for (key, value) in user_defined {
if let Some(suffix) = normalize_internal_identity_suffix(key) {
if identity
.internal
.insert(suffix, value.clone())
.is_some_and(|previous| previous != *value)
{
return None;
}
} else {
identity.other.insert(key.clone(), value.clone());
}
}
Some(identity)
}
fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
match (
normalize_user_defined_identity(&left.user_defined),
normalize_user_defined_identity(&right.user_defined),
) {
(Some(left), Some(right)) => left == right,
_ => false,
}
}
/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
/// differ for the same object version while the selected winner still carries
/// the chosen pool's layout. `put_object_reader` is also intentionally
/// excluded because it is a transient request handle that `ObjectInfo::clone`
/// drops. Every other ObjectInfo field is part of the production-visible
/// identity and must agree before the pool index can provide a deterministic
/// tie-break.
fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
left.bucket == right.bucket
&& left.name == right.name
&& left.storage_class == right.storage_class
&& left.mod_time == right.mod_time
&& left.size == right.size
&& left.actual_size == right.actual_size
&& left.is_dir == right.is_dir
&& same_user_defined_identity(left, right)
&& left.user_tags == right.user_tags
&& left.version_id == right.version_id
&& left.data_dir == right.data_dir
&& left.delete_marker == right.delete_marker
&& same_transition_identity(left, right)
&& left.restore_ongoing == right.restore_ongoing
&& left.restore_expires == right.restore_expires
&& left.parts == right.parts
&& left.is_latest == right.is_latest
&& left.content_type == right.content_type
&& left.content_encoding == right.content_encoding
&& left.expires == right.expires
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.etag == right.etag
&& left.inlined == right.inlined
&& left.metadata_only == right.metadata_only
&& left.version_only == right.version_only
&& left.replication_status_internal == right.replication_status_internal
&& left.replication_status == right.replication_status
&& left.version_purge_status_internal == right.version_purge_status_internal
&& left.version_purge_status == right.version_purge_status
&& left.replication_decision == right.replication_decision
&& left.checksum == right.checksum
}
pub(super) fn resolve_latest_object_info_candidates(
mut candidates: Vec<LatestObjectInfoCandidate>,
candidates: Vec<LatestObjectInfoCandidate>,
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Result<(ObjectInfo, usize)> {
candidates.sort_by(|a, b| {
let a_mod = if let Some(info) = &a.info {
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
OffsetDateTime::UNIX_EPOCH
let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max();
if let Some(latest_mod_time) = latest_mod_time {
let mut latest_candidates = candidates
.into_iter()
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
.collect::<Vec<_>>();
latest_candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.idx));
let Some(winner) = latest_candidates.first() else {
return Err(Error::ErasureReadQuorum);
};
let Some(winner_info) = winner.info.as_ref() else {
return Err(Error::ErasureReadQuorum);
};
let b_mod = if let Some(info) = &b.info {
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else {
OffsetDateTime::UNIX_EPOCH
};
if a_mod == b_mod {
return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less };
if latest_candidates.iter().skip(1).any(|candidate| {
candidate
.info
.as_ref()
.is_none_or(|info| !same_latest_object_info_identity(winner_info, info))
}) {
return Err(Error::ErasureReadQuorum);
}
b_mod.cmp(&a_mod)
});
return Ok((winner_info.clone(), winner.idx));
}
for candidate in candidates {
if let Some(info) = candidate.info {
return Ok((info, candidate.idx));
}
if let Some(err) = candidate.err
&& !is_err_object_not_found(&err)
&& !is_err_version_not_found(&err)
@@ -191,6 +316,18 @@ mod tests {
assert_eq!(err, Error::PreconditionFailed);
}
#[test]
fn rebalance_delete_result_preserves_prefix_access_denied() {
let err = resolve_rebalance_delete_from_all_pools_result(
Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
"bucket",
"object",
)
.expect_err("prefix access denial should remain structured");
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
}
#[test]
fn rebalance_delete_pool_result_preserves_precondition_failed() {
let err = resolve_rebalance_delete_from_all_pools_results(
@@ -205,4 +342,19 @@ mod tests {
assert_eq!(err, Error::PreconditionFailed);
}
#[test]
fn rebalance_delete_pool_result_preserves_prefix_access_denied() {
let err = resolve_rebalance_delete_from_all_pools_results(
vec![RebalanceDeletePoolResult {
pool_idx: 0,
result: Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
}],
"bucket",
"object",
)
.expect_err("prefix access denial should remain structured");
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
}
}