mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-04 18:13:57 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| faf636e1c7 | |||
| 2401a8d8ac | |||
| 0aa9ba496b | |||
| d15dde7bb5 | |||
| 0c56ea85db | |||
| 4d46bd25f8 | |||
| 3483a48fcb |
Generated
+3
@@ -10276,6 +10276,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"parking_lot 0.13.0",
|
||||
"rand 0.10.3",
|
||||
"rustfs-common",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
@@ -10788,8 +10789,10 @@ version = "1.0.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"csv-core",
|
||||
"datafusion",
|
||||
"derive_builder",
|
||||
"flate2",
|
||||
"futures",
|
||||
"hotpath",
|
||||
"parking_lot 0.13.0",
|
||||
|
||||
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
|
||||
@@ -94,3 +94,12 @@ pub(crate) mod scanner_peer_bucket {
|
||||
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
|
||||
pub(crate) use ecstore_rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod bucket_tag_fixture {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
|
||||
}
|
||||
|
||||
@@ -360,12 +360,26 @@ impl OpaMock {
|
||||
|
||||
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, action, bucket, department);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
|
||||
assert_eq!(payload["input"]["resource"]["object"], object);
|
||||
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
|
||||
let input = &payload["input"];
|
||||
assert_eq!(input["identity"]["account"], "opabuckettags");
|
||||
assert_eq!(input["identity"]["is_owner"], false);
|
||||
assert_eq!(input["action"], action);
|
||||
assert_eq!(input["resource"]["bucket"], bucket);
|
||||
let expected = department.map(|value| serde_json::json!([value]));
|
||||
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_validation(&mut self) -> TestResult {
|
||||
@@ -678,22 +692,61 @@ async fn test_list_buckets_opa_contract() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop and reap the directory's owning server before opening the fixture store.
|
||||
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
|
||||
use crate::storage_api::bucket_tag_fixture::{
|
||||
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
|
||||
};
|
||||
|
||||
let server = env
|
||||
.process
|
||||
.as_mut()
|
||||
.ok_or("missing server process before offline fixture write")?;
|
||||
server.kill()?;
|
||||
server.wait()?;
|
||||
env.process = None;
|
||||
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
|
||||
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
|
||||
let context = std::sync::Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
|
||||
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
|
||||
let _cancel = context
|
||||
.background_cancel_token()
|
||||
.expect("store binds its shutdown token")
|
||||
.drop_guard();
|
||||
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
|
||||
let incarnation = metadata.bucket_incarnation_id;
|
||||
let policy = metadata.policy_config_json.clone();
|
||||
assert!(!incarnation.is_nil());
|
||||
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
|
||||
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
|
||||
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
|
||||
metadata.tagging_config_xml = xml.to_vec();
|
||||
metadata.save_with_store(store.clone()).await?;
|
||||
let persisted = load_bucket_metadata(store, bucket).await?;
|
||||
assert_eq!(persisted.bucket_incarnation_id, incarnation);
|
||||
assert_eq!(persisted.policy_config_json, policy);
|
||||
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
|
||||
};
|
||||
|
||||
init_logging();
|
||||
let mut opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
let opa_url = opa.url.clone();
|
||||
let server_env = [
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
])
|
||||
.await?;
|
||||
];
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
let secret = uuid::Uuid::new_v4().to_string();
|
||||
create_user(&env, "opabuckettags", &secret).await?;
|
||||
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
|
||||
@@ -733,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
client
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
@@ -741,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
assert_eq!(head.tag_count(), Some(1));
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
client.list_objects_v2().bucket(source).send().await?;
|
||||
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
|
||||
client.list_object_versions().bucket(source).send().await?;
|
||||
@@ -750,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
admin.create_bucket().bucket(destination).send().await?;
|
||||
set_department(&admin, destination, "engineering").await?;
|
||||
let copy_source = format!("{source}/report?versionId={version}");
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("denied COPY and REPLACE must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
set_department(&admin, destination, "finance").await?;
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
// The access hook checks preliminary tags, then the copy use case checks
|
||||
// the exact source reader's tags before publishing the destination.
|
||||
for _ in 0..2 {
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
}
|
||||
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
|
||||
assert_eq!(tags.tag_set().len(), 1);
|
||||
assert_eq!(tags.tag_set()[0].key(), "department");
|
||||
assert_eq!(
|
||||
tags.tag_set()[0].value(),
|
||||
if directive == TaggingDirective::Copy {
|
||||
"engineering"
|
||||
} else {
|
||||
"legal"
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
@@ -847,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("denied-source")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied source must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
set_department(&client, source, "finance")
|
||||
.await
|
||||
.expect_err("requested tags must not authorize their own mutation");
|
||||
@@ -885,7 +985,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
admin
|
||||
let invalid_write = admin
|
||||
.put_bucket_tagging()
|
||||
.bucket(destination)
|
||||
.tagging(
|
||||
@@ -895,7 +995,25 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("new duplicate tags must be rejected before publication");
|
||||
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
|
||||
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
|
||||
set_department(&admin, source, "finance").await?;
|
||||
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
|
||||
let read = client
|
||||
.get_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(destination)
|
||||
|
||||
@@ -192,6 +192,7 @@ pub mod bucket {
|
||||
table_catalog_path_hash, unreadable_config_refusal,
|
||||
};
|
||||
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
|
||||
}
|
||||
|
||||
pub mod config_parse_mode {
|
||||
@@ -331,8 +332,8 @@ pub mod bucket {
|
||||
pub mod utils {
|
||||
pub use crate::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
|
||||
serialize,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
|
||||
is_meta_bucketname, is_valid_object_prefix, serialize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -451,9 +452,15 @@ pub mod disk {
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
|
||||
pub use crate::disk::multipart_index::{
|
||||
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
|
||||
MultipartIndexRpcRequest,
|
||||
};
|
||||
pub use crate::disk::multipart_source::{
|
||||
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
|
||||
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
|
||||
@@ -466,7 +473,9 @@ pub mod disk {
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
|
||||
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
|
||||
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
@@ -617,7 +626,7 @@ pub mod rebalance {
|
||||
pub mod rio {
|
||||
pub use crate::io_support::rio::{
|
||||
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
|
||||
boxed_reader, compression_metadata_value, wrap_reader,
|
||||
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -654,7 +663,9 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::bucket::quota::reservation::{
|
||||
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
|
||||
};
|
||||
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
|
||||
@@ -16,7 +16,8 @@ use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::get_bucket_targets_config;
|
||||
use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
|
||||
UploadResponseDeadline, build_remote_s3_client,
|
||||
};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
@@ -119,6 +120,7 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
|
||||
// Replication has no retry budget of its own on the request path,
|
||||
// so it keeps the SDK's standard three attempts; stating it here
|
||||
// pins the behaviour to this line instead of an SDK default.
|
||||
@@ -1191,6 +1193,7 @@ impl BucketTargetSys {
|
||||
health_check_duration: target.health_check_duration,
|
||||
replicate_sync: target.replication_sync,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::from_env(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1922,6 +1925,83 @@ impl std::fmt::Display for S3ClientError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication request budgets, snapshotted when a target client is built.
|
||||
/// Streamed GET bodies are not subject to the upload-response budget.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TargetRequestTimeouts {
|
||||
request: Duration,
|
||||
complete: Duration,
|
||||
minimum_transfer_rate: u64,
|
||||
}
|
||||
|
||||
impl Default for TargetRequestTimeouts {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
request: Duration::from_secs(60),
|
||||
complete: Duration::from_secs(15 * 60),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TargetRequestTimeouts {
|
||||
fn from_env() -> Self {
|
||||
let defaults = Self::default();
|
||||
let seconds = |name: &str, fallback: Duration| {
|
||||
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
|
||||
Duration::from_secs(if (1..=86400).contains(&value) {
|
||||
value
|
||||
} else {
|
||||
fallback.as_secs()
|
||||
})
|
||||
};
|
||||
let minimum_transfer_rate =
|
||||
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
|
||||
Self {
|
||||
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
|
||||
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
|
||||
minimum_transfer_rate: if minimum_transfer_rate == 0 {
|
||||
defaults.minimum_transfer_rate
|
||||
} else {
|
||||
minimum_transfer_rate
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
|
||||
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
|
||||
// The SDK can make three attempts. Leave transmission headroom for
|
||||
// all three; the separate response deadline still bounds a trickling
|
||||
// target after each upload finishes.
|
||||
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
|
||||
RemoteS3RetryPolicy::Disabled => 1,
|
||||
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
|
||||
};
|
||||
let seconds = size
|
||||
.div_ceil(self.minimum_transfer_rate)
|
||||
.checked_mul(attempts)
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
let timeout = self
|
||||
.request
|
||||
.checked_add(Duration::from_secs(seconds))
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
if Instant::now().checked_add(timeout).is_none() {
|
||||
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
|
||||
}
|
||||
Ok(timeout)
|
||||
}
|
||||
|
||||
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
|
||||
Ok(aws_sdk_s3::config::Builder::new()
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.upload_timeout(size)?)
|
||||
.build(),
|
||||
)
|
||||
.interceptor(UploadResponseDeadline(self.request)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TargetClient {
|
||||
pub endpoint: String,
|
||||
@@ -1935,6 +2015,7 @@ pub struct TargetClient {
|
||||
pub health_check_duration: Duration,
|
||||
pub replicate_sync: bool,
|
||||
pub client: Arc<S3Client>,
|
||||
pub timeouts: TargetRequestTimeouts,
|
||||
}
|
||||
|
||||
impl TargetClient {
|
||||
@@ -2360,12 +2441,11 @@ impl TargetClient {
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
});
|
||||
let mut config = self.timeouts.upload_override(size)?;
|
||||
if forwards_source_checksum {
|
||||
operation = operation.config_override(
|
||||
aws_sdk_s3::config::Builder::new()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
|
||||
);
|
||||
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
|
||||
}
|
||||
operation = operation.config_override(config);
|
||||
match operation.send().await {
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
@@ -2501,6 +2581,7 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(self.timeouts.upload_override(size)?)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2538,6 +2619,13 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(
|
||||
aws_sdk_s3::config::Builder::new().timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.timeouts.complete)
|
||||
.build(),
|
||||
),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2966,11 +3054,241 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_headers,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TimedResponseConnector {
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: bytes::Bytes,
|
||||
frames: Arc<std::sync::atomic::AtomicUsize>,
|
||||
dropped: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl Drop for ResponseDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for TimedResponseConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let settings = self.clone();
|
||||
HttpConnectorFuture::new(async move {
|
||||
http_body_util::BodyExt::collect(request.into_body())
|
||||
.await
|
||||
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
|
||||
tokio::time::sleep(settings.upload_delay).await;
|
||||
let guard = ResponseDrop(Arc::clone(&settings.dropped));
|
||||
let stream = futures::stream::unfold(
|
||||
(settings.prefix_bytes, Some(settings.response), true, guard),
|
||||
move |(remaining, mut body, first, guard)| {
|
||||
let frames = Arc::clone(&settings.frames);
|
||||
async move {
|
||||
if remaining == 0 && body.is_none() {
|
||||
return None;
|
||||
}
|
||||
if !first {
|
||||
tokio::time::sleep(settings.response_delay).await;
|
||||
}
|
||||
let bytes = if remaining > 0 {
|
||||
bytes::Bytes::from_static(b" ")
|
||||
} else {
|
||||
body.take()?
|
||||
};
|
||||
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Some((
|
||||
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
|
||||
(remaining.saturating_sub(1), body, false, guard),
|
||||
))
|
||||
}
|
||||
},
|
||||
)
|
||||
.fuse();
|
||||
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
|
||||
let mut response =
|
||||
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
|
||||
response.headers_mut().insert("etag", "\"test-etag\"");
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn timed_response_target(
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: &'static [u8],
|
||||
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
|
||||
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let connector = SharedHttpConnector::new(TimedResponseConnector {
|
||||
upload_delay,
|
||||
response_delay,
|
||||
prefix_bytes,
|
||||
response: bytes::Bytes::from_static(response),
|
||||
frames: Arc::clone(&frames),
|
||||
dropped: Arc::clone(&dropped),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let (mut target, _) = header_recording_target_client(Vec::new());
|
||||
target.timeouts = TargetRequestTimeouts {
|
||||
request: Duration::from_secs(2),
|
||||
complete: Duration::from_secs(10),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
};
|
||||
target.client = Arc::new(S3Client::from_conf(
|
||||
target
|
||||
.client
|
||||
.config()
|
||||
.to_builder()
|
||||
.http_client(http_client)
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(target.timeouts.request)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
));
|
||||
(target, frames, dropped)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
|
||||
let policy = TargetRequestTimeouts::default();
|
||||
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
|
||||
assert_eq!(
|
||||
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
|
||||
Duration::from_secs(15420)
|
||||
);
|
||||
assert!(policy.upload_timeout(-1).is_err());
|
||||
let slow = TargetRequestTimeouts {
|
||||
minimum_transfer_rate: 1,
|
||||
..policy
|
||||
};
|
||||
assert!(slow.upload_timeout(i64::MAX).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
|
||||
for operation in ["put", "forwarded-checksum", "part"] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
let started = tokio::time::Instant::now();
|
||||
let error = if operation == "part" {
|
||||
target
|
||||
.put_object_part(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
1,
|
||||
size,
|
||||
ByteStream::from(body),
|
||||
&PutObjectPartOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
} else {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
if operation == "forwarded-checksum" {
|
||||
opts.user_metadata.insert(
|
||||
"x-amz-checksum-sha256".to_owned(),
|
||||
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
|
||||
);
|
||||
}
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
};
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_secs(6),
|
||||
"upload gets its full independent window: {error}"
|
||||
);
|
||||
assert!(
|
||||
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
|
||||
"response must end before the larger total deadline"
|
||||
);
|
||||
assert!(
|
||||
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
|
||||
"trickle delivered progress before failure"
|
||||
);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
|
||||
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("slow upload, prompt response");
|
||||
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
|
||||
let response = target
|
||||
.get_object("bucket", "key", None, None, None, HeaderMap::new())
|
||||
.await
|
||||
.expect("GET headers");
|
||||
let started = tokio::time::Instant::now();
|
||||
let body = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("streamed GET remains outside upload deadline")
|
||||
.into_bytes();
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert_eq!(&body[..], b" payload");
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
|
||||
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
|
||||
for prefix in [5, 100] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = target
|
||||
.complete_multipart_upload(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
if prefix == 5 {
|
||||
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
|
||||
} else {
|
||||
assert!(result.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(11));
|
||||
}
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_control_operation_deadline_covers_response_trickle() {
|
||||
let (target, frames, dropped) =
|
||||
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
|
||||
let started = tokio::time::Instant::now();
|
||||
assert!(target.get_bucket_versioning("bucket").await.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(3));
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
|
||||
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
|
||||
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
|
||||
@@ -3279,6 +3597,7 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_uris,
|
||||
)
|
||||
@@ -3591,6 +3910,7 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -5487,6 +5487,14 @@ pub async fn put_restore_opts(
|
||||
if !oi.user_tags.is_empty() {
|
||||
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
|
||||
}
|
||||
if let Some(expires) = oi.expires {
|
||||
meta.insert(
|
||||
"expires".to_string(),
|
||||
expires
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(std::io::Error::other)?,
|
||||
);
|
||||
}
|
||||
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
|
||||
meta.insert(
|
||||
metadata_keys::RESTORE.to_string(),
|
||||
@@ -5506,7 +5514,6 @@ pub async fn put_restore_opts(
|
||||
// computed MD5 is not the object's public plaintext ETag.
|
||||
preserve_etag: oi.etag.clone(),
|
||||
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
|
||||
//expires: oi.expires,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
@@ -7144,6 +7151,25 @@ mod tests {
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
let unread = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("open an unread transitioned response");
|
||||
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
|
||||
drop(unread);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"canceling an unread response must release the tier lease"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -256,6 +256,38 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
|
||||
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
|
||||
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
|
||||
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
|
||||
/// Shared tag-entry validation for object requests and persisted bucket tags.
|
||||
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
|
||||
fn is_valid_tag_text(value: &str) -> bool {
|
||||
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
|
||||
VALID_TAG_TEXT
|
||||
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
|
||||
.as_ref()
|
||||
.is_ok_and(|pattern| pattern.is_match(value))
|
||||
}
|
||||
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
|
||||
for tag in tag_set {
|
||||
let key = tag
|
||||
.key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or("Tag key cannot be empty")?;
|
||||
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
|
||||
return Err("The TagKey you have provided is invalid");
|
||||
}
|
||||
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
|
||||
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
|
||||
return Err("The TagValue you have provided is invalid");
|
||||
}
|
||||
if !keys.insert(key) {
|
||||
return Err("Cannot provide multiple Tags with the same key");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub const MAX_BUCKET_TAGS: usize = 50;
|
||||
|
||||
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
|
||||
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
|
||||
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
|
||||
@@ -990,6 +1022,20 @@ impl BucketMetadata {
|
||||
self.encryption_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_TAGGING_CONFIG => {
|
||||
if !data.is_empty() {
|
||||
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
|
||||
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
|
||||
})?;
|
||||
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
|
||||
return Err(Error::InvalidArgument(
|
||||
self.name.clone(),
|
||||
String::new(),
|
||||
"Cannot have more than 50 tags per bucket".to_owned(),
|
||||
));
|
||||
}
|
||||
validate_tag_set_entries(&tagging.tag_set)
|
||||
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
|
||||
}
|
||||
self.tagging_config_xml = data;
|
||||
// Drop the parsed form (like lifecycle above) so clearing the
|
||||
// payload can't leave stale parsed tags to be cached.
|
||||
@@ -1784,6 +1830,31 @@ mod test {
|
||||
/// backlog#2292: a replicated config is stamped with the source
|
||||
/// `updated_at` it was given, not the local clock, while the plain
|
||||
/// `update_config` entry keeps stamping the local clock.
|
||||
#[test]
|
||||
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
|
||||
let mut metadata = BucketMetadata::new("validated-tags");
|
||||
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
|
||||
.expect("empty value is valid");
|
||||
let updated = metadata.tagging_config_updated_at;
|
||||
for invalid in [
|
||||
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
|
||||
] {
|
||||
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
|
||||
assert_eq!(metadata.tagging_config_xml, valid);
|
||||
assert_eq!(metadata.tagging_config_updated_at, updated);
|
||||
}
|
||||
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
|
||||
.expect("deletion repairs legacy corruption");
|
||||
assert!(metadata.tagging_config_xml.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
|
||||
|
||||
@@ -3428,7 +3428,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
),
|
||||
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
|
||||
(
|
||||
@@ -4556,8 +4556,13 @@ mod tests {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
|
||||
// Model legacy persisted corruption without passing the write-time validator.
|
||||
bm.tagging_config_xml = raw.to_vec();
|
||||
} else {
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
}
|
||||
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
|
||||
}
|
||||
|
||||
@@ -5738,7 +5743,8 @@ mod tests {
|
||||
// update only needs one interleaving to show up.
|
||||
const ROUNDS: usize = 8;
|
||||
for round in 0..ROUNDS {
|
||||
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
|
||||
let tagging =
|
||||
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
|
||||
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
|
||||
|
||||
let start = Arc::new(tokio::sync::Barrier::new(2));
|
||||
@@ -5811,7 +5817,7 @@ mod tests {
|
||||
|
||||
let blocked = tokio::spawn({
|
||||
let node_b = node_b.clone();
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
|
||||
});
|
||||
|
||||
// Long enough for the write to have finished had it not waited: the
|
||||
|
||||
@@ -119,6 +119,21 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
// Exercise the real storage entry point before testing the new
|
||||
// reporting semantics: the same ledger must admit and refuse
|
||||
// these writes even when a scanner floor claims a million bytes.
|
||||
if protocol == 1 {
|
||||
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::SlowDown)
|
||||
),
|
||||
"a missing authoritative ledger must not admit a write from scanner reporting"
|
||||
);
|
||||
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative usage through the production worker");
|
||||
}
|
||||
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
|
||||
assert!(
|
||||
matches!(
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::data_usage::compute_bucket_usage;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::disk::{DiskAPI, error::DiskError};
|
||||
use crate::error::{Result, StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, QuotaAdmission};
|
||||
use crate::object_api::{ObjectInfo, ObjectLockConfigSnapshot, ObjectOptions, QuotaAdmission};
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectOperations};
|
||||
@@ -28,7 +28,7 @@ use futures::{StreamExt, stream};
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
@@ -36,6 +36,8 @@ use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
const QUOTA_LEDGER_FORMAT_VERSION: u8 = 1;
|
||||
const RECONCILE_RETRY_SECONDS: i64 = 30;
|
||||
const EVENT_QUOTA_RECONCILIATION: &str = "quota_reconciliation";
|
||||
const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
|
||||
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
|
||||
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
|
||||
@@ -96,6 +98,10 @@ struct QuotaLedger {
|
||||
version: u8,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision_unix_nanos: i128,
|
||||
/// Changes on every durable write, including zero-growth reservations.
|
||||
/// Missing on an older writer's output, which invalidates a concurrent scan.
|
||||
#[serde(default)]
|
||||
writer_revision: Uuid,
|
||||
accounted_usage: u64,
|
||||
reservations: BTreeMap<Uuid, PersistedReservation>,
|
||||
#[serde(default)]
|
||||
@@ -110,6 +116,7 @@ impl QuotaLedger {
|
||||
version: QUOTA_LEDGER_FORMAT_VERSION,
|
||||
bucket_incarnation,
|
||||
quota_revision_unix_nanos: quota_revision.unix_timestamp_nanos(),
|
||||
writer_revision: Uuid::nil(),
|
||||
accounted_usage,
|
||||
reservations: BTreeMap::new(),
|
||||
reconcile_required: false,
|
||||
@@ -485,22 +492,23 @@ impl QuotaContext {
|
||||
let ledger_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_data.ledger_object).await?;
|
||||
let ledger_guard = Arc::new(ledger_lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &ledger_data.ledger_object, None).await?;
|
||||
let mut ledger = load_current_ledger_locked(
|
||||
Arc::clone(&store),
|
||||
&ledger_data.bucket,
|
||||
&ledger_data.ledger_object,
|
||||
bucket_incarnation,
|
||||
quota_revision,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut ledger =
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, bucket_incarnation, quota_revision)
|
||||
.await?;
|
||||
let job = ReconcileJob::new(Arc::clone(&store), ledger_data.bucket.clone(), bucket_incarnation, quota_revision);
|
||||
let Some(mut ledger) = ledger.take() else {
|
||||
// Bootstrap has no authoritative usage yet. Release every
|
||||
// request-owned fence by returning; the worker owns no request.
|
||||
job.schedule();
|
||||
return Err(StorageError::SlowDown);
|
||||
};
|
||||
let growth = reservation.growth();
|
||||
let mut current_usage = ledger.admitted_usage()?;
|
||||
let mut expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
if growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial() {
|
||||
reconcile_exact(&store, &ledger_data.bucket, &mut ledger).await?;
|
||||
current_usage = ledger.admitted_usage()?;
|
||||
expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
let current_usage = ledger.admitted_usage()?;
|
||||
let expected_usage = current_usage.checked_add(growth).ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
if ledger.reconcile_required
|
||||
|| (growth > 0 && expected_usage > quota_limit && growth <= quota_limit && ledger.should_reconcile_after_denial())
|
||||
{
|
||||
job.schedule();
|
||||
}
|
||||
if growth > 0 && expected_usage > quota_limit {
|
||||
return Err(StorageError::QuotaExceeded {
|
||||
@@ -518,7 +526,7 @@ impl QuotaContext {
|
||||
});
|
||||
}
|
||||
ledger.reserve(operation_id, reservation)?;
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &ledger, &ledger_guard).await?;
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_data.ledger_object, &mut ledger, &ledger_guard).await?;
|
||||
|
||||
Ok(QuotaReservation {
|
||||
ledger: Some(ReservationLedgerData::Single(ledger_data)),
|
||||
@@ -1082,7 +1090,7 @@ async fn mark_commit_started(data: &ReservationLedgerData) -> Result<()> {
|
||||
fence_namespace_mutations(&data.store, RUSTFS_META_BUCKET, &data.ledger_object, None).await?;
|
||||
let mut ledger = load_ledger_locked(Arc::clone(&data.store), &data.ledger_object).await?;
|
||||
ledger.mark_commit_started(data.operation_id, &data.reservation)?;
|
||||
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &ledger, &ledger_guard).await
|
||||
save_ledger_locked(Arc::clone(&data.store), &data.ledger_object, &mut ledger, &ledger_guard).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other_with_context("quota commit marker task failed", err))?
|
||||
@@ -1113,7 +1121,7 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
|
||||
} else {
|
||||
ledger.abort(operation_id, &reservation)?;
|
||||
}
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_object, &ledger, &ledger_guard).await
|
||||
save_ledger_locked(Arc::clone(&store), &ledger_object, &mut ledger, &ledger_guard).await
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other_with_context("quota ledger settlement task failed", err))?
|
||||
@@ -1121,22 +1129,15 @@ async fn settle(data: &ReservationLedgerData, committed: bool) -> Result<()> {
|
||||
|
||||
async fn load_current_ledger_locked(
|
||||
store: Arc<ECStore>,
|
||||
bucket: &str,
|
||||
ledger_object: &str,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision: OffsetDateTime,
|
||||
) -> Result<QuotaLedger> {
|
||||
match load_ledger_locked(Arc::clone(&store), ledger_object).await {
|
||||
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(ledger),
|
||||
Ok(ledger) if ledger.reservations.is_empty() && !ledger.reconcile_required => {
|
||||
let usage = exact_bucket_usage(&store, bucket).await?;
|
||||
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
|
||||
}
|
||||
) -> Result<Option<QuotaLedger>> {
|
||||
match load_ledger_locked(store, ledger_object).await {
|
||||
Ok(ledger) if ledger.matches(bucket_incarnation, quota_revision) => Ok(Some(ledger)),
|
||||
Ok(ledger) if ledger.reservations.is_empty() => Ok(None),
|
||||
Ok(_) => Err(StorageError::PartMissingOrCorrupt),
|
||||
Err(StorageError::ConfigNotFound) => {
|
||||
let usage = exact_bucket_usage(&store, bucket).await?;
|
||||
Ok(QuotaLedger::new(bucket_incarnation, quota_revision, usage))
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
@@ -1245,15 +1246,11 @@ async fn reap_stale_reservations(store: Arc<ECStore>, bucket: &str, ledger_objec
|
||||
for (object, target) in orphan_commit_targets {
|
||||
fence_namespace_mutations(&store, bucket, &object, target).await?;
|
||||
}
|
||||
let mut removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
|
||||
if ledger.reservations.is_empty() && ledger.reconcile_required {
|
||||
reconcile_exact(&store, bucket, &mut ledger).await?;
|
||||
removed = true;
|
||||
}
|
||||
let removed = remove_orphan_reservations(&mut ledger, &orphan_ids)?;
|
||||
if !removed && !cursor_changed {
|
||||
return Ok(());
|
||||
}
|
||||
save_ledger_locked(store, ledger_object, &ledger, &ledger_guard).await
|
||||
save_ledger_locked(store, ledger_object, &mut ledger, &ledger_guard).await
|
||||
}
|
||||
|
||||
fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid]) -> Result<bool> {
|
||||
@@ -1273,13 +1270,217 @@ fn remove_orphan_reservations(ledger: &mut QuotaLedger, operation_ids: &[Uuid])
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
async fn reconcile_exact(store: &Arc<ECStore>, bucket: &str, ledger: &mut QuotaLedger) -> Result<()> {
|
||||
if !ledger.reservations.is_empty() {
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct ReconcileControl {
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision_unix_nanos: i128,
|
||||
next_attempt_at: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ReconcileJob {
|
||||
store: Arc<ECStore>,
|
||||
bucket: String,
|
||||
bucket_incarnation: Uuid,
|
||||
quota_revision: OffsetDateTime,
|
||||
}
|
||||
|
||||
type ReconcileKey = (Uuid, String, Uuid, i128);
|
||||
|
||||
fn active_reconciliations() -> &'static std::sync::Mutex<HashSet<ReconcileKey>> {
|
||||
static ACTIVE: std::sync::OnceLock<std::sync::Mutex<HashSet<ReconcileKey>>> = std::sync::OnceLock::new();
|
||||
ACTIVE.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
struct ActiveReconciliation(ReconcileKey);
|
||||
|
||||
impl Drop for ActiveReconciliation {
|
||||
fn drop(&mut self) {
|
||||
active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
impl ReconcileJob {
|
||||
fn new(store: Arc<ECStore>, bucket: String, bucket_incarnation: Uuid, quota_revision: OffsetDateTime) -> Self {
|
||||
Self {
|
||||
store,
|
||||
bucket,
|
||||
bucket_incarnation,
|
||||
quota_revision,
|
||||
}
|
||||
}
|
||||
|
||||
fn schedule(self) {
|
||||
let key = (
|
||||
self.store.id,
|
||||
self.bucket.clone(),
|
||||
self.bucket_incarnation,
|
||||
self.quota_revision.unix_timestamp_nanos(),
|
||||
);
|
||||
if !active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(key.clone())
|
||||
{
|
||||
return;
|
||||
}
|
||||
let owner = ActiveReconciliation(key);
|
||||
// The owner carries no object, metadata, lifecycle or reservation guard.
|
||||
// Cancellation of the rejected request cannot cancel its reconciliation.
|
||||
tokio::spawn(async move {
|
||||
let _owner = owner;
|
||||
if let Err(error) = self.run().await {
|
||||
warn!(event = EVENT_QUOTA_RECONCILIATION, component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_QUOTA, state = "deferred", bucket = %self.bucket,
|
||||
error = %error, "Quota reconciliation deferred");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn control_object(&self) -> String {
|
||||
format!(
|
||||
"{}.reconcile/{}/{}",
|
||||
ledger_object(&self.bucket),
|
||||
self.bucket_incarnation,
|
||||
self.quota_revision.unix_timestamp_nanos()
|
||||
)
|
||||
}
|
||||
|
||||
async fn save_control(&self, object: &str, control: &ReconcileControl, guard: &Arc<NamespaceLockGuard>) -> Result<()> {
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(guard), RUSTFS_META_BUCKET, object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
save_config_with_opts(Arc::clone(&self.store), object, serde_json::to_vec(control)?, &opts).await
|
||||
}
|
||||
|
||||
async fn run(&self) -> Result<()> {
|
||||
let control_object = self.control_object();
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &control_object).await?;
|
||||
let Ok(guard) = lock.get_write_lock_quiet(Duration::from_millis(50)).await else {
|
||||
return Ok(());
|
||||
};
|
||||
let guard = Arc::new(guard);
|
||||
let now = now_unix();
|
||||
match read_config_no_lock(Arc::clone(&self.store), &control_object).await {
|
||||
Ok(bytes) => {
|
||||
let control: ReconcileControl = serde_json::from_slice(&bytes)?;
|
||||
if control.bucket_incarnation != self.bucket_incarnation
|
||||
|| control.quota_revision_unix_nanos != self.quota_revision.unix_timestamp_nanos()
|
||||
{
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
if control.next_attempt_at > now {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &control_object, None).await?;
|
||||
let mut control = ReconcileControl {
|
||||
bucket_incarnation: self.bucket_incarnation,
|
||||
quota_revision_unix_nanos: self.quota_revision.unix_timestamp_nanos(),
|
||||
next_attempt_at: now.saturating_add(RECONCILE_RETRY_SECONDS),
|
||||
};
|
||||
// Persist the throttle before work, including bootstrap and failed scans.
|
||||
self.save_control(&control_object, &control, &guard).await?;
|
||||
let result = self.scan_and_publish(&guard).await;
|
||||
control.next_attempt_at = now_unix().saturating_add(RECONCILE_RETRY_SECONDS);
|
||||
self.save_control(&control_object, &control, &guard).await?;
|
||||
result
|
||||
}
|
||||
|
||||
/// Take a short authoritative configuration snapshot without recursively
|
||||
/// acquiring a reader behind a queued metadata/lifecycle writer.
|
||||
async fn configuration_snapshot(&self) -> Result<Arc<ObjectLockConfigSnapshot>> {
|
||||
let snapshot = self.store.object_lock_config_snapshot_for_single_put(&self.bucket).await?;
|
||||
let guard = snapshot
|
||||
.metadata_transaction_guard_for(self.store.id, &self.bucket, Some(self.bucket_incarnation))
|
||||
.ok_or(StorageError::PreconditionFailed)?;
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(self.bucket_incarnation),
|
||||
object_lock_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
};
|
||||
let (quota, incarnation, revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(&self.store.ctx, &self.bucket, &opts, &guard)
|
||||
.await?;
|
||||
if incarnation != self.bucket_incarnation
|
||||
|| revision != self.quota_revision
|
||||
|| !quota.as_ref().is_some_and(|quota| {
|
||||
quota.uses_durable_reservations() && quota.reservation_protocol != Some(QUOTA_RESERVATION_PROTOCOL_V2)
|
||||
})
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
async fn scan_and_publish(&self, owner: &NamespaceLockGuard) -> Result<()> {
|
||||
let ledger_object = ledger_object(&self.bucket);
|
||||
let expected_revision = {
|
||||
// Lock order: reconcile owner, lifecycle, metadata, ledger.
|
||||
// Request paths never wait for the reconcile owner.
|
||||
let _configuration = self.configuration_snapshot().await?;
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
|
||||
Ok(mut ledger) => {
|
||||
if !ledger.reservations.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
// Seed a non-nil token even for a legacy ledger. An old
|
||||
// node drops this field on write and cannot pass the CAS.
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
|
||||
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await?;
|
||||
Some(ledger.writer_revision)
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
};
|
||||
#[cfg(test)]
|
||||
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforeScan).await?;
|
||||
// Only the independent reconcile owner is held during this whole-bucket
|
||||
// walk. Concurrent reserve/settle/reap writes invalidate its result.
|
||||
let usage = exact_bucket_usage(&self.store, &self.bucket).await?;
|
||||
#[cfg(test)]
|
||||
tests::pause_reconciliation(&self.bucket, tests::ReconcilePauseStage::BeforePublish).await?;
|
||||
let configuration = self.configuration_snapshot().await?;
|
||||
let lock = self.store.new_ns_lock(RUSTFS_META_BUCKET, &ledger_object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
let unchanged = match load_ledger_locked(Arc::clone(&self.store), &ledger_object).await {
|
||||
Ok(ledger) => {
|
||||
expected_revision == Some(ledger.writer_revision)
|
||||
&& !ledger.writer_revision.is_nil()
|
||||
&& ledger.reservations.is_empty()
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => expected_revision.is_none(),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
}
|
||||
if owner.is_lock_lost()
|
||||
|| !configuration.is_valid_for_destructive_put(self.store.id, &self.bucket, self.bucket_incarnation)
|
||||
{
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
fence_namespace_mutations(&self.store, RUSTFS_META_BUCKET, &ledger_object, None).await?;
|
||||
let mut ledger = QuotaLedger::new(self.bucket_incarnation, self.quota_revision, usage);
|
||||
save_ledger_locked(Arc::clone(&self.store), &ledger_object, &mut ledger, &guard).await
|
||||
}
|
||||
ledger.accounted_usage = exact_bucket_usage(store, bucket).await?;
|
||||
ledger.reconcile_required = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn exact_bucket_usage(store: &Arc<ECStore>, bucket: &str) -> Result<u64> {
|
||||
@@ -1935,7 +2136,7 @@ async fn load_ledger_locked(store: Arc<ECStore>, ledger_object: &str) -> Result<
|
||||
async fn save_ledger_locked(
|
||||
store: Arc<ECStore>,
|
||||
ledger_object: &str,
|
||||
ledger: &QuotaLedger,
|
||||
ledger: &mut QuotaLedger,
|
||||
ledger_guard: &Arc<NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
if ledger_guard.is_lock_lost() {
|
||||
@@ -1959,9 +2160,71 @@ async fn save_ledger_locked(
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(ledger_guard), RUSTFS_META_BUCKET, ledger_object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
ledger.writer_revision = Uuid::new_v4();
|
||||
save_config_with_opts(store, ledger_object, serde_json::to_vec(ledger)?, &opts).await
|
||||
}
|
||||
|
||||
/// Advance the persisted reconciliation retry deadline for deterministic fixtures.
|
||||
/// The production reaper, ownership fence, scan and compare-and-swap still run.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn retry_quota_reconciliation_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
let (_, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
|
||||
let job = ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision);
|
||||
let key = (store.id, bucket.to_owned(), incarnation, revision.unix_timestamp_nanos());
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let active = active_reconciliations()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.contains(&key);
|
||||
if !active {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| StorageError::SlowDown)?;
|
||||
reap_stale_reservations(Arc::clone(&store), bucket, &ledger_object(bucket)).await?;
|
||||
let object = job.control_object();
|
||||
{
|
||||
let lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await?);
|
||||
match read_config_no_lock(Arc::clone(&store), &object).await {
|
||||
Ok(bytes) => {
|
||||
let mut control: ReconcileControl = serde_json::from_slice(&bytes)?;
|
||||
control.next_attempt_at = 0;
|
||||
fence_namespace_mutations(&store, RUSTFS_META_BUCKET, &object, None).await?;
|
||||
job.save_control(&object, &control, &guard).await?;
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
job.run().await
|
||||
}
|
||||
|
||||
/// Bootstrap a test bucket through the real background reconciliation path.
|
||||
/// Fixtures call this after enabling V1 quota, before issuing writes whose
|
||||
/// contract is independent of the initial retryable admission response.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn initialize_quota_ledger_for_test(store: Arc<ECStore>, bucket: &str) -> Result<()> {
|
||||
let (quota, incarnation, revision) = store.bucket_quota_and_incarnation_from_disk(bucket).await?;
|
||||
if !quota
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.uses_durable_reservations() && !quota.uses_sharded_reservations())
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
ReconcileJob::new(Arc::clone(&store), bucket.to_owned(), incarnation, revision)
|
||||
.run()
|
||||
.await?;
|
||||
match load_current_ledger_locked(store, &ledger_object(bucket), incarnation, revision).await? {
|
||||
Some(_) => Ok(()),
|
||||
None => Err(StorageError::SlowDown),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub fn fail_next_quota_ledger_save_for_test() {
|
||||
@@ -2003,6 +2266,434 @@ mod tests {
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use serial_test::serial;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum ReconcilePauseStage {
|
||||
BeforeScan,
|
||||
BeforePublish,
|
||||
}
|
||||
|
||||
struct ReconcilePause {
|
||||
bucket: String,
|
||||
stage: ReconcilePauseStage,
|
||||
armed: std::sync::atomic::AtomicBool,
|
||||
scans: std::sync::atomic::AtomicUsize,
|
||||
fail_scan: std::sync::atomic::AtomicBool,
|
||||
reached: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
static RECONCILE_PAUSE: std::sync::Mutex<Option<Arc<ReconcilePause>>> = std::sync::Mutex::new(None);
|
||||
|
||||
struct ReconcilePauseGuard(Arc<ReconcilePause>);
|
||||
|
||||
impl ReconcilePauseGuard {
|
||||
fn install(bucket: &str, stage: ReconcilePauseStage) -> Self {
|
||||
let pause = Arc::new(ReconcilePause {
|
||||
bucket: bucket.to_owned(),
|
||||
stage,
|
||||
armed: true.into(),
|
||||
scans: 0.into(),
|
||||
fail_scan: false.into(),
|
||||
reached: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
*RECONCILE_PAUSE.lock().expect("reconcile hooks") = Some(Arc::clone(&pause));
|
||||
Self(pause)
|
||||
}
|
||||
|
||||
async fn reached(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(10), self.0.reached.notified())
|
||||
.await
|
||||
.expect("reconciliation reached barrier");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReconcilePauseGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.release.notify_one();
|
||||
let mut hook = RECONCILE_PAUSE.lock().expect("reconcile hooks");
|
||||
if hook.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
|
||||
*hook = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pause_reconciliation(bucket: &str, stage: ReconcilePauseStage) -> Result<()> {
|
||||
use std::sync::atomic::Ordering;
|
||||
let hook = RECONCILE_PAUSE
|
||||
.lock()
|
||||
.expect("reconcile hooks")
|
||||
.as_ref()
|
||||
.filter(|hook| hook.bucket == bucket)
|
||||
.cloned();
|
||||
if let Some(hook) = hook {
|
||||
if stage == ReconcilePauseStage::BeforeScan {
|
||||
hook.scans.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
if stage == hook.stage && hook.armed.swap(false, Ordering::SeqCst) {
|
||||
hook.reached.notify_one();
|
||||
hook.release.notified().await;
|
||||
}
|
||||
if stage == ReconcilePauseStage::BeforeScan && hook.fail_scan.load(Ordering::SeqCst) {
|
||||
return Err(StorageError::other("injected quota usage failure"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reconciliation_fixture(limit: u64) -> (Vec<tempfile::TempDir>, ReconcileJob) {
|
||||
use crate::object_api::PutObjReader;
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
let (directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
|
||||
metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
|
||||
let bucket = format!("quota-reconcile-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let mut seed = PutObjReader::from_vec(vec![1; 10]);
|
||||
store
|
||||
.put_object(&bucket, "seed", &mut seed, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("existing usage");
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
&bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(limit))).expect("quota JSON"),
|
||||
)
|
||||
.await
|
||||
.expect("enable quota");
|
||||
let (_, incarnation, revision) = store
|
||||
.bucket_quota_and_incarnation_from_disk(&bucket)
|
||||
.await
|
||||
.expect("quota identity");
|
||||
(directories, ReconcileJob::new(store, bucket, incarnation, revision))
|
||||
}
|
||||
|
||||
async fn quota_reservation(job: &ReconcileJob, old: u64, new: u64) -> Result<QuotaReservation> {
|
||||
begin(&job.store.ctx, &job.bucket, "candidate", &ObjectOptions::default(), 0, 0)
|
||||
.await?
|
||||
.reserve(old, new)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_test_ledger(job: &ReconcileJob) -> QuotaLedger {
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket))
|
||||
.await
|
||||
.expect("ledger")
|
||||
}
|
||||
|
||||
async fn drain_reconciliation_scheduler(job: &ReconcileJob) {
|
||||
let key = (
|
||||
job.store.id,
|
||||
job.bucket.clone(),
|
||||
job.bucket_incarnation,
|
||||
job.quota_revision.unix_timestamp_nanos(),
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while active_reconciliations().lock().expect("active workers").contains(&key) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned reconciliation finished");
|
||||
}
|
||||
|
||||
async fn expire_reconciliation_backoff(job: &ReconcileJob) {
|
||||
drain_reconciliation_scheduler(job).await;
|
||||
let object = job.control_object();
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("owner guard"));
|
||||
let mut control: ReconcileControl =
|
||||
serde_json::from_slice(&read_config_no_lock(Arc::clone(&job.store), &object).await.expect("control"))
|
||||
.expect("control JSON");
|
||||
control.next_attempt_at = 0;
|
||||
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
|
||||
.await
|
||||
.expect("control fence");
|
||||
job.save_control(&object, &control, &guard).await.expect("advance test clock");
|
||||
}
|
||||
|
||||
async fn wait_for_reconcile_owner(job: &ReconcileJob) {
|
||||
let object = job.control_object();
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("owner lock");
|
||||
let _guard = tokio::time::timeout(Duration::from_secs(10), lock.get_write_lock(get_lock_acquire_timeout()))
|
||||
.await
|
||||
.expect("worker finished")
|
||||
.expect("owner guard");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_bootstrap_is_single_flight_without_request_guards() {
|
||||
use std::sync::atomic::Ordering;
|
||||
let (_directories, job) = reconciliation_fixture(10).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
assert!(
|
||||
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)),
|
||||
"unknown usage cannot admit a reservation"
|
||||
);
|
||||
pause.reached().await;
|
||||
assert!(
|
||||
matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
),
|
||||
"bootstrap must not publish zero usage"
|
||||
);
|
||||
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::SlowDown)));
|
||||
// Bypass the local scheduler, exercising the cross-node ownership boundary.
|
||||
tokio::time::timeout(Duration::from_secs(2), job.run())
|
||||
.await
|
||||
.expect("contender must not wait for a scan")
|
||||
.expect("another owner is active");
|
||||
let lifecycle = tokio::time::timeout(Duration::from_secs(2), job.store.acquire_bucket_lifecycle_write_lock(&job.bucket))
|
||||
.await
|
||||
.expect("scan does not retain lifecycle readers")
|
||||
.expect("lifecycle writer");
|
||||
let metadata = tokio::time::timeout(
|
||||
Duration::from_secs(2),
|
||||
metadata_sys::acquire_bucket_metadata_transaction_lock_in(&job.store.ctx, &job.bucket),
|
||||
)
|
||||
.await
|
||||
.expect("scan does not retain metadata readers")
|
||||
.expect("metadata writer");
|
||||
drop(metadata);
|
||||
drop(lifecycle);
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(&job).await;
|
||||
let ledger = read_test_ledger(&job).await;
|
||||
assert_eq!(ledger.accounted_usage, 10);
|
||||
assert!(ledger.reservations.is_empty());
|
||||
assert!(!ledger.writer_revision.is_nil());
|
||||
for _ in 0..3 {
|
||||
assert!(matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })));
|
||||
job.run().await.expect("persisted cooldown");
|
||||
}
|
||||
drain_reconciliation_scheduler(&job).await;
|
||||
assert_eq!(
|
||||
pause.0.scans.load(Ordering::SeqCst),
|
||||
1,
|
||||
"denied requests cannot repeatedly scan a full bucket"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_rejects_concurrent_zero_growth_settle_reap_and_legacy_writes() {
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
job.run().await.expect("bootstrap");
|
||||
for mutation in ["abort", "commit", "reap", "legacy"] {
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let worker = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move { job.run().await }
|
||||
});
|
||||
pause.reached().await;
|
||||
let initial = read_test_ledger(&job).await.writer_revision;
|
||||
if mutation == "legacy" {
|
||||
let object = ledger_object(&job.bucket);
|
||||
let lock = job.store.new_ns_lock(RUSTFS_META_BUCKET, &object).await.expect("ledger lock");
|
||||
let guard = Arc::new(lock.get_write_lock(get_lock_acquire_timeout()).await.expect("ledger writer"));
|
||||
let mut document = serde_json::to_value(read_test_ledger(&job).await).expect("ledger JSON");
|
||||
document.as_object_mut().expect("ledger map").remove("writer_revision");
|
||||
fence_namespace_mutations(&job.store, RUSTFS_META_BUCKET, &object, None)
|
||||
.await
|
||||
.expect("ledger fence");
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = opts.set_quota_admission(0, u64::MAX);
|
||||
opts.add_owned_write_lock(Arc::clone(&guard), RUSTFS_META_BUCKET, &object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrained;
|
||||
save_config_with_opts(
|
||||
Arc::clone(&job.store),
|
||||
&object,
|
||||
serde_json::to_vec(&document).expect("old node JSON"),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("old node drops new field");
|
||||
} else {
|
||||
let mut reservation = quota_reservation(&job, 10, 10)
|
||||
.await
|
||||
.expect("zero-growth admission while scanning");
|
||||
let reserved = read_test_ledger(&job).await.writer_revision;
|
||||
assert_ne!(reserved, initial, "even zero-growth reserve must invalidate a scan");
|
||||
reservation.mark_commit_started().await.expect("commit marker");
|
||||
assert_ne!(read_test_ledger(&job).await.writer_revision, reserved);
|
||||
match mutation {
|
||||
"abort" => reservation.abort().await,
|
||||
"commit" => reservation.commit().await,
|
||||
"reap" => {
|
||||
reservation.defer_after_fence();
|
||||
reap_stale_reservations(Arc::clone(&job.store), &job.bucket, &ledger_object(&job.bucket))
|
||||
.await
|
||||
.expect("reap orphan");
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
let written = read_test_ledger(&job).await;
|
||||
assert!(written.reservations.is_empty());
|
||||
assert_ne!(written.writer_revision, initial);
|
||||
pause.0.release.notify_one();
|
||||
worker.await.expect("worker join").expect("stale scan discarded");
|
||||
let current = read_test_ledger(&job).await;
|
||||
assert_eq!(
|
||||
current.writer_revision, written.writer_revision,
|
||||
"{mutation}: stale usage must not replace a concurrent writer"
|
||||
);
|
||||
assert_eq!(current.accounted_usage, written.accounted_usage);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_failure_is_throttled_and_deletions_eventually_reclaim_usage() {
|
||||
use std::sync::atomic::Ordering;
|
||||
let (_directories, job) = reconciliation_fixture(10).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
pause.0.fail_scan.store(true, Ordering::SeqCst);
|
||||
pause.0.release.notify_one();
|
||||
assert!(job.run().await.is_err());
|
||||
assert!(matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
job.run().await.expect("failure cooldown");
|
||||
assert_eq!(pause.0.scans.load(Ordering::SeqCst), 1);
|
||||
pause.0.fail_scan.store(false, Ordering::SeqCst);
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
job.run().await.expect("recover after failure");
|
||||
assert_eq!(read_test_ledger(&job).await.accounted_usage, 10);
|
||||
job.store
|
||||
.delete_object(&job.bucket, "seed", ObjectOptions::default())
|
||||
.await
|
||||
.expect("delete stored usage");
|
||||
assert!(
|
||||
matches!(quota_reservation(&job, 0, 1).await, Err(StorageError::QuotaExceeded { .. })),
|
||||
"stale usage remains conservative during cooldown"
|
||||
);
|
||||
expire_reconciliation_backoff(&job).await;
|
||||
job.run().await.expect("refresh deleted usage");
|
||||
assert_eq!(read_test_ledger(&job).await.accounted_usage, 0);
|
||||
quota_reservation(&job, 0, 1)
|
||||
.await
|
||||
.expect("delete must not leave bucket permanently full")
|
||||
.abort()
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_survives_waiter_cancellation_and_rechecks_configuration() {
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let caller = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move {
|
||||
job.schedule();
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
});
|
||||
pause.reached().await;
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel caller").is_cancelled());
|
||||
job.store
|
||||
.update_bucket_metadata_config(
|
||||
&job.bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(200))).expect("updated quota"),
|
||||
)
|
||||
.await
|
||||
.expect("quota changes while scanning");
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(&job).await;
|
||||
assert!(
|
||||
matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
),
|
||||
"old config cannot publish bootstrap usage"
|
||||
);
|
||||
drop(pause);
|
||||
let (_, incarnation, revision) = job
|
||||
.store
|
||||
.bucket_quota_and_incarnation_from_disk(&job.bucket)
|
||||
.await
|
||||
.expect("current configuration");
|
||||
let current = ReconcileJob::new(Arc::clone(&job.store), job.bucket.clone(), incarnation, revision);
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforeScan);
|
||||
let caller = tokio::spawn({
|
||||
let job = current.clone();
|
||||
async move {
|
||||
job.schedule();
|
||||
std::future::pending::<()>().await
|
||||
}
|
||||
});
|
||||
pause.reached().await;
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("cancel new caller").is_cancelled());
|
||||
pause.0.release.notify_one();
|
||||
wait_for_reconcile_owner(¤t).await;
|
||||
assert!(
|
||||
read_test_ledger(¤t).await.matches(incarnation, revision),
|
||||
"owned scan publishes after its waiter is canceled"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_reconciliation_cannot_publish_into_recreated_bucket() {
|
||||
use crate::storage_api_contracts::bucket::DeleteBucketOptions;
|
||||
let (_directories, job) = reconciliation_fixture(100).await;
|
||||
let pause = ReconcilePauseGuard::install(&job.bucket, ReconcilePauseStage::BeforePublish);
|
||||
let worker = tokio::spawn({
|
||||
let job = job.clone();
|
||||
async move { job.run().await }
|
||||
});
|
||||
pause.reached().await;
|
||||
job.store
|
||||
.delete_object(&job.bucket, "seed", ObjectOptions::default())
|
||||
.await
|
||||
.expect("empty old bucket");
|
||||
job.store
|
||||
.delete_bucket(&job.bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("retire bucket while scan is paused");
|
||||
job.store
|
||||
.make_bucket(&job.bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("replacement bucket");
|
||||
job.store
|
||||
.update_bucket_metadata_config(
|
||||
&job.bucket,
|
||||
"quota.json",
|
||||
serde_json::to_vec(&super::super::BucketQuota::new(Some(100))).expect("new quota"),
|
||||
)
|
||||
.await
|
||||
.expect("replacement quota");
|
||||
pause.0.release.notify_one();
|
||||
assert!(worker.await.expect("old worker join").is_err(), "old incarnation must not publish");
|
||||
assert!(matches!(
|
||||
load_ledger_locked(Arc::clone(&job.store), &ledger_object(&job.bucket)).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
drop(pause);
|
||||
initialize_quota_ledger_for_test(Arc::clone(&job.store), &job.bucket)
|
||||
.await
|
||||
.expect("replacement can bootstrap independently");
|
||||
let replacement = read_test_ledger(&job).await;
|
||||
assert_ne!(replacement.bucket_incarnation, job.bucket_incarnation);
|
||||
assert_eq!(replacement.accounted_usage, 0);
|
||||
}
|
||||
|
||||
fn ledger(accounted_usage: u64) -> QuotaLedger {
|
||||
QuotaLedger::new(Uuid::new_v4(), OffsetDateTime::now_utc(), accounted_usage)
|
||||
}
|
||||
@@ -2090,8 +2781,7 @@ mod tests {
|
||||
.expect("persist crash-left reservation fixture");
|
||||
|
||||
assert!(matches!(
|
||||
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
.await,
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc()).await,
|
||||
Err(StorageError::PartMissingOrCorrupt)
|
||||
));
|
||||
assert!(matches!(
|
||||
@@ -2109,10 +2799,10 @@ mod tests {
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
let replacement =
|
||||
load_current_ledger_locked(Arc::clone(&store), &bucket, &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
load_current_ledger_locked(Arc::clone(&store), &ledger_path, new_incarnation, OffsetDateTime::now_utc())
|
||||
.await
|
||||
.expect("new quota admission must not inherit a fresh stale reservation");
|
||||
assert!(replacement.reservations.is_empty());
|
||||
assert!(replacement.is_none(), "replacement must await authoritative bootstrap before admission");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -40,7 +40,9 @@ use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::interceptors::Intercept;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::{
|
||||
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
|
||||
@@ -155,6 +157,8 @@ pub struct RemoteS3EndpointSpec {
|
||||
pub ca_cert_pem: Option<String>,
|
||||
pub connect_timeout: Option<Duration>,
|
||||
pub read_timeout: Option<Duration>,
|
||||
/// Entire SDK call, including non-streaming response bodies and retries.
|
||||
pub operation_timeout: Option<Duration>,
|
||||
/// How many wire requests one logical call may cost. Every consumer
|
||||
/// declares it; see [`RemoteS3RetryPolicy`].
|
||||
pub retry: RemoteS3RetryPolicy,
|
||||
@@ -252,6 +256,78 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Only installed on non-streaming upload operations. The budget begins at
|
||||
/// response headers, independently of the potentially long request upload.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct UploadResponseDeadline(pub Duration);
|
||||
|
||||
impl Intercept for UploadResponseDeadline {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsUploadResponseDeadline"
|
||||
}
|
||||
|
||||
fn modify_before_deserialization(
|
||||
&self,
|
||||
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let response = context.response_mut();
|
||||
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
|
||||
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
|
||||
inner: Some(inner),
|
||||
deadline: Box::pin(tokio::time::sleep(self.0)),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct UploadResponseBody {
|
||||
inner: Option<SdkBody>,
|
||||
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
|
||||
}
|
||||
|
||||
impl http_body::Body for UploadResponseBody {
|
||||
type Data = bytes::Bytes;
|
||||
type Error = BoxError;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
||||
use std::future::Future as _;
|
||||
use std::task::Poll;
|
||||
if self.inner.is_none() {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
// The absolute response budget never renews on data, empty frames,
|
||||
// or trailers. A retry gets its own bounded response window.
|
||||
if self.deadline.as_mut().poll(cx).is_ready() {
|
||||
self.inner.take();
|
||||
return Poll::Ready(Some(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"replication response body deadline exceeded",
|
||||
)
|
||||
.into())));
|
||||
}
|
||||
let Some(inner) = self.inner.as_mut() else {
|
||||
return Poll::Ready(None);
|
||||
};
|
||||
let result = std::pin::Pin::new(inner).poll_frame(cx);
|
||||
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
|
||||
self.inner.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
|
||||
}
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
|
||||
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
|
||||
/// signature stays valid.
|
||||
@@ -309,7 +385,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
config_builder = config_builder.http_client(http_client);
|
||||
}
|
||||
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
|
||||
let mut timeouts = TimeoutConfig::builder();
|
||||
if let Some(connect_timeout) = spec.connect_timeout {
|
||||
timeouts = timeouts.connect_timeout(connect_timeout);
|
||||
@@ -317,6 +393,9 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
if let Some(read_timeout) = spec.read_timeout {
|
||||
timeouts = timeouts.read_timeout(read_timeout);
|
||||
}
|
||||
if let Some(operation_timeout) = spec.operation_timeout {
|
||||
timeouts = timeouts.operation_timeout(operation_timeout);
|
||||
}
|
||||
config_builder = config_builder.timeout_config(timeouts.build());
|
||||
}
|
||||
|
||||
@@ -674,6 +753,7 @@ mod tests {
|
||||
ca_cert_pem: None,
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: None,
|
||||
retry: RemoteS3RetryPolicy::Disabled,
|
||||
user_agent_suffix: "",
|
||||
}
|
||||
@@ -1027,11 +1107,16 @@ mod tests {
|
||||
spec.user_agent_suffix = "RustFS-Test/0.0";
|
||||
spec.connect_timeout = Some(Duration::from_secs(5));
|
||||
spec.read_timeout = Some(Duration::from_secs(5));
|
||||
spec.operation_timeout = Some(Duration::from_secs(12));
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("spec should build")
|
||||
.http_client(http_client)
|
||||
.build();
|
||||
assert_eq!(
|
||||
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
|
||||
Some(Duration::from_secs(12))
|
||||
);
|
||||
S3Client::from_conf(config)
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
|
||||
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_filemeta_boundary::{
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
|
||||
replication_statuses_map, version_purge_statuses_map,
|
||||
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
|
||||
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
|
||||
@@ -1581,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues a replica task
|
||||
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
|
||||
// A version that is already queued or being uploaded is not driven a
|
||||
// A source generation already queued or being uploaded is not driven a
|
||||
// second time: the scanner heal pass sees it as PENDING until the
|
||||
// first upload lands and would otherwise re-queue it every cycle
|
||||
// (backlog#2362). The key is released when the worker finishes, or
|
||||
@@ -2404,28 +2404,47 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Object versions currently queued or being uploaded, keyed by bucket,
|
||||
/// object name and version. `queue_replica_task` admits a version only once
|
||||
/// object name, version and source generation. `queue_replica_task` admits a generation only once
|
||||
/// while it is in flight; the scanner heal pass and MRF replays that arrive
|
||||
/// in the meantime are `Skipped` instead of driving a second complete upload
|
||||
/// (backlog#2362). Entries are removed when the worker finishes the task or
|
||||
/// when no worker accepted it.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct ReplicationInFlight {
|
||||
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
|
||||
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
struct ReplicationInFlightKey {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<uuid::Uuid>,
|
||||
generation: ReplicationGenerationSnapshot,
|
||||
}
|
||||
|
||||
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
|
||||
fn from(info: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
bucket: info.bucket.clone(),
|
||||
object: info.name.clone(),
|
||||
version_id: info.version_id,
|
||||
generation: info.replication_generation.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReplicationInFlight {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
|
||||
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
/// Claim `ri`; `false` when the same version is already in flight.
|
||||
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
|
||||
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
|
||||
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
|
||||
self.lock().insert(ReplicationInFlightKey::from(ri))
|
||||
}
|
||||
|
||||
fn finish(&self, ri: &ReplicateObjectInfo) {
|
||||
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
|
||||
self.lock().remove(&ReplicationInFlightKey::from(ri));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2438,26 +2457,21 @@ impl ReplicationInFlight {
|
||||
/// including when replication panics.
|
||||
struct ReplicationInFlightGuard {
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
key: ReplicateObjectInfo,
|
||||
key: ReplicationInFlightKey,
|
||||
}
|
||||
|
||||
impl ReplicationInFlightGuard {
|
||||
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
in_flight,
|
||||
key: ReplicateObjectInfo {
|
||||
bucket: ri.bucket.clone(),
|
||||
name: ri.name.clone(),
|
||||
version_id: ri.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
key: ReplicationInFlightKey::from(ri),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReplicationInFlightGuard {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.finish(&self.key);
|
||||
self.in_flight.lock().remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4054,6 +4068,48 @@ mod tests {
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, mut rx) = mpsc::channel(4);
|
||||
pool.workers.write().await.push(tx);
|
||||
let initial = ReplicateObjectInfo {
|
||||
bucket: "pending-tags-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let metadata = ReplicateObjectInfo {
|
||||
op_type: ReplicationType::Metadata,
|
||||
user_tags: String::new(),
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..initial.clone()
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
assert_eq!(pool.in_flight.len(), 2);
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
|
||||
assert_eq!(pool.in_flight.len(), 1);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
for expected in [&initial, &metadata] {
|
||||
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
|
||||
panic!("expected object replication");
|
||||
};
|
||||
assert_eq!(queued.replication_generation, expected.replication_generation);
|
||||
}
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
|
||||
assert_eq!(pool.in_flight.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
|
||||
@@ -6609,6 +6609,7 @@ mod tests {
|
||||
health_check_duration: std::time::Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
|
||||
timeouts: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -259,6 +259,10 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
|
||||
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
|
||||
check_put_object_args(bucket, object)?;
|
||||
|
||||
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
|
||||
}
|
||||
|
||||
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
@@ -270,6 +274,14 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// New public keys must remain representable in an unencoded XML listing.
|
||||
/// Existing keys still use the read/delete validators so they can be removed.
|
||||
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
|
||||
object
|
||||
.chars()
|
||||
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
|
||||
}
|
||||
|
||||
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
|
||||
check_bucket_and_object_names(bucket, object)
|
||||
}
|
||||
|
||||
@@ -224,6 +224,8 @@ pub struct ListPathRawOptions {
|
||||
pub recursive: bool,
|
||||
pub incl_deleted: bool,
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
pub hidden_entry_budget: usize,
|
||||
pub pending_prefix: Option<String>,
|
||||
pub filter_prefix: Option<String>,
|
||||
pub forward_to: Option<String>,
|
||||
pub min_disks: usize,
|
||||
@@ -260,6 +262,8 @@ impl Clone for ListPathRawOptions {
|
||||
recursive: self.recursive,
|
||||
incl_deleted: self.incl_deleted,
|
||||
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: self.hidden_entry_budget,
|
||||
pending_prefix: self.pending_prefix.clone(),
|
||||
filter_prefix: self.filter_prefix.clone(),
|
||||
forward_to: self.forward_to.clone(),
|
||||
min_disks: self.min_disks,
|
||||
@@ -289,6 +293,8 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
recursive: opts.recursive,
|
||||
incl_deleted: opts.incl_deleted,
|
||||
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: opts.hidden_entry_budget,
|
||||
pending_prefix: opts.pending_prefix.clone(),
|
||||
report_notfound: opts.report_not_found,
|
||||
filter_prefix: opts.filter_prefix.clone(),
|
||||
forward_to: opts.forward_to.clone(),
|
||||
@@ -301,13 +307,15 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
}
|
||||
|
||||
/// How a merged raw listing stream ended.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ListPathRawOutcome {
|
||||
/// At least one producer stopped at its per-disk limit instead of at the
|
||||
/// end of its namespace, so the closed stream is not proof of exhaustion.
|
||||
/// Carried in-band by the metacache stream trailer, so remote producers
|
||||
/// report it as well.
|
||||
pub limit_reached: bool,
|
||||
/// Inclusive sorted boundary proved by every reader before a bounded stop.
|
||||
pub scan_cut: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
|
||||
@@ -722,6 +730,8 @@ async fn list_path_raw_inner(
|
||||
// Sticky across rounds: once any reader's trailer says its producer
|
||||
// stopped at the per-disk limit, the merged stream may hide more keys.
|
||||
let mut limit_reached = false;
|
||||
let mut scan_cut = None;
|
||||
let mut last_scanned_key: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -733,7 +743,7 @@ async fn list_path_raw_inner(
|
||||
// );
|
||||
|
||||
if revjob_rx.is_cancelled() {
|
||||
return Ok(ListPathRawOutcome { limit_reached });
|
||||
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
|
||||
}
|
||||
|
||||
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
@@ -793,7 +803,12 @@ async fn list_path_raw_inner(
|
||||
continue;
|
||||
}
|
||||
// eof
|
||||
limit_reached |= r.limit_reached() == Some(true);
|
||||
if r.limit_reached() == Some(true) {
|
||||
limit_reached = true;
|
||||
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
|
||||
DiskError::other("bounded listing reader ended without a scan boundary")
|
||||
})?);
|
||||
}
|
||||
at_eof += 1;
|
||||
// warn!("list_path_raw: peek eof, disk: {}", i);
|
||||
continue;
|
||||
@@ -867,7 +882,7 @@ async fn list_path_raw_inner(
|
||||
*r = MetacacheReader::new(detached_rd);
|
||||
continue;
|
||||
}
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -926,7 +941,7 @@ async fn list_path_raw_inner(
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -976,6 +991,14 @@ async fn list_path_raw_inner(
|
||||
return Err(classify_listing_quorum_failure(&failures));
|
||||
}
|
||||
|
||||
// The next head from a bounded reader is unknown. The previous
|
||||
// globally consumed group is a safe cut; consuming another reader's
|
||||
// later head would turn this absence of evidence into a skipped key.
|
||||
// Check quorum failures above before accepting the bounded result.
|
||||
if limit_reached {
|
||||
break;
|
||||
}
|
||||
|
||||
// Break if all at EOF or error.
|
||||
if at_eof + has_err == readers.len() {
|
||||
if has_err > 0
|
||||
@@ -983,7 +1006,7 @@ async fn list_path_raw_inner(
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -994,6 +1017,7 @@ async fn list_path_raw_inner(
|
||||
break;
|
||||
}
|
||||
|
||||
last_scanned_key = Some(current.name.clone());
|
||||
if agree == readers.len() && !opts.preserve_replica_metadata {
|
||||
for r in readers.iter_mut() {
|
||||
if !r.skip_current() {
|
||||
@@ -1005,7 +1029,7 @@ async fn list_path_raw_inner(
|
||||
// warn!("list_path_raw: agreed_fn start, current: {:?}", ¤t.name);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
result = agreed_fn(current) => result?
|
||||
}
|
||||
// warn!("list_path_raw: agreed_fn done");
|
||||
@@ -1029,12 +1053,12 @@ async fn list_path_raw_inner(
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ListPathRawOutcome { limit_reached })
|
||||
Ok(ListPathRawOutcome { limit_reached, scan_cut })
|
||||
});
|
||||
|
||||
let merge_started = std::time::Instant::now();
|
||||
@@ -1199,6 +1223,24 @@ mod tests {
|
||||
assert!(!options.skip_total_timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
|
||||
let request = ListPathRawOptions {
|
||||
bucket: "versioned".to_owned(),
|
||||
hidden_entry_budget: 100,
|
||||
pending_prefix: Some("pending/".to_owned()),
|
||||
forward_to: Some("pending/0099".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
// Primary and fallback readers both construct their walk from the
|
||||
// per-job clone, rather than directly from the request options.
|
||||
let job = request.clone();
|
||||
let walk = walk_dir_options(&job);
|
||||
assert_eq!(walk.hidden_entry_budget, 100);
|
||||
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
|
||||
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_empty_disks_returns_read_quorum() {
|
||||
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
|
||||
@@ -1316,6 +1358,7 @@ mod tests {
|
||||
outcome.limit_reached,
|
||||
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
|
||||
);
|
||||
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1588,7 +1588,13 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)?;
|
||||
// A body-bound request also carries a usable v2 signature. Consume both
|
||||
// nonces so stripping the replay-scope headers cannot authorize it again.
|
||||
if content_sha256 != UNSIGNED_PAYLOAD {
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
@@ -2579,6 +2585,43 @@ mod tests {
|
||||
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_consumes_the_body_bound_v2_nonce() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "node-a:9000";
|
||||
let path = "/node_service.NodeService/RecoverDeleteTransaction";
|
||||
let digest = "a".repeat(64);
|
||||
for v3_first in [true, false] {
|
||||
let mut headers =
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
|
||||
.expect("body-bound v2 headers should build");
|
||||
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers(audience, path, ×tamp, &digest, tonic_rpc_boot_epoch())
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
let mut downgraded = headers.clone();
|
||||
for header in [
|
||||
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
RPC_BOOT_EPOCH_HEADER,
|
||||
] {
|
||||
downgraded.remove(header);
|
||||
}
|
||||
let (first, replay) = if v3_first {
|
||||
(&headers, &downgraded)
|
||||
} else {
|
||||
(&downgraded, &headers)
|
||||
};
|
||||
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
|
||||
.expect("the first delivery must be accepted");
|
||||
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
|
||||
.expect_err("a mutation cannot replay through the other signature generation");
|
||||
assert_eq!(error.to_string(), "RPC request replay detected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -2014,9 +2014,21 @@ impl PeerRestClient {
|
||||
}
|
||||
result
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
|
||||
self.finalize_result(result).await
|
||||
.await;
|
||||
self.settle_bucket_metadata_reload(bucket, result).await
|
||||
}
|
||||
|
||||
async fn settle_bucket_metadata_reload(
|
||||
&self,
|
||||
bucket: &str,
|
||||
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
|
||||
) -> Result<()> {
|
||||
match result {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
// A local operation deadline does not establish that the peer is
|
||||
// unreachable. In particular, it must not gate IAM revocations.
|
||||
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
@@ -4753,6 +4765,31 @@ mod tests {
|
||||
assert!(client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
|
||||
let client = test_peer_client();
|
||||
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("slow-policy", elapsed)
|
||||
.await
|
||||
.expect_err("local budget expires");
|
||||
assert!(error.to_string().contains("timed out"));
|
||||
assert!(
|
||||
!client.offline.load(Ordering::Acquire),
|
||||
"local work must not suppress subsequent IAM RPCs"
|
||||
);
|
||||
client
|
||||
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
|
||||
.await
|
||||
.expect("peer reply");
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
|
||||
.await
|
||||
.expect_err("real transport failure");
|
||||
assert!(error.to_string().contains("connection refused"));
|
||||
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
|
||||
let client = test_peer_client();
|
||||
|
||||
@@ -1082,7 +1082,10 @@ impl RemoteDisk {
|
||||
if payload.len() > operation.max_wire_payload_bytes() {
|
||||
return Err(Error::other("delete recovery operation exceeds payload limit"));
|
||||
}
|
||||
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
|
||||
let settle_timeout = if matches!(
|
||||
&operation,
|
||||
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
|
||||
) {
|
||||
Some(crate::disk::get_delete_recovery_timeout()?)
|
||||
} else {
|
||||
None
|
||||
@@ -2716,6 +2719,54 @@ impl RemoteDisk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for RemoteDisk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
|
||||
let expected_disk = self
|
||||
.id
|
||||
.lock()
|
||||
.await
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::DiskNotFound)?;
|
||||
let query = MultipartIndexRpcRequest::new(bucket, marker);
|
||||
let endpoint = self.endpoint.to_string();
|
||||
let canonical = query.canonical_request(&endpoint)?;
|
||||
let opts = serde_json::to_string(&query)?;
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DiskInfoRequest {
|
||||
disk: endpoint.clone(),
|
||||
opts: opts.clone(),
|
||||
});
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = client.disk_info(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// An old server ignores the additive query and returns ordinary DiskInfo.
|
||||
// It cannot satisfy the dedicated reply schema or the fresh challenge.
|
||||
let reply: MultipartIndexRpcReply =
|
||||
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
|
||||
reply.validate(&query, expected_disk)?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&query.canonical_response(&endpoint, &reply.page)?,
|
||||
&reply.proof,
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
Ok(reply.page)
|
||||
},
|
||||
get_drive_disk_info_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
|
||||
if !response.errors.is_empty() || !response.item_errors.is_empty() {
|
||||
@@ -4299,6 +4350,35 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
// Older peers reject this variant. Do not retry through the unbounded protocol.
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::DiscoverBounded {
|
||||
object: object.to_owned(),
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
Err(Error::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4334,6 +4414,58 @@ impl DiskAPI for RemoteDisk {
|
||||
_ => Err(Error::other("unexpected delete discovery reply")),
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
match self
|
||||
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
Err(Error::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::InspectBounded {
|
||||
object: object.to_owned(),
|
||||
transaction,
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
Err(Error::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4854,6 +4986,113 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("test peer must bind");
|
||||
let directory = tempfile::tempdir().expect("temporary disk");
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("local disk"),
|
||||
);
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
let prefix = directory.path().join("bucket/prefix");
|
||||
std::fs::create_dir(&prefix).expect("prefix");
|
||||
for index in 0..64 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
|
||||
}
|
||||
let fixture = Arc::new(TestBoundedDiscovery {
|
||||
disk: Arc::clone(&disk),
|
||||
response_gate: TestFormatReadGate::default(),
|
||||
});
|
||||
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
|
||||
.await
|
||||
.expect("disk completed bounded scan");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("caller cancelled").is_cancelled());
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
fixture.response_gate.release.cancel();
|
||||
peer.stop().await;
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
assert!(prefix.join("child-63").is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
remote
|
||||
.set_disk_id(Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("expected disk identity");
|
||||
assert!(matches!(
|
||||
remote.list_multipart_index_roots("bucket", None).await,
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
));
|
||||
assert_eq!(
|
||||
peer.peer.disk_info_calls.load(Ordering::Acquire),
|
||||
1,
|
||||
"exercise the actual legacy JSON handler"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
assert_eq!(
|
||||
remote
|
||||
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
|
||||
.await
|
||||
.expect_err("old peer cannot prove a bounded inventory"),
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
);
|
||||
let transaction = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
remote
|
||||
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
|
||||
.await
|
||||
.expect_err("old peer cannot inspect with a budget"),
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
);
|
||||
assert_eq!(
|
||||
remote
|
||||
.settle_delete_transaction_bounded(
|
||||
"bucket",
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "prefix".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: [0; 32],
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries: 65536,
|
||||
},
|
||||
None
|
||||
)
|
||||
.await
|
||||
.expect_err("old peer cannot settle with a budget"),
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
);
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods"),
|
||||
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -5746,9 +5985,16 @@ mod tests {
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestBoundedDiscovery {
|
||||
disk: Arc<crate::disk::local::LocalDisk>,
|
||||
response_gate: TestFormatReadGate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TestFormatReadGate {
|
||||
entered: tokio::sync::Notify,
|
||||
@@ -5765,6 +6011,7 @@ mod tests {
|
||||
format_data,
|
||||
read_all_data,
|
||||
next_format_read_gate: Arc::default(),
|
||||
bounded_discovery: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
@@ -5864,6 +6111,56 @@ mod tests {
|
||||
Ok(grpc.unary(method, request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/RecoverDeleteTransaction"
|
||||
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
|
||||
{
|
||||
#[derive(Clone)]
|
||||
struct BoundedSvc(Arc<TestBoundedDiscovery>);
|
||||
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
|
||||
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
|
||||
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
|
||||
let fixture = Arc::clone(&self.0);
|
||||
Box::pin(async move {
|
||||
let request = request.into_inner();
|
||||
let operation: DeleteRecoveryRpcOperation =
|
||||
rustfs_protos::decode_delete_recovery_payload(&request.operation)
|
||||
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
|
||||
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
|
||||
return Err(Status::invalid_argument("unexpected fallback"));
|
||||
};
|
||||
let page = fixture
|
||||
.disk
|
||||
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
|
||||
.await
|
||||
.map_err(|_| Status::internal("bounded discovery failed"))?;
|
||||
// Lose the response only after the real disk operation has completed.
|
||||
fixture.response_gate.entered.notify_one();
|
||||
fixture.response_gate.release.cancelled().await;
|
||||
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
|
||||
.map_err(|_| Status::internal("encode bounded reply"))?;
|
||||
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
|
||||
success: true,
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
result: result.into(),
|
||||
error: None,
|
||||
}))
|
||||
})
|
||||
}
|
||||
}
|
||||
let fixture = self
|
||||
.peer
|
||||
.bounded_discovery
|
||||
.lock()
|
||||
.expect("bounded fixture")
|
||||
.clone()
|
||||
.expect("configured fixture");
|
||||
Box::pin(async move {
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec);
|
||||
Ok(grpc.unary(BoundedSvc(fixture), request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/ReadAll" => {
|
||||
#[derive(Clone)]
|
||||
struct ReadAllSvc(AuthenticatedReadPeer);
|
||||
|
||||
@@ -317,8 +317,9 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
|
||||
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn stop_endpoint_monitor_for_test(&self) {
|
||||
if let Some(exit_signal) = &self.exit_signal {
|
||||
let _ = exit_signal.send(());
|
||||
// Wait for an in-flight reconnect pass as well as its receiver to exit.
|
||||
|
||||
@@ -55,6 +55,20 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
|
||||
/// Completion may mean legacy fallback after a failed scan; callers must assert
|
||||
/// the expected index result separately. Remote disks have no local worker.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
|
||||
match disk.as_ref() {
|
||||
crate::disk::Disk::Local(local) => {
|
||||
local.disk.wait_for_multipart_index_bootstrap().await;
|
||||
Ok(())
|
||||
}
|
||||
crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Disk health status constants
|
||||
const DISK_HEALTH_OK: u32 = 0;
|
||||
const DISK_HEALTH_FAULTY: u32 = 1;
|
||||
@@ -2443,6 +2457,7 @@ impl LocalDiskWrapper {
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2450,7 +2465,7 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2466,6 +2481,7 @@ impl LocalDiskWrapper {
|
||||
transaction: uuid::Uuid,
|
||||
journal: &crate::disk::DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2473,7 +2489,14 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2728,6 +2751,18 @@ impl LocalDiskWrapper {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDiskWrapper {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.list_multipart_index_roots(bucket, marker).await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
// The local implementation shares this binary's capability. Availability
|
||||
// and disk identity remain mutation checks and write-quorum inputs.
|
||||
@@ -2781,6 +2816,62 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
self.track_disk_health_with_op(
|
||||
"discover_delete_transactions",
|
||||
|| async {
|
||||
self.disk
|
||||
.discover_delete_transactions_bounded(volume, object, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
self.track_disk_health_with_op(
|
||||
"inspect_delete_transaction",
|
||||
|| async {
|
||||
self.disk
|
||||
.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
self.disk
|
||||
.settle_delete_transaction_bounded(volume, request, namespace_owner)
|
||||
.await
|
||||
},
|
||||
crate::disk::get_delete_recovery_timeout()?,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -3749,7 +3840,11 @@ mod tests {
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
|
||||
assert!(
|
||||
!logs.contents().contains("disk_operation_slow"),
|
||||
"normal disk latency should not warn: {}",
|
||||
logs.contents()
|
||||
);
|
||||
time::advance(Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
let warning = logs.contents();
|
||||
|
||||
@@ -172,6 +172,8 @@ pub enum DiskError {
|
||||
|
||||
#[error("guarded batch deletion is not supported by this disk")]
|
||||
GuardedDeleteUnsupported,
|
||||
#[error("foreground delete recovery work limit reached")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
|
||||
#[error("erasure write quorum")]
|
||||
ErasureWriteQuorum,
|
||||
@@ -770,6 +772,7 @@ impl Clone for DiskError {
|
||||
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
||||
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
||||
DiskError::ShortWrite => DiskError::ShortWrite,
|
||||
@@ -829,6 +832,7 @@ impl DiskError {
|
||||
DiskError::RemoteClientUnavailable(_) => 0x2B,
|
||||
DiskError::ConditionalDeleteConflict => 0x2C,
|
||||
DiskError::GuardedDeleteUnsupported => 0x2D,
|
||||
DiskError::DeleteRecoveryWorkLimit => 0x2E,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -879,6 +883,7 @@ impl DiskError {
|
||||
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
|
||||
0x2C => Some(DiskError::ConditionalDeleteConflict),
|
||||
0x2D => Some(DiskError::GuardedDeleteUnsupported),
|
||||
0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -1662,6 +1667,7 @@ mod tests {
|
||||
for (error, code) in [
|
||||
(DiskError::ConditionalDeleteConflict, 0x2C),
|
||||
(DiskError::GuardedDeleteUnsupported, 0x2D),
|
||||
(DiskError::DeleteRecoveryWorkLimit, 0x2E),
|
||||
] {
|
||||
assert_eq!(error.to_u32(), code);
|
||||
assert_eq!(DiskError::from_u32(code), Some(error.clone()));
|
||||
|
||||
@@ -25,6 +25,7 @@ pub(super) mod delete_recovery;
|
||||
#[cfg(all(unix, test))]
|
||||
#[path = "fd_cache_tests.rs"]
|
||||
mod fd_cache_tests;
|
||||
mod multipart_index;
|
||||
pub(in crate::disk) mod multipart_source_abort;
|
||||
mod multipart_source_recovery;
|
||||
pub(in crate::disk) mod multipart_source_state;
|
||||
@@ -185,6 +186,12 @@ const DELETE_RESIDUE_PROBE_LIMIT: i32 = 8;
|
||||
const DELETE_RESIDUE_PROBE_READ_BUDGET: usize = 32;
|
||||
|
||||
/// One directory read owed by the delete-residue probe.
|
||||
enum PrefixListingProbe {
|
||||
Visible,
|
||||
Empty,
|
||||
Cut(Vec<MetaCacheEntry>),
|
||||
}
|
||||
|
||||
enum ResidueProbeStep {
|
||||
/// Read a bounded batch of `dir` and probe what it holds.
|
||||
Sample(String),
|
||||
@@ -644,6 +651,7 @@ async fn write_delete_rollback_files(
|
||||
let object_dir_buf = object_dir.to_path_buf();
|
||||
let volume_str = volume.to_owned();
|
||||
let inspect_object = inspect_object.map(str::to_owned);
|
||||
let recovery_scope = delete_recovery::recovery_operation_scope();
|
||||
os::run_blocking_namespace_operation_with_sync(first_lease, sync, move || {
|
||||
for (path, _) in &removed_entries {
|
||||
reject_local_disk_symlink_components(&io_root, path).map_err(std::io::Error::from)?;
|
||||
@@ -702,10 +710,21 @@ async fn write_delete_rollback_files(
|
||||
}
|
||||
}
|
||||
drop(staging_dir);
|
||||
if entries.iter().any(|(name, _, _, _)| *name == DELETE_QUORUM_COMMITTED_FILE)
|
||||
&& let Some(scope) = recovery_scope.as_ref()
|
||||
{
|
||||
scope.receipt_published();
|
||||
}
|
||||
let inspected = inspect_object
|
||||
.map(|object| {
|
||||
delete_recovery::inspection_with_markers(&object_dir_buf, &volume_str, &object, rollback_dir)
|
||||
.map(|(state, _)| state)
|
||||
delete_recovery::inspection_with_markers_bounded(
|
||||
&object_dir_buf,
|
||||
&volume_str,
|
||||
&object,
|
||||
rollback_dir,
|
||||
recovery_scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
)
|
||||
.map(|(state, _)| state)
|
||||
})
|
||||
.transpose();
|
||||
drop((removed_entries, entries, inspect_lease));
|
||||
@@ -777,7 +796,10 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
let mut remove_new_metadata = false;
|
||||
match fs::read_dir(&rollback_path).await {
|
||||
Ok(mut entries) => {
|
||||
while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
|
||||
while let Some(entry) = delete_recovery::next_recovery_entry(&mut entries)
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
{
|
||||
let name = entry.file_name();
|
||||
if name == DELETE_MARKER_ROLLBACK_FILE {
|
||||
remove_new_metadata = true;
|
||||
@@ -811,7 +833,10 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
if recovery_journal.is_some() {
|
||||
let mut entries = fs::read_dir(&src).await.map_err(to_file_error)?;
|
||||
let mut files = Vec::new();
|
||||
while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
|
||||
while let Some(entry) = delete_recovery::next_recovery_entry(&mut entries)
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
{
|
||||
let name = entry.file_name();
|
||||
let name = name.to_str().ok_or(DiskError::FileCorrupt)?;
|
||||
if !entry.file_type().await.map_err(to_file_error)?.is_file()
|
||||
@@ -909,10 +934,16 @@ async fn restore_delete_rollback_with_namespace_owner(
|
||||
match restored {
|
||||
Ok(()) => {}
|
||||
Err(DiskError::FileNotFound) if remove_new_metadata => {
|
||||
match os::remove_file_with_owner(xl_path, namespace_owner.clone()).await {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
let preserves_child_prefix = recovery_journal
|
||||
.as_ref()
|
||||
.is_some_and(delete_recovery::missing_marker_preserves_child_prefix)
|
||||
&& fs::symlink_metadata(xl_path).await.is_ok_and(|metadata| metadata.is_dir());
|
||||
if !preserves_child_prefix {
|
||||
match os::remove_file_with_owner(xl_path, namespace_owner.clone()).await {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(DiskError::FileNotFound) if had_staged_paths => return Err(DiskError::FileNotFound),
|
||||
@@ -962,7 +993,10 @@ async fn remove_version_delete_markers(
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
};
|
||||
while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
|
||||
while let Some(entry) = delete_recovery::next_recovery_entry(&mut entries)
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
{
|
||||
if !entry.file_type().await.map_err(to_file_error)?.is_dir()
|
||||
|| !entry.file_name().to_str().is_some_and(|name| Uuid::parse_str(name).is_ok())
|
||||
{
|
||||
@@ -7476,6 +7510,7 @@ pub struct LocalDisk {
|
||||
pub root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
multipart_proof_authority: OnceLock<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
multipart_index: Arc<multipart_index::IndexOwner>,
|
||||
/// I/O root pinned to the mount instance that was opened while the disk
|
||||
/// was initialized. On Linux this is `/proc/self/fd/<dirfd>/.`; resolving
|
||||
/// paths beneath it keeps repair I/O on that mount even if the configured
|
||||
@@ -8018,7 +8053,10 @@ impl LocalDisk {
|
||||
|
||||
// TODO(backlog): add O_DIRECT I/O support for performance-critical paths
|
||||
// TODO(backlog): populate DiskInfo in constructor
|
||||
let multipart_index = multipart_index::IndexOwner::acquire(&publication_root, &root, &io_root)?;
|
||||
let blocking_queue = multipart_index.attach_blocking_queue(&root, &io_root);
|
||||
let mut disk = Self {
|
||||
multipart_index,
|
||||
root: root.clone(),
|
||||
publication_root,
|
||||
multipart_proof_authority: OnceLock::new(),
|
||||
@@ -8054,7 +8092,7 @@ impl LocalDisk {
|
||||
)
|
||||
.await,
|
||||
file_sync_permits: os::disk_file_sync_limiter(&root),
|
||||
blocking_queue: os::register_disk_blocking_queue(&root, &io_root),
|
||||
blocking_queue,
|
||||
snapshot_leases: Arc::new(ParkingLotMutex::new(SnapshotLeaseRegistry::default())),
|
||||
#[cfg(unix)]
|
||||
pool_meta_durable_head: ParkingLotMutex::new(None),
|
||||
@@ -8097,6 +8135,8 @@ impl LocalDisk {
|
||||
Self::check_startup_writable(&disk.publication_root, &disk.root.join(directory)).await?;
|
||||
}
|
||||
|
||||
disk.multipart_index.start();
|
||||
|
||||
let (exit_tx, exit_rx) = tokio::sync::broadcast::channel(1);
|
||||
disk.exit_signal = Some(exit_tx);
|
||||
|
||||
@@ -9909,7 +9949,10 @@ impl LocalDisk {
|
||||
let mut transaction_cleanup = false;
|
||||
match fs::read_dir(&transaction_path).await {
|
||||
Ok(mut entries) => {
|
||||
while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
|
||||
while let Some(entry) = delete_recovery::next_recovery_entry(&mut entries)
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
{
|
||||
let name = entry.file_name();
|
||||
if !entry.file_type().await.map_err(to_file_error)?.is_file()
|
||||
|| (name != STORAGE_FORMAT_FILE_BACKUP
|
||||
@@ -10125,6 +10168,7 @@ impl LocalDisk {
|
||||
let data_path = file_path.clone();
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let keep_rollback_witnesses = opt.recoverable_delete;
|
||||
let recovery_scope = delete_recovery::recovery_operation_scope();
|
||||
os::run_blocking_namespace_operation_with_sync(lease, sync, move || {
|
||||
if let Some(check) = &multipart_check {
|
||||
check.callback()()?;
|
||||
@@ -10142,7 +10186,9 @@ impl LocalDisk {
|
||||
let mut parts = Vec::new();
|
||||
let mut markers = Vec::new();
|
||||
let mut has_committed_marker = false;
|
||||
for entry in entries {
|
||||
for entry in
|
||||
delete_recovery::recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref()))
|
||||
{
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name = name.to_str().ok_or_else(corrupt)?;
|
||||
@@ -10500,7 +10546,6 @@ impl LocalDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn read_listing_metadata(&self, volume: &str, object_name: &str) -> Result<ListingMetadataRead> {
|
||||
let mut batch = self
|
||||
.spawn_listing_metadata_batch(volume, vec![object_name.to_owned()], None)
|
||||
@@ -10647,6 +10692,14 @@ impl LocalDisk {
|
||||
let rollback_dir = opts.old_data_dir;
|
||||
let incarnation = opts.delete_recovery_bucket_incarnation;
|
||||
let xl_path = object_dir.join(STORAGE_FORMAT_FILE);
|
||||
// An independent child key owns this directory. Reject before creating
|
||||
// recovery evidence that could otherwise make a harmless prefix unhealable.
|
||||
match fs::symlink_metadata(&xl_path).await {
|
||||
Ok(metadata) if metadata.is_dir() => return Err(DiskError::IsNotRegular),
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
}
|
||||
let mut recovery_target = None;
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
if opts.recoverable_delete {
|
||||
@@ -11351,6 +11404,9 @@ impl LocalDisk {
|
||||
admission: Option<multipart_staging::MultipartSourceAdmission>,
|
||||
) -> Result<()> {
|
||||
multipart_staging::reject_reserved_anchor(path)?;
|
||||
if volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_head(path, &buf).await?;
|
||||
}
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
@@ -11771,9 +11827,9 @@ impl LocalDisk {
|
||||
continue;
|
||||
}
|
||||
|
||||
let metadata =
|
||||
with_walk_stall_timeout(stall, self.read_metadata(bucket, format!("{}/{}", current, entry).as_str()))
|
||||
.await?;
|
||||
let metadata = with_walk_stall_timeout(stall, self.read_listing_metadata(bucket, ¤t))
|
||||
.await?
|
||||
.bytes;
|
||||
|
||||
let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
|
||||
let name = entry.trim_end_matches(SLASH_SEPARATOR);
|
||||
@@ -11817,19 +11873,27 @@ impl LocalDisk {
|
||||
});
|
||||
}
|
||||
|
||||
let mut dir_stack: Vec<(String, bool, Option<HashSet<String>>, bool)> = Vec::with_capacity(5);
|
||||
type PendingDirectory = (String, bool, Option<HashSet<String>>, bool, bool);
|
||||
let mut dir_stack: Vec<PendingDirectory> = Vec::with_capacity(5);
|
||||
// Explicit directory markers and real directories can resolve to the same logical path.
|
||||
let schedule_dir = |dir_stack: &mut Vec<(String, bool, Option<HashSet<String>>, bool)>,
|
||||
let schedule_dir = |dir_stack: &mut Vec<PendingDirectory>,
|
||||
dir_name: String,
|
||||
skip_object: bool,
|
||||
dir_to_skip: Option<HashSet<String>>,
|
||||
scan_required: bool| {
|
||||
if let Some((last_dir_name, existing_skip_object, existing_dir_to_skip, existing_scan_required)) =
|
||||
dir_stack.last_mut()
|
||||
scan_required: bool,
|
||||
verify_prefix: bool| {
|
||||
if let Some((
|
||||
last_dir_name,
|
||||
existing_skip_object,
|
||||
existing_dir_to_skip,
|
||||
existing_scan_required,
|
||||
existing_verify_prefix,
|
||||
)) = dir_stack.last_mut()
|
||||
&& *last_dir_name == dir_name
|
||||
{
|
||||
*existing_skip_object |= skip_object;
|
||||
*existing_scan_required |= scan_required;
|
||||
*existing_verify_prefix |= verify_prefix;
|
||||
if let Some(existing_dir_to_skip) = existing_dir_to_skip {
|
||||
if let Some(new_dir_to_skip) = &dir_to_skip {
|
||||
existing_dir_to_skip.extend(new_dir_to_skip.iter().cloned());
|
||||
@@ -11838,9 +11902,12 @@ impl LocalDisk {
|
||||
*existing_dir_to_skip = dir_to_skip;
|
||||
}
|
||||
} else {
|
||||
dir_stack.push((dir_name, skip_object, dir_to_skip, scan_required));
|
||||
dir_stack.push((dir_name, skip_object, dir_to_skip, scan_required, verify_prefix));
|
||||
}
|
||||
};
|
||||
let bounded_probe =
|
||||
!opts.recursive && !opts.incl_deleted && !opts.skip_hidden_prefix_check && opts.hidden_entry_budget > 0;
|
||||
let mut remaining_hidden = opts.hidden_entry_budget;
|
||||
prefix = "".to_owned();
|
||||
let mut prefetched_batch: Option<ListingMetadataBatch> = None;
|
||||
let mut adaptive_batch_cap = WALK_DIR_BATCH_SIZE;
|
||||
@@ -11856,7 +11923,7 @@ impl LocalDisk {
|
||||
|
||||
let name = path_join_buf(&[current.as_str(), entry.as_str()]);
|
||||
|
||||
while let Some((last_name, _, _, _)) = dir_stack.last()
|
||||
while let Some((last_name, _, _, _, _)) = dir_stack.last()
|
||||
&& *last_name < name
|
||||
{
|
||||
// A prior iteration of this same loop may have just recursed
|
||||
@@ -11870,7 +11937,20 @@ impl LocalDisk {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let (pop, skip_object, dir_to_skip, scan_required) = dir_stack.pop().expect("operation should succeed");
|
||||
let (pop, skip_object, dir_to_skip, scan_required, verify_prefix) =
|
||||
dir_stack.pop().expect("operation should succeed");
|
||||
if verify_prefix {
|
||||
match self.probe_listing_prefix(opts, &pop, &mut remaining_hidden).await? {
|
||||
PrefixListingProbe::Empty => continue,
|
||||
PrefixListingProbe::Visible => {}
|
||||
PrefixListingProbe::Cut(entries) => {
|
||||
for entry in entries {
|
||||
write_metacache_obj(out, &entry).await?;
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
let more_ready_in_batch = !(opts.recursive && scan_required)
|
||||
&& prefetched_batch.as_ref().is_some_and(|b| b.remaining > 0 && !b.rx.is_empty());
|
||||
if more_ready_in_batch {
|
||||
@@ -11978,7 +12058,7 @@ impl LocalDisk {
|
||||
};
|
||||
let batch_names = if batch_limit > 1
|
||||
&& dir_objes.is_empty()
|
||||
&& dir_stack.iter().all(|(_, _, _, scan_required)| !*scan_required)
|
||||
&& dir_stack.iter().all(|(_, _, _, scan_required, _)| !*scan_required)
|
||||
{
|
||||
entries[entry_idx..]
|
||||
.iter()
|
||||
@@ -12001,23 +12081,7 @@ impl LocalDisk {
|
||||
}
|
||||
.await
|
||||
} else {
|
||||
let fname = format!("{}/{}", meta.name, STORAGE_FORMAT_FILE);
|
||||
with_walk_stall_timeout(stall, self.read_metadata(&opts.bucket, fname.as_str()))
|
||||
.await
|
||||
.map(|metadata| {
|
||||
let bytes = Vec::from(metadata);
|
||||
let (file_meta, data_dirs, counts_toward_limit) = parse_listing_metadata_bytes(&bytes);
|
||||
#[cfg(not(test))]
|
||||
let _ = file_meta;
|
||||
ListingMetadataRead {
|
||||
bytes,
|
||||
#[cfg(test)]
|
||||
file_meta,
|
||||
data_dirs,
|
||||
has_namespace_child_candidate: true,
|
||||
counts_toward_limit,
|
||||
}
|
||||
})
|
||||
with_walk_stall_timeout(stall, self.read_listing_metadata(&opts.bucket, &meta.name)).await
|
||||
};
|
||||
|
||||
match metadata_read {
|
||||
@@ -12063,12 +12127,20 @@ impl LocalDisk {
|
||||
true,
|
||||
if dir_to_skip.is_empty() { None } else { Some(dir_to_skip) },
|
||||
has_namespace_child_candidate,
|
||||
false,
|
||||
);
|
||||
} else if !is_dir_obj
|
||||
&& has_namespace_child_candidate
|
||||
&& self
|
||||
.object_dir_has_listable_child(&opts.bucket, &meta.name, &dir_to_skip, opts.incl_deleted, stall)
|
||||
.await?
|
||||
&& (bounded_probe
|
||||
|| self
|
||||
.object_dir_has_listable_child(
|
||||
&opts.bucket,
|
||||
&meta.name,
|
||||
&dir_to_skip,
|
||||
opts.incl_deleted,
|
||||
stall,
|
||||
)
|
||||
.await?)
|
||||
{
|
||||
// A plain object `a` shares its backing directory with any
|
||||
// children `a/...`, and non-recursive walks never descend into
|
||||
@@ -12078,7 +12150,7 @@ impl LocalDisk {
|
||||
// separate real directory entry handled above.
|
||||
let mut dir_name = meta.name.clone();
|
||||
dir_name.push_str(SLASH_SEPARATOR);
|
||||
schedule_dir(&mut dir_stack, dir_name, true, None, true);
|
||||
schedule_dir(&mut dir_stack, dir_name, true, None, true, bounded_probe);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -12094,7 +12166,7 @@ impl LocalDisk {
|
||||
meta.name.push_str(SLASH_SEPARATOR);
|
||||
// Conservative listings verify physical prefixes. Never-versioned
|
||||
// buckets first rule out the data dirs a deleted version leaves behind.
|
||||
let listable = if opts.recursive || opts.incl_deleted {
|
||||
let listable = if opts.recursive || opts.incl_deleted || bounded_probe {
|
||||
true
|
||||
} else if opts.skip_hidden_prefix_check {
|
||||
!self.directory_is_delete_residue(&opts.bucket, &meta.name, stall).await?
|
||||
@@ -12103,7 +12175,7 @@ impl LocalDisk {
|
||||
.await?
|
||||
};
|
||||
if listable {
|
||||
schedule_dir(&mut dir_stack, meta.name, false, None, true);
|
||||
schedule_dir(&mut dir_stack, meta.name, false, None, true, bounded_probe);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12125,11 +12197,23 @@ impl LocalDisk {
|
||||
};
|
||||
}
|
||||
|
||||
while let Some((dir, skip_object, dir_to_skip, scan_required)) = dir_stack.pop() {
|
||||
while let Some((dir, skip_object, dir_to_skip, scan_required, verify_prefix)) = dir_stack.pop() {
|
||||
if opts.limit > 0 && *objs_returned >= opts.limit {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
if verify_prefix {
|
||||
match self.probe_listing_prefix(opts, &dir, &mut remaining_hidden).await? {
|
||||
PrefixListingProbe::Empty => continue,
|
||||
PrefixListingProbe::Visible => {}
|
||||
PrefixListingProbe::Cut(entries) => {
|
||||
for entry in entries {
|
||||
write_metacache_obj(out, &entry).await?;
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
write_metacache_obj(
|
||||
out,
|
||||
&MetaCacheEntry {
|
||||
@@ -12347,6 +12431,86 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe a prefix in the same byte order as the listing stream. Keep the
|
||||
/// ordinary one-prefix result when a live object is found; only publish
|
||||
/// hidden entries when their budget forces a resumable stream cut.
|
||||
async fn probe_listing_prefix(
|
||||
&self,
|
||||
opts: &WalkDirOptions,
|
||||
prefix: &str,
|
||||
remaining: &mut usize,
|
||||
) -> Result<PrefixListingProbe> {
|
||||
if opts.forward_to.as_ref().is_some_and(|marker| prefix <= marker.as_str())
|
||||
&& opts.pending_prefix.as_deref() != Some(prefix)
|
||||
{
|
||||
return Ok(PrefixListingProbe::Empty);
|
||||
}
|
||||
let root = prefix.trim_end_matches('/').to_owned();
|
||||
let mut pending = std::collections::BTreeMap::from([(root.clone(), root.clone())]);
|
||||
let mut hidden = Vec::new();
|
||||
let mut bytes = 0usize;
|
||||
let stall = opts.stall_timeout_duration();
|
||||
while let Some((name, path)) = pending.pop_first() {
|
||||
#[cfg(test)]
|
||||
let _previous_probe_count =
|
||||
DIRECTORY_LISTING_ENTRY_PROBE_COUNT.try_with(|count| count.fetch_add(1, Ordering::Relaxed));
|
||||
let mut data_dirs = HashSet::new();
|
||||
match with_walk_stall_timeout(stall, self.read_listing_metadata(&opts.bucket, &path)).await {
|
||||
Ok(read) => {
|
||||
data_dirs.extend(read.data_dirs.into_iter().map(|id| id.to_string()));
|
||||
// The root object's visibility does not prove that its
|
||||
// child prefix exists. Ancestor metadata still identifies
|
||||
// internal data directories that must never be traversed.
|
||||
if path != root && opts.forward_to.as_ref().is_none_or(|marker| &name > marker) {
|
||||
if read.counts_toward_limit {
|
||||
return Ok(PrefixListingProbe::Visible);
|
||||
}
|
||||
bytes = bytes.saturating_add(read.bytes.len());
|
||||
hidden.push(MetaCacheEntry {
|
||||
name: name.clone(),
|
||||
metadata: read.bytes,
|
||||
..Default::default()
|
||||
});
|
||||
*remaining = remaining.saturating_sub(1);
|
||||
if *remaining == 0 || bytes >= LISTING_META_BATCH_MAX_BYTES {
|
||||
return Ok(PrefixListingProbe::Cut(hidden));
|
||||
}
|
||||
}
|
||||
if !read.has_namespace_child_candidate {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Err(Error::FileNotFound | Error::IsNotRegular) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
let children = match with_walk_stall_timeout(stall, self.list_dir("", &opts.bucket, &path, -1)).await {
|
||||
Ok(children) => children,
|
||||
Err(Error::FileNotFound | Error::VolumeNotFound) => continue,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
for child in children {
|
||||
let Some(child) = child.strip_suffix('/') else { continue };
|
||||
if child.is_empty() || data_dirs.contains(child) {
|
||||
continue;
|
||||
}
|
||||
let path = path_join_buf(&[&path, child]);
|
||||
let name = decode_dir_object(&path);
|
||||
if name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if opts
|
||||
.forward_to
|
||||
.as_ref()
|
||||
.is_some_and(|marker| &name <= marker && !marker.starts_with(&format!("{name}/")))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
pending.insert(name, path);
|
||||
}
|
||||
}
|
||||
Ok(PrefixListingProbe::Empty)
|
||||
}
|
||||
|
||||
/// Whether anything under `dir_name` would appear in a listing. With
|
||||
/// `incl_deleted`, any `xl.meta` counts (versioned listings surface
|
||||
/// delete-marker-only objects too); otherwise the metadata must hold a
|
||||
@@ -12389,10 +12553,9 @@ impl LocalDisk {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let metadata_path = path_join_buf(&[current.as_str(), STORAGE_FORMAT_FILE]);
|
||||
match with_walk_stall_timeout(stall, self.read_metadata(bucket, metadata_path.as_str())).await {
|
||||
match with_walk_stall_timeout(stall, self.read_listing_metadata(bucket, ¤t)).await {
|
||||
Ok(metadata) => {
|
||||
let file_meta = match FileMeta::load(&metadata) {
|
||||
let file_meta = match FileMeta::load(&metadata.bytes) {
|
||||
Ok(file_meta) => file_meta,
|
||||
Err(_) => return Ok(true),
|
||||
};
|
||||
@@ -12976,6 +13139,7 @@ impl LocalDisk {
|
||||
let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}");
|
||||
let scan_object_path = object_path.clone();
|
||||
let scan_reserved_name = reserved_name.clone();
|
||||
let recovery_scope = delete_recovery::recovery_operation_scope();
|
||||
let (matching_dirs, recovery_owned) = self
|
||||
.blocking_queue
|
||||
.run_cancellable(move |is_cancelled| {
|
||||
@@ -12985,7 +13149,10 @@ impl LocalDisk {
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let mut matching = Vec::new();
|
||||
for (idx, entry) in entries.enumerate() {
|
||||
for (idx, entry) in
|
||||
delete_recovery::recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref()))
|
||||
.enumerate()
|
||||
{
|
||||
if idx & 63 == 0 && is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
@@ -13287,6 +13454,7 @@ impl LocalDisk {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let recovery_scope = delete_recovery::recovery_operation_scope();
|
||||
let matching_data_dirs = self
|
||||
.blocking_queue
|
||||
.run_cancellable(move |is_cancelled| {
|
||||
@@ -13306,7 +13474,10 @@ impl LocalDisk {
|
||||
};
|
||||
let marker_name = format!("{prefix}{rollback_dir}");
|
||||
let mut matched = Vec::new();
|
||||
for (idx, entry) in entries.enumerate() {
|
||||
for (idx, entry) in
|
||||
delete_recovery::recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref()))
|
||||
.enumerate()
|
||||
{
|
||||
if idx & 63 == 0 && is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
@@ -13571,6 +13742,14 @@ fn measure_local_batch_stage<T>(
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDisk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
self.multipart_index_page(bucket, marker).await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -13616,6 +13795,14 @@ impl DiskAPI for LocalDisk {
|
||||
async fn has_delete_transaction_evidence(&self, volume: &str, object: &str) -> Result<bool> {
|
||||
self.delete_recovery_has_evidence(volume, object).await
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<super::BoundedDeleteRecoveryDiscovery> {
|
||||
self.delete_recovery_discover_bounded(volume, object, max_entries).await
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -13625,6 +13812,24 @@ impl DiskAPI for LocalDisk {
|
||||
) -> Result<super::DeleteRecoveryPage> {
|
||||
self.delete_recovery_discover(volume, object, cursor, limit).await
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: super::BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<super::BoundedDeleteRecoverySettlementReply> {
|
||||
self.delete_recovery_settle_bounded(volume, request, namespace_owner).await
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<super::BoundedDeleteRecoveryInspection> {
|
||||
self.delete_recovery_inspect_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -15090,20 +15295,16 @@ impl DiskAPI for LocalDisk {
|
||||
if opts.base_dir.ends_with(SLASH_SEPARATOR) {
|
||||
if let Ok(data) = with_walk_stall_timeout(
|
||||
stall,
|
||||
self.read_metadata(
|
||||
self.read_listing_metadata(
|
||||
&opts.bucket,
|
||||
path_join_buf(&[
|
||||
format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX).as_str(),
|
||||
STORAGE_FORMAT_FILE,
|
||||
])
|
||||
.as_str(),
|
||||
&format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let meta = MetaCacheEntry {
|
||||
name: opts.base_dir.clone(),
|
||||
metadata: data.to_vec(),
|
||||
metadata: data.bytes,
|
||||
..Default::default()
|
||||
};
|
||||
write_metacache_obj(&mut out, &meta).await?;
|
||||
@@ -15121,15 +15322,9 @@ impl DiskAPI for LocalDisk {
|
||||
} && meta.is_file()
|
||||
{
|
||||
skip_current_dir_object = true;
|
||||
if let Ok(meta_bytes) = with_walk_stall_deadline(
|
||||
stall,
|
||||
self.read_metadata(
|
||||
opts.bucket.as_str(),
|
||||
path_join_buf(&[opts.base_dir.as_str(), STORAGE_FORMAT_FILE]).as_str(),
|
||||
),
|
||||
)
|
||||
.await?
|
||||
&& let Ok(file_meta) = FileMeta::load(&meta_bytes)
|
||||
if let Ok(meta_bytes) =
|
||||
with_walk_stall_deadline(stall, self.read_listing_metadata(&opts.bucket, &opts.base_dir)).await?
|
||||
&& let Ok(file_meta) = FileMeta::load(&meta_bytes.bytes)
|
||||
&& let Ok(data_dirs) = file_meta.get_data_dirs()
|
||||
{
|
||||
for data_dir in data_dirs.iter().flatten() {
|
||||
@@ -15486,7 +15681,10 @@ impl DiskAPI for LocalDisk {
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
|
||||
let xl_rel_path = format!("{path}/{STORAGE_FORMAT_FILE}");
|
||||
if self.multipart_source_admission(volume, &xl_rel_path, true, false)?.is_none()
|
||||
// The common publisher indexes the merged owner metadata before
|
||||
// a multipart head becomes durable, including legacy source heads.
|
||||
if volume != RUSTFS_META_MULTIPART_BUCKET
|
||||
&& self.multipart_source_admission(volume, &xl_rel_path, true, false)?.is_none()
|
||||
&& !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, &xl_rel_path)
|
||||
&& !has_delete_version_test_failpoints()
|
||||
&& !has_rename_data_test_failpoints(&xl_rel_path)
|
||||
@@ -26663,6 +26861,162 @@ mod test {
|
||||
assert!(FileMeta::load(&corrupt_read.bytes).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_prefix_probe_resumes_hidden_subtrees_and_keeps_live_prefix_fast_path() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
let directory = tempfile::tempdir().expect("disk");
|
||||
let bucket = "bounded-prefix-probe";
|
||||
let modified = OffsetDateTime::from_unix_timestamp(1_705_312_500).expect("time");
|
||||
for index in 0..600 {
|
||||
let name = format!("prefix/{index:04}");
|
||||
let mut info = if index < 250 {
|
||||
FileInfo {
|
||||
name: name.clone(),
|
||||
deleted: true,
|
||||
..Default::default()
|
||||
}
|
||||
} else {
|
||||
FileInfo::new(&name, 1, 1)
|
||||
};
|
||||
info.version_id = Some(Uuid::new_v4());
|
||||
info.mod_time = Some(modified);
|
||||
let mut metadata = FileMeta::default();
|
||||
metadata.add_version(info).expect("version");
|
||||
assert_eq!(file_meta_counts_toward_limit(&metadata), index >= 250);
|
||||
let path = directory.path().join(bucket).join(&name);
|
||||
fs::create_dir_all(&path).await.expect("object directory");
|
||||
fs::write(path.join(STORAGE_FORMAT_FILE), metadata.marshal_msg().expect("metadata"))
|
||||
.await
|
||||
.expect("object metadata");
|
||||
}
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("path")).expect("endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("disk");
|
||||
let mut marker = None;
|
||||
for page in 0..3 {
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let (reader, mut writer) = tokio::io::duplex(1024 * 1024);
|
||||
DIRECTORY_LISTING_ENTRY_PROBE_COUNT
|
||||
.scope(
|
||||
count.clone(),
|
||||
disk.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
recursive: false,
|
||||
hidden_entry_budget: 100,
|
||||
forward_to: marker.clone(),
|
||||
pending_prefix: marker.as_ref().map(|_| "prefix/".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
&mut writer,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("bounded walk");
|
||||
drop(writer);
|
||||
let mut reader = MetacacheReader::new(reader);
|
||||
let mut entries = reader.read_all().await.expect("stream");
|
||||
assert!(count.load(Ordering::Relaxed) <= 102, "one page must not read the entire hidden subtree");
|
||||
if page < 2 {
|
||||
assert_eq!(entries.len(), 100);
|
||||
assert_eq!(reader.limit_reached(), Some(true));
|
||||
assert!(entries.iter_mut().all(|entry| entry.is_latest_delete_marker()));
|
||||
marker = entries.last().map(|entry| entry.name.clone());
|
||||
assert_eq!(marker, Some(format!("prefix/{:04}", (page + 1) * 100 - 1)));
|
||||
} else {
|
||||
assert_eq!(entries.iter().map(|entry| entry.name.as_str()).collect::<Vec<_>>(), ["prefix/"]);
|
||||
assert_eq!(reader.limit_reached(), Some(false));
|
||||
assert!(
|
||||
count.load(Ordering::Relaxed) < 60,
|
||||
"a live witness must keep the whole remaining subtree out of the probe"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn listing_strips_inline_bodies_from_base_objects_and_directory_markers() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
let directory = tempfile::tempdir().expect("disk directory");
|
||||
let bucket = "inline-listing";
|
||||
let root = directory.path().join(bucket);
|
||||
let mut expected = std::collections::HashMap::new();
|
||||
for name in ["parent", "parent/child", "folder/", "folder/child"] {
|
||||
let object_dir = root.join(encode_dir_object(name));
|
||||
fs::create_dir_all(&object_dir).await.expect("object directory");
|
||||
let mut info = FileInfo::new(name, 1, 1);
|
||||
info.size = 128 * 1024;
|
||||
info.data_dir = Some(Uuid::new_v4());
|
||||
info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
info.data = Some(Bytes::from(vec![42; 128 * 1024]));
|
||||
info.set_inline_data();
|
||||
let mut meta = FileMeta::default();
|
||||
meta.add_version(info).expect("inline metadata");
|
||||
let full = meta.marshal_msg().expect("serialize metadata");
|
||||
let envelope = read_xl_meta_no_data_sync(&mut std::io::Cursor::new(&full), full.len()).expect("metadata envelope");
|
||||
assert!(full.len() > envelope.len() + 120 * 1024);
|
||||
expected.insert(name.to_string(), envelope);
|
||||
fs::write(object_dir.join(STORAGE_FORMAT_FILE), full)
|
||||
.await
|
||||
.expect("inline xl.meta");
|
||||
}
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().unwrap()).unwrap();
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
|
||||
for (base, recursive) in [
|
||||
("", true),
|
||||
("", false),
|
||||
("parent", true),
|
||||
("parent/", true),
|
||||
("folder/", true),
|
||||
] {
|
||||
let (reader, mut writer) = tokio::io::duplex(1024 * 1024);
|
||||
disk.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: bucket.into(),
|
||||
base_dir: base.into(),
|
||||
recursive,
|
||||
..Default::default()
|
||||
},
|
||||
&mut writer,
|
||||
)
|
||||
.await
|
||||
.expect("walk");
|
||||
drop(writer);
|
||||
let entries = MetacacheReader::new(reader).read_all().await.expect("listing stream");
|
||||
let objects: Vec<_> = entries.iter().filter(|entry| !entry.metadata.is_empty()).collect();
|
||||
let expected_names: &[&str] = match (base, recursive) {
|
||||
("", true) => &["folder/", "folder/child", "parent", "parent/child"],
|
||||
("", false) => &["folder/", "parent"],
|
||||
("parent", true) => &["parent/", "parent/child"],
|
||||
("parent/", true) => &["parent/child"],
|
||||
("folder/", true) => &["folder/", "folder/child"],
|
||||
_ => unreachable!(),
|
||||
};
|
||||
assert_eq!(
|
||||
objects.iter().map(|entry| entry.name.as_str()).collect::<Vec<_>>(),
|
||||
expected_names,
|
||||
"base {base}, recursive {recursive}"
|
||||
);
|
||||
for entry in objects {
|
||||
// A walk rooted at a plain object's backing directory emits its
|
||||
// own metadata with a trailing slash, unlike a root-level walk.
|
||||
let physical_name = if base == "parent" && entry.name == "parent/" {
|
||||
"parent"
|
||||
} else {
|
||||
entry.name.as_str()
|
||||
};
|
||||
assert_eq!(
|
||||
&entry.metadata,
|
||||
expected.get(physical_name).expect("known object"),
|
||||
"inline data leaked into listing of {} from base {base}, recursive {recursive}",
|
||||
entry.name
|
||||
);
|
||||
}
|
||||
}
|
||||
// Raw metadata is still required by non-listing recovery consumers.
|
||||
let raw = disk.read_metadata(bucket, "parent/xl.meta").await.expect("raw metadata");
|
||||
assert!(raw.len() > 128 * 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_listing_metadata_bytes_covers_large_envelope_versions_and_non_recursive_walk_coexistence() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
|
||||
@@ -486,6 +486,9 @@ impl LocalDisk {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_info(dst_path, &fi).await?;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
|
||||
@@ -5,6 +5,28 @@ use crate::disk::{
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub(super) struct RecoveryOperationScope {
|
||||
pub(super) budget: Arc<crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
may_publish_receipt: bool,
|
||||
receipt_published: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static RECOVERY_OPERATION: Arc<RecoveryOperationScope>;
|
||||
}
|
||||
|
||||
pub(super) fn recovery_operation_scope() -> Option<Arc<RecoveryOperationScope>> {
|
||||
RECOVERY_OPERATION.try_with(Arc::clone).ok()
|
||||
}
|
||||
|
||||
impl RecoveryOperationScope {
|
||||
pub(super) fn receipt_published(&self) {
|
||||
if self.may_publish_receipt {
|
||||
self.receipt_published.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) const PREPARED: &str = "delete-prepared";
|
||||
pub(super) const TARGET: &str = "delete-target";
|
||||
pub(super) const FINISHED: &str = "delete-commit-finished";
|
||||
@@ -21,22 +43,27 @@ struct DiscoverySession {
|
||||
|
||||
impl DiscoverySession {
|
||||
/// Consume one bounded step, returning ownership only when traversal remains.
|
||||
fn advance(mut self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
|
||||
fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
|
||||
self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
|
||||
.map(|(transactions, next, _)| (transactions, next))
|
||||
}
|
||||
|
||||
fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
|
||||
if self.frames.is_empty() {
|
||||
match std::fs::read_dir(&self.root) {
|
||||
Ok(entries) => self.frames.push((String::new(), entries)),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
return Ok((Vec::new(), None));
|
||||
return Ok((Vec::new(), None, 0));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
let mut transactions = BTreeSet::new();
|
||||
let mut seen_candidates = HashSet::new();
|
||||
let mut visited = 0usize;
|
||||
let mut visited = 0u32;
|
||||
while !self.frames.is_empty()
|
||||
&& transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
|
||||
&& visited < 4096
|
||||
&& visited < max_entries
|
||||
{
|
||||
let (parent, entries) = self
|
||||
.frames
|
||||
@@ -156,7 +183,7 @@ impl DiscoverySession {
|
||||
.map(|(object, transaction)| DeleteTransactionId { object, transaction })
|
||||
.collect();
|
||||
let next = (!self.frames.is_empty()).then_some(self);
|
||||
Ok((transactions, next))
|
||||
Ok((transactions, next, visited))
|
||||
}
|
||||
|
||||
async fn has_evidence(mut self, disk: &LocalDisk, path: &Path) -> Result<bool> {
|
||||
@@ -580,12 +607,41 @@ fn explicit_delete_referenced_data_dirs(
|
||||
Ok(Some(referenced))
|
||||
}
|
||||
|
||||
fn explicit_delete_plan(
|
||||
pub(super) async fn next_recovery_entry(entries: &mut tokio::fs::ReadDir) -> std::io::Result<Option<tokio::fs::DirEntry>> {
|
||||
if let Some(scope) = recovery_operation_scope() {
|
||||
scope.budget.reserve(1).map_err(std::io::Error::from)?;
|
||||
}
|
||||
entries.next_entry().await
|
||||
}
|
||||
|
||||
pub(super) fn recovery_entries(
|
||||
mut entries: std::fs::ReadDir,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> impl Iterator<Item = std::io::Result<std::fs::DirEntry>> + '_ {
|
||||
let mut finished = false;
|
||||
std::iter::from_fn(move || {
|
||||
if finished {
|
||||
return None;
|
||||
}
|
||||
if let Some(budget) = budget
|
||||
&& let Err(error) = budget.reserve(1)
|
||||
{
|
||||
finished = true;
|
||||
return Some(Err(error.into()));
|
||||
}
|
||||
let entry = entries.next();
|
||||
finished = entry.is_none();
|
||||
entry
|
||||
})
|
||||
}
|
||||
|
||||
fn explicit_delete_plan_bounded(
|
||||
root: &Path,
|
||||
transaction: Uuid,
|
||||
original: Option<&FileMeta>,
|
||||
journal: Option<&DeleteRecoveryJournal>,
|
||||
current_missing: bool,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Result<Option<ExplicitDeletePlan>> {
|
||||
let Some(referenced) = explicit_delete_referenced_data_dirs(original, journal, current_missing)? else {
|
||||
return Ok(None);
|
||||
@@ -602,7 +658,7 @@ fn explicit_delete_plan(
|
||||
let mut candidates = Vec::new();
|
||||
match std::fs::read_dir(&rollback) {
|
||||
Ok(entries) => {
|
||||
for entry in entries {
|
||||
for entry in recovery_entries(entries, budget) {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let kind = entry.file_type().map_err(to_file_error)?;
|
||||
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
|
||||
@@ -634,7 +690,7 @@ fn explicit_delete_plan(
|
||||
}
|
||||
match std::fs::read_dir(root) {
|
||||
Ok(entries) => {
|
||||
for entry in entries {
|
||||
for entry in recovery_entries(entries, budget) {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let name = entry.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
|
||||
let Some(id) = canonical_uuid(&name).filter(|id| !id.is_nil()) else {
|
||||
@@ -658,7 +714,7 @@ fn explicit_delete_plan(
|
||||
candidates.dedup();
|
||||
for dir in candidates {
|
||||
let mut has_parts = false;
|
||||
for entry in std::fs::read_dir(&dir).map_err(to_file_error)? {
|
||||
for entry in recovery_entries(std::fs::read_dir(&dir).map_err(to_file_error)?, budget) {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
if !entry.file_type().map_err(to_file_error)?.is_file() {
|
||||
return Ok(None);
|
||||
@@ -697,15 +753,27 @@ fn canonical_uuid(name: &str) -> Option<Uuid> {
|
||||
Uuid::parse_str(name).ok().filter(|id| name == id.hyphenated().to_string())
|
||||
}
|
||||
|
||||
pub(super) fn inspection_with_markers(
|
||||
pub(super) fn missing_marker_preserves_child_prefix(journal: &DeleteRecoveryJournal) -> bool {
|
||||
journal.original_digest.is_none()
|
||||
&& journal.original_versions.is_empty()
|
||||
&& journal.data_dirs.is_empty()
|
||||
&& journal.target_digest.is_some()
|
||||
&& !journal.requests.is_empty()
|
||||
&& journal
|
||||
.requests
|
||||
.iter()
|
||||
.all(|request| request.deleted && request.version_id.is_some_and(|id| !id.is_nil()))
|
||||
}
|
||||
|
||||
pub(super) fn inspection_with_markers_bounded(
|
||||
root: &Path,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Result<(DeleteRecoveryInspection, bool)> {
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let before = regular_metadata_bytes(&rollback.join(STORAGE_FORMAT_FILE_BACKUP))?;
|
||||
let current = regular_metadata_bytes(&root.join(STORAGE_FORMAT_FILE))?;
|
||||
let prepared = regular_bytes(&rollback.join(PREPARED))?;
|
||||
let target = regular_metadata_bytes(&rollback.join(TARGET))?;
|
||||
let receipt_bytes = regular_bytes(&rollback.join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
@@ -739,6 +807,20 @@ pub(super) fn inspection_with_markers(
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// A failed creation of a marker over a child named `xl.meta` did not
|
||||
// replace that directory. Older releases nevertheless left a journal.
|
||||
// Only this bound, originally absent generation may treat it as absence.
|
||||
let current_path = root.join(STORAGE_FORMAT_FILE);
|
||||
let current = if !committed
|
||||
&& before.is_none()
|
||||
&& (journal.as_ref().is_some_and(missing_marker_preserves_child_prefix)
|
||||
|| (journal.is_none() && target.is_none() && receipt_bytes.is_none()))
|
||||
&& std::fs::symlink_metadata(¤t_path).is_ok_and(|metadata| metadata.is_dir())
|
||||
{
|
||||
None
|
||||
} else {
|
||||
regular_metadata_bytes(¤t_path)?
|
||||
};
|
||||
let backup_digest = before.as_deref().map(digest);
|
||||
let current_digest = current.as_deref().map(digest);
|
||||
let prepared_digest = prepared.as_deref().map(digest);
|
||||
@@ -781,7 +863,7 @@ pub(super) fn inspection_with_markers(
|
||||
Err(err) => return Err(to_file_error(err).into()),
|
||||
};
|
||||
if let Some(entries) = entries {
|
||||
for entry in entries {
|
||||
for entry in recovery_entries(entries, budget) {
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let kind = entry.file_type().map_err(to_file_error)?;
|
||||
if !kind.is_file() && !kind.is_dir() {
|
||||
@@ -798,7 +880,7 @@ pub(super) fn inspection_with_markers(
|
||||
descriptions.push((name.clone(), kind.is_dir(), entry.metadata().map_err(to_file_error)?.len()));
|
||||
if kind.is_dir() && id.is_some_and(|id| !id.is_nil()) {
|
||||
has_other_uuid_dir |= id != Some(transaction);
|
||||
for child in std::fs::read_dir(entry.path()).map_err(to_file_error)? {
|
||||
for child in recovery_entries(std::fs::read_dir(entry.path()).map_err(to_file_error)?, budget) {
|
||||
let child = child.map_err(to_file_error)?;
|
||||
let kind = child.file_type().map_err(to_file_error)?;
|
||||
if !kind.is_dir() && !kind.is_file() {
|
||||
@@ -844,7 +926,7 @@ pub(super) fn inspection_with_markers(
|
||||
}
|
||||
if name == transaction.to_string() && kind.is_dir() {
|
||||
has_staged_dir = true;
|
||||
for file in std::fs::read_dir(child.path()).map_err(to_file_error)? {
|
||||
for file in recovery_entries(std::fs::read_dir(child.path()).map_err(to_file_error)?, budget) {
|
||||
let file = file.map_err(to_file_error)?;
|
||||
let kind = file.file_type().map_err(to_file_error)?;
|
||||
let file_name = file.file_name().into_string().map_err(|_| DiskError::FileCorrupt)?;
|
||||
@@ -968,7 +1050,8 @@ pub(super) fn inspection_with_markers(
|
||||
.is_some_and(|referenced| !referenced.contains(&transaction))
|
||||
}
|
||||
DeleteRecoveryState::LegacyAmbiguous | DeleteRecoveryState::NewPrepared => {
|
||||
explicit_delete_plan(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none())?.is_some()
|
||||
explicit_delete_plan_bounded(root, transaction, before_meta.as_ref(), journal.as_ref(), current.is_none(), budget)?
|
||||
.is_some()
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
@@ -1039,6 +1122,7 @@ impl LocalDisk {
|
||||
let volume = volume.to_owned();
|
||||
let object = object.to_owned();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
|
||||
let recovery_scope = recovery_operation_scope();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
Ok((|| {
|
||||
let entries = match std::fs::read_dir(&root) {
|
||||
@@ -1048,7 +1132,7 @@ impl LocalDisk {
|
||||
};
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut visited = 0usize;
|
||||
for entry in entries {
|
||||
for entry in recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())) {
|
||||
visited += 1;
|
||||
if visited > 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
@@ -1201,6 +1285,42 @@ impl LocalDisk {
|
||||
.map_err(|error| to_file_error(error).into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn public_delete_discovery_sessions_for_test(&self) -> usize {
|
||||
DISCOVERY_SESSIONS
|
||||
.lock()
|
||||
.values()
|
||||
.filter(|session| session.disk_root == self.io_root())
|
||||
.count()
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_discover_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<super::super::BoundedDeleteRecoveryDiscovery> {
|
||||
if !(1..=super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES).contains(&max_entries) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let path = self.delete_discovery_path(volume, Some(object))?;
|
||||
let session = self.new_delete_discovery_session(volume, Some(object), &path)?;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
|
||||
self.validate_delete_discovery_owner(&session)?;
|
||||
// The blocking owner drops every directory handle even if the RPC waiter disappears.
|
||||
// A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
|
||||
Ok(super::super::BoundedDeleteRecoveryDiscovery {
|
||||
transactions: if next.is_none() { transactions } else { Vec::new() },
|
||||
visited,
|
||||
complete: next.is_none(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|error| to_file_error(error).into())
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_discover(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1248,6 +1368,39 @@ impl LocalDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_inspect_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<super::super::BoundedDeleteRecoveryInspection> {
|
||||
if !safe_path(object, false)
|
||||
|| transaction.is_nil()
|
||||
|| !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&max_entries)
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let root = self.get_object_path(volume, object)?;
|
||||
let owner = os::acquire_metadata_mutation_lease(&root, None).await;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, Some(owner)).await;
|
||||
let volume = volume.to_owned();
|
||||
let object = object.to_owned();
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let budget = crate::object_api::DeleteRecoveryWorkBudget::new(max_entries);
|
||||
Ok(
|
||||
inspection_with_markers_bounded(&root, &volume, &object, transaction, Some(&budget)).map(|(inspection, _)| {
|
||||
super::super::BoundedDeleteRecoveryInspection {
|
||||
inspection,
|
||||
visited: max_entries - budget.remaining(),
|
||||
}
|
||||
}),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_inspect(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1287,9 +1440,22 @@ impl LocalDisk {
|
||||
let volume = volume.to_owned();
|
||||
let object = object.to_owned();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner).await;
|
||||
os::run_blocking_namespace_operation(lease, move || Ok(inspection_with_markers(&root, &volume, &object, transaction)))
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
let scope = recovery_operation_scope();
|
||||
// Journal inspections ride the durable-mutation lane (the fused delete
|
||||
// prepare already does): a bulk delete publishes one transaction per
|
||||
// key, and the one-worker foreground namespace lane would serialize
|
||||
// those per-drive publishes behind every other mutation.
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
Ok(inspection_with_markers_bounded(
|
||||
&root,
|
||||
&volume,
|
||||
&object,
|
||||
transaction,
|
||||
scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
))
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
|
||||
async fn activate_delete_preparation(
|
||||
@@ -1487,6 +1653,49 @@ impl LocalDisk {
|
||||
self.activate_delete_preparation(&journal, original, target, owner).await
|
||||
}
|
||||
|
||||
pub(super) async fn delete_recovery_settle_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: super::super::BoundedDeleteRecoverySettlement,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<super::super::BoundedDeleteRecoverySettlementReply> {
|
||||
if !(1..=super::super::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES).contains(&request.max_entries) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let scope = Arc::new(RecoveryOperationScope {
|
||||
budget: Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(request.max_entries)),
|
||||
may_publish_receipt: matches!(
|
||||
&request.action,
|
||||
DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. }
|
||||
),
|
||||
receipt_published: std::sync::atomic::AtomicBool::new(false),
|
||||
});
|
||||
let result = RECOVERY_OPERATION
|
||||
.scope(
|
||||
Arc::clone(&scope),
|
||||
self.delete_recovery_settle(
|
||||
volume,
|
||||
&request.object,
|
||||
request.transaction,
|
||||
request.expected_fingerprint,
|
||||
request.action,
|
||||
owner,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
let outcome = match result {
|
||||
Ok(inspection) => super::super::DeleteRecoverySettlementOutcome::Inspected(Box::new(inspection)),
|
||||
Err(DiskError::DeleteRecoveryWorkLimit) if scope.receipt_published.load(std::sync::atomic::Ordering::Acquire) => {
|
||||
super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
Ok(super::super::BoundedDeleteRecoverySettlementReply {
|
||||
outcome,
|
||||
visited: request.max_entries - scope.budget.remaining(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_recovery_settle(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1685,14 +1894,26 @@ impl LocalDisk {
|
||||
let plan_root = root.clone();
|
||||
let missing = observed.current_digest.is_none();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&root, owner.clone()).await;
|
||||
let plan = os::run_blocking_namespace_operation(lease, move || {
|
||||
let scope = recovery_operation_scope();
|
||||
// Same durable-mutation lane as the journal inspection above: this
|
||||
// plan closure runs inside every settle on the bulk-delete publish
|
||||
// path.
|
||||
let plan = os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
Ok((|| {
|
||||
let before =
|
||||
regular_metadata_bytes(&plan_root.join(transaction.to_string()).join(STORAGE_FORMAT_FILE_BACKUP))?
|
||||
.as_deref()
|
||||
.map(FileMeta::load)
|
||||
.transpose()?;
|
||||
explicit_delete_plan(&plan_root, transaction, before.as_ref(), None, missing)?.ok_or(DiskError::FileCorrupt)
|
||||
explicit_delete_plan_bounded(
|
||||
&plan_root,
|
||||
transaction,
|
||||
before.as_ref(),
|
||||
None,
|
||||
missing,
|
||||
scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
)?
|
||||
.ok_or(DiskError::FileCorrupt)
|
||||
})())
|
||||
})
|
||||
.await
|
||||
@@ -1957,6 +2178,7 @@ impl LocalDisk {
|
||||
let (rollback_buf, root_buf, volume_str, object_str) =
|
||||
(rollback.clone(), root.clone(), volume.to_owned(), object.to_owned());
|
||||
let publication_root = self.publication_root.clone();
|
||||
let scope = recovery_operation_scope();
|
||||
return os::run_blocking_namespace_operation_with_sync(receipt_lease, sync, move || {
|
||||
let _publication_root = publication_root;
|
||||
for (path, _) in &file_leases {
|
||||
@@ -1989,8 +2211,14 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
let inspected =
|
||||
inspection_with_markers(&root_buf, &volume_str, &object_str, transaction).map(|(state, _)| state);
|
||||
let inspected = inspection_with_markers_bounded(
|
||||
&root_buf,
|
||||
&volume_str,
|
||||
&object_str,
|
||||
transaction,
|
||||
scope.as_ref().map(|scope| scope.budget.as_ref()),
|
||||
)
|
||||
.map(|(state, _)| state);
|
||||
drop((file_leases, prune_dirs));
|
||||
Ok(inspected)
|
||||
})
|
||||
@@ -2147,6 +2375,7 @@ impl LocalDisk {
|
||||
let volume_str = volume.to_owned();
|
||||
let object_str = object.to_owned();
|
||||
|
||||
let recovery_scope = recovery_operation_scope();
|
||||
let res = os::run_blocking_namespace_operation_with_sync(
|
||||
lease,
|
||||
sync,
|
||||
@@ -2193,7 +2422,9 @@ impl LocalDisk {
|
||||
let mut predecessors = BTreeSet::new();
|
||||
let mut sole_transaction = true;
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
for (visited, entry) in
|
||||
recovery_entries(entries, recovery_scope.as_ref().map(|scope| scope.budget.as_ref())).enumerate()
|
||||
{
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
@@ -2406,12 +2637,25 @@ impl LocalDisk {
|
||||
Ok(Some(()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local_with_budget(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) || transaction.is_nil() {
|
||||
@@ -2490,7 +2734,7 @@ impl LocalDisk {
|
||||
if sole_transaction {
|
||||
match std::fs::read_dir(&root) {
|
||||
Ok(entries) => {
|
||||
for entry in entries {
|
||||
for entry in recovery_entries(entries, work_budget.as_deref()) {
|
||||
let entry = entry?;
|
||||
let Ok(name) = entry.file_name().into_string() else {
|
||||
sole_transaction = false;
|
||||
@@ -2538,6 +2782,7 @@ impl LocalDisk {
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2545,6 +2790,26 @@ impl LocalDisk {
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<bool> {
|
||||
self.try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
|
||||
match res {
|
||||
@@ -2611,7 +2876,7 @@ impl LocalDisk {
|
||||
had_physical_data_dirs = true;
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in entries {
|
||||
for entry in recovery_entries(entries, work_budget.as_deref()) {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else {
|
||||
@@ -4115,6 +4380,100 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_marker_collision_rejects_new_journals_and_recovers_legacy_proof() {
|
||||
let (temp, disk, _info, original) = fixture().await;
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
let staged_child = temp.path().join("child-fixture");
|
||||
fs::rename(&root, &staged_child).await.expect("move intact object");
|
||||
fs::create_dir(&root).await.expect("create parent prefix");
|
||||
fs::rename(staged_child, root.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("seed an independent object named object/xl.meta");
|
||||
let transaction = Uuid::new_v4();
|
||||
let marker = FileInfo {
|
||||
volume: "recovery".to_owned(),
|
||||
name: "object".to_owned(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
};
|
||||
disk.delete_version_recoverable(
|
||||
"recovery",
|
||||
"object",
|
||||
marker.clone(),
|
||||
true,
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("a child xl.meta directory cannot be replaced by a marker");
|
||||
assert!(!root.join(transaction.to_string()).exists(), "rejection must precede journal publication");
|
||||
|
||||
// Reproduce the durable evidence written by releases that prepared a
|
||||
// missing-object marker before attempting to replace the directory.
|
||||
disk.prepare_delete_recovery(
|
||||
"recovery",
|
||||
"object",
|
||||
None,
|
||||
&[marker],
|
||||
None,
|
||||
DeletePreparationContext {
|
||||
transaction,
|
||||
bucket_incarnation: None,
|
||||
namespace_owner: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed legacy prepared marker");
|
||||
let journal_path = root.join(transaction.to_string()).join(PREPARED);
|
||||
let journal_bytes = fs::read(&journal_path).await.expect("prepared journal");
|
||||
let mut incompatible: DeleteRecoveryJournal = serde_json::from_slice(&journal_bytes).expect("journal");
|
||||
incompatible.original_digest = Some([7; 32]);
|
||||
fs::write(&journal_path, serde_json::to_vec(&incompatible).expect("serialize journal"))
|
||||
.await
|
||||
.expect("write incompatible evidence");
|
||||
assert!(
|
||||
disk.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.is_err(),
|
||||
"a journal that claims existing metadata cannot ignore a directory collision"
|
||||
);
|
||||
fs::write(&journal_path, journal_bytes)
|
||||
.await
|
||||
.expect("restore exact legacy proof");
|
||||
let observed = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("inspect legacy proof");
|
||||
assert_eq!(observed.state, DeleteRecoveryState::NewPrepared);
|
||||
let recovered = disk
|
||||
.settle_delete_transaction("recovery", "object", transaction, observed.fingerprint, DeleteRecoveryAction::Rollback)
|
||||
.await
|
||||
.expect("rollback legacy marker without unlinking the child directory");
|
||||
assert_eq!(recovered.state, DeleteRecoveryState::Absent);
|
||||
assert!(!root.join(transaction.to_string()).exists());
|
||||
assert_eq!(
|
||||
disk.read_all("recovery", "object/xl.meta/xl.meta")
|
||||
.await
|
||||
.expect("child metadata")
|
||||
.as_ref(),
|
||||
original
|
||||
);
|
||||
assert!(
|
||||
disk.read_version("", "recovery", "object/xl.meta", "", &Default::default())
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
discover_all_keys(&disk, Some("object")).await.is_empty(),
|
||||
"no proof may keep replacement heal pending"
|
||||
);
|
||||
}
|
||||
|
||||
// Data movement copies a delete marker onto a target that holds no
|
||||
// metadata yet. Rolling back that sub-quorum write removes the marker it
|
||||
// published, and must not report the backup it never wrote as missing:
|
||||
@@ -5758,6 +6117,151 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_inspection_charges_root_and_uuid_children() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
apply(&disk, transaction).await;
|
||||
let full = disk
|
||||
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
|
||||
.await
|
||||
.expect("full bounded inspection");
|
||||
assert!(full.visited > 1, "both root and transaction directory entries are charged");
|
||||
let error = disk
|
||||
.inspect_delete_transaction_bounded("recovery", "object", transaction, full.visited - 1)
|
||||
.await
|
||||
.expect_err("nested entry exceeds allowance");
|
||||
assert_eq!(error, DiskError::DeleteRecoveryWorkLimit);
|
||||
let unchanged = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("unchanged proof");
|
||||
assert_eq!(full.inspection.fingerprint, unchanged.fingerprint);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_receipt_publication_distinguishes_deferred_inspection_from_write_failure() {
|
||||
for write_fails in [false, true] {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
apply(&disk, transaction).await;
|
||||
let full = disk
|
||||
.inspect_delete_transaction_bounded("recovery", "object", transaction, 65536)
|
||||
.await
|
||||
.expect("measure preparation work");
|
||||
let receipt = disk
|
||||
.io_get_object_path("recovery", &format!("object/{transaction}/{DELETE_QUORUM_COMMITTED_FILE}"))
|
||||
.expect("receipt path");
|
||||
if write_fails {
|
||||
let destination = receipt.clone();
|
||||
set_owned_file_before_open(&receipt, move || std::fs::create_dir(&destination).expect("reject receipt rename"));
|
||||
}
|
||||
let result = disk
|
||||
.settle_delete_transaction_bounded(
|
||||
"recovery",
|
||||
super::super::super::BoundedDeleteRecoverySettlement {
|
||||
object: "object".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: full.inspection.fingerprint,
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries: full.visited,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
if write_fails {
|
||||
assert!(result.is_err(), "a failed receipt write never counts as publication");
|
||||
assert!(receipt.is_dir());
|
||||
} else {
|
||||
let result = result.expect("persisted receipt is acknowledged independently of post-write inspection");
|
||||
assert!(matches!(
|
||||
result.outcome,
|
||||
super::super::super::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred
|
||||
));
|
||||
assert_eq!(result.visited, full.visited);
|
||||
let state = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("durable receipt");
|
||||
assert_eq!(state.state, DeleteRecoveryState::QuorumCommitted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
|
||||
fs::create_dir_all(&path).await.expect("physical prefix");
|
||||
for index in 0..64 {
|
||||
fs::create_dir(path.join(format!("child-{index}")))
|
||||
.await
|
||||
.expect("legitimate child");
|
||||
}
|
||||
for _ in 0..140 {
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "prefix", 8)
|
||||
.await
|
||||
.expect("bounded probe");
|
||||
assert_eq!(page.visited, 8);
|
||||
assert!(!page.complete, "an empty partial page cannot establish absence");
|
||||
assert!(page.transactions.is_empty());
|
||||
}
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "prefix", 128)
|
||||
.await
|
||||
.expect("complete probe");
|
||||
assert!(page.complete);
|
||||
assert_eq!(page.visited, 64);
|
||||
assert!(page.transactions.is_empty());
|
||||
let transaction = Uuid::new_v4();
|
||||
apply(&disk, transaction).await;
|
||||
let before = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("before probe");
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "object", 4096)
|
||||
.await
|
||||
.expect("complete evidence");
|
||||
assert!(page.complete);
|
||||
assert!(page.transactions.iter().any(|id| id.transaction == transaction));
|
||||
let after = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("after probe");
|
||||
assert_eq!(before.fingerprint, after.fingerprint);
|
||||
assert!(
|
||||
DISCOVERY_SESSIONS
|
||||
.lock()
|
||||
.values()
|
||||
.all(|session| session.disk_root != disk.io_root())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_cancelled_bounded_discovery_drops_private_session() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("object")).expect("probe scope");
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
|
||||
let mut probe = Box::pin(disk.discover_delete_transactions_bounded("recovery", "object", 4096));
|
||||
assert!(futures::poll!(probe.as_mut()).is_pending());
|
||||
drop(probe);
|
||||
assert!(
|
||||
DISCOVERY_SESSIONS
|
||||
.lock()
|
||||
.values()
|
||||
.all(|session| session.disk_root != disk.io_root())
|
||||
);
|
||||
drop(lease);
|
||||
assert!(
|
||||
disk.discover_delete_transactions_bounded("recovery", "object", 4096)
|
||||
.await
|
||||
.expect("subsequent probe")
|
||||
.complete
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_evidence_probes_do_not_register_public_cursors() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -117,6 +117,7 @@ pub(super) struct LoadedSource {
|
||||
}
|
||||
|
||||
pub(super) struct SourceStore {
|
||||
multipart_index: Arc<super::multipart_index::IndexOwner>,
|
||||
pub disk_id: Uuid,
|
||||
pub physical_slot: usize,
|
||||
pub disk_ids: Vec<Uuid>,
|
||||
@@ -186,6 +187,7 @@ impl LocalDisk {
|
||||
let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
|
||||
Ok(SourceStore {
|
||||
multipart_index: self.multipart_index.clone(),
|
||||
disk_id,
|
||||
physical_slot,
|
||||
disk_ids,
|
||||
@@ -755,6 +757,9 @@ impl SourceStore {
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
self.multipart_index
|
||||
.record_head(&format!("{}/xl.meta", self.path), bytes)
|
||||
.await?;
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
let path = self.source.join(STORAGE_FORMAT_FILE);
|
||||
let base = self.base.clone();
|
||||
|
||||
@@ -27,6 +27,7 @@ mod io_admission;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle_test_io;
|
||||
pub mod local;
|
||||
pub mod multipart_index;
|
||||
pub mod multipart_source;
|
||||
pub mod os;
|
||||
|
||||
@@ -414,10 +415,11 @@ impl Disk {
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(None),
|
||||
@@ -431,11 +433,19 @@ impl Disk {
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.await
|
||||
disk.try_finish_and_finalize_prepared_delete_local(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(false),
|
||||
}
|
||||
@@ -512,6 +522,17 @@ impl Disk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for Disk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.supports_guarded_delete_versions().await,
|
||||
@@ -555,6 +576,17 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -567,6 +599,35 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1513,6 +1574,15 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
fn host_name(&self) -> String;
|
||||
fn endpoint(&self) -> Endpoint;
|
||||
async fn close(&self) -> Result<()>;
|
||||
/// Return candidates only while this disk maintains complete boot-local coverage.
|
||||
/// Unsupported or unready disks must never return an authoritative empty page.
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
|
||||
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
|
||||
|
||||
@@ -1690,6 +1760,33 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
) -> Result<DeleteRecoveryPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
/// Bounded foreground inventory owns all traversal state until return or cancellation.
|
||||
/// Unsupported peers must fail closed; callers must never fall back to paginated discovery.
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_transaction: Uuid,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_request: BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
/// A read-only capability check; unsupported implementations must fail closed.
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
Err(DiskError::GuardedDeleteUnsupported)
|
||||
@@ -2075,6 +2172,13 @@ pub struct WalkDirOptions {
|
||||
#[serde(default)]
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
|
||||
/// Bound hidden metadata read while proving a non-recursive prefix visible.
|
||||
#[serde(default)]
|
||||
pub hidden_entry_budget: usize,
|
||||
/// A prefix whose visibility was not established before the preceding cut.
|
||||
#[serde(default)]
|
||||
pub pending_prefix: Option<String>,
|
||||
|
||||
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
|
||||
pub report_notfound: bool,
|
||||
|
||||
@@ -2407,9 +2511,62 @@ pub enum DeleteRecoveryAction {
|
||||
AdoptCommit { authority: Box<DeleteRecoveryJournal> },
|
||||
}
|
||||
|
||||
/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryDiscovery {
|
||||
pub transactions: Vec<DeleteTransactionId>,
|
||||
pub visited: u32,
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
|
||||
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlement {
|
||||
pub object: String,
|
||||
pub transaction: Uuid,
|
||||
pub expected_fingerprint: DeleteRecoveryDigest,
|
||||
pub action: DeleteRecoveryAction,
|
||||
pub max_entries: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoverySettlementOutcome {
|
||||
Inspected(Box<DeleteRecoveryInspection>),
|
||||
ReceiptPublishedInspectionDeferred,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlementReply {
|
||||
pub outcome: DeleteRecoverySettlementOutcome,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryInspection {
|
||||
pub inspection: DeleteRecoveryInspection,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoveryRpcOperation {
|
||||
SettleBounded(BoundedDeleteRecoverySettlement),
|
||||
InspectBounded {
|
||||
object: String,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
},
|
||||
DiscoverBounded {
|
||||
object: String,
|
||||
max_entries: u32,
|
||||
},
|
||||
InspectOrphanDataDir {
|
||||
path: String,
|
||||
},
|
||||
@@ -2447,7 +2604,11 @@ impl DeleteRecoveryRpcOperation {
|
||||
/// Full metadata application and adoption share the existing mutation envelope.
|
||||
pub fn max_wire_payload_bytes(&self) -> usize {
|
||||
match self {
|
||||
Self::ApplyVersion { .. }
|
||||
Self::SettleBounded(BoundedDeleteRecoverySettlement {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
..
|
||||
})
|
||||
| Self::ApplyVersion { .. }
|
||||
| Self::ApplyVersions { .. }
|
||||
| Self::Settle {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
@@ -2463,8 +2624,11 @@ impl DeleteRecoveryRpcOperation {
|
||||
pub enum DeleteRecoveryRpcReply {
|
||||
OrphanDataDir { plain: bool },
|
||||
Discovered(DeleteRecoveryPage),
|
||||
DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
|
||||
Inspected(DeleteRecoveryInspection),
|
||||
InspectedBounded(BoundedDeleteRecoveryInspection),
|
||||
Settled(DeleteRecoveryInspection),
|
||||
SettledBounded(BoundedDeleteRecoverySettlementReply),
|
||||
AppliedVersion,
|
||||
AppliedVersions(Vec<Option<u32>>),
|
||||
Evidence { present: bool },
|
||||
@@ -2942,6 +3106,8 @@ mod tests {
|
||||
recursive: true,
|
||||
incl_deleted: false,
|
||||
skip_hidden_prefix_check: false,
|
||||
hidden_entry_budget: 0,
|
||||
pending_prefix: None,
|
||||
report_notfound: false,
|
||||
filter_prefix: Some("prefix_".to_string()),
|
||||
forward_to: Some("object/path".to_string()),
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! A candidate index never replaces multipart metadata or quorum validation.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||
|
||||
pub const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
|
||||
|
||||
/// A bounded page from one disk's continuously maintained, boot-local index.
|
||||
/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
|
||||
/// pages from different epochs or physical disk identities.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexPage {
|
||||
pub bucket: String,
|
||||
pub disk_id: Uuid,
|
||||
pub epoch: Uuid,
|
||||
pub roots: Vec<String>,
|
||||
pub next_marker: Option<String>,
|
||||
}
|
||||
|
||||
pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
|
||||
pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
|
||||
|
||||
/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
|
||||
/// Old peers may ignore this field, but their ordinary response cannot decode
|
||||
/// as the required signed, challenge-bound MultipartIndexRpcReply.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcRequest {
|
||||
pub disk_id: String,
|
||||
pub metrics: bool,
|
||||
pub noop: bool,
|
||||
pub multipart_index: MultipartIndexQuery,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexQuery {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub bucket: String,
|
||||
pub marker: Option<String>,
|
||||
pub page_cap: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcReply {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub page: MultipartIndexPage,
|
||||
pub proof: Vec<u8>,
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcRequest {
|
||||
pub fn new(bucket: &str, marker: Option<&str>) -> Self {
|
||||
Self {
|
||||
disk_id: String::new(),
|
||||
metrics: false,
|
||||
noop: true,
|
||||
multipart_index: MultipartIndexQuery {
|
||||
operation: "list_multipart_index_roots".into(),
|
||||
version: 1,
|
||||
challenge: Uuid::new_v4(),
|
||||
bucket: bucket.to_owned(),
|
||||
marker: marker.map(str::to_owned),
|
||||
page_cap: MULTIPART_INDEX_PAGE_SIZE,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> crate::disk::error::Result<()> {
|
||||
let query = &self.multipart_index;
|
||||
if !self.disk_id.is_empty()
|
||||
|| self.metrics
|
||||
|| !self.noop
|
||||
|| query.operation != "list_multipart_index_roots"
|
||||
|| query.version != 1
|
||||
|| query.challenge.is_nil()
|
||||
|| query.bucket.is_empty()
|
||||
|| query.bucket.len() > 255
|
||||
|| query.bucket.contains('/')
|
||||
|| matches!(query.bucket.as_str(), "." | "..")
|
||||
|| query.page_cap != MULTIPART_INDEX_PAGE_SIZE
|
||||
|| query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::MethodNotAllowed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
|
||||
}
|
||||
|
||||
pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_root(root: &str) -> bool {
|
||||
root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
impl MultipartIndexPage {
|
||||
pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
if expected_disk.is_nil()
|
||||
|| self.disk_id != expected_disk
|
||||
|| self.epoch.is_nil()
|
||||
|| self.bucket != bucket
|
||||
|| self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
|
||||
|| self
|
||||
.roots
|
||||
.iter()
|
||||
.any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
|
||||
|| self.roots.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
|| self
|
||||
.next_marker
|
||||
.as_ref()
|
||||
.is_some_and(|marker| self.roots.last() != Some(marker) || self.roots.len() != MULTIPART_INDEX_PAGE_SIZE)
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcReply {
|
||||
/// This is structural validation only; the caller must also verify `proof`.
|
||||
pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
let query = &request.multipart_index;
|
||||
request.validate()?;
|
||||
if self.operation != query.operation
|
||||
|| self.version != query.version
|
||||
|| self.challenge != query.challenge
|
||||
|| self.proof.len() != 32
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
|
||||
}
|
||||
}
|
||||
|
||||
/// A disk can attest coverage only after every page belongs to the same physical
|
||||
/// identity and index generation. An interrupted/changed stream contributes no
|
||||
/// partial candidates to the erasure set's coverage count.
|
||||
pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
|
||||
where
|
||||
F: FnMut(Option<String>) -> Fut,
|
||||
Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
|
||||
{
|
||||
let mut epoch = None;
|
||||
let mut marker = None;
|
||||
let mut roots = Vec::new();
|
||||
loop {
|
||||
let page = fetch(marker.clone()).await?;
|
||||
page.validate(bucket, marker.as_deref(), identity)?;
|
||||
if epoch.is_some_and(|epoch| epoch != page.epoch) {
|
||||
return Err(crate::disk::error::DiskError::DiskNotFound);
|
||||
}
|
||||
epoch = Some(page.epoch);
|
||||
roots.extend(page.roots);
|
||||
marker = page.next_marker;
|
||||
if marker.is_none() {
|
||||
return Ok(roots);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
|
||||
let request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let id = Uuid::new_v4();
|
||||
let mut reply = MultipartIndexRpcReply {
|
||||
operation: request.multipart_index.operation.clone(),
|
||||
version: 1,
|
||||
challenge: request.multipart_index.challenge,
|
||||
page: MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec!["a".repeat(64)],
|
||||
next_marker: None,
|
||||
},
|
||||
proof: vec![0; 32],
|
||||
};
|
||||
reply.validate(&request, id).expect("valid structural envelope");
|
||||
assert!(
|
||||
reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
|
||||
"old challenge cannot replay"
|
||||
);
|
||||
assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
|
||||
reply.page.next_marker = Some("a".repeat(64));
|
||||
assert!(reply.validate(&request, id).is_err(), "short page cannot claim a continuation");
|
||||
reply.page.next_marker = None;
|
||||
reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
|
||||
assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_index_response_proof_binds_the_complete_query_and_page() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let mut request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let mut page = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: Uuid::new_v4(),
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let canonical = request.canonical_response("disk", &page).expect("canonical response");
|
||||
let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
|
||||
let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&request.canonical_response("disk", page).expect("canonical"),
|
||||
&proof,
|
||||
)
|
||||
};
|
||||
verify(&request, &page).expect("exact response");
|
||||
request.multipart_index.marker = Some("0".repeat(64));
|
||||
assert!(verify(&request, &page).is_err(), "cursor is signed");
|
||||
request.multipart_index.marker = None;
|
||||
page.epoch = Uuid::new_v4();
|
||||
assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
|
||||
let id = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
for replace_disk in [false, true] {
|
||||
let first = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch,
|
||||
roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
|
||||
next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
|
||||
};
|
||||
let second = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: if replace_disk { Uuid::new_v4() } else { id },
|
||||
epoch: if replace_disk { epoch } else { Uuid::new_v4() },
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let mut replies = std::collections::VecDeque::from([first, second]);
|
||||
let result = collect_pages("bucket", id, |marker| {
|
||||
let page = replies.pop_front().expect("bounded fixture pages");
|
||||
if marker.is_some() {
|
||||
assert!(page.roots.is_empty());
|
||||
}
|
||||
std::future::ready(Ok(page))
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "changed physical source cannot contribute partial coverage");
|
||||
assert!(replies.is_empty(), "exercise the continuation response");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2731,6 +2731,11 @@ pub(crate) struct NamespaceMutationLease {
|
||||
}
|
||||
|
||||
impl NamespaceMutationLease {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn blocking_queue_for_test(&self) -> Arc<DiskBlockingQueue> {
|
||||
self.blocking_queue.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn attach_external_guard(&self, guard: Arc<dyn Send + Sync>) {
|
||||
*self.external_guard.lock() = Some(guard);
|
||||
}
|
||||
@@ -5382,6 +5387,29 @@ impl PublicationRoot {
|
||||
})
|
||||
}
|
||||
|
||||
/// Distinguish reconnects from replacement mounts before reusing disk-local state.
|
||||
pub(crate) fn physical_identity(&self) -> io::Result<(u64, [u8; 16])> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
#[cfg(target_os = "linux")]
|
||||
let metadata = self.directory.metadata()?;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let metadata = std::fs::metadata(&*self.path)?;
|
||||
let mut inode = [0; 16];
|
||||
inode[..8].copy_from_slice(&metadata.ino().to_le_bytes());
|
||||
Ok((metadata.dev(), inode))
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
windows_file_identity(&self.directory.handle)
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
Err(io::Error::new(io::ErrorKind::Unsupported, "physical root identity unavailable"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
@@ -269,6 +269,11 @@ pub enum StorageError {
|
||||
/// retained for operator inspection.
|
||||
#[error("delete recovery found an ambiguous legacy residue; operator inspection is required")]
|
||||
DeleteRecoveryLegacyAmbiguous,
|
||||
/// Delete recovery exhausted the per-request work budget before it could
|
||||
/// classify every residue. Retrying later makes progress, so callers treat
|
||||
/// it like `SlowDown`; it stays distinct so the disk error round-trips.
|
||||
#[error("delete recovery work limit reached; please retry")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
#[error("Decommission not started")]
|
||||
DecommissionNotStarted,
|
||||
#[error("Decommission already running")]
|
||||
@@ -480,6 +485,7 @@ impl From<DiskError> for StorageError {
|
||||
DiskError::InvalidPath => StorageError::InvalidPath,
|
||||
DiskError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
|
||||
DiskError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail),
|
||||
}
|
||||
}
|
||||
@@ -507,6 +513,7 @@ impl StorageError {
|
||||
StorageError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
StorageError::StorageFull => DiskError::DiskFull,
|
||||
StorageError::SlowDown => DiskError::TooManyOpenFiles,
|
||||
StorageError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
StorageError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
|
||||
StorageError::InconsistentDisk => DiskError::InconsistentDisk,
|
||||
StorageError::UnsupportedDisk => DiskError::UnsupportedDisk,
|
||||
@@ -710,6 +717,7 @@ impl Clone for StorageError {
|
||||
StorageError::StorageFull => StorageError::StorageFull,
|
||||
StorageError::SlowDown => StorageError::SlowDown,
|
||||
StorageError::DeleteRecoveryLegacyAmbiguous => StorageError::DeleteRecoveryLegacyAmbiguous,
|
||||
StorageError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
|
||||
StorageError::PrefixAccessDenied(a, b) => StorageError::PrefixAccessDenied(a.clone(), b.clone()),
|
||||
StorageError::InvalidUploadIDKeyCombination(a, b) => {
|
||||
StorageError::InvalidUploadIDKeyCombination(a.clone(), b.clone())
|
||||
@@ -852,6 +860,7 @@ impl StorageError {
|
||||
StorageError::StorageFull => StorageErrorCode::StorageFull,
|
||||
StorageError::SlowDown => StorageErrorCode::SlowDown,
|
||||
StorageError::DeleteRecoveryLegacyAmbiguous => StorageErrorCode::DeleteRecoveryLegacyAmbiguous,
|
||||
StorageError::DeleteRecoveryWorkLimit => StorageErrorCode::DeleteRecoveryWorkLimit,
|
||||
StorageError::PrefixAccessDenied(_, _) => StorageErrorCode::PrefixAccessDenied,
|
||||
StorageError::InvalidUploadIDKeyCombination(_, _) => StorageErrorCode::InvalidUploadIDKeyCombination,
|
||||
StorageError::MalformedUploadID(_) => StorageErrorCode::MalformedUploadID,
|
||||
@@ -958,6 +967,7 @@ impl StorageError {
|
||||
StorageErrorCode::StorageFull => Some(StorageError::StorageFull),
|
||||
StorageErrorCode::SlowDown => Some(StorageError::SlowDown),
|
||||
StorageErrorCode::DeleteRecoveryLegacyAmbiguous => Some(StorageError::DeleteRecoveryLegacyAmbiguous),
|
||||
StorageErrorCode::DeleteRecoveryWorkLimit => Some(StorageError::DeleteRecoveryWorkLimit),
|
||||
StorageErrorCode::PrefixAccessDenied => {
|
||||
Some(StorageError::PrefixAccessDenied(Default::default(), Default::default()))
|
||||
}
|
||||
|
||||
@@ -79,6 +79,14 @@ pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(Compres
|
||||
Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
|
||||
}
|
||||
|
||||
/// Validate that replicated compression metadata can be read by this build.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error for an unsupported compression scheme.
|
||||
pub fn validate_compression_scheme(scheme: &str) -> std::io::Result<()> {
|
||||
compression_scheme_to_read_plan(scheme).map(|_| ())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReadEncryptionBackend {
|
||||
Legacy,
|
||||
@@ -444,10 +452,12 @@ impl WritePlan {
|
||||
|
||||
if let Some(encryption) = self.encryption {
|
||||
let etag_nonce = match encryption.mode {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
|
||||
base_nonce
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
WriteEncryptionMode::Singlepart { base_nonce } => base_nonce,
|
||||
WriteEncryptionMode::SinglepartObjectKey => [0u8; 12],
|
||||
// Each part write needs an independent fingerprint, including a
|
||||
// retry at the same part number. This salt affects only the ETag
|
||||
// MAC; encryption framing and its data nonce stay unchanged.
|
||||
WriteEncryptionMode::MultipartLegacy { .. } | WriteEncryptionMode::MultipartObjectKey { .. } => rand::random(),
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
|
||||
.with_inner_etag_delegation(delegate_inner_etag);
|
||||
@@ -558,6 +568,54 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_etags_are_unique_per_write_without_changing_data_encryption() {
|
||||
let plaintext = b"same confidential part";
|
||||
let size = i64::try_from(plaintext.len()).expect("test length");
|
||||
for object_key_mode in [false, true] {
|
||||
let mut tags = std::collections::HashSet::new();
|
||||
for part_number in [1, 2, 1] {
|
||||
let encryption = if object_key_mode {
|
||||
WriteEncryption::multipart_object_key([7; 32], part_number)
|
||||
} else {
|
||||
WriteEncryption::multipart([7; 32], [9; 12], part_number as usize)
|
||||
};
|
||||
let source = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext");
|
||||
let mut reader = WritePlan::new()
|
||||
.with_encryption(encryption)
|
||||
.apply(source, size)
|
||||
.expect("part plan");
|
||||
let mut ciphertext = Vec::new();
|
||||
reader.read_to_end(&mut ciphertext).await.expect("encrypt part");
|
||||
let tag = reader.try_resolve_etag().expect("part fingerprint");
|
||||
assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag), "reading a stored result stays stable");
|
||||
assert!(
|
||||
tags.insert(tag),
|
||||
"different part numbers and retries must not identify matching plaintext"
|
||||
);
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let mut decrypt = if object_key_mode {
|
||||
DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), [7; 32], vec![part_number as usize])
|
||||
} else {
|
||||
DecryptReader::new_multipart(Cursor::new(ciphertext), [7; 32], [9; 12], vec![part_number as usize])
|
||||
};
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let mut decrypt = DecryptReader::new_multipart(
|
||||
Cursor::new(ciphertext),
|
||||
[7; 32],
|
||||
if object_key_mode { [0; 12] } else { [9; 12] },
|
||||
vec![part_number as usize],
|
||||
);
|
||||
let mut decoded = Vec::new();
|
||||
decrypt
|
||||
.read_to_end(&mut decoded)
|
||||
.await
|
||||
.expect("existing data nonce still decrypts");
|
||||
assert_eq!(decoded, plaintext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
|
||||
let plaintext = b"checksum-context-through-write-plan".repeat(256);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
|
||||
@@ -949,8 +949,65 @@ pub enum ShardIntegrityWriteMode {
|
||||
Protected,
|
||||
}
|
||||
|
||||
/// Shared foreground recovery allowance. Clones of ObjectOptions retain one request budget.
|
||||
/// Credits cover directory entries (including a minimum charge for an empty probe) and
|
||||
/// transaction inspections. Failed/cancelled probes keep their worst-case reservation.
|
||||
#[derive(Debug)]
|
||||
pub struct DeleteRecoveryWorkBudget {
|
||||
remaining: std::sync::atomic::AtomicU32,
|
||||
}
|
||||
|
||||
/// Entries one disk may reserve up front per recovery step (discovery plus
|
||||
/// inspection). Every disk of every set a request fans out to reserves this
|
||||
/// much before any refund arrives, so a budget must cover that many disks.
|
||||
const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
|
||||
|
||||
impl Default for DeleteRecoveryWorkBudget {
|
||||
/// Covers a full 16-drive set with headroom; `for_disks` scales it when a
|
||||
/// request spans more drives than that.
|
||||
fn default() -> Self {
|
||||
Self::new(16 * 1024 * 1024)
|
||||
}
|
||||
}
|
||||
|
||||
impl DeleteRecoveryWorkBudget {
|
||||
/// A per-request budget sized for `disks` concurrent up-front reservations.
|
||||
pub(crate) fn for_disks(disks: usize) -> Self {
|
||||
let per_disk = u64::from(DELETE_RECOVERY_RESERVATION_PER_DISK) * 2;
|
||||
let scaled = u64::try_from(disks).unwrap_or(u64::MAX).saturating_mul(per_disk);
|
||||
let default = u64::from(Self::default().remaining());
|
||||
Self::new(u32::try_from(scaled.max(default)).unwrap_or(u32::MAX))
|
||||
}
|
||||
|
||||
pub(crate) fn new(entries: u32) -> Self {
|
||||
Self {
|
||||
remaining: std::sync::atomic::AtomicU32::new(entries),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reserve(&self, maximum: u32) -> crate::disk::error::Result<u32> {
|
||||
use std::sync::atomic::Ordering;
|
||||
self.remaining
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
|
||||
(remaining > 0).then(|| remaining.saturating_sub(maximum))
|
||||
})
|
||||
.map(|remaining| remaining.min(maximum))
|
||||
.map_err(|_| crate::disk::error::DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
|
||||
pub(crate) fn remaining(&self) -> u32 {
|
||||
self.remaining.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn refund(&self, unused: u32) {
|
||||
self.remaining.fetch_add(unused, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
#[doc(hidden)]
|
||||
pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
pub max_parity: bool,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
@@ -1102,6 +1159,12 @@ pub struct ObjectOptions {
|
||||
pub expiration: ExpirationOptions,
|
||||
pub lifecycle_audit_event: LcAuditEvent,
|
||||
|
||||
/// In-process request gate evaluated on authoritative completion metadata.
|
||||
/// Not persisted or sent over RPC. The trait deliberately has no Debug bound:
|
||||
/// implementations may own customer encryption keys.
|
||||
#[doc(hidden)]
|
||||
pub multipart_completion_authorization:
|
||||
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
|
||||
pub eval_metadata: Option<HashMap<String, String>>,
|
||||
/// Internal compare-and-set condition for replication workers publishing
|
||||
/// terminal status after remote I/O. Storage validates it while holding
|
||||
@@ -1115,6 +1178,10 @@ pub struct ObjectOptions {
|
||||
/// Authoritative bucket Object Lock snapshot installed inside `ECStore`
|
||||
/// before a destructive commit reaches the set layer.
|
||||
pub object_lock_config_snapshot: Option<Arc<ObjectLockConfigSnapshot>>,
|
||||
/// ECStore single PUTs acquire their destructive-write configuration guards
|
||||
/// only after staging the client body. External lock owners supply a snapshot.
|
||||
#[doc(hidden)]
|
||||
pub defer_put_object_lock_snapshot: bool,
|
||||
|
||||
pub want_checksum: Option<Checksum>,
|
||||
pub skip_verify_bitrot: bool,
|
||||
@@ -1146,6 +1213,11 @@ pub enum ReplicationStatusWritebackMode {
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
pub(crate) fn ensure_delete_recovery_work_budget(&mut self) {
|
||||
self.delete_recovery_work_budget
|
||||
.get_or_insert_with(|| Arc::new(DeleteRecoveryWorkBudget::default()));
|
||||
}
|
||||
|
||||
pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
|
||||
match self.shard_integrity_write_mode {
|
||||
Some(ShardIntegrityWriteMode::Protected) => true,
|
||||
|
||||
@@ -786,7 +786,7 @@ impl WarmBackend for MockWarmBackend {
|
||||
bytes.len()
|
||||
};
|
||||
|
||||
Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
|
||||
Ok(ReadCloser::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
|
||||
@@ -37,7 +37,6 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::{
|
||||
select,
|
||||
sync::{Notify, RwLock},
|
||||
@@ -8183,7 +8182,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
|
||||
@@ -8248,7 +8247,7 @@ mod tests {
|
||||
_opts: WarmBackendGetOpts,
|
||||
) -> std::result::Result<ReadCloser, std::io::Error> {
|
||||
if self.healthy {
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
} else {
|
||||
Err(std::io::Error::other("mock get failed"))
|
||||
}
|
||||
@@ -9895,7 +9894,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
|
||||
|
||||
@@ -195,7 +195,14 @@ impl SetDisks {
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
return Ok(None);
|
||||
// With no decodable version, the per-version quorum loop cannot
|
||||
// reject corrupt or unavailable replicas. Only complete, readable
|
||||
// evidence may establish that this key has no authoritative version.
|
||||
return if complete {
|
||||
Ok(None)
|
||||
} else {
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
};
|
||||
}
|
||||
Ok(Some(MetaCacheEntry {
|
||||
name,
|
||||
|
||||
@@ -357,8 +357,7 @@ use crate::io_support::rio::HashReader;
|
||||
|
||||
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
|
||||
pub const MAX_PARTS_COUNT: usize = 10000;
|
||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||
pub(crate) use crate::disk::multipart_index::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement";
|
||||
const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
|
||||
|
||||
|
||||
@@ -141,6 +141,7 @@ pub(crate) struct DeleteRecoverySummary {
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DeleteRecoveryContext<'a> {
|
||||
pub incarnation: Option<Uuid>,
|
||||
pub work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
pub dry_run: bool,
|
||||
pub explicit_delete: bool,
|
||||
/// Only the locked heal path may repair a live marker-creation source.
|
||||
@@ -156,6 +157,7 @@ impl<'a> DeleteRecoveryContext<'a> {
|
||||
pub(crate) fn new(incarnation: Option<Uuid>, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self {
|
||||
Self {
|
||||
incarnation,
|
||||
work_budget: None,
|
||||
dry_run: false,
|
||||
explicit_delete: false,
|
||||
repair_marker_source: false,
|
||||
@@ -230,19 +232,54 @@ async fn inspect(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Vec<Result<DeleteRecoveryInspection>> {
|
||||
join_all(disks.iter().map(|disk| async move {
|
||||
let state = disk
|
||||
.as_ref()
|
||||
.ok_or(DiskError::DiskNotFound)?
|
||||
.inspect_delete_transaction(bucket, object, transaction)
|
||||
.await?;
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
let state = if let Some(budget) = budget {
|
||||
let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
|
||||
let reply = disk
|
||||
.inspect_delete_transaction_bounded(bucket, object, transaction, reserved)
|
||||
.await?;
|
||||
if reply.visited > reserved {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.refund(reserved - reply.visited.max(1));
|
||||
reply.inspection
|
||||
} else {
|
||||
disk.inspect_delete_transaction(bucket, object, transaction).await?
|
||||
};
|
||||
validate(&state, object, transaction)?;
|
||||
Ok(state)
|
||||
}))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_bounded(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
budget: &crate::object_api::DeleteRecoveryWorkBudget,
|
||||
) -> Result<Vec<DeleteTransactionId>> {
|
||||
let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
|
||||
let page = disk.discover_delete_transactions_bounded(bucket, object, reserved).await?;
|
||||
if page.visited > reserved
|
||||
|| page.transactions.len() > 256
|
||||
|| page
|
||||
.transactions
|
||||
.iter()
|
||||
.any(|id| id.object != object || id.transaction.is_nil())
|
||||
|| (!page.complete && !page.transactions.is_empty())
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.refund(reserved - page.visited.max(1));
|
||||
if !page.complete {
|
||||
return Err(DiskError::DeleteRecoveryWorkLimit);
|
||||
}
|
||||
Ok(page.transactions)
|
||||
}
|
||||
|
||||
async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result<Vec<DeleteTransactionId>> {
|
||||
let mut cursor = None;
|
||||
let mut ids = BTreeSet::new();
|
||||
@@ -273,30 +310,92 @@ async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Resul
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct RecoveryWorkScope<'a> {
|
||||
namespace_owner: Option<&'a Arc<dyn Send + Sync>>,
|
||||
budget: Option<&'a crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
}
|
||||
|
||||
async fn settle_outcome(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
state: &DeleteRecoveryInspection,
|
||||
action: DeleteRecoveryAction,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
) -> Result<crate::disk::DeleteRecoverySettlementOutcome> {
|
||||
use crate::disk::DeleteRecoverySettlementOutcome;
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let outcome = if let Some(budget) = scope.budget {
|
||||
let reserved = budget.reserve(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
|
||||
let may_publish = matches!(&action, DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. });
|
||||
let reply = disk
|
||||
.settle_delete_transaction_bounded(
|
||||
bucket,
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: state.id.object.clone(),
|
||||
transaction: state.id.transaction,
|
||||
expected_fingerprint: state.fingerprint,
|
||||
action,
|
||||
max_entries: reserved,
|
||||
},
|
||||
scope.namespace_owner.cloned(),
|
||||
)
|
||||
.await?;
|
||||
if reply.visited > reserved
|
||||
|| (!may_publish && matches!(&reply.outcome, DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred))
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.refund(reserved - reply.visited.max(1));
|
||||
reply.outcome
|
||||
} else {
|
||||
DeleteRecoverySettlementOutcome::Inspected(Box::new(
|
||||
disk.settle_delete_transaction_with_namespace_owner(
|
||||
bucket,
|
||||
&state.id.object,
|
||||
state.id.transaction,
|
||||
state.fingerprint,
|
||||
action,
|
||||
scope.namespace_owner.cloned(),
|
||||
)
|
||||
.await?,
|
||||
))
|
||||
};
|
||||
if let DeleteRecoverySettlementOutcome::Inspected(next) = &outcome {
|
||||
validate(next, &state.id.object, state.id.transaction)?;
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn settle(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
state: &DeleteRecoveryInspection,
|
||||
action: DeleteRecoveryAction,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
namespace_owner: Option<&Arc<dyn Send + Sync>>,
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
match settle_outcome(disk, bucket, state, action, can_commit, scope).await? {
|
||||
crate::disk::DeleteRecoverySettlementOutcome::Inspected(inspection) => Ok(*inspection),
|
||||
crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred => {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_receipt_quorum(outcomes: &[Result<crate::disk::DeleteRecoverySettlementOutcome>], quorum: usize) -> bool {
|
||||
outcomes.iter().filter(|outcome| published_receipt(outcome)).count() >= quorum
|
||||
}
|
||||
|
||||
fn published_receipt(outcome: &Result<crate::disk::DeleteRecoverySettlementOutcome>) -> bool {
|
||||
match outcome {
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => receipt(state.state),
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => true,
|
||||
Err(_) => false,
|
||||
}
|
||||
let result = disk
|
||||
.settle_delete_transaction_with_namespace_owner(
|
||||
bucket,
|
||||
&state.id.object,
|
||||
state.id.transaction,
|
||||
state.fingerprint,
|
||||
action,
|
||||
namespace_owner.cloned(),
|
||||
)
|
||||
.await;
|
||||
let next = result?;
|
||||
validate(&next, &state.id.object, state.id.transaction)?;
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
/// Each phase fans out across independent drives, and awaits all started work
|
||||
@@ -307,23 +406,16 @@ async fn settle_group(
|
||||
copies: &mut [DeleteRecoveryInspection],
|
||||
action: impl Fn(&DeleteRecoveryInspection) -> Option<DeleteRecoveryAction> + Sync,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
namespace_owner: Option<&Arc<dyn Send + Sync>>,
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
removed_versions: &mut HashSet<Uuid>,
|
||||
) -> Result<bool> {
|
||||
let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| {
|
||||
let action = action(copy);
|
||||
async move {
|
||||
match action {
|
||||
Some(action) => settle(
|
||||
disk.as_ref().ok_or(DiskError::DiskNotFound)?,
|
||||
bucket,
|
||||
copy,
|
||||
action,
|
||||
can_commit,
|
||||
namespace_owner,
|
||||
)
|
||||
.await
|
||||
.map(Some),
|
||||
Some(action) => settle(disk.as_ref().ok_or(DiskError::DiskNotFound)?, bucket, copy, action, can_commit, scope)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
@@ -439,16 +531,31 @@ impl SetDisks {
|
||||
true
|
||||
}
|
||||
};
|
||||
let mut fast_receipt_quorum = false;
|
||||
if can_try_local_fast_path && can_commit() {
|
||||
let fast_published = join_all(disks.iter().map(|disk| async {
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
disk.try_publish_prepared_delete_local(bucket, object, transaction, context.namespace_owner.clone())
|
||||
.await
|
||||
disk.try_publish_prepared_delete_local(
|
||||
bucket,
|
||||
object,
|
||||
transaction,
|
||||
context.namespace_owner.clone(),
|
||||
context.work_budget.clone(),
|
||||
)
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
fast_receipt_quorum = fast_published.iter().filter(|result| matches!(result, Ok(Some(_)))).count() >= write_quorum;
|
||||
if fast_receipt_quorum
|
||||
&& fast_published
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let all_fast_published = fast_published.len() == self.set_drive_count
|
||||
&& fast_published.iter().all(|res| matches!(res, Ok(Some((_, true, true)))));
|
||||
if all_fast_published && can_commit() {
|
||||
@@ -456,6 +563,7 @@ impl SetDisks {
|
||||
test_hooks::record_cleanup_attempt(bucket, object);
|
||||
let finalized = join_all(disks.iter().zip(fast_published.iter()).map(|(disk, pub_res)| {
|
||||
let ns_owner = context.namespace_owner.clone();
|
||||
let work_budget = context.work_budget.clone();
|
||||
async move {
|
||||
if !can_commit() {
|
||||
return Ok(false);
|
||||
@@ -466,11 +574,24 @@ impl SetDisks {
|
||||
let Ok(Some((journal, _, _))) = pub_res else {
|
||||
return Ok(false);
|
||||
};
|
||||
disk.try_finish_and_finalize_prepared_delete_local(bucket, object, transaction, journal, ns_owner)
|
||||
.await
|
||||
disk.try_finish_and_finalize_prepared_delete_local(
|
||||
bucket,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
ns_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
if finalized
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if finalized.iter().all(|res| matches!(res, Ok(true))) {
|
||||
if let Some(ticket) = context.recovery_ticket.as_ref() {
|
||||
self.ctx.delete_recovery_hints().ack(ticket);
|
||||
@@ -479,7 +600,17 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
}
|
||||
let states = inspect(disks, bucket, object, transaction).await;
|
||||
let states = inspect(disks, bucket, object, transaction, context.work_budget.as_deref()).await;
|
||||
if states
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return if fast_receipt_quorum {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
};
|
||||
}
|
||||
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
|
||||
if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum
|
||||
&& states.iter().all(|state| {
|
||||
@@ -499,25 +630,50 @@ impl SetDisks {
|
||||
if state.state == DeleteRecoveryState::LegacyAmbiguous {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
settle(
|
||||
settle_outcome(
|
||||
disk,
|
||||
bucket,
|
||||
state,
|
||||
DeleteRecoveryAction::PublishCommit,
|
||||
can_commit,
|
||||
context.namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: context.namespace_owner.as_ref(),
|
||||
budget: context.work_budget.as_deref(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
if published
|
||||
.iter()
|
||||
.filter(|state| matches!(state, Ok(state) if receipt(state.state)))
|
||||
.count()
|
||||
< write_quorum
|
||||
{
|
||||
return Err(DiskError::ErasureWriteQuorum);
|
||||
if !has_receipt_quorum(&published, write_quorum) {
|
||||
return Err(
|
||||
if published
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
} else {
|
||||
DiskError::ErasureWriteQuorum
|
||||
},
|
||||
);
|
||||
}
|
||||
if published.iter().any(|outcome| {
|
||||
matches!(
|
||||
outcome,
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
|
||||
)
|
||||
}) {
|
||||
return Ok(());
|
||||
}
|
||||
let published = published
|
||||
.into_iter()
|
||||
.map(|outcome| match outcome {
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => Ok(*state),
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
// Physical cleanup is retryable after the certified decision. An
|
||||
// unavailable participant retains the decision for its eventual return.
|
||||
let ticket = context.recovery_ticket.clone();
|
||||
@@ -670,6 +826,7 @@ impl SetDisks {
|
||||
}
|
||||
let DeleteRecoveryContext {
|
||||
incarnation,
|
||||
work_budget,
|
||||
dry_run,
|
||||
explicit_delete,
|
||||
repair_marker_source,
|
||||
@@ -681,7 +838,13 @@ impl SetDisks {
|
||||
} = context;
|
||||
let disks = self.disk_inventory().await;
|
||||
let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
|
||||
let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await;
|
||||
let pages = join_all(disks.iter().flatten().map(|disk| async {
|
||||
match work_budget.as_deref() {
|
||||
Some(budget) => discover_bounded(disk, bucket, object, budget).await,
|
||||
None => discover(disk, bucket, Some(object)).await,
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
let mut transactions = BTreeSet::new();
|
||||
let mut inspected = 0;
|
||||
let mut offline = !complete_inventory;
|
||||
@@ -712,6 +875,7 @@ impl SetDisks {
|
||||
..Default::default()
|
||||
};
|
||||
let mut all_states = BTreeMap::new();
|
||||
let mut registered_evidence_hint = false;
|
||||
for transaction in transactions {
|
||||
let mut complete = Vec::with_capacity(disks.len());
|
||||
let inspections = if let Some((published_tx, published_states)) = published_hint
|
||||
@@ -723,8 +887,14 @@ impl SetDisks {
|
||||
{
|
||||
published_states.to_vec()
|
||||
} else {
|
||||
inspect(&disks, bucket, object, transaction).await
|
||||
inspect(&disks, bucket, object, transaction, work_budget.as_deref()).await
|
||||
};
|
||||
if inspections
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Err(DiskError::DeleteRecoveryWorkLimit);
|
||||
}
|
||||
for state in inspections {
|
||||
match state {
|
||||
Ok(state) => {
|
||||
@@ -736,6 +906,33 @@ impl SetDisks {
|
||||
summary.blocked += 1;
|
||||
return Ok(summary);
|
||||
}
|
||||
// Candidate UUID directories are not evidence. Only a bound, inspected
|
||||
// journal can create a foreground hint; empty/partial scans rely on the
|
||||
// existing raw anti-entropy obligation instead of queuing arbitrary prefixes.
|
||||
if work_budget.is_some()
|
||||
&& !registered_evidence_hint
|
||||
&& journal.protocol_version == DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
&& journal.volume == bucket
|
||||
&& journal.object == object
|
||||
&& journal.transaction == transaction
|
||||
&& matches!(
|
||||
state.state,
|
||||
DeleteRecoveryState::NewPrepared
|
||||
| DeleteRecoveryState::QuorumCommitted
|
||||
| DeleteRecoveryState::CommitFinished
|
||||
)
|
||||
{
|
||||
self.ctx.delete_recovery_hints().register(
|
||||
crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: self.pool_index,
|
||||
set: self.set_index,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
incarnation,
|
||||
},
|
||||
);
|
||||
registered_evidence_hint = true;
|
||||
}
|
||||
}
|
||||
complete.push(state);
|
||||
}
|
||||
@@ -767,7 +964,7 @@ impl SetDisks {
|
||||
};
|
||||
if let Some((transaction, source)) = prerequisite {
|
||||
Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit, namespace_owner.as_ref())).await?;
|
||||
let refreshed = inspect(&disks, bucket, object, transaction)
|
||||
let refreshed = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -853,7 +1050,7 @@ impl SetDisks {
|
||||
if mutated {
|
||||
// A sibling may have restored metadata or reclaimed shared
|
||||
// evidence. Refresh fingerprints before acting on this group.
|
||||
*copies = inspect(&disks, bucket, object, transaction)
|
||||
*copies = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -876,7 +1073,10 @@ impl SetDisks {
|
||||
(copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy)
|
||||
},
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -987,7 +1187,10 @@ impl SetDisks {
|
||||
)
|
||||
},
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?;
|
||||
@@ -1001,7 +1204,10 @@ impl SetDisks {
|
||||
.then_some(DeleteRecoveryAction::FinishCommit)
|
||||
},
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?;
|
||||
@@ -1048,7 +1254,10 @@ impl SetDisks {
|
||||
copies,
|
||||
|copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback),
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1076,7 +1285,7 @@ impl SetDisks {
|
||||
settled.push(copies);
|
||||
} else {
|
||||
for transaction in all_states.keys() {
|
||||
let fresh = inspect(&disks, bucket, object, *transaction).await;
|
||||
let fresh = inspect(&disks, bucket, object, *transaction, work_budget.as_deref()).await;
|
||||
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
|
||||
summary.pending += 1;
|
||||
return Ok(summary);
|
||||
@@ -1086,7 +1295,7 @@ impl SetDisks {
|
||||
}
|
||||
for (index, mut copies) in settled.into_iter().enumerate() {
|
||||
if index > 0 {
|
||||
copies = inspect(&disks, bucket, object, copies[0].id.transaction)
|
||||
copies = inspect(&disks, bucket, object, copies[0].id.transaction, work_budget.as_deref())
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -1106,7 +1315,10 @@ impl SetDisks {
|
||||
&mut copies,
|
||||
|copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit),
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1358,6 +1570,457 @@ mod tests {
|
||||
body
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_mixed_inspected_and_deferred_receipts_count_only_real_quorum() {
|
||||
let bucket = "recover-bounded-mixed-quorum";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![73; 4096], false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let mut outcomes = Vec::new();
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let state = disk
|
||||
.inspect_delete_transaction_bounded(bucket, "object", transaction, 65536)
|
||||
.await
|
||||
.expect("preparation inspection");
|
||||
let max_entries = match index {
|
||||
0 => 65536,
|
||||
1 | 2 => state.visited,
|
||||
_ => 1,
|
||||
};
|
||||
outcomes.push(
|
||||
disk.settle_delete_transaction_bounded(
|
||||
bucket,
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "object".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: state.inspection.fingerprint,
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map(|reply| reply.outcome),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
matches!(&outcomes[0], Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) if receipt(state.state))
|
||||
);
|
||||
assert!(matches!(
|
||||
&outcomes[1],
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
|
||||
));
|
||||
assert!(outcomes[3].is_err());
|
||||
assert!(has_receipt_quorum(&outcomes, 3));
|
||||
assert!(!has_receipt_quorum(&outcomes, 4));
|
||||
outcomes[2] = Err(DiskError::FileAccessDenied);
|
||||
assert!(
|
||||
!has_receipt_quorum(&outcomes, 3),
|
||||
"a write failure cannot substitute for a durable receipt"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_single_and_batch_share_bounded_work_across_versions_and_retries() {
|
||||
use crate::object_api::DeleteRecoveryWorkBudget;
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
let bucket = "recover-bounded-request";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
for dir in &dirs {
|
||||
let prefix = dir.path().join(bucket).join("prefix");
|
||||
std::fs::create_dir_all(&prefix).expect("physical prefix");
|
||||
for index in 0..64 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
|
||||
}
|
||||
}
|
||||
let error = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"prefix",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(DeleteRecoveryWorkBudget::new(8))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("incomplete discovery is not absence");
|
||||
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
|
||||
for incarnation in [
|
||||
None,
|
||||
Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation"),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "prefix".to_owned(),
|
||||
incarnation,
|
||||
}),
|
||||
"an unproven prefix must not move its scan into background recovery"
|
||||
);
|
||||
}
|
||||
for distinct_versions in [false, true] {
|
||||
let budget = Arc::new(DeleteRecoveryWorkBudget::new(32_768));
|
||||
let objects = (0..1000)
|
||||
.map(|_| ObjectToDelete {
|
||||
object_name: "prefix".to_owned(),
|
||||
version_id: distinct_versions.then(Uuid::new_v4),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
objects,
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::clone(&budget)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors
|
||||
.iter()
|
||||
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
|
||||
"repeated actions must not each obtain a new allowance: {distinct_versions}"
|
||||
);
|
||||
assert!(
|
||||
dirs.iter()
|
||||
.all(|dir| dir.path().join(bucket).join("prefix/child-63").is_dir())
|
||||
);
|
||||
}
|
||||
let error = store
|
||||
.delete_object(bucket, "prefix", ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("ordinary missing key");
|
||||
assert!(
|
||||
crate::error::is_err_object_not_found(&error),
|
||||
"small complete inventory retains no-op semantics: {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
let mut dirs = Vec::new();
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for set_index in 0..2 {
|
||||
for disk_index in 0..4 {
|
||||
let dir = tempfile::tempdir().expect("temporary disk");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(set_index);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
dirs.push(dir);
|
||||
}
|
||||
}
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 2,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("bounded-recovery-{pool_index}"),
|
||||
platform: "test".to_owned(),
|
||||
});
|
||||
}
|
||||
let endpoints = EndpointServerPools(pools);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
|
||||
.await
|
||||
.expect("local disks");
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("address"), endpoints, cancel.clone(), ctx)
|
||||
.await
|
||||
.expect("two pools with two sets");
|
||||
cancel.cancel();
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "recover-bounded-fanout";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let candidates = (0..100).map(|index| format!("prefix-{index}")).collect::<Vec<_>>();
|
||||
let mut keys = BTreeSet::new();
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
keys.insert(
|
||||
candidates
|
||||
.iter()
|
||||
.find(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set))
|
||||
.expect("fixture key for set")
|
||||
.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
let keys = keys.into_iter().collect::<Vec<_>>();
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
assert!(
|
||||
keys.iter().any(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set)),
|
||||
"fixture must cover every set"
|
||||
);
|
||||
}
|
||||
}
|
||||
for dir in &dirs {
|
||||
for key in &keys {
|
||||
let prefix = dir.path().join(bucket).join(key);
|
||||
std::fs::create_dir_all(&prefix).expect("physical prefix");
|
||||
for child in 0..8 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{child}"))).expect("child");
|
||||
}
|
||||
}
|
||||
}
|
||||
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(32_768));
|
||||
let objects = (0..1000)
|
||||
.map(|index| crate::storage_api_contracts::object::ObjectToDelete {
|
||||
object_name: keys[index % keys.len()].clone(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
objects,
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::clone(&budget)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors
|
||||
.iter()
|
||||
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
|
||||
"all pools and sets must share the finite allowance"
|
||||
);
|
||||
assert!(budget.remaining() < 32_768, "fanout must debit the original request allowance");
|
||||
assert!(
|
||||
dirs.iter()
|
||||
.all(|dir| dir.path().join(bucket).join(&keys[0]).join("child-7").is_dir())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fast_commit_keeps_success_when_cleanup_budget_is_empty() {
|
||||
let bucket = "recover-bounded-fast-commit";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![74; 2 * 1024 * 1024], false).await;
|
||||
// Inline cleanup needs no directory enumeration and may legitimately
|
||||
// finish with zero scan credits. Require real part files here.
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, "object", "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read non-inline source");
|
||||
assert!(!info.inline_data(), "the fixture must require physical part cleanup");
|
||||
let data_dir = info.data_dir.expect("physical data directory");
|
||||
let part = std::path::PathBuf::from(disk.endpoint().to_string())
|
||||
.join(bucket)
|
||||
.join("object")
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1");
|
||||
assert!(part.is_file(), "the cleanup scan must encounter a real part");
|
||||
}
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(0))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a cached prepared receipt commits independently of cleanup budget");
|
||||
assert!(
|
||||
store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "object".to_owned(),
|
||||
incarnation: Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation")
|
||||
),
|
||||
})
|
||||
);
|
||||
assert!(
|
||||
!disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 128)
|
||||
.await
|
||||
.expect("retained recovery proof")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("background finishes committed cleanup");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_unverified_uuid_candidate_does_not_enqueue_background_work() {
|
||||
let bucket = "recover-bounded-candidate";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
let transaction = Uuid::new_v4();
|
||||
for directory in &dirs {
|
||||
let candidate = directory.path().join(bucket).join("candidate").join(transaction.to_string());
|
||||
std::fs::create_dir_all(&candidate).expect("candidate path");
|
||||
std::fs::write(candidate.join("delete-prepared"), b"not a bound journal").expect("unverified candidate");
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.delete_object(bucket, "candidate", ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for incarnation in [
|
||||
None,
|
||||
Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation"),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "candidate".to_owned(),
|
||||
incarnation,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_budget_refusal_preserves_evidence_for_background_replay() {
|
||||
let bucket = "recover-bounded-evidence";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
let payload = vec![71; 4096];
|
||||
put(&store, bucket, &payload, false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let before = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("before refusal");
|
||||
let error = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(1))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("no incomplete recovery admission");
|
||||
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
|
||||
let after = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("after refusal");
|
||||
assert_eq!(before.fingerprint, after.fingerprint);
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("background-compatible recovery");
|
||||
assert_eq!(body(&store, bucket, None).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_committed_delete_defers_exhausted_cleanup_without_rollback() {
|
||||
let bucket = "recover-bounded-commit";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![72; 4096], false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let key = crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "object".to_owned(),
|
||||
incarnation: None,
|
||||
};
|
||||
let ticket = set.ctx.delete_recovery_hints().register(key.clone());
|
||||
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::default());
|
||||
let exhausted_after_commit = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
// This hook disables local fast cleanup and runs only after receipt
|
||||
// publication and the shared recovery path's key-wide GC inspection.
|
||||
// Exhausting preparation instead would correctly reject the delete.
|
||||
test_hooks::before_gc_finalize(bucket, "object", {
|
||||
let budget = Arc::clone(&budget);
|
||||
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
|
||||
Arc::new(move || {
|
||||
let budget = Arc::clone(&budget);
|
||||
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
|
||||
Box::pin(async move {
|
||||
assert!(budget.reserve(u32::MAX).expect("credits remain until committed cleanup") > 0);
|
||||
exhausted_after_commit.store(true, std::sync::atomic::Ordering::Release);
|
||||
})
|
||||
})
|
||||
});
|
||||
set.publish_delete_commit(
|
||||
bucket,
|
||||
"object",
|
||||
transaction,
|
||||
&set.disk_inventory().await,
|
||||
set.set_drive_count / 2 + 1,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: Some(Arc::clone(&budget)),
|
||||
recovery_ticket: ticket,
|
||||
..DeleteRecoveryContext::new(None, &|| true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the durable decision still succeeds");
|
||||
assert!(exhausted_after_commit.load(std::sync::atomic::Ordering::Acquire));
|
||||
assert_eq!(budget.remaining(), 0);
|
||||
assert!(set.ctx.delete_recovery_hints().is_pending(&key));
|
||||
let state = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("retained decision");
|
||||
assert!(receipt(state.state));
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("eventual cleanup");
|
||||
assert!(
|
||||
disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 128)
|
||||
.await
|
||||
.expect("cleaned proofs")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_set_rolls_back_undecided_inline_bytes() {
|
||||
let bucket = "recover-undecided";
|
||||
|
||||
@@ -95,13 +95,8 @@ impl SetDisks {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
let resource = ObjectKey::with_version(
|
||||
bucket,
|
||||
object_name_disk_identity(object).as_ref(),
|
||||
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
|
||||
);
|
||||
let guard = self
|
||||
.namespace_lock_for_key(resource)
|
||||
.namespace_lock_for_key(self.delete_transaction_resource(bucket, object))
|
||||
.await
|
||||
.get_write_lock(super::super::get_lock_acquire_timeout())
|
||||
.await
|
||||
@@ -111,6 +106,94 @@ impl SetDisks {
|
||||
Ok(Arc::new(guard))
|
||||
}
|
||||
|
||||
fn delete_transaction_resource(&self, bucket: &str, object: &str) -> ObjectKey {
|
||||
ObjectKey::with_version(
|
||||
bucket,
|
||||
object_name_disk_identity(object).as_ref(),
|
||||
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
|
||||
)
|
||||
}
|
||||
|
||||
/// Batched write-lock acquisition for the whole key set of one bulk
|
||||
/// mutation: one lock round instead of one quorum round-trip per key.
|
||||
///
|
||||
/// Bulk delete is the only multi-key write-lock holder. The batched round
|
||||
/// is deadlock-free by construction (a single `lock_batch` RPC per lock
|
||||
/// client, and every lock server orders concurrent batches by the same
|
||||
/// global (shard, key) order), so unlike an optimistic probe wave it
|
||||
/// needs no canonical serial fallback. Outcomes are per key and use the
|
||||
/// same error shape as the single-key paths.
|
||||
async fn namespace_write_locks_batch(
|
||||
&self,
|
||||
bucket: &str,
|
||||
resources: &[ObjectKey],
|
||||
timeout: std::time::Duration,
|
||||
mode: &'static str,
|
||||
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
|
||||
if resources.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let lock = self.namespace_lock_for_key(resources[0].clone()).await;
|
||||
lock.get_write_locks_batch(resources, timeout)
|
||||
.await
|
||||
.into_iter()
|
||||
.zip(resources)
|
||||
.map(|(result, resource)| {
|
||||
result.map_err(|error| self.map_namespace_lock_error(bucket, resource.object.as_ref(), mode, error))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Batched raw namespace write locks for a bulk delete's key set. The
|
||||
/// resource keys use the same disk-identity folding as `new_ns_lock`, so
|
||||
/// aliases (`a\b` vs `a/b`) still share one lock.
|
||||
pub(crate) async fn acquire_object_write_locks_batch(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: &[&str],
|
||||
timeout: std::time::Duration,
|
||||
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
|
||||
let resources: Vec<ObjectKey> = objects
|
||||
.iter()
|
||||
.map(|object| ObjectKey {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object_name_disk_identity(object).as_ref()),
|
||||
version: None,
|
||||
})
|
||||
.collect();
|
||||
self.namespace_write_locks_batch(bucket, &resources, timeout, "write").await
|
||||
}
|
||||
|
||||
/// Batched delete-transaction locks for a bulk delete's key set; see
|
||||
/// [`Self::acquire_delete_transaction_lock`] for the resource shape.
|
||||
pub(in crate::set_disk) async fn acquire_delete_transaction_locks_batch(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: &[&str],
|
||||
timeout: std::time::Duration,
|
||||
) -> Vec<Result<Arc<rustfs_lock::NamespaceLockGuard>>> {
|
||||
let resources: Vec<ObjectKey> = objects
|
||||
.iter()
|
||||
.map(|object| self.delete_transaction_resource(bucket, object))
|
||||
.collect();
|
||||
let mut guards = Vec::with_capacity(resources.len());
|
||||
for (object, result) in objects.iter().zip(
|
||||
self.namespace_write_locks_batch(bucket, &resources, timeout, "delete_transaction")
|
||||
.await,
|
||||
) {
|
||||
match result {
|
||||
Ok(guard) => {
|
||||
#[cfg(test)]
|
||||
super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
|
||||
let _ = object;
|
||||
guards.push(Ok(Arc::new(guard)));
|
||||
}
|
||||
Err(error) => guards.push(Err(error)),
|
||||
}
|
||||
}
|
||||
guards
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
||||
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
||||
match err {
|
||||
|
||||
@@ -99,6 +99,7 @@ use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
|
||||
#[cfg(test)]
|
||||
use rustfs_utils::http::SUFFIX_COMPRESSION;
|
||||
use rustfs_utils::http::{SUFFIX_INLINE_DATA, SUFFIX_MAX_TOTAL_OBJECT_SIZE, get_consistent_str};
|
||||
use rustfs_utils::path::{decode_dir_object, encode_dir_object};
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::AtomicBool;
|
||||
@@ -972,12 +973,48 @@ fn multipart_upload_leaf_is_safe(leaf: &str) -> bool {
|
||||
!leaf.is_empty() && leaf != "." && leaf != ".." && !leaf.contains(['/', '\\', '\0'])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static MULTIPART_DISCOVERY_DIRECTORIES: std::sync::LazyLock<std::sync::Mutex<HashMap<String, Vec<String>>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(test)]
|
||||
fn observe_multipart_discovery_directory(bucket: &str, directory: &str) {
|
||||
if let Some(directories) = MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("multipart discovery observer lock")
|
||||
.get_mut(bucket)
|
||||
{
|
||||
directories.push(directory.to_owned());
|
||||
}
|
||||
}
|
||||
|
||||
async fn multipart_index_roots_on_disk(disk: DiskStore, bucket: &str) -> disk::error::Result<(Uuid, Vec<String>)> {
|
||||
if !disk.is_online().await {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
let identity = disk
|
||||
.get_disk_id()
|
||||
.await?
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::DiskNotFound)?;
|
||||
let query_bucket = bucket.to_owned();
|
||||
let roots = crate::disk::multipart_index::collect_pages(bucket, identity, move |marker| {
|
||||
let disk = disk.clone();
|
||||
let bucket = query_bucket.clone();
|
||||
async move { disk.list_multipart_index_roots(&bucket, marker.as_deref()).await }
|
||||
})
|
||||
.await?;
|
||||
Ok((identity, roots))
|
||||
}
|
||||
|
||||
async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_prefix: &str) -> disk::error::Result<Vec<String>> {
|
||||
if !disk.is_online().await {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
|
||||
#[cfg(test)]
|
||||
observe_multipart_discovery_directory(bucket, root_prefix);
|
||||
let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -992,6 +1029,8 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.collect());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
observe_multipart_discovery_directory(bucket, root_prefix);
|
||||
let sha_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -1014,6 +1053,8 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.map(|sha_dir| {
|
||||
let disk = disk.clone();
|
||||
async move {
|
||||
#[cfg(test)]
|
||||
observe_multipart_discovery_directory(bucket, &sha_dir);
|
||||
match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &sha_dir, -1).await {
|
||||
Ok(entries) => Ok((sha_dir, entries)),
|
||||
Err(DiskError::FileNotFound) => Ok((sha_dir, Vec::new())),
|
||||
@@ -1038,6 +1079,12 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
Ok(upload_paths)
|
||||
}
|
||||
|
||||
struct MultipartUploadListScope<'a> {
|
||||
bucket: &'a str,
|
||||
prefix: &'a str,
|
||||
exact_key: bool,
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
fn validated_multipart_upload_path(bucket: &str, object: &str, upload_id: &str, data_movement: bool) -> Result<String> {
|
||||
let path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
@@ -1134,13 +1181,25 @@ impl SetDisks {
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.is_empty() {
|
||||
self.discover_multipart_upload_paths_on_disks(disks, orig_bucket, error_path, root_prefix, require_all_disks)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_multipart_upload_paths_on_disks(
|
||||
&self,
|
||||
disks: Vec<Option<DiskStore>>,
|
||||
orig_bucket: &str,
|
||||
error_path: &str,
|
||||
root_prefix: &str,
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
if disks.len() != self.set_drive_count || self.set_drive_count == 0 {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
let discovery_quorum = if self.default_parity_count == 0 {
|
||||
disks.len()
|
||||
self.set_drive_count
|
||||
} else {
|
||||
(disks.len() / 2).max(1)
|
||||
(self.set_drive_count / 2).max(1)
|
||||
};
|
||||
let mut discovery_errors = (0..disks.len()).map(|_| Some(DiskError::DiskNotFound)).collect::<Vec<_>>();
|
||||
let mut candidate_counts = HashMap::<String, usize>::new();
|
||||
@@ -1819,8 +1878,127 @@ impl SetDisks {
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let (disks, candidate_paths, discovery_quorum, _) =
|
||||
self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
|
||||
self.list_multipart_uploads_in_scope(
|
||||
MultipartUploadListScope {
|
||||
bucket,
|
||||
prefix,
|
||||
exact_key: false,
|
||||
},
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
delimiter,
|
||||
max_uploads,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Probe only the legacy staging directory belonging to this encoded key.
|
||||
pub(crate) async fn has_multipart_upload_for_key(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<bool> {
|
||||
let key = decode_dir_object(object);
|
||||
let result = self
|
||||
.list_multipart_uploads_in_scope(
|
||||
MultipartUploadListScope {
|
||||
bucket,
|
||||
prefix: &key,
|
||||
exact_key: true,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
1,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
Ok(!result.uploads.is_empty())
|
||||
}
|
||||
|
||||
/// Index coverage only locates candidates. Every candidate still traverses
|
||||
/// the original directory and metadata quorum checks on this disk snapshot.
|
||||
async fn multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Option<(Vec<Option<DiskStore>>, std::collections::BTreeSet<String>, usize)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
// Coverage describes configured physical slots, not the size of a
|
||||
// transient inventory. Missing slots must remain explicit None entries.
|
||||
if disks.len() != self.set_drive_count || self.set_drive_count == 0 {
|
||||
return None;
|
||||
}
|
||||
let quorum = if self.default_parity_count == 0 {
|
||||
self.set_drive_count
|
||||
} else {
|
||||
(self.set_drive_count / 2).max(1)
|
||||
};
|
||||
let required = self.set_drive_count.checked_sub(quorum)?.checked_add(1)?.max(quorum);
|
||||
let mut tasks = JoinSet::new();
|
||||
for disk in disks.iter().flatten().cloned() {
|
||||
let bucket = bucket.to_owned();
|
||||
tasks.spawn(async move { multipart_index_roots_on_disk(disk, &bucket).await });
|
||||
}
|
||||
let mut covered_ids = HashSet::new();
|
||||
let mut roots = std::collections::BTreeSet::new();
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
if let Ok(Ok((disk_id, candidates))) = result {
|
||||
// Duplicate physical identities cannot count as independent slots.
|
||||
if !covered_ids.insert(disk_id) {
|
||||
return None;
|
||||
}
|
||||
roots.extend(candidates);
|
||||
}
|
||||
}
|
||||
// Every returnable metadata read quorum has at least `quorum` slots.
|
||||
// N-D+1 independently indexed slots intersect every such quorum, even
|
||||
// when an old coordinator wrote the upload without knowing this index.
|
||||
(covered_ids.len() >= required).then_some((disks, roots, quorum))
|
||||
}
|
||||
|
||||
async fn list_multipart_uploads_in_scope(
|
||||
&self,
|
||||
scope: MultipartUploadListScope<'_>,
|
||||
key_marker: Option<String>,
|
||||
upload_id_marker: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let MultipartUploadListScope {
|
||||
bucket,
|
||||
prefix,
|
||||
exact_key,
|
||||
} = scope;
|
||||
let root = if exact_key {
|
||||
Self::get_multipart_sha_dir(bucket, &encode_dir_object(prefix))
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let covered = if exact_key {
|
||||
None
|
||||
} else {
|
||||
self.multipart_index_roots(bucket).await
|
||||
};
|
||||
let (disks, candidate_paths, discovery_quorum, _) = if let Some((disks, mut roots, quorum)) = covered {
|
||||
// Legacy no-incarnation callers can list an exact-prefix orphan after
|
||||
// completion moved xl.meta. Its SHA is known without foreign scanning.
|
||||
if expected_incarnation_id.is_none() && !prefix.is_empty() {
|
||||
roots.insert(Self::get_multipart_sha_dir(bucket, &encode_dir_object(prefix)));
|
||||
}
|
||||
let mut candidates = Vec::new();
|
||||
for root in roots {
|
||||
let (_, paths, _, _) = self
|
||||
.discover_multipart_upload_paths_on_disks(disks.clone(), bucket, prefix, &root, false)
|
||||
.await?;
|
||||
candidates.extend(paths);
|
||||
}
|
||||
(disks, candidates, quorum, false)
|
||||
} else {
|
||||
self.discover_multipart_upload_paths(bucket, prefix, &root, false).await?
|
||||
};
|
||||
let listed_uploads = stream::iter(candidate_paths)
|
||||
.map(|upload_path| {
|
||||
let disks = &disks;
|
||||
@@ -1844,7 +2022,11 @@ impl SetDisks {
|
||||
.iter()
|
||||
.filter(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::VolumeNotFound)))
|
||||
.count();
|
||||
if missing_metadata >= discovery_quorum {
|
||||
// At half-set parity, both successful heads and missing or corrupt
|
||||
// replies can reach discovery quorum. Resolve the surviving heads
|
||||
// through the authoritative layout and identity checks in that tie.
|
||||
let below_discovery_quorum = errs.iter().filter(|err| err.is_none()).count() < discovery_quorum;
|
||||
if below_discovery_quorum && missing_metadata >= discovery_quorum {
|
||||
if expected_incarnation_id.is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -1853,7 +2035,7 @@ impl SetDisks {
|
||||
// crash in that window intentionally leaves a reclaimable
|
||||
// upload directory whose object name can still be proven for
|
||||
// an exact-key listing by matching the namespace hash.
|
||||
if !prefix.is_empty() && sha_dir == Self::get_multipart_sha_dir(bucket, prefix) {
|
||||
if !prefix.is_empty() && sha_dir == Self::get_multipart_sha_dir(bucket, &encode_dir_object(prefix)) {
|
||||
let initiated = raw_upload_id
|
||||
.rsplit_once('x')
|
||||
.and_then(|(_, timestamp)| timestamp.parse::<i128>().ok())
|
||||
@@ -1882,7 +2064,7 @@ impl SetDisks {
|
||||
.iter()
|
||||
.filter(|err| err.as_ref().is_some_and(is_corrupt_upload_metadata_error))
|
||||
.count();
|
||||
if corrupt_metadata > 0 && missing_metadata + corrupt_metadata >= discovery_quorum {
|
||||
if below_discovery_quorum && corrupt_metadata > 0 && missing_metadata + corrupt_metadata >= discovery_quorum {
|
||||
debug!(
|
||||
bucket,
|
||||
upload_path = %upload_path,
|
||||
@@ -1947,7 +2129,8 @@ impl SetDisks {
|
||||
// retry instead of silently losing entries.
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if !object.starts_with(prefix) {
|
||||
let object = decode_dir_object(&object);
|
||||
if (exact_key && object != prefix) || !object.starts_with(prefix) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
@@ -2051,14 +2234,9 @@ impl SetDisks {
|
||||
.zip(errors)
|
||||
.map(|(fi, error)| error.is_none().then_some(fi))
|
||||
.collect();
|
||||
let (completed, mut matching_minority) = self.completed_multipart_write_quorum_candidate(
|
||||
(bucket, object),
|
||||
upload_id,
|
||||
uploaded_parts,
|
||||
source,
|
||||
candidates,
|
||||
opts,
|
||||
)?;
|
||||
let (completed, mut matching_minority) = self
|
||||
.completed_multipart_write_quorum_candidate((bucket, object), upload_id, uploaded_parts, source, candidates, opts)
|
||||
.await?;
|
||||
if let Some(completed) = completed {
|
||||
return self
|
||||
.acknowledge_completed_multipart(&disks, bucket, object, upload_id, completed, opts)
|
||||
@@ -2108,14 +2286,9 @@ impl SetDisks {
|
||||
}
|
||||
candidates.push(candidate);
|
||||
}
|
||||
let (completed, historical_minority) = self.completed_multipart_write_quorum_candidate(
|
||||
(bucket, object),
|
||||
upload_id,
|
||||
uploaded_parts,
|
||||
source,
|
||||
candidates,
|
||||
opts,
|
||||
)?;
|
||||
let (completed, historical_minority) = self
|
||||
.completed_multipart_write_quorum_candidate((bucket, object), upload_id, uploaded_parts, source, candidates, opts)
|
||||
.await?;
|
||||
matching_minority |= historical_minority;
|
||||
if let Some(completed) = completed {
|
||||
return self
|
||||
@@ -2170,7 +2343,7 @@ impl SetDisks {
|
||||
/// Count exact completion identities once per physical slot. Each call
|
||||
/// evaluates one complete fanout; observations from different reads never
|
||||
/// combine to manufacture a write quorum.
|
||||
fn completed_multipart_write_quorum_candidate(
|
||||
async fn completed_multipart_write_quorum_candidate(
|
||||
&self,
|
||||
target: (&str, &str),
|
||||
upload_id: &str,
|
||||
@@ -2215,6 +2388,16 @@ impl SetDisks {
|
||||
matching_minority |= count < required
|
||||
&& rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_MULTIPART_UPLOAD_ID)
|
||||
== Some(upload_id);
|
||||
// Select a write-quorum candidate belonging to this upload before
|
||||
// authorizing it; never compare requested parts or confirm/reclaim a
|
||||
// completed object using a different upload's authorization evidence.
|
||||
if count >= required
|
||||
&& rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_MULTIPART_UPLOAD_ID)
|
||||
== Some(upload_id)
|
||||
&& let Some(authorization) = &opts.multipart_completion_authorization
|
||||
{
|
||||
authorization.authorize(&fi.metadata).await?;
|
||||
}
|
||||
if count >= required
|
||||
&& completed_multipart_upload_matches(
|
||||
&fi.metadata,
|
||||
@@ -2809,7 +2992,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let mut ret = ListPartsInfo {
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
object: decode_dir_object(object),
|
||||
upload_id: upload_id.to_owned(),
|
||||
storage_class,
|
||||
max_parts,
|
||||
@@ -3188,7 +3371,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
Ok(MultipartInfo {
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
object: decode_dir_object(object),
|
||||
upload_id: upload_id.to_owned(),
|
||||
user_defined: {
|
||||
strip_internal_multipart_metadata(&mut fi.metadata);
|
||||
@@ -3402,6 +3585,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
|
||||
.await?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
// Recovery can restore a source after the request's metadata preflight
|
||||
// missed it. Reauthorize that exact source before checksum comparison or
|
||||
// publication while retaining the original object/upload locks.
|
||||
if let Some(authorization) = &opts.multipart_completion_authorization {
|
||||
authorization.authorize(&fi.metadata).await?;
|
||||
}
|
||||
if !initial_multipart_source_observations(&fi, &files_metas)
|
||||
&& let Some(completed) = self
|
||||
.replay_completed_multipart_upload(bucket, object, upload_id, &uploaded_parts, Some(&fi), opts)
|
||||
@@ -5253,6 +5442,45 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_upload_paths_reject_aliases_before_disk_or_lock_use() {
|
||||
let bucket = "multipart-path-owner";
|
||||
let key = "allowed-key";
|
||||
let leaf = format!("{}x1000000000", Uuid::new_v4());
|
||||
let encode = |deployment: &str, suffix: &str| {
|
||||
base64_simd::URL_SAFE_NO_PAD.encode_to_string(format!("{deployment}.{suffix}").as_bytes())
|
||||
};
|
||||
for data_movement in [false, true] {
|
||||
let canonical = SetDisks::validated_multipart_upload_path(bucket, key, &encode("deployment", &leaf), data_movement)
|
||||
.expect("issued upload leaf is accepted");
|
||||
assert_eq!(
|
||||
canonical,
|
||||
SetDisks::validated_multipart_upload_path(bucket, key, &encode("another-deployment", &leaf), data_movement)
|
||||
.expect("deployment aliases resolve to the identical disk and lock path")
|
||||
);
|
||||
for suffix in [
|
||||
format!("{leaf}/"),
|
||||
format!("{leaf}//"),
|
||||
format!("./{leaf}"),
|
||||
format!("../{}/{leaf}", SetDisks::get_multipart_sha_dir(bucket, "denied-key")),
|
||||
format!("{leaf}\\"),
|
||||
format!("{leaf}\0"),
|
||||
] {
|
||||
assert!(
|
||||
matches!(
|
||||
SetDisks::validated_multipart_upload_path(bucket, key, &encode("deployment", &suffix), data_movement),
|
||||
Err(Error::InvalidUploadID(..))
|
||||
),
|
||||
"unsafe upload suffix must not reach path normalization: {suffix:?}"
|
||||
);
|
||||
}
|
||||
assert!(matches!(
|
||||
SetDisks::validated_multipart_upload_path(bucket, key, "not-base64!", data_movement),
|
||||
Err(Error::InvalidUploadID(..))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_multipart_identity_requires_canonical_upload_id() {
|
||||
let created = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10);
|
||||
@@ -8798,6 +9026,82 @@ mod tests {
|
||||
put_object_part_fences_upload_lock_loss_before_rename_case(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_part_retry_persists_new_etag_and_rejects_the_previous_receipt() {
|
||||
use crate::io_support::rio::{WriteEncryption, WritePlan};
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "encrypted-etag-retry";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let opts = ObjectOptions {
|
||||
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, "object", &opts)
|
||||
.await
|
||||
.expect("create encrypted upload");
|
||||
let mut receipts = Vec::new();
|
||||
for part_number in [1, 2, 1, 3] {
|
||||
let bytes = b"same confidential part";
|
||||
let size = i64::try_from(bytes.len()).expect("test length");
|
||||
let source = HashReader::from_stream(Cursor::new(bytes.to_vec()), size, size, None, None, false).expect("source");
|
||||
let encrypted = WritePlan::new()
|
||||
.with_encryption(WriteEncryption::multipart([7; 32], [9; 12], part_number))
|
||||
.apply(source, size)
|
||||
.expect("encrypt part");
|
||||
let mut reader = PutObjReader::new(encrypted);
|
||||
let mut part_opts = opts.clone();
|
||||
if part_number == 3 {
|
||||
part_opts.preserve_etag = Some("0123456789abcdef0123456789abcdef".to_string());
|
||||
}
|
||||
let part = set
|
||||
.put_object_part(bucket, "object", &upload.upload_id, part_number, &mut reader, &part_opts)
|
||||
.await
|
||||
.expect("persist encrypted part");
|
||||
if part_number == 3 {
|
||||
assert_eq!(part.etag, part_opts.preserve_etag);
|
||||
} else {
|
||||
receipts.push(part.etag);
|
||||
}
|
||||
}
|
||||
assert_ne!(receipts[0], receipts[1], "spare parts must not identify another part's plaintext");
|
||||
assert_ne!(receipts[0], receipts[2], "same-number retries get a new receipt");
|
||||
for _ in 0..2 {
|
||||
let listed = set
|
||||
.list_object_parts(bucket, "object", &upload.upload_id, None, 10, &opts)
|
||||
.await
|
||||
.expect("list parts");
|
||||
assert_eq!(listed.parts[0].etag, receipts[2], "stored ETags do not change on reads");
|
||||
}
|
||||
let stale = CompletePart {
|
||||
part_num: 1,
|
||||
etag: receipts[0].clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let error = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload.upload_id, vec![stale], &opts)
|
||||
.await
|
||||
.expect_err("the old part receipt must not complete a rewritten part");
|
||||
assert!(matches!(error, Error::InvalidPart(..)), "{error:?}");
|
||||
let current = CompletePart {
|
||||
part_num: 1,
|
||||
etag: receipts[2].clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let committed = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload.upload_id, vec![current.clone()], &opts)
|
||||
.await
|
||||
.expect("complete with the stored receipt");
|
||||
let replay = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload.upload_id, vec![current], &opts)
|
||||
.await
|
||||
.expect("committed completion replay is stable");
|
||||
assert_eq!(replay.etag, committed.etag);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_encrypted_multipart_marks_quorum_validated_layout() {
|
||||
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
|
||||
@@ -11177,6 +11481,237 @@ mod tests {
|
||||
.unwrap_or(decoded)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_shortened_inventory_cannot_lower_configured_quorum() {
|
||||
let (_roots, disks, set) = hermetic_set_disks(4).await;
|
||||
for disk in &disks {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::disk::disk_store::wait_for_multipart_index_bootstrap(disk),
|
||||
)
|
||||
.await
|
||||
.expect("bootstrap completes")
|
||||
.expect("local bootstrap worker");
|
||||
}
|
||||
let bucket = "multipart-index-short-inventory";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let full = set.disks.read().await.clone();
|
||||
assert_eq!(set.set_drive_count, 4);
|
||||
for available in 0..4 {
|
||||
*set.disks.write().await = full[..available].to_vec();
|
||||
assert!(
|
||||
set.multipart_index_roots(bucket).await.is_none(),
|
||||
"{available} inventory slots cannot redefine configured N"
|
||||
);
|
||||
assert!(matches!(
|
||||
set.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await,
|
||||
Err(Error::ErasureReadQuorum)
|
||||
));
|
||||
}
|
||||
*set.disks.write().await = full.clone();
|
||||
set.disks.write().await[3] = None;
|
||||
assert!(
|
||||
set.multipart_index_roots(bucket).await.is_some(),
|
||||
"three covered slots still satisfy configured 4-slot coverage"
|
||||
);
|
||||
set.disks.write().await[2] = None;
|
||||
assert!(set.multipart_index_roots(bucket).await.is_none());
|
||||
let listed = set
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("two explicit available slots still satisfy legacy D=2");
|
||||
assert!(listed.uploads.is_empty());
|
||||
*set.disks.write().await = full;
|
||||
assert!(set.multipart_index_roots(bucket).await.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_covering_set_finds_old_write_on_minimum_metadata_quorum() {
|
||||
let (_roots, disks, set) = hermetic_set_disks(4).await;
|
||||
for disk in &disks {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::disk::disk_store::wait_for_multipart_index_bootstrap(disk),
|
||||
)
|
||||
.await
|
||||
.expect("bootstrap completes")
|
||||
.expect("local bootstrap worker");
|
||||
}
|
||||
let bucket = "multipart-index-minimum-quorum";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("fixture bucket");
|
||||
}
|
||||
let key = "old-writer";
|
||||
let upload = format!("{}x1", Uuid::new_v4());
|
||||
let path = format!("{}/{upload}", SetDisks::get_multipart_sha_dir(bucket, key));
|
||||
let mut metadata = FileInfo::new(&path, 2, 2);
|
||||
metadata.data_dir = Some(Uuid::new_v4());
|
||||
metadata.mod_time = Some(OffsetDateTime::now_utc());
|
||||
metadata.fresh = true;
|
||||
metadata.metadata.insert(RUSTFS_MULTIPART_BUCKET_KEY.into(), bucket.into());
|
||||
metadata.metadata.insert(RUSTFS_MULTIPART_OBJECT_KEY.into(), key.into());
|
||||
// Exactly one of the two surviving source copies intersects the three
|
||||
// covered disks. No coordinator-side index update is involved.
|
||||
for (index, disk) in disks.iter().take(2).enumerate() {
|
||||
let mut info = metadata.clone();
|
||||
info.erasure.index = metadata.erasure.distribution[index];
|
||||
disk.write_metadata(bucket, RUSTFS_META_MULTIPART_BUCKET, &path, info)
|
||||
.await
|
||||
.expect("legacy metadata publication");
|
||||
}
|
||||
// Raw head replacement requires an existing source. Initialize a separate
|
||||
// upload before damaging it so the two readable target copies stay intact.
|
||||
let damaged_key = "damaged-source";
|
||||
let damaged_path = format!("{}/{}x1", SetDisks::get_multipart_sha_dir(bucket, damaged_key), Uuid::new_v4());
|
||||
let mut damaged = metadata.clone();
|
||||
damaged.name.clone_from(&damaged_path);
|
||||
damaged.data_dir = Some(Uuid::new_v4());
|
||||
damaged.erasure.index = damaged.erasure.distribution[0];
|
||||
damaged
|
||||
.metadata
|
||||
.insert(RUSTFS_MULTIPART_OBJECT_KEY.into(), damaged_key.into());
|
||||
disks[0]
|
||||
.write_metadata(bucket, RUSTFS_META_MULTIPART_BUCKET, &damaged_path, damaged)
|
||||
.await
|
||||
.expect("initialize the source that will lose index coverage");
|
||||
disks[0]
|
||||
.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&format!("{damaged_path}/xl.meta"),
|
||||
Bytes::from_static(b"old damaged head"),
|
||||
)
|
||||
.await
|
||||
.expect("one uncovered disk");
|
||||
assert!(matches!(
|
||||
multipart_index_roots_on_disk(disks[0].clone(), bucket).await,
|
||||
Err(disk::error::DiskError::MethodNotAllowed)
|
||||
));
|
||||
let target_root = SetDisks::get_multipart_sha_dir(bucket, key);
|
||||
for (index, disk) in disks.iter().enumerate().skip(1) {
|
||||
let (_, roots) = multipart_index_roots_on_disk(disk.clone(), bucket)
|
||||
.await
|
||||
.expect("the other three physical disks attest coverage");
|
||||
assert_eq!(roots.contains(&target_root), index == 1, "only covered disk 1 contains the target source");
|
||||
}
|
||||
let (_, roots, quorum) = set
|
||||
.multipart_index_roots(bucket)
|
||||
.await
|
||||
.expect("three covered disks intersect D=2");
|
||||
assert_eq!(quorum, 2);
|
||||
assert_eq!(roots, std::collections::BTreeSet::from([target_root]));
|
||||
let (head, _) = set
|
||||
.check_upload_id_exists(bucket, key, &runtime_sources::deployment_upload_id(&upload), false)
|
||||
.await
|
||||
.expect("the two surviving legacy source heads satisfy an ordinary metadata read");
|
||||
assert_eq!(head.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).map(String::as_str), Some(key));
|
||||
let listed = set
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("minimum readable source quorum remains discoverable");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(listed.uploads[0].object, key);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_covering_set_avoids_foreign_scans_and_falls_back_without_coverage() {
|
||||
let (_roots, disks, set) = hermetic_set_disks(4).await;
|
||||
for disk in &disks {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::disk::disk_store::wait_for_multipart_index_bootstrap(disk),
|
||||
)
|
||||
.await
|
||||
.expect("bootstrap completes")
|
||||
.expect("local bootstrap worker");
|
||||
}
|
||||
let bucket = "multipart-index-covered-bucket";
|
||||
let foreign = "multipart-index-foreign-bucket";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("target bucket");
|
||||
disk.make_volume(foreign).await.expect("foreign bucket");
|
||||
}
|
||||
let mine = set
|
||||
.new_multipart_upload(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("target upload");
|
||||
let other = set
|
||||
.new_multipart_upload(foreign, "other", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("foreign upload");
|
||||
let foreign_path = SetDisks::get_upload_id_dir(foreign, "other", &other.upload_id);
|
||||
let target_root = SetDisks::get_multipart_sha_dir(bucket, "object");
|
||||
// Model one disk that cannot attest coverage. The real raw legacy write
|
||||
// revokes only that physical disk's index, without changing the target.
|
||||
disks[0]
|
||||
.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&format!("{foreign_path}/xl.meta"),
|
||||
Bytes::from_static(b"legacy damaged head"),
|
||||
)
|
||||
.await
|
||||
.expect("unindexable legacy publication");
|
||||
assert!(
|
||||
set.multipart_index_roots(bucket).await.is_some(),
|
||||
"3 of 4 intersects every 2-copy metadata quorum"
|
||||
);
|
||||
MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer")
|
||||
.insert(bucket.into(), Vec::new());
|
||||
let listed = set
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("covered listing");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&mine.upload_id));
|
||||
let directories = MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer")
|
||||
.remove(bucket)
|
||||
.expect("observed discovery");
|
||||
assert!(!directories.is_empty());
|
||||
assert!(
|
||||
directories.iter().all(|directory| directory == &target_root),
|
||||
"covered requests cannot enumerate the staging root or foreign uploads"
|
||||
);
|
||||
|
||||
disks[1]
|
||||
.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&format!("{foreign_path}/xl.meta"),
|
||||
Bytes::from_static(b"another legacy damaged head"),
|
||||
)
|
||||
.await
|
||||
.expect("second unavailable index");
|
||||
assert!(
|
||||
set.multipart_index_roots(bucket).await.is_none(),
|
||||
"2 of 4 does not cover the disjoint 2-copy quorum"
|
||||
);
|
||||
MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer")
|
||||
.insert(bucket.into(), Vec::new());
|
||||
let listed = set
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("complete legacy fallback");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&mine.upload_id));
|
||||
let directories = MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer")
|
||||
.remove(bucket)
|
||||
.expect("observed fallback");
|
||||
assert!(
|
||||
directories.iter().any(String::is_empty),
|
||||
"fallback uses the complete old discovery algorithm"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): the multipart staging namespace is flat —
|
||||
/// `sha256(bucket/object)` directories from every bucket share one volume —
|
||||
/// so a bucket-scoped listing reads metadata belonging to other buckets'
|
||||
@@ -11345,6 +11880,66 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_multipart_pool_probe_is_scoped_to_bucket_and_encoded_key() {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-exact-probe";
|
||||
let other_bucket = "multipart-other-probe";
|
||||
for disk in &disks {
|
||||
for bucket in [bucket, other_bucket] {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
}
|
||||
for (upload_bucket, key) in [(bucket, "object-long"), (bucket, "object/"), (other_bucket, "object")] {
|
||||
set.new_multipart_upload(upload_bucket, &encode_dir_object(key), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create unrelated upload");
|
||||
}
|
||||
// An unrelated broken SHA directory makes a full staging-root walk fail.
|
||||
// It must not be touched by an exact-key pool probe.
|
||||
let unrelated = SetDisks::get_multipart_sha_dir(other_bucket, "broken");
|
||||
for dir in &dirs {
|
||||
tokio::fs::write(dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&unrelated), b"not a directory")
|
||||
.await
|
||||
.expect("seed unrelated discovery failure");
|
||||
}
|
||||
MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer lock")
|
||||
.insert(bucket.to_owned(), Vec::new());
|
||||
assert!(
|
||||
!set.has_multipart_upload_for_key(bucket, "object", None)
|
||||
.await
|
||||
.expect("unrelated keys cannot match")
|
||||
);
|
||||
set.new_multipart_upload(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create exact legacy-layout upload");
|
||||
assert!(
|
||||
set.has_multipart_upload_for_key(bucket, "object", None)
|
||||
.await
|
||||
.expect("exact upload remains discoverable")
|
||||
);
|
||||
let probes = MULTIPART_DISCOVERY_DIRECTORIES
|
||||
.lock()
|
||||
.expect("observer lock")
|
||||
.remove(bucket)
|
||||
.expect("recorded probes");
|
||||
assert_eq!(probes.len(), 2 * disks.len(), "one directory per disk and probe");
|
||||
let expected = SetDisks::get_multipart_sha_dir(bucket, "object");
|
||||
assert!(probes.iter().all(|directory| directory == &expected));
|
||||
assert!(
|
||||
set.has_multipart_upload_for_key(bucket, &encode_dir_object("object/"), None)
|
||||
.await
|
||||
.expect("directory key keeps its encoded staging layout")
|
||||
);
|
||||
set.disks.write().await.fill(None);
|
||||
assert!(
|
||||
set.has_multipart_upload_for_key(bucket, "object", None).await.is_err(),
|
||||
"a failed discovery quorum must not become a negative pool match"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_requires_every_disk_online() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -11534,7 +12129,7 @@ mod tests {
|
||||
/// surviving disks still identify it, so it stays listed.
|
||||
#[tokio::test]
|
||||
async fn list_multipart_uploads_survives_sub_quorum_corruption() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-partial-corrupt-bucket";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
@@ -11545,32 +12140,29 @@ mod tests {
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
|
||||
// Corrupt the upload's metadata on exactly one disk (discovery quorum
|
||||
// on this 4-disk set is 2): the other replicas keep it identifiable.
|
||||
let sha = SetDisks::get_multipart_sha_dir(bucket, "partial/object.bin");
|
||||
let mut corrupted = 0usize;
|
||||
for temp_dir in &temp_dirs {
|
||||
let sha_dir = temp_dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&sha);
|
||||
for meta in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
if meta.starts_with(sha_dir.to_string_lossy().as_ref()) {
|
||||
tokio::fs::write(&meta, b"not an xl.meta")
|
||||
.await
|
||||
.expect("corrupting xl.meta should succeed");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
if corrupted > 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert_eq!(corrupted, 1, "exactly one disk's metadata should be corrupted");
|
||||
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "partial/", None, None, None, 1000, None)
|
||||
// Exercise both a minority of corrupt replies and a tie between corruption
|
||||
// and the remaining readable metadata. Each list must resolve the real ID.
|
||||
let path = SetDisks::get_upload_id_dir(bucket, "partial/object.bin", &upload.upload_id);
|
||||
for disk in disk_stores.iter().take(2) {
|
||||
disk.write_all(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&format!("{path}/xl.meta"),
|
||||
Bytes::from_static(b"not an xl.meta"),
|
||||
)
|
||||
.await
|
||||
.expect("sub-quorum corruption must not fail the listing");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&upload.upload_id));
|
||||
.expect("replace one existing head and invalidate its cached metadata");
|
||||
let error = disk
|
||||
.read_version(bucket, RUSTFS_META_MULTIPART_BUCKET, &path, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect_err("the damaged head must no longer return cached metadata");
|
||||
assert!(is_corrupt_upload_metadata_error(&error), "unexpected damage classification: {error:?}");
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "partial/", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("a readable upload survives corrupt replies");
|
||||
assert_eq!(listed.uploads.len(), 1);
|
||||
assert_eq!(upload_uuid_suffix(&listed.uploads[0].upload_id), upload_uuid_suffix(&upload.upload_id));
|
||||
}
|
||||
}
|
||||
|
||||
/// Pins the degrade-vs-propagate classification (issue #5716 review): only
|
||||
@@ -12002,6 +12594,108 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
struct TestCompletionAuthorization {
|
||||
allowed: std::sync::atomic::AtomicBool,
|
||||
calls: std::sync::atomic::AtomicUsize,
|
||||
expected_marker: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::multipart::MultipartCompletionAuthorization for TestCompletionAuthorization {
|
||||
type Error = Error;
|
||||
|
||||
async fn authorize(&self, metadata: &HashMap<String, String>) -> Result<()> {
|
||||
if let Some(expected) = self.expected_marker {
|
||||
assert_eq!(metadata.get("authorization-source").map(String::as_str), Some(expected));
|
||||
}
|
||||
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
if self.allowed.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::PrefixAccessDenied("bucket".to_string(), "object".to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_authorizes_selected_source_and_historical_replay_before_part_comparison() {
|
||||
use std::sync::atomic::Ordering;
|
||||
for versioned in [false, true] {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "completion-authorization";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let create_opts = ObjectOptions {
|
||||
versioned,
|
||||
user_defined: HashMap::from([("authorization-source".to_string(), "original".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
let (upload, parts) = stage_upload_with_create_opts(&set, bucket, object, &[0x61; 4096], &create_opts).await;
|
||||
let gate = Arc::new(TestCompletionAuthorization {
|
||||
allowed: std::sync::atomic::AtomicBool::new(false),
|
||||
calls: std::sync::atomic::AtomicUsize::new(0),
|
||||
expected_marker: Some("original"),
|
||||
});
|
||||
let opts = ObjectOptions {
|
||||
multipart_completion_authorization: Some(gate.clone()),
|
||||
..create_opts.clone()
|
||||
};
|
||||
let denied = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts.clone(), &opts)
|
||||
.await;
|
||||
assert!(matches!(denied, Err(Error::PrefixAccessDenied(..))));
|
||||
assert!(set.get_multipart_info(bucket, object, &upload, &create_opts).await.is_ok());
|
||||
gate.allowed.store(true, Ordering::SeqCst);
|
||||
let committed = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts.clone(), &opts)
|
||||
.await
|
||||
.expect("authorized commit");
|
||||
if versioned {
|
||||
let other_opts = ObjectOptions {
|
||||
user_defined: HashMap::from([("authorization-source".to_string(), "other".to_string())]),
|
||||
..create_opts.clone()
|
||||
};
|
||||
let (other_upload, other_parts) =
|
||||
stage_upload_with_create_opts(&set, bucket, object, &[0x62; 4096], &other_opts).await;
|
||||
set.clone()
|
||||
.complete_multipart_upload(bucket, object, &other_upload, other_parts, &other_opts)
|
||||
.await
|
||||
.expect("new current version");
|
||||
}
|
||||
gate.allowed.store(false, Ordering::SeqCst);
|
||||
let mut guessed_parts = parts.clone();
|
||||
guessed_parts[0].etag = Some("guessed-part-etag".to_string());
|
||||
let denied = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, guessed_parts, &opts)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(denied, Err(Error::PrefixAccessDenied(..))),
|
||||
"authorization must precede InvalidPart disclosure"
|
||||
);
|
||||
gate.allowed.store(true, Ordering::SeqCst);
|
||||
let replay_opts = ObjectOptions {
|
||||
want_checksum: Some(rustfs_rio::Checksum {
|
||||
checksum_type: rustfs_rio::ChecksumType::SHA256,
|
||||
encoded: "a-guess-is-not-compared-on-replay".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
..opts
|
||||
};
|
||||
let replay = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts, &replay_opts)
|
||||
.await
|
||||
.expect("authorized replay");
|
||||
assert!(replay.multipart_completion_replayed);
|
||||
assert_eq!(replay.version_id, committed.version_id);
|
||||
assert_eq!(replay.etag, committed.etag);
|
||||
assert!(gate.calls.load(Ordering::SeqCst) >= 4);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_replay_requires_write_quorum_after_uncertain_publication() {
|
||||
use crate::set_disk::core::io_primitives::{rename_fault_injection, rollback_fault_injection};
|
||||
@@ -14112,9 +14806,30 @@ mod tests {
|
||||
}
|
||||
assert_eq!(seals, 2, "two local seals do not establish a terminal quorum");
|
||||
assert_completion_files_preserved(&disks, RUSTFS_META_MULTIPART_BUCKET, &source_files).await;
|
||||
let retried = complete(&set, bucket, object, &upload, parts)
|
||||
let gate = Arc::new(TestCompletionAuthorization {
|
||||
allowed: std::sync::atomic::AtomicBool::new(false),
|
||||
calls: std::sync::atomic::AtomicUsize::new(0),
|
||||
expected_marker: None,
|
||||
});
|
||||
let guarded_opts = ObjectOptions {
|
||||
multipart_completion_authorization: Some(gate.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let denied = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts.clone(), &guarded_opts)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(denied, Err(Error::PrefixAccessDenied(..))),
|
||||
"restored source must be authorized before publication"
|
||||
);
|
||||
gate.allowed.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
let retried = set
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload, parts, &guarded_opts)
|
||||
.await
|
||||
.expect("H remaining present heads authorize all-N blocked restore and activation");
|
||||
.expect("authorized recovery retains all-N blocked restore and activation");
|
||||
assert!(gate.calls.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert!(
|
||||
!retried.multipart_completion_replayed,
|
||||
"a partial seal cannot acknowledge the old minority"
|
||||
|
||||
@@ -3914,6 +3914,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
explicit_delete: true,
|
||||
transaction_guard: transaction_guard.cloned(),
|
||||
namespace_owner: Some(Arc::new((
|
||||
@@ -4789,6 +4790,27 @@ impl SetDisks {
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Lock order: lifecycle, metadata transaction, then object namespace.
|
||||
// No body reads occur below this point. The snapshot must describe the
|
||||
// configuration at commit, not the pre-body advisory object lookup.
|
||||
let mut commit_opts;
|
||||
let opts = if opts.defer_put_object_lock_snapshot {
|
||||
commit_opts = opts.clone();
|
||||
let expected = source_bucket_incarnation_id
|
||||
.ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
let snapshot = store.object_lock_config_snapshot_for_single_put(bucket).await?;
|
||||
if !snapshot.is_valid_for_destructive_put(store.id, bucket, expected) {
|
||||
return Err(Error::other("PUT bucket generation or configuration changed while staging"));
|
||||
}
|
||||
snapshot.add_lock_fences(&mut commit_opts);
|
||||
commit_opts.object_lock_config_snapshot = Some(snapshot);
|
||||
commit_opts.defer_put_object_lock_snapshot = false;
|
||||
&commit_opts
|
||||
} else {
|
||||
opts
|
||||
};
|
||||
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -5282,6 +5304,7 @@ impl SetDisks {
|
||||
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
|
||||
let commit_publication_guard = publication_commit_guard.take();
|
||||
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
|
||||
let commit_object_lock_config_snapshot = opts.object_lock_config_snapshot.clone();
|
||||
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
|
||||
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
|
||||
// A scanner publication scope owns the movement permit until the
|
||||
@@ -5344,6 +5367,7 @@ impl SetDisks {
|
||||
let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard;
|
||||
let mut _publication_guard = commit_publication_guard;
|
||||
let mut _bucket_lifecycle_guard = commit_bucket_lifecycle_guard;
|
||||
let mut _object_lock_config_snapshot = commit_object_lock_config_snapshot;
|
||||
let mut _decommission_capacity_guard = commit_decommission_capacity_guard;
|
||||
let mut quota_reservation = quota_reservation;
|
||||
let rename_stage_start = Instant::now();
|
||||
@@ -5677,6 +5701,7 @@ impl SetDisks {
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let publication_guard = _publication_guard.take();
|
||||
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
|
||||
let object_lock_config_snapshot = _object_lock_config_snapshot.take();
|
||||
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
|
||||
let decommission_capacity_guard = _decommission_capacity_guard.take();
|
||||
let cleanup_bucket = commit_bucket.clone();
|
||||
@@ -5699,6 +5724,7 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
object_lock_config_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5720,6 +5746,7 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
object_lock_config_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5733,6 +5760,7 @@ impl SetDisks {
|
||||
drop(write_context);
|
||||
drop(publication_guard);
|
||||
drop(bucket_lifecycle_guard);
|
||||
drop(object_lock_config_snapshot);
|
||||
cleanup_runtime::run(cleanup_reservation, async move {
|
||||
let zero_target_cleanup = targets.is_empty();
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
@@ -9147,6 +9175,9 @@ impl SetDisks {
|
||||
&disks,
|
||||
write_quorum,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: commit_context
|
||||
.options
|
||||
.and_then(|options| options.delete_recovery_work_budget.clone()),
|
||||
recovery_ticket,
|
||||
transaction_guard: commit_context.transaction_guard.clone(),
|
||||
namespace_owner: namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
@@ -9408,6 +9439,76 @@ where
|
||||
futures::future::try_join_all(probes).await.map(|_| ()).map_err(Error::from)
|
||||
}
|
||||
|
||||
/// Per-key source-read inputs for the bulk-delete admission loop. The bounded
|
||||
/// metadata pre-fetch and the loop itself must resolve these identically, so
|
||||
/// both sides derive them here.
|
||||
struct BatchDeleteSourceInputs {
|
||||
replication_object_name: String,
|
||||
explicit_null_version: bool,
|
||||
version_id: Option<Uuid>,
|
||||
versioned: bool,
|
||||
version_suspended: bool,
|
||||
check_opts: ObjectOptions,
|
||||
object_lock_check_required: bool,
|
||||
replicate_delete: bool,
|
||||
needs_source_read: bool,
|
||||
}
|
||||
|
||||
fn batch_delete_source_inputs(
|
||||
bucket: &str,
|
||||
dobj: &ObjectToDelete,
|
||||
opts: &ObjectOptions,
|
||||
delete_config_snapshot: &DeleteReplicationConfigSnapshot,
|
||||
object_lock_config: &Arc<ObjectLockConfigSnapshot>,
|
||||
guarded: bool,
|
||||
) -> BatchDeleteSourceInputs {
|
||||
let replication_object_name = decode_dir_object(&dobj.object_name);
|
||||
let explicit_null_version = is_explicit_null_version(dobj.version_id);
|
||||
let version_id = delete_file_info_version_id(dobj.version_id);
|
||||
let (versioned, version_suspended) = delete_config_snapshot
|
||||
.versioning_config()
|
||||
.delete_state(replication_object_name.as_str());
|
||||
let check_opts = ObjectOptions {
|
||||
version_id: dobj.version_id.map(|version_id| version_id.to_string()),
|
||||
versioned,
|
||||
version_suspended,
|
||||
object_lock_delete: opts.object_lock_delete.clone(),
|
||||
object_lock_config_snapshot: Some(Arc::clone(object_lock_config)),
|
||||
no_lock: true,
|
||||
suppress_read_repair: guarded,
|
||||
..Default::default()
|
||||
};
|
||||
// A missing bucket Object Lock configuration does not prove that
|
||||
// persisted object metadata is clean: migrated or corrupt data may
|
||||
// still carry an explicit retention or legal hold. Only a real
|
||||
// delete-marker creation can skip the object-level WORM check.
|
||||
let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
|
||||
let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
|
||||
let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
|
||||
let replication_needs_source = replicate_delete
|
||||
&& (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
|
||||
// Guarded explicit-version deletes are rejected before any read and must
|
||||
// not join the pre-fetch wave.
|
||||
let admission_error = guarded && dobj.version_id.is_some();
|
||||
let needs_source_read = !admission_error
|
||||
&& (object_lock_check_required
|
||||
|| replication_needs_source
|
||||
|| opts.tier_delete_journal_api.is_some()
|
||||
|| dobj.expected_identity.is_some()
|
||||
|| dobj.version_id.is_some());
|
||||
BatchDeleteSourceInputs {
|
||||
replication_object_name,
|
||||
explicit_null_version,
|
||||
version_id,
|
||||
versioned,
|
||||
version_suspended,
|
||||
check_opts,
|
||||
object_lock_check_required,
|
||||
replicate_delete,
|
||||
needs_source_read,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
type Error = Error;
|
||||
@@ -9640,9 +9741,26 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
if let Some(etag) = &src_info.etag {
|
||||
replacement_metadata.insert("etag".to_owned(), etag.clone());
|
||||
}
|
||||
// ObjectInfo externalizes these fields; copy callers supply the effective
|
||||
// values, including empty tags/absent expiry for explicit replacement.
|
||||
if !src_info.user_tags.is_empty() {
|
||||
replacement_metadata
|
||||
.entry(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string())
|
||||
.or_insert_with(|| (*src_info.user_tags).clone());
|
||||
}
|
||||
if let Some(expires) = src_info.expires
|
||||
&& !replacement_metadata.contains_key("expires")
|
||||
{
|
||||
replacement_metadata.insert(
|
||||
"expires".to_string(),
|
||||
expires
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(Error::other)?,
|
||||
);
|
||||
}
|
||||
fi.metadata = replacement_metadata.clone();
|
||||
|
||||
let mod_time = OffsetDateTime::now_utc();
|
||||
let mod_time = dst_opts.mod_time.unwrap_or_else(OffsetDateTime::now_utc);
|
||||
fi.mod_time = Some(mod_time);
|
||||
fi.version_id = version_id;
|
||||
fi.versioned = src_opts.versioned || src_opts.version_suspended;
|
||||
@@ -9765,6 +9883,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
objects: Vec<ObjectToDelete>,
|
||||
mut opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
opts.ensure_delete_recovery_work_budget();
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
let mut accounting = vec![None; objects.len()];
|
||||
let delete_config_snapshot = opts
|
||||
@@ -9806,7 +9925,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let unique_lock_count = batch.requests.len();
|
||||
|
||||
let mut failed_map = HashMap::new();
|
||||
let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
|
||||
let mut local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
|
||||
let mut locked_objects = HashSet::new();
|
||||
|
||||
// Instance-scoped, not the ambient facade (backlog#1052) — see
|
||||
@@ -9823,7 +9942,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
(failed_map, locked_objects, dist_batch_lock_ids) = self.acquire_dist_delete_object_locks_batch(&batch).await;
|
||||
} else {
|
||||
let batch_result = self.local_lock_manager.acquire_locks_batch(batch).await;
|
||||
_local_batch_guards = batch_result.guards;
|
||||
local_batch_guards = batch_result.guards;
|
||||
|
||||
for key in batch_result.successful_locks {
|
||||
locked_objects.insert(key.object.as_ref().to_string());
|
||||
@@ -9865,8 +9984,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// Every raw batch lock is acquired first. Canonical sorting also
|
||||
// gives aliases of one disk key exactly one transaction owner.
|
||||
let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
|
||||
for key in keys {
|
||||
match self.acquire_delete_transaction_lock(bucket, &key).await {
|
||||
// Bulk delete is the only multi-key transaction-lock holder; one
|
||||
// batched lock round replaces one quorum round-trip per key and is
|
||||
// deadlock-free by construction (a single `lock_batch` RPC per lock
|
||||
// client, global (shard, key) order at each server). Per-key
|
||||
// failures keep the set layer's per-key error reporting.
|
||||
let key_names: Vec<&str> = keys.iter().map(|key| key.as_str()).collect();
|
||||
let acquired = self
|
||||
.acquire_delete_transaction_locks_batch(bucket, &key_names, super::super::get_lock_acquire_timeout())
|
||||
.await;
|
||||
for (key, result) in keys.into_iter().zip(acquired) {
|
||||
match result {
|
||||
Ok(guard) => {
|
||||
opts.add_namespace_lock_guard(&guard);
|
||||
transaction_guards.insert(key, guard);
|
||||
@@ -9902,6 +10030,54 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
},
|
||||
}
|
||||
};
|
||||
// Per-key read inputs are resolved exactly once; the pre-fetch and the
|
||||
// admission loop below consume this table so the two cannot disagree
|
||||
// on eligibility or read options.
|
||||
let mut source_inputs: Vec<Option<BatchDeleteSourceInputs>> = Vec::with_capacity(objects.len());
|
||||
for dobj in &objects {
|
||||
let guarded = current_generations.is_some_and(|generations| generations.contains_key(&dobj.object_name));
|
||||
source_inputs.push(Some(batch_delete_source_inputs(
|
||||
bucket,
|
||||
dobj,
|
||||
&opts,
|
||||
&delete_config_snapshot,
|
||||
&object_lock_config,
|
||||
guarded,
|
||||
)));
|
||||
}
|
||||
// Bounded-parallel pre-fetch of the source metadata each admitted key
|
||||
// needs below. These reads are pure quorum lookups under the locks
|
||||
// already held; sequencing one metadata fan-out per key made a
|
||||
// 1000-key batch pay a full round of drive reads at a time.
|
||||
const DELETE_OBJECTS_SOURCE_READ_CONCURRENCY: usize = 16;
|
||||
let mut source_reads: HashMap<usize, (ObjectInfo, FileInfo, Option<Error>)> = HashMap::with_capacity(objects.len());
|
||||
{
|
||||
let mut pending = Vec::new();
|
||||
for (i, source) in source_inputs.iter().enumerate() {
|
||||
if del_errs[i].is_some() {
|
||||
continue;
|
||||
}
|
||||
let Some(source) = source else {
|
||||
continue;
|
||||
};
|
||||
if !source.needs_source_read {
|
||||
continue;
|
||||
}
|
||||
let object_name = objects[i].object_name.clone();
|
||||
let check_opts = &source.check_opts;
|
||||
pending.push(async move {
|
||||
let (goi, file_info, _write_quorum, gerr) = self
|
||||
.get_object_info_fileinfo_and_quorum(bucket, &object_name, check_opts)
|
||||
.await;
|
||||
(i, (goi, file_info, gerr))
|
||||
});
|
||||
}
|
||||
let reads = futures::stream::iter(pending)
|
||||
.buffer_unordered(DELETE_OBJECTS_SOURCE_READ_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
source_reads.extend(reads);
|
||||
}
|
||||
let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
|
||||
let mut tier_reference_leases: Vec<(usize, String, Option<TierDestinationId>)> = Vec::new();
|
||||
let mut tier_free_version_receipt_candidates: HashMap<usize, TierFreeVersionReceiptCandidate> = HashMap::new();
|
||||
@@ -9923,41 +10099,30 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
));
|
||||
continue;
|
||||
}
|
||||
let replication_object_name = decode_dir_object(&dobj.object_name);
|
||||
let explicit_null_version = is_explicit_null_version(dobj.version_id);
|
||||
let version_id = delete_file_info_version_id(dobj.version_id);
|
||||
let (versioned, version_suspended) = delete_config_snapshot
|
||||
.versioning_config()
|
||||
.delete_state(replication_object_name.as_str());
|
||||
let check_opts = ObjectOptions {
|
||||
version_id: dobj.version_id.map(|version_id| version_id.to_string()),
|
||||
let Some(source) = source_inputs[i].take() else {
|
||||
continue;
|
||||
};
|
||||
let BatchDeleteSourceInputs {
|
||||
replication_object_name,
|
||||
explicit_null_version,
|
||||
version_id,
|
||||
versioned,
|
||||
version_suspended,
|
||||
object_lock_delete: opts.object_lock_delete.clone(),
|
||||
object_lock_config_snapshot: Some(Arc::clone(&object_lock_config)),
|
||||
no_lock: true,
|
||||
suppress_read_repair: guarded,
|
||||
..Default::default()
|
||||
};
|
||||
// A missing bucket Object Lock configuration does not prove that
|
||||
// persisted object metadata is clean: migrated or corrupt data may
|
||||
// still carry an explicit retention or legal hold. Only a real
|
||||
// delete-marker creation can skip the object-level WORM check.
|
||||
let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts);
|
||||
let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name);
|
||||
let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
|
||||
let replication_needs_source = replicate_delete
|
||||
&& (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
|
||||
let (mut goi, mut authoritative_file_info, mut gerr) = if object_lock_check_required
|
||||
|| replication_needs_source
|
||||
|| opts.tier_delete_journal_api.is_some()
|
||||
|| dobj.expected_identity.is_some()
|
||||
|| dobj.version_id.is_some()
|
||||
{
|
||||
let (goi, file_info, _write_quorum, gerr) = self
|
||||
.get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts)
|
||||
.await;
|
||||
(goi, file_info, gerr)
|
||||
check_opts,
|
||||
object_lock_check_required,
|
||||
replicate_delete,
|
||||
needs_source_read,
|
||||
} = source;
|
||||
let (mut goi, mut authoritative_file_info, mut gerr) = if needs_source_read {
|
||||
match source_reads.remove(&i) {
|
||||
Some(read) => read,
|
||||
None => {
|
||||
let (goi, file_info, _write_quorum, gerr) = self
|
||||
.get_object_info_fileinfo_and_quorum(bucket, &dobj.object_name, &check_opts)
|
||||
.await;
|
||||
(goi, file_info, gerr)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(ObjectInfo::default(), FileInfo::default(), None)
|
||||
};
|
||||
@@ -9974,11 +10139,23 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
&& !guarded
|
||||
&& goi.name.is_empty()
|
||||
{
|
||||
// Recovery can restore or retire versions of this key
|
||||
// (rolling back an interrupted prepared delete brings a
|
||||
// retained version back). A pre-fetched read for a later
|
||||
// duplicate of the same name would still report the stale
|
||||
// miss and skip its object-lock check — bypassing retention —
|
||||
// so evict those entries and let the loop re-read them live.
|
||||
for (other_index, other) in objects.iter().enumerate() {
|
||||
if other_index != i && other.object_name == dobj.object_name {
|
||||
source_reads.remove(&other_index);
|
||||
}
|
||||
}
|
||||
let recovery = self
|
||||
.recover_delete_key(
|
||||
bucket,
|
||||
&dobj.object_name,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
dry_run: !opts.no_lock,
|
||||
transaction_guard: transaction_guards.get(lock_names[i].as_ref()).cloned(),
|
||||
namespace_owner: opts.delete_mutation_owner.clone(),
|
||||
@@ -10558,6 +10735,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// response can hide a persisted decision and must never roll back.
|
||||
let groups = groups.collect::<Vec<_>>();
|
||||
published_groups.extend(groups.iter().map(|index| vers[*index].name.clone()));
|
||||
// Journal publish fans out per object per drive; keep enough
|
||||
// publications in flight to feed the drives' durable-mutation
|
||||
// lanes (8 workers each) across a 16-drive set.
|
||||
const DELETE_JOURNAL_PUBLISH_CONCURRENCY: usize = 32;
|
||||
let publications = futures::stream::iter(groups.into_iter().map(|index: usize| {
|
||||
let group = &vers[index];
|
||||
let recovery_ticket = recovery_tickets[index].clone();
|
||||
@@ -10583,6 +10764,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
disks,
|
||||
self.set_drive_count / 2 + 1,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
recovery_ticket,
|
||||
namespace_owner,
|
||||
transaction_guard,
|
||||
@@ -10593,7 +10775,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
(index, result)
|
||||
}
|
||||
}))
|
||||
.buffer_unordered(16)
|
||||
.buffer_unordered(DELETE_JOURNAL_PUBLISH_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
for (index, result) in publications {
|
||||
@@ -10733,9 +10915,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// Deferred recovery takes fresh raw and transaction locks. Discard the
|
||||
// completed leases' deadlines while retaining every caller-owned fence.
|
||||
opts.namespace_lock_fence = outer_namespace_lock_fence;
|
||||
// Release inner owners before reacquiring any raw key lock.
|
||||
// Release inner owners before reacquiring any raw key lock. The raw
|
||||
// batch guards must go too: deferred recovery takes a fresh write lock
|
||||
// on the same keys below, and a still-held local batch guard would
|
||||
// deadlock it against itself (the distributed path releases its batch
|
||||
// ids in the same region).
|
||||
drop(namespace_owner);
|
||||
drop(transaction_guards);
|
||||
drop(local_batch_guards);
|
||||
|
||||
// TODO(backlog): support partial object deletion for multi-part objects
|
||||
|
||||
@@ -10761,6 +10948,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
bucket,
|
||||
&object.object_name,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
explicit_delete,
|
||||
namespace_owner: Some(Arc::new((
|
||||
self.ctx.begin_namespace_commit(),
|
||||
@@ -10819,6 +11007,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self, opts))]
|
||||
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
|
||||
opts.ensure_delete_recovery_work_budget();
|
||||
let _scope_outcome_guard = opts
|
||||
.scanner_publication_commit_scope
|
||||
.clone()
|
||||
@@ -11115,6 +11304,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
transaction_guard: transaction_guard.clone(),
|
||||
namespace_owner: Some(Arc::new((
|
||||
self.ctx.begin_namespace_commit(),
|
||||
@@ -11726,7 +11916,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// as a second object header set: forwarding SSE intent or wrapped DEKs
|
||||
// would request a second encryption pass and disclose local envelope
|
||||
// material to the remote provider.
|
||||
transition_meta.retain(|key, _| !rustfs_utils::http::is_replication_stripped_encryption_key(key));
|
||||
transition_meta.retain(|key, _| {
|
||||
!rustfs_utils::http::is_replication_stripped_encryption_key(key)
|
||||
&& !rustfs_utils::http::is_snowball_transport_key(key)
|
||||
});
|
||||
transition_meta.insert("name".to_string(), object.to_string());
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut transition_meta,
|
||||
@@ -16202,6 +16395,170 @@ mod metadata_mutation_generation_tests {
|
||||
assert_retired(&set_disks, &metadata_key).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_metadata_copy_preserves_tags_expiry_and_version_order() {
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use s3s::dto::RestoreRequest;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "restore-metadata-preservation";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::days(1);
|
||||
let latest_time = old_time + time::Duration::days(1);
|
||||
let expires = latest_time + time::Duration::days(365);
|
||||
for deleted in [false, true] {
|
||||
let object = if deleted { "deleted" } else { "live" };
|
||||
let tags = "classification=secret&owner=alice";
|
||||
let old = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(b"old protected content".to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
user_defined: HashMap::from([
|
||||
(AMZ_OBJECT_TAGGING.to_string(), tags.to_string()),
|
||||
("x-amz-tagging".to_string(), tags.to_string()),
|
||||
("expires".to_string(), expires.format(&Rfc3339).expect("expiry")),
|
||||
]),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let latest = if deleted {
|
||||
let marker = FileInfo {
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
mod_time: Some(latest_time),
|
||||
deleted: true,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
for disk in &disks {
|
||||
disk.write_metadata(bucket, bucket, object, marker.clone())
|
||||
.await
|
||||
.expect("publish current delete marker");
|
||||
}
|
||||
marker.version_id
|
||||
} else {
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(b"latest content".to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(latest_time),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write latest version")
|
||||
.version_id
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
version_id: old.version_id.map(|id| id.to_string()),
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
..Default::default()
|
||||
};
|
||||
let mut source = set.get_object_info(bucket, object, &opts).await.expect("read old metadata");
|
||||
assert_eq!(source.user_tags.as_str(), tags);
|
||||
assert_eq!(source.expires, Some(expires));
|
||||
assert!(!source.user_defined.contains_key(AMZ_OBJECT_TAGGING));
|
||||
assert!(source.user_defined.contains_key("x-amz-tagging"));
|
||||
source.metadata_only = true;
|
||||
Arc::make_mut(&mut source.user_defined)
|
||||
.insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"true\"".to_string());
|
||||
set.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
|
||||
.await
|
||||
.expect("restore metadata update");
|
||||
let restored = set
|
||||
.get_object_info(bucket, object, &opts)
|
||||
.await
|
||||
.expect("read restored metadata");
|
||||
assert_eq!(restored.user_tags.as_str(), tags);
|
||||
assert_eq!(restored.expires, Some(expires));
|
||||
assert_eq!(restored.mod_time, Some(old_time));
|
||||
for disk in &disks {
|
||||
let bytes = disk
|
||||
.read_all(bucket, &format!("{object}/xl.meta"))
|
||||
.await
|
||||
.expect("persisted metadata");
|
||||
let meta = FileMeta::load(&bytes).expect("decode versions");
|
||||
let versions = meta.list_versions(bucket, object, false).expect("list versions");
|
||||
assert_eq!(versions.len(), 2);
|
||||
assert_eq!(versions[0].version_id, latest);
|
||||
assert_eq!(versions[0].mod_time, Some(latest_time));
|
||||
assert_eq!(versions[0].deleted, deleted);
|
||||
assert_eq!(versions[1].version_id, old.version_id);
|
||||
assert_eq!(versions[1].mod_time, Some(old_time));
|
||||
}
|
||||
let restore_opts = put_restore_opts(bucket, object, &RestoreRequest::default(), &restored)
|
||||
.await
|
||||
.expect("background restore options");
|
||||
assert_eq!(restore_opts.user_defined.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some(tags));
|
||||
assert_eq!(restore_opts.user_defined.get("expires"), Some(&expires.format(&Rfc3339).expect("expiry")));
|
||||
assert_eq!(restore_opts.mod_time, Some(old_time));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_copy_honors_explicit_tag_and_expiry_replacement() {
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "copy-metadata-replacement";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let old_time = OffsetDateTime::UNIX_EPOCH + time::Duration::days(1);
|
||||
let expires = old_time + time::Duration::days(365);
|
||||
set.put_object(
|
||||
bucket,
|
||||
"object",
|
||||
&mut PutObjReader::from_vec(b"content".to_vec()),
|
||||
&ObjectOptions {
|
||||
mod_time: Some(old_time),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original");
|
||||
let opts = ObjectOptions::default();
|
||||
let mut source = set.get_object_info(bucket, "object", &opts).await.expect("original metadata");
|
||||
source.metadata_only = true;
|
||||
source.user_tags = Arc::new("old=value".to_string());
|
||||
source.expires = Some(expires - time::Duration::days(1));
|
||||
let metadata = Arc::make_mut(&mut source.user_defined);
|
||||
metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "new=value".to_string());
|
||||
metadata.insert("expires".to_string(), expires.format(&Rfc3339).expect("expiry"));
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut source, &opts, &opts)
|
||||
.await
|
||||
.expect("explicit metadata wins over externalized fields");
|
||||
let mut copied = set.get_object_info(bucket, "object", &opts).await.expect("copied metadata");
|
||||
assert_eq!(copied.user_tags.as_str(), "new=value");
|
||||
assert_eq!(copied.expires, Some(expires));
|
||||
assert!(copied.mod_time.is_some_and(|time| time > old_time));
|
||||
|
||||
copied.metadata_only = true;
|
||||
copied.user_tags = Arc::new(String::new());
|
||||
copied.expires = None;
|
||||
set.copy_object(bucket, "object", bucket, "object", &mut copied, &opts, &opts)
|
||||
.await
|
||||
.expect("explicit empty replacement");
|
||||
let cleared = set.get_object_info(bucket, "object", &opts).await.expect("cleared metadata");
|
||||
assert!(cleared.user_tags.is_empty());
|
||||
assert!(cleared.expires.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_only_copy_preserves_valid_part_checksums_and_rejects_conflicting_aliases() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -16654,6 +17011,19 @@ mod transition_commit_failure_tests {
|
||||
.collect();
|
||||
user_defined.insert("x-amz-meta-owner".to_string(), "finance".to_string());
|
||||
|
||||
let snowball_keys = [
|
||||
"snowball-auto-extract",
|
||||
"x-amz-meta-snowball-auto-extract",
|
||||
"x-amz-snowball-auto-extract",
|
||||
"snowball-prefix",
|
||||
"x-amz-meta-rustfs-snowball-prefix",
|
||||
"x-amz-meta-minio-snowball-ignore-dirs",
|
||||
"other-vendor-snowball-ignore-errors",
|
||||
];
|
||||
for key in snowball_keys {
|
||||
user_defined.insert(key.to_string(), "true".to_string());
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"stored bytes the tier keeps opaque ".repeat(64));
|
||||
set_disks
|
||||
.put_object(
|
||||
@@ -16703,6 +17073,10 @@ mod transition_commit_failure_tests {
|
||||
.await
|
||||
.expect("the tier must have received the object");
|
||||
|
||||
for key in snowball_keys {
|
||||
assert!(!remote_metadata.contains_key(key), "tier must receive opaque bytes, not {key}");
|
||||
assert_eq!(transitioned.user_defined.get(key).map(String::as_str), Some("true"));
|
||||
}
|
||||
for (key, _) in encryption_metadata {
|
||||
assert!(
|
||||
!remote_metadata.keys().any(|stored| stored.eq_ignore_ascii_case(key)),
|
||||
@@ -25075,6 +25449,131 @@ mod delete_objects_lock_gating_tests {
|
||||
.expect("plain object should be written");
|
||||
}
|
||||
|
||||
/// Regression for the PR9155 review finding: a pre-fetched source-missing
|
||||
/// read of a later duplicate must not skip the object-lock check after an
|
||||
/// earlier entry's recovery rolls back an interrupted prepared delete and
|
||||
/// restores a GOVERNANCE-retained version. Without the eviction the later
|
||||
/// duplicate deletes the restored version, bypassing retention.
|
||||
#[tokio::test]
|
||||
async fn duplicate_entries_recheck_object_lock_after_recovery_restores_retained_version() {
|
||||
let (_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "recovery-retention-dup";
|
||||
let object = "retained.bin";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
|
||||
// A versioned put, so the batch can address the version explicitly.
|
||||
let mut reader = PutObjReader::from_vec(vec![7u8; 256]);
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned object should be written");
|
||||
let version = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read back the version identity")
|
||||
.version_id
|
||||
.expect("versioned put assigns a version id");
|
||||
|
||||
// GOVERNANCE retention on that version, persisted the way
|
||||
// PutObjectRetention persists it.
|
||||
let retain_until = OffsetDateTime::now_utc() + time::Duration::days(60);
|
||||
set_disks
|
||||
.put_object_metadata(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
version_id: Some(version.to_string()),
|
||||
eval_metadata: Some(HashMap::from([
|
||||
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string()),
|
||||
(
|
||||
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
|
||||
retain_until
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.expect("retention date formats"),
|
||||
),
|
||||
])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("governance retention should persist");
|
||||
|
||||
// Interrupted authorized delete: the recoverable per-disk delete
|
||||
// applies the version removal and leaves the prepared journal
|
||||
// undecided — exactly a crash between apply and publish.
|
||||
let transaction = Uuid::new_v4();
|
||||
for disk in &disk_stores {
|
||||
let fi = disk
|
||||
.read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await
|
||||
.expect("version exists before the interrupted delete");
|
||||
let errors = disk
|
||||
.delete_versions_recoverable(
|
||||
bucket,
|
||||
vec![FileInfoVersions {
|
||||
name: object.to_string(),
|
||||
versions: vec![fi],
|
||||
..Default::default()
|
||||
}],
|
||||
DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "the plant must apply: {errors:?}");
|
||||
}
|
||||
|
||||
// A no-bypass batch naming the retained version twice: the first
|
||||
// entry rolls the prepared delete back (restoring the retained
|
||||
// version) and must be denied; the second must re-read and stay
|
||||
// denied too instead of reusing the stale pre-fetched miss.
|
||||
let victim = ObjectToDelete {
|
||||
object_name: object.to_string(),
|
||||
version_id: Some(version),
|
||||
..Default::default()
|
||||
};
|
||||
let (_deleted, errors) = set_disks
|
||||
.delete_objects(
|
||||
bucket,
|
||||
vec![victim.clone(), victim],
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors.iter().all(|error| error
|
||||
.as_ref()
|
||||
.is_some_and(|error| matches!(error, StorageError::PrefixAccessDenied(_, _)))),
|
||||
"both duplicates must be denied by the restored retention: {errors:?}"
|
||||
);
|
||||
|
||||
// The retained version survives the batch.
|
||||
for disk in &disk_stores {
|
||||
disk.read_version("", bucket, object, &version.to_string(), &crate::disk::ReadOptions::default())
|
||||
.await
|
||||
.expect("the retained version must survive the batch");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(shard_integrity_rollout)]
|
||||
async fn conditional_delete_snapshot_preserves_multipart_etag() {
|
||||
|
||||
@@ -52,8 +52,8 @@ pub(crate) mod list {
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
|
||||
MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartCompletionAuthorization, MultipartInfo, MultipartOperations,
|
||||
MultipartUploadInspection, MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::{
|
||||
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState, table_bucket_catalog_metadata_prefix},
|
||||
metadata::{BUCKET_TABLE_RESERVED_PREFIX, BUCKET_TAGGING_CONFIG, ConfigState},
|
||||
utils::is_meta_bucketname,
|
||||
};
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
@@ -145,11 +145,41 @@ async fn table_bucket_enabled_in(ctx: &crate::runtime::instance::InstanceContext
|
||||
}
|
||||
}
|
||||
|
||||
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
|
||||
[
|
||||
table_bucket_catalog_metadata_prefix(bucket),
|
||||
format!("{BUCKET_META_PREFIX}/{bucket}"),
|
||||
]
|
||||
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> Vec<String> {
|
||||
use crate::bucket::metadata::*;
|
||||
|
||||
let mut prefixes = vec![table_bucket_catalog_metadata_prefix(bucket)];
|
||||
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
|
||||
if bucket != "ahm-replacement" {
|
||||
prefixes.push(root);
|
||||
return prefixes;
|
||||
}
|
||||
|
||||
// This valid user bucket shares its metadata directory with raw drive
|
||||
// replacement intents, seals and recovery proofs. Delete only bucket-owned
|
||||
// paths, including legacy standalone configuration, never the shared root.
|
||||
// S3 objects live in the separate user volume; configuration APIs write the
|
||||
// aggregate metadata file, so clients cannot populate arbitrary siblings.
|
||||
prefixes.extend(
|
||||
[
|
||||
BUCKET_METADATA_FILE,
|
||||
BUCKET_INCARNATION_FILE,
|
||||
BUCKET_POLICY_CONFIG,
|
||||
BUCKET_QUOTA_CONFIG_FILE,
|
||||
BUCKET_TARGETS_FILE,
|
||||
BUCKET_ACL_CONFIG,
|
||||
BUCKET_TABLE_CONFIG,
|
||||
BUCKET_DURABILITY_CONFIG,
|
||||
BUCKET_ON_DEMAND_MIGRATION_CONFIG,
|
||||
".replication",
|
||||
"integrity-jobs",
|
||||
crate::data_usage::DATA_USAGE_CACHE_NAME,
|
||||
]
|
||||
.into_iter()
|
||||
.chain(XML_BUCKET_CONFIG_FILES)
|
||||
.map(|file| format!("{root}/{file}")),
|
||||
);
|
||||
prefixes
|
||||
}
|
||||
|
||||
fn bucket_deleted_marker_prefix(bucket: &str) -> String {
|
||||
@@ -160,41 +190,48 @@ fn bucket_deleted_marker_volume(bucket: &str) -> String {
|
||||
format!("{RUSTFS_META_BUCKET}/{}", bucket_deleted_marker_prefix(bucket))
|
||||
}
|
||||
|
||||
pub(crate) async fn await_bucket_namespace_operation<T, F>(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &str,
|
||||
// Box the child before constructing either lease wrapper. Keeping F inline in
|
||||
// nested select futures multiplies poll-frame temporaries along DELETE -> quota
|
||||
// cleanup -> metadata GET. This boundary must not inline back into that chain.
|
||||
#[inline(never)]
|
||||
pub(crate) fn await_bucket_namespace_operation<'a, T: 'a, F>(
|
||||
guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &'a str,
|
||||
operation: &'static str,
|
||||
future: F,
|
||||
) -> Result<T>
|
||||
) -> impl Future<Output = Result<T>> + 'a
|
||||
where
|
||||
F: Future<Output = Result<T>>,
|
||||
F: Future<Output = Result<T>> + 'a,
|
||||
{
|
||||
let Some(guard) = guard else {
|
||||
return future.await;
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost before {operation}: {bucket}"
|
||||
)));
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost during {operation}: {bucket}"
|
||||
))),
|
||||
result = future => result,
|
||||
let future = Box::pin(future);
|
||||
async move {
|
||||
let Some(guard) = guard else {
|
||||
return future.await;
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost before {operation}: {bucket}"
|
||||
)));
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = guard.lock_lost_notified() => Err(StorageError::other(format!(
|
||||
"bucket namespace lock was lost during {operation}: {bucket}"
|
||||
))),
|
||||
result = future => result,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn await_bucket_lifecycle_operation<T, F>(
|
||||
lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &str,
|
||||
fn await_bucket_lifecycle_operation<'a, T: 'a, F>(
|
||||
lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
namespace_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
bucket: &'a str,
|
||||
operation: &'static str,
|
||||
future: F,
|
||||
) -> Result<T>
|
||||
) -> impl Future<Output = Result<T>> + 'a
|
||||
where
|
||||
F: Future<Output = Result<T>>,
|
||||
F: Future<Output = Result<T>> + 'a,
|
||||
{
|
||||
await_bucket_namespace_operation(
|
||||
lifecycle_guard,
|
||||
@@ -202,7 +239,6 @@ where
|
||||
operation,
|
||||
await_bucket_namespace_operation(namespace_guard, bucket, operation, future),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn bucket_recovery_lock_unavailable(bucket: &str, object: &str) -> StorageError {
|
||||
@@ -478,10 +514,18 @@ impl ECStore {
|
||||
let mut tags = HashMap::with_capacity(tagging.tag_set.len());
|
||||
for tag in &tagging.tag_set {
|
||||
let (Some(key), Some(value)) = (&tag.key, &tag.value) else {
|
||||
return Err(Error::other("stored bucket tag is missing a key or value"));
|
||||
return Err(crate::bucket::metadata::unreadable_config_error(
|
||||
bucket,
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
metadata.tagging_config_xml.len(),
|
||||
));
|
||||
};
|
||||
if key.is_empty() || tags.insert(key.clone(), value.clone()).is_some() {
|
||||
return Err(Error::other("stored bucket tags contain empty or duplicate keys"));
|
||||
return Err(crate::bucket::metadata::unreadable_config_error(
|
||||
bucket,
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
metadata.tagging_config_xml.len(),
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(tags)
|
||||
@@ -1673,7 +1717,17 @@ impl ECStore {
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"bucket retirement publication",
|
||||
crate::bucket::retirement::commit_retirement(store, bucket, incarnation, &record_opts),
|
||||
crate::bucket::retirement::commit_retirement(store.clone(), bucket, incarnation, &record_opts),
|
||||
)
|
||||
.await?;
|
||||
// The lifecycle fence excludes live reservation owners. A crash-left
|
||||
// V1 ledger must not block quota admission after same-name recreation.
|
||||
await_bucket_lifecycle_operation(
|
||||
bucket_lifecycle_guard.as_ref(),
|
||||
ns_guard.as_ref(),
|
||||
bucket,
|
||||
"retired bucket quota ledger cleanup",
|
||||
crate::bucket::quota::reservation::cleanup_retired_bucket_ledger(store, bucket, incarnation),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1824,6 +1878,20 @@ mod tests {
|
||||
assert!(!xlmeta.diagnostic_truncated);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_namespace_operation_keeps_nested_future_state_off_the_poll_stack() {
|
||||
let payload = std::hint::black_box([7u8; 128 * 1024]);
|
||||
let operation = super::await_bucket_lifecycle_operation(None, None, "bucket", "large metadata operation", async move {
|
||||
tokio::task::yield_now().await;
|
||||
Ok(std::hint::black_box(payload)[0])
|
||||
});
|
||||
assert!(
|
||||
std::mem::size_of_val(&operation) < 1024,
|
||||
"nested lease wrappers must not retain the concrete storage future inline"
|
||||
);
|
||||
assert_eq!(operation.await.expect("the boxed operation must still execute"), 7);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bucket_namespace_operation_fails_closed_after_lease_expiry() {
|
||||
let ttl = Duration::from_millis(20);
|
||||
@@ -2240,7 +2308,13 @@ mod tests {
|
||||
let result = store.get_bucket_tags_for_policy(bucket).await;
|
||||
match expected {
|
||||
Some(tags) => assert_eq!(&result.expect("valid tag configuration"), tags),
|
||||
None => assert!(result.is_err(), "corrupt tags must not become an untagged bucket: {xml:?}"),
|
||||
None => assert!(
|
||||
crate::bucket::metadata::unreadable_config_refusal(
|
||||
&result.expect_err("corrupt tags must not become an untagged bucket")
|
||||
)
|
||||
.is_some_and(|failure| failure.config_file == super::BUCKET_TAGGING_CONFIG),
|
||||
"corrupt tags must carry a typed tagging refusal: {xml:?}"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2442,6 +2516,187 @@ mod tests {
|
||||
assert!(prefixes.contains(&"buckets/analytics".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replacement_named_bucket_delete_preserves_recovery_and_recreates_clean_metadata() {
|
||||
use crate::bucket::metadata::{BUCKET_QUOTA_CONFIG_FILE, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG};
|
||||
|
||||
let (_temp, store) = setup_bucket_quorum_test_env(&[4], None).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "ahm-replacement";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create legal bucket");
|
||||
let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
|
||||
for (config, bytes) in [
|
||||
(BUCKET_QUOTA_CONFIG_FILE, br#"{"quota":1024,"quota_type":"Hard"}"#.as_slice()),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
b"<Tagging><TagSet><Tag><Key>old</Key><Value>value</Value></Tag></TagSet></Tagging>".as_slice(),
|
||||
),
|
||||
(
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".as_slice(),
|
||||
),
|
||||
] {
|
||||
metadata_sys::update_in(&store.ctx, bucket, config, bytes.to_vec())
|
||||
.await
|
||||
.expect("persist old configuration");
|
||||
}
|
||||
let ledger_path = format!("config/quota-ledger/{bucket}.json");
|
||||
// V1 precharges outstanding growth: four live bytes plus the
|
||||
// crash-left reservation's 100 bytes must already be accounted.
|
||||
let ledger = serde_json::to_vec(&serde_json::json!({
|
||||
"version": 1, "bucket_incarnation": old_incarnation,
|
||||
"quota_revision_unix_nanos": 0, "accounted_usage": 104,
|
||||
"reservations": {Uuid::new_v4().to_string(): {
|
||||
"object": "crashed-put", "old_size": 0, "new_size": 100,
|
||||
"created_at": 0, "pool_index": 0, "set_index": 0, "commit_started": false
|
||||
}}, "reconcile_required": true
|
||||
}))
|
||||
.expect("encode crash-left ledger");
|
||||
|
||||
// The storage API requires the same legacy-quota admission supplied
|
||||
// by the S3 handler, even for this small fixture object.
|
||||
let mut live_options = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(live_options.set_quota_admission(0, 1024));
|
||||
// Failed deletion must leave both the live incarnation and its ledger alone.
|
||||
store
|
||||
.put_object(bucket, "live", &mut PutObjReader::from_vec(b"live".to_vec()), &live_options)
|
||||
.await
|
||||
.expect("write live object");
|
||||
crate::config::com::save_config(store.clone(), &ledger_path, ledger.clone())
|
||||
.await
|
||||
.expect("persist old ledger");
|
||||
assert!(matches!(
|
||||
store.delete_bucket(bucket, &DeleteBucketOptions::default()).await,
|
||||
Err(StorageError::BucketNotEmpty(_))
|
||||
));
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(store.clone(), &ledger_path)
|
||||
.await
|
||||
.expect("live ledger remains"),
|
||||
ledger
|
||||
);
|
||||
// Remove every version; the configured bucket is otherwise still empty.
|
||||
let versions = store
|
||||
.clone()
|
||||
.list_object_versions(bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("list live versions");
|
||||
for object in versions.objects {
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
&object.name,
|
||||
ObjectOptions {
|
||||
version_id: object.version_id.map(|id| id.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove live version");
|
||||
}
|
||||
|
||||
let root = format!("{BUCKET_META_PREFIX}/{bucket}");
|
||||
let task = Uuid::new_v4();
|
||||
let recovery_names = [
|
||||
"ahm_replacement_intent.json",
|
||||
"ahm_replacement_intent_seal",
|
||||
"ahm_replacement_completion_proof.json",
|
||||
]
|
||||
.map(|suffix| format!("{task}_{suffix}"));
|
||||
let disk_paths = store.pools[0].disk_set[0]
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|disk| PathBuf::from(disk.endpoint().to_string()))
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disk_paths {
|
||||
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
|
||||
for name in &recovery_names {
|
||||
tokio::fs::write(directory.join(name), b"durable heal record")
|
||||
.await
|
||||
.expect("seed raw recovery record");
|
||||
}
|
||||
// Legacy standalone settings and bucket-owned subtrees must not survive.
|
||||
for name in [
|
||||
"quota.json",
|
||||
"versioning.xml",
|
||||
"policy.json",
|
||||
"bucket-targets.json",
|
||||
"integrity-jobs/job.json",
|
||||
".replication/resync.bin",
|
||||
".usage-cache.bin",
|
||||
] {
|
||||
let path = directory.join(name);
|
||||
tokio::fs::create_dir_all(path.parent().expect("fixture parent"))
|
||||
.await
|
||||
.expect("legacy parent");
|
||||
tokio::fs::write(path, b"old bucket configuration")
|
||||
.await
|
||||
.expect("seed legacy configuration");
|
||||
}
|
||||
}
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete legal same-name bucket");
|
||||
assert!(matches!(
|
||||
crate::config::com::read_config(store.clone(), &ledger_path).await,
|
||||
Err(StorageError::ConfigNotFound)
|
||||
));
|
||||
for disk in &disk_paths {
|
||||
let directory = disk.join(RUSTFS_META_BUCKET).join(&root);
|
||||
for name in &recovery_names {
|
||||
assert_eq!(
|
||||
tokio::fs::read(directory.join(name)).await.expect("recovery record survives"),
|
||||
b"durable heal record"
|
||||
);
|
||||
}
|
||||
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket)
|
||||
.into_iter()
|
||||
.filter(|p| p.starts_with(&root))
|
||||
{
|
||||
assert!(
|
||||
!disk.join(RUSTFS_META_BUCKET).join(prefix).exists(),
|
||||
"bucket-owned metadata must be removed"
|
||||
);
|
||||
}
|
||||
}
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate legal bucket");
|
||||
let fresh = metadata_sys::get_in(&store.ctx, bucket).await.expect("fresh metadata");
|
||||
assert_ne!(fresh.bucket_incarnation_id, old_incarnation);
|
||||
assert!(fresh.quota_config_json.is_empty());
|
||||
assert!(fresh.tagging_config_xml.is_empty());
|
||||
assert!(fresh.versioning_config_xml.is_empty());
|
||||
assert!(fresh.policy_config_json.is_empty());
|
||||
// The strict purge/recovery cleanup must share the same ownership boundary.
|
||||
store
|
||||
.cleanup_deleted_bucket_metadata_strict(bucket, true)
|
||||
.await
|
||||
.expect("strict metadata cleanup");
|
||||
for disk in &disk_paths {
|
||||
for name in &recovery_names {
|
||||
assert_eq!(
|
||||
tokio::fs::read(disk.join(RUSTFS_META_BUCKET).join(&root).join(name))
|
||||
.await
|
||||
.expect("strict cleanup preserves recovery"),
|
||||
b"durable heal record"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_delete_marker_path_uses_internal_deleted_bucket_metadata_prefix() {
|
||||
assert_eq!(bucket_deleted_marker_prefix("analytics"), "buckets/.deleted/analytics");
|
||||
|
||||
@@ -368,7 +368,7 @@ impl ECStore {
|
||||
Some(expected) => expected,
|
||||
None => self.bucket_incarnation_id_from_disk(bucket).await?,
|
||||
};
|
||||
let object = object.map(str::to_owned);
|
||||
let object = object.map(encode_dir_object);
|
||||
let prefix = prefix.map(str::to_owned);
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
|
||||
let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
|
||||
@@ -430,7 +430,7 @@ impl ECStore {
|
||||
HealItemType::Bucket
|
||||
}
|
||||
.to_string(),
|
||||
object: object.or(prefix).unwrap_or_default(),
|
||||
object: object.map(|object| decode_dir_object(&object)).or(prefix).unwrap_or_default(),
|
||||
bucket,
|
||||
detail: format!(
|
||||
"delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}, failed={}",
|
||||
@@ -656,12 +656,48 @@ impl ECStore {
|
||||
Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
|
||||
}
|
||||
|
||||
/// Read the published bucket identity without fabricating or migrating metadata.
|
||||
/// Absence is distinct from quorum, corruption and transaction-fence failures.
|
||||
pub async fn replacement_bucket_incarnation_from_disk(self: &Arc<Self>, bucket: &str) -> Result<Option<uuid::Uuid>> {
|
||||
let transaction =
|
||||
crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
let incarnation = crate::bucket::metadata::load_bucket_incarnation(self.clone(), bucket).await?;
|
||||
if transaction.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(incarnation)
|
||||
}
|
||||
|
||||
/// Restore the bucket configuration and incarnation before retiring a replacement marker.
|
||||
pub async fn heal_replacement_bucket_metadata(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
// Reject an invalid request before touching the drives so callers keep
|
||||
// the PreconditionFailed contract for bad buckets and options.
|
||||
if opts.pool.is_none()
|
||||
|| opts.set.is_none()
|
||||
|| targets.is_empty()
|
||||
|| opts.dry_run
|
||||
|| opts.no_lock
|
||||
|| check_valid_bucket_name_strict(bucket).is_err()
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let expected = self.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
self.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, opts, targets)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Keep a replacement pass bound to its admitted bucket through metadata commit.
|
||||
pub async fn heal_replacement_bucket_metadata_at_incarnation(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
expected: uuid::Uuid,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
|
||||
use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
|
||||
@@ -698,9 +734,8 @@ impl ECStore {
|
||||
|
||||
// Read the persisted identity without lazy migration: a replacement
|
||||
// must never turn lost configuration into newly fabricated defaults.
|
||||
let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let targets = targets.to_vec();
|
||||
self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, move |store, bucket, opts| async move {
|
||||
// Match config writers: lifecycle, metadata transaction, object locks.
|
||||
// Keep the transaction stable through target readback, including when
|
||||
// the caller is cancelled and the storage owner finishes its write.
|
||||
@@ -1022,6 +1057,7 @@ impl ECStore {
|
||||
set_index: usize,
|
||||
targets: &[String],
|
||||
) -> Result<bool> {
|
||||
let object = encode_dir_object(object);
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -1034,7 +1070,7 @@ impl ECStore {
|
||||
)
|
||||
})?;
|
||||
|
||||
set.replacement_targets_have_version(bucket, object, version_id, targets)
|
||||
set.replacement_targets_have_version(bucket, &object, version_id, targets)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
@@ -2859,6 +2895,105 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn replacement_directory_object_readback_and_exact_recovery_use_encoded_key() {
|
||||
let (_root, store, shutdown) = heal_store_with_pool_count(1).await;
|
||||
let bucket = "directory-replacement-readback";
|
||||
let object = "folder/";
|
||||
let encoded = encode_dir_object(object);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(Vec::new()),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write directory object through the public store entry");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let targets = vec![set.set_endpoints[0].to_string()];
|
||||
assert!(
|
||||
store
|
||||
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("readback")
|
||||
);
|
||||
let disk = set.disks.read().await[0].clone().expect("target disk");
|
||||
disk.delete(
|
||||
bucket,
|
||||
&encoded,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove target directory shard");
|
||||
assert!(
|
||||
!store
|
||||
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("missing target")
|
||||
);
|
||||
let opts = HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
let (result, error) = store
|
||||
.heal_object(bucket, object, "", &opts)
|
||||
.await
|
||||
.expect("heal directory object");
|
||||
assert!(error.is_none(), "heal failed: {error:?}");
|
||||
assert_eq!(result.object, object);
|
||||
assert!(
|
||||
store
|
||||
.replacement_targets_have_version(bucket, object, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("healed readback")
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.replacement_targets_have_version(bucket, &encoded, "", 0, 0, &targets)
|
||||
.await
|
||||
.expect("encoded readback")
|
||||
);
|
||||
|
||||
// Exact-object recovery must inspect the same physical namespace and
|
||||
// reject corrupt proof rather than silently report no transactions.
|
||||
let metadata = disk
|
||||
.read_all(bucket, &format!("{encoded}/xl.meta"))
|
||||
.await
|
||||
.expect("healed metadata");
|
||||
disk.write_all(
|
||||
bucket,
|
||||
&format!("{encoded}/{}/delete-prepared", Uuid::new_v4()),
|
||||
b"invalid proof".to_vec().into(),
|
||||
)
|
||||
.await
|
||||
.expect("seed corrupt recovery proof");
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some(object), None, &HealOpts { dry_run: true, ..opts })
|
||||
.await
|
||||
.expect_err("directory recovery must discover and reject the corrupt proof");
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, &format!("{encoded}/xl.meta"))
|
||||
.await
|
||||
.expect("unchanged metadata"),
|
||||
metadata
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
|
||||
+298
-278
@@ -5155,6 +5155,11 @@ mod tests {
|
||||
let offline_disk = force_set_disk_range_offline_for_test(set, 0..1).await;
|
||||
let _without_proof = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
|
||||
assert!(crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_none());
|
||||
if !sharded {
|
||||
crate::bucket::quota::reservation::initialize_quota_ledger_for_test(Arc::clone(&store), &bucket)
|
||||
.await
|
||||
.expect("bootstrap quota with one disk offline");
|
||||
}
|
||||
|
||||
let payload = vec![0x51; 4096];
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
@@ -21007,6 +21012,299 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn seed_retained_null_version(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
protection: &str,
|
||||
latest_is_marker: bool,
|
||||
) {
|
||||
let metadata = if protection == "legal-hold" {
|
||||
HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
|
||||
} else {
|
||||
HashMap::from([
|
||||
("x-amz-object-lock-mode".to_string(), protection.to_string()),
|
||||
("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
|
||||
])
|
||||
};
|
||||
let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut original,
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed retained null version");
|
||||
let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut latest,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed newer unprotected UUID version");
|
||||
if latest_is_marker {
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a UUID delete marker above the retained null version");
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retained null version must remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read retained null version");
|
||||
assert_eq!(body, b"protected null version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_delete_checks_actual_target() {
|
||||
use crate::bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
|
||||
|
||||
let temp = tempfile::tempdir().expect("create null-version delete store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
let configs = [
|
||||
VersioningConfiguration::default(),
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
for (case, config) in configs.into_iter().enumerate() {
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("archive/{case}-{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
|
||||
"batch delete must judge the retained null version: {object}: {errors:?}"
|
||||
);
|
||||
let error = store
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&object,
|
||||
ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("suspended single delete must also judge the retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
if protection == "GOVERNANCE" {
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
|
||||
bypass_governance: true,
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
|
||||
let removed = store
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
&object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
match removed {
|
||||
Ok(marker) => {
|
||||
assert!(marker.delete_marker, "only a null delete marker may replace the payload");
|
||||
assert_eq!(marker.size, 0);
|
||||
assert!(marker.data_dir.is_none());
|
||||
}
|
||||
Err(error) => assert!(
|
||||
matches!(
|
||||
error,
|
||||
StorageError::MethodNotAllowed
|
||||
| StorageError::ObjectNotFound(_, _)
|
||||
| StorageError::VersionNotFound(_, _, _)
|
||||
),
|
||||
"unexpected bypass result: {error:?}"
|
||||
),
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.get_object_reader(
|
||||
&bucket,
|
||||
&object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"authorized deletion must make the retained null payload unreadable"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
|
||||
let temp = tempfile::tempdir().expect("create null-version overwrite store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
|
||||
let error = store
|
||||
.put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("PUT must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
|
||||
let upload = store
|
||||
.clone()
|
||||
.new_multipart_upload(&bucket, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
|
||||
let part = store
|
||||
.clone()
|
||||
.put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart replacement");
|
||||
let error = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("multipart completion must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
|
||||
let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
&object,
|
||||
&mut next_version,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a new UUID version must remain allowed");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() {
|
||||
@@ -27352,282 +27650,4 @@ mod tests {
|
||||
.expect("recovery must preserve the local source object");
|
||||
assert_eq!(source_after.size, i64::try_from(payload.len()).expect("payload length should fit i64"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn seed_retained_null_version(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
protection: &str,
|
||||
latest_is_marker: bool,
|
||||
) {
|
||||
let metadata = if protection == "legal-hold" {
|
||||
HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())])
|
||||
} else {
|
||||
HashMap::from([
|
||||
("x-amz-object-lock-mode".to_string(), protection.to_string()),
|
||||
("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()),
|
||||
])
|
||||
};
|
||||
let mut original = PutObjReader::from_vec(b"protected null version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut original,
|
||||
&ObjectOptions {
|
||||
user_defined: metadata,
|
||||
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed retained null version");
|
||||
let mut latest = PutObjReader::from_vec(b"unprotected latest version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut latest,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed newer unprotected UUID version");
|
||||
if latest_is_marker {
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create a UUID delete marker above the retained null version");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn assert_retained_null_version_body(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("retained null version must remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read retained null version");
|
||||
assert_eq!(body, b"protected null version");
|
||||
}
|
||||
|
||||
/// A versionless delete removes the null version, so the Object Lock check
|
||||
/// must judge that version even when a UUID version or delete marker is
|
||||
/// the key's latest entry (rustfs/backlog#3017).
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_delete_checks_actual_target() {
|
||||
use crate::bucket::replication::DeleteReplicationConfigSnapshot;
|
||||
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
|
||||
|
||||
let temp = tempfile::tempdir().expect("create null-version delete store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-delete-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-delete-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
let configs = [
|
||||
VersioningConfiguration::default(),
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
excluded_prefixes: Some(vec![ExcludedPrefix {
|
||||
prefix: Some("archive/".to_string()),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
for (case, config) in configs.into_iter().enumerate() {
|
||||
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(config, None));
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("archive/{case}-{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(errors.as_slice(), [Some(StorageError::PrefixAccessDenied(_, _))]),
|
||||
"batch delete must judge the retained null version: {object}: {errors:?}"
|
||||
);
|
||||
let error = store
|
||||
.delete_object(
|
||||
&bucket,
|
||||
&object,
|
||||
ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("suspended single delete must also judge the retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
if protection == "GOVERNANCE" {
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&snapshot)),
|
||||
object_lock_delete: Some(crate::storage_api_contracts::object::ObjectLockDeleteOptions {
|
||||
bypass_governance: true,
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "authorized governance bypass: {errors:?}");
|
||||
assert!(
|
||||
store
|
||||
.get_object_reader(
|
||||
&bucket,
|
||||
&object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(Uuid::nil().to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"authorized deletion must make the retained null payload unreadable"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A versionless PUT or multipart completion replaces the null version in
|
||||
/// place, so the commit-time WORM gate must judge that version even when a
|
||||
/// UUID version or delete marker is the key's latest entry
|
||||
/// (rustfs/backlog#3019).
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn null_version_object_lock_put_and_multipart_preserve_retained_target() {
|
||||
let temp = tempfile::tempdir().expect("create null-version overwrite store");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp.path(), "null-version-overwrite-lock", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("null-overwrite-lock-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create Object Lock bucket");
|
||||
for protection in ["COMPLIANCE", "GOVERNANCE", "legal-hold"] {
|
||||
for latest_is_marker in [false, true] {
|
||||
let object = format!("{protection}-{latest_is_marker}");
|
||||
seed_retained_null_version(&store, &bucket, &object, protection, latest_is_marker).await;
|
||||
let mut replacement = PutObjReader::from_vec(b"replacement".to_vec());
|
||||
let error = store
|
||||
.put_object(&bucket, &object, &mut replacement, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("PUT must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
|
||||
let upload = store
|
||||
.clone()
|
||||
.new_multipart_upload(&bucket, &object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(b"multipart replacement".to_vec());
|
||||
let part = store
|
||||
.clone()
|
||||
.put_object_part(&bucket, &object, &upload.upload_id, 1, &mut part_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart replacement");
|
||||
let error = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
&object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("multipart completion must reject replacement of a retained null version");
|
||||
assert!(matches!(error, StorageError::PrefixAccessDenied(_, _)), "{object}: {error:?}");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
|
||||
let mut next_version = PutObjReader::from_vec(b"new UUID version".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
&object,
|
||||
&mut next_version,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a new UUID version must remain allowed");
|
||||
assert_retained_null_version_body(&store, &bucket, &object).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -423,8 +423,6 @@ fn should_enqueue_transition_immediately(oi: &ObjectInfo) -> bool {
|
||||
!is_meta_bucketname(&oi.bucket)
|
||||
}
|
||||
|
||||
const MAX_UPLOADS_LIST: usize = 10000;
|
||||
|
||||
mod bucket;
|
||||
mod bucket_fence;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
|
||||
@@ -273,6 +273,28 @@ async fn pause_data_movement_multipart_before_selected_completion(bucket: &str)
|
||||
}
|
||||
}
|
||||
|
||||
async fn pool_has_multipart_upload_for_key(
|
||||
pool: &crate::core::sets::Sets,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<bool> {
|
||||
let results = stream::iter(pool.disk_set.iter().cloned())
|
||||
.map(|set| async move {
|
||||
set.has_multipart_upload_for_key(bucket, object, expected_incarnation_id)
|
||||
.await
|
||||
})
|
||||
.buffer_unordered(MULTIPART_LIST_SET_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
let mut found = false;
|
||||
for result in results {
|
||||
// Preserve errors from every set instead of hiding them behind a hit.
|
||||
found |= result?;
|
||||
}
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
async fn list_pool_multipart_uploads_for_incarnation(
|
||||
pool: &crate::core::sets::Sets,
|
||||
bucket: &str,
|
||||
@@ -490,6 +512,8 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ListPartsInfo> {
|
||||
check_list_parts_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
|
||||
@@ -620,6 +644,8 @@ impl ECStore {
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
|
||||
check_new_object_args(bucket, object, opts.data_movement)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
if self.single_pool() {
|
||||
@@ -675,21 +701,7 @@ impl ECStore {
|
||||
if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
}
|
||||
let res = list_pool_multipart_uploads_for_incarnation(
|
||||
pool,
|
||||
bucket,
|
||||
&MultipartUploadListRequest {
|
||||
prefix: object.to_string(),
|
||||
key_marker: None,
|
||||
upload_id_marker: None,
|
||||
delimiter: None,
|
||||
max_uploads: MAX_UPLOADS_LIST,
|
||||
expected_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !res.uploads.is_empty() {
|
||||
if pool_has_multipart_upload_for_key(pool, bucket, object, opts.expected_bucket_incarnation_id).await? {
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
|
||||
@@ -772,6 +784,8 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if let Some(guard) = bucket_lifecycle_guard {
|
||||
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
|
||||
@@ -806,6 +820,8 @@ impl ECStore {
|
||||
.part_number
|
||||
.ok_or_else(|| Error::other("targeted multipart upload requires a part number"))?;
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart upload requires data_movement options"));
|
||||
}
|
||||
@@ -838,6 +854,8 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<MultipartInfo> {
|
||||
check_list_parts_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
if self.single_pool() {
|
||||
@@ -871,6 +889,8 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
@@ -1052,6 +1072,8 @@ impl ECStore {
|
||||
for upload_id in upload_ids {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
}
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart abort requires data_movement options"));
|
||||
}
|
||||
@@ -1346,6 +1368,8 @@ impl ECStore {
|
||||
let request_opts = opts;
|
||||
let (target_pool_idx, mutation_fence) = target;
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
let object = encoded_object.as_ref();
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||
}
|
||||
@@ -1467,6 +1491,362 @@ mod tests {
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_multipart_pool_selection_uses_only_an_exact_key_upload() {
|
||||
use crate::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
|
||||
let mut dirs = Vec::new();
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let directory = tempfile::tempdir().expect("temporary disk");
|
||||
let mut endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
dirs.push(directory);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("exact-multipart-pool-{pool_index}"),
|
||||
platform: "test".to_owned(),
|
||||
});
|
||||
}
|
||||
let endpoints = EndpointServerPools(pools);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
|
||||
.await
|
||||
.expect("initialize local disks");
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("test address"), endpoints, cancel.clone(), ctx)
|
||||
.await
|
||||
.expect("initialize two pools");
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "exact-pool-selection";
|
||||
let other_bucket = "other-pool-selection";
|
||||
for bucket in [bucket, other_bucket] {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
for (upload_bucket, key) in [(bucket, "report-long"), (bucket, "report/"), (other_bucket, "report")] {
|
||||
let (opts, _guard) = store
|
||||
.guard_multipart_bucket_incarnation(upload_bucket, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("pin upload incarnation");
|
||||
store.pools[0]
|
||||
.new_multipart_upload(upload_bucket, &encode_dir_object(key), &opts)
|
||||
.await
|
||||
.expect("seed nonmatching upload in first pool");
|
||||
}
|
||||
let (opts, _guard) = store
|
||||
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("pin target incarnation");
|
||||
assert!(
|
||||
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", opts.expected_bucket_incarnation_id)
|
||||
.await
|
||||
.expect("first pool contains only other keys")
|
||||
);
|
||||
store.pools[1]
|
||||
.new_multipart_upload(bucket, "report", &opts)
|
||||
.await
|
||||
.expect("seed exact upload in second pool");
|
||||
let (created, pool_index, incarnation) = store
|
||||
.handle_new_multipart_upload_with_pool_idx(bucket, "report", &ObjectOptions::default(), None)
|
||||
.await
|
||||
.expect("create another upload for the exact key");
|
||||
assert_eq!(pool_index, 1, "prefix matches in the first pool must not change placement");
|
||||
assert_eq!(incarnation, opts.expected_bucket_incarnation_id);
|
||||
assert!(!created.upload_id.is_empty());
|
||||
assert!(
|
||||
!pool_has_multipart_upload_for_key(&store.pools[0], bucket, "report", incarnation)
|
||||
.await
|
||||
.expect("new upload must not appear in first pool")
|
||||
);
|
||||
assert!(
|
||||
!pool_has_multipart_upload_for_key(&store.pools[1], bucket, "report", Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("old bucket incarnations must not affect placement")
|
||||
);
|
||||
cancel.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_directory_keys_preserve_the_bare_object_and_round_trip() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use sha2::Digest;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
for versioned in [false, true] {
|
||||
let bucket = if versioned {
|
||||
"directory-mpu-versioned"
|
||||
} else {
|
||||
"directory-mpu-unversioned"
|
||||
};
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
if versioned {
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
bucket,
|
||||
"versioning.xml",
|
||||
br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("enable versioning");
|
||||
}
|
||||
let opts = ObjectOptions {
|
||||
versioned,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
let original = store
|
||||
.put_object(
|
||||
bucket,
|
||||
"protected",
|
||||
&mut PutObjReader::from_vec(b"private original".to_vec()),
|
||||
&ObjectOptions {
|
||||
user_defined: HashMap::from([(
|
||||
rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(),
|
||||
"classification=secret".to_string(),
|
||||
)]),
|
||||
..opts.clone()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write protected bare object");
|
||||
let upload = store
|
||||
.new_multipart_upload(bucket, "protected/", &opts)
|
||||
.await
|
||||
.expect("create directory upload");
|
||||
let part = store
|
||||
.put_object_part(
|
||||
bucket,
|
||||
"protected/",
|
||||
&upload.upload_id,
|
||||
1,
|
||||
&mut PutObjReader::from_vec(b"directory payload".to_vec()),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("write directory part");
|
||||
store
|
||||
.put_object_part(
|
||||
bucket,
|
||||
"protected/",
|
||||
&upload.upload_id,
|
||||
2,
|
||||
&mut PutObjReader::from_vec(b"unselected private bytes".to_vec()),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("write an unselected part");
|
||||
let upload_sha = hex_simd::encode_to_string(
|
||||
sha2::Sha256::digest(format!("{bucket}/protected__XLDIR__").as_bytes()),
|
||||
hex_simd::AsciiCase::Lower,
|
||||
);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
for dir in &dirs {
|
||||
let staging = dir
|
||||
.path()
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid);
|
||||
let metadata = rustfs_filemeta::FileMeta::load(&std::fs::read(staging.join("xl.meta")).expect("upload metadata"))
|
||||
.expect("decode upload metadata");
|
||||
let versions = metadata
|
||||
.get_file_info_versions(bucket, "protected__XLDIR__", true)
|
||||
.expect("upload file info");
|
||||
let data_dir = versions
|
||||
.versions
|
||||
.first()
|
||||
.expect("upload version")
|
||||
.data_dir
|
||||
.expect("staging data directory");
|
||||
assert!(
|
||||
staging.join(data_dir.to_string()).join("part.2").is_file(),
|
||||
"unselected part exists before completion"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
store
|
||||
.get_multipart_info(bucket, "protected/", &upload.upload_id, &opts)
|
||||
.await
|
||||
.expect("upload info")
|
||||
.object,
|
||||
"protected/"
|
||||
);
|
||||
assert_eq!(
|
||||
store
|
||||
.list_object_parts(bucket, "protected/", &upload.upload_id, None, 10, &opts)
|
||||
.await
|
||||
.expect("list parts")
|
||||
.object,
|
||||
"protected/"
|
||||
);
|
||||
let uploads = store
|
||||
.list_multipart_uploads(bucket, "protected/", None, None, None, 10)
|
||||
.await
|
||||
.expect("list logical prefix");
|
||||
assert_eq!(uploads.uploads.len(), 1);
|
||||
assert_eq!(uploads.uploads[0].object, "protected/");
|
||||
assert_eq!(crate::runtime::sources::upload_uuid_suffix(&uploads.uploads[0].upload_id), upload_uuid,);
|
||||
let completed = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
"protected/",
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("complete directory upload");
|
||||
assert_eq!(completed.name, "protected/");
|
||||
for (key, expected) in [
|
||||
("protected", b"private original".as_slice()),
|
||||
("protected/", b"directory payload".as_slice()),
|
||||
] {
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, key, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read distinct object");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read body");
|
||||
assert_eq!(bytes, expected);
|
||||
}
|
||||
let preserved = store
|
||||
.get_object_info(bucket, "protected", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("bare metadata");
|
||||
assert_eq!(preserved.version_id, original.version_id);
|
||||
assert_eq!(preserved.etag, original.etag);
|
||||
assert_eq!(preserved.user_tags.as_str(), "classification=secret");
|
||||
for dir in &dirs {
|
||||
assert!(dir.path().join(bucket).join("protected/xl.meta").is_file());
|
||||
assert!(dir.path().join(bucket).join("protected__XLDIR__/xl.meta").is_file());
|
||||
let data = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("protected__XLDIR__")
|
||||
.join(completed.data_dir.expect("multipart data directory").to_string());
|
||||
assert!(data.join("part.1").is_file());
|
||||
assert!(!data.join("part.2").exists(), "unselected part must never be published");
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid)
|
||||
.exists(),
|
||||
"original staging must be cleaned"
|
||||
);
|
||||
}
|
||||
let abandoned = store
|
||||
.new_multipart_upload(bucket, "protected/", &opts)
|
||||
.await
|
||||
.expect("create abort control");
|
||||
store
|
||||
.abort_multipart_upload(bucket, "protected/", &abandoned.upload_id, &opts)
|
||||
.await
|
||||
.expect("abort encoded upload");
|
||||
assert!(
|
||||
store
|
||||
.get_multipart_info(bucket, "protected/", &abandoned.upload_id, &opts)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(bucket, "protected/", &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn new_control_character_keys_are_rejected_without_trapping_legacy_objects() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "xml-key-admission";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
for byte in (1u8..32).filter(|byte| !matches!(byte, 9 | 10 | 13)) {
|
||||
let key = format!("bad-{}", char::from(byte));
|
||||
assert!(
|
||||
store
|
||||
.put_object(bucket, &key, &mut PutObjReader::from_vec(b"body".to_vec()), &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.new_multipart_upload(bucket, &key, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for dir in &dirs {
|
||||
assert!(!dir.path().join(bucket).join(&key).exists());
|
||||
}
|
||||
}
|
||||
let legacy_key = "legacy-\u{1}";
|
||||
store.pools[0]
|
||||
.get_disks_by_key(legacy_key)
|
||||
.put_object(
|
||||
bucket,
|
||||
legacy_key,
|
||||
&mut PutObjReader::from_vec(b"legacy payload".to_vec()),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed a pre-upgrade object below new-key admission");
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, legacy_key, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy GET stays usable");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("legacy bytes");
|
||||
assert_eq!(bytes, b"legacy payload");
|
||||
drop(reader);
|
||||
store
|
||||
.delete_object(bucket, legacy_key, ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy DELETE stays usable");
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(bucket, legacy_key, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
fn mp(object: &str, upload_id: &str) -> MultipartInfo {
|
||||
MultipartInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
|
||||
+616
-164
File diff suppressed because it is too large
Load Diff
@@ -20,14 +20,230 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use storage_api::contract_compat::ECStore;
|
||||
use storage_api::metadata_lock::{
|
||||
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
|
||||
isolated_store_over_temp_disks,
|
||||
BucketOperations, CompletePart, DeleteBucketOptions, Error, HashReader, MakeBucketOptions, MultipartOperations,
|
||||
NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause,
|
||||
init_bucket_metadata_sys, isolated_store_over_temp_disks,
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct ObservedPutBody {
|
||||
inner: tokio::io::DuplexStream,
|
||||
polled: Arc<tokio::sync::Notify>,
|
||||
}
|
||||
|
||||
impl tokio::io::AsyncRead for ObservedPutBody {
|
||||
fn poll_read(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
buf: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
self.polled.notify_one();
|
||||
std::pin::Pin::new(&mut self.inner).poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn destructive_put_takes_bucket_locks_only_after_its_body_is_staged() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
for (object, version_id) in [("object", None), ("directory/", Some(Uuid::nil().to_string()))] {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "deferred-put-locks";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let encoded_object = if object.ends_with('/') { "directory__XLDIR__" } else { object };
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, encoded_object, PutObjectCommitPause::BeforeQuotaRename);
|
||||
let (mut sender, inner) = tokio::io::duplex(16);
|
||||
let polled = Arc::new(tokio::sync::Notify::new());
|
||||
let body = ObservedPutBody {
|
||||
inner,
|
||||
polled: polled.clone(),
|
||||
};
|
||||
let put_store = store.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
|
||||
put_store
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
version_id,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(5), polled.notified())
|
||||
.await
|
||||
.expect("PUT must be waiting for client data");
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("metadata lock");
|
||||
drop(
|
||||
timeout(Duration::from_secs(2), lock.get_write_lock(Duration::from_secs(5)))
|
||||
.await
|
||||
.expect("body must not block metadata writers")
|
||||
.expect("metadata write guard"),
|
||||
);
|
||||
drop(
|
||||
timeout(Duration::from_secs(2), store.lock_bucket_for_recursive_delete(bucket))
|
||||
.await
|
||||
.expect("body must not retain lifecycle guard")
|
||||
.expect("lifecycle writer"),
|
||||
);
|
||||
sender.write_all(b"body").await.expect("finish body");
|
||||
sender.shutdown().await.expect("EOF");
|
||||
timeout(Duration::from_secs(5), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("staged PUT reaches commit");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
assert!(futures::poll!(&mut writer).is_pending(), "commit must fence metadata updates");
|
||||
let lifecycle = store.lock_bucket_for_recursive_delete(bucket);
|
||||
tokio::pin!(lifecycle);
|
||||
assert!(futures::poll!(&mut lifecycle).is_pending(), "commit must fence bucket deletion");
|
||||
barrier.release();
|
||||
let release_metadata = async {
|
||||
drop(writer.await.expect("metadata writer"));
|
||||
};
|
||||
let release_lifecycle = async {
|
||||
drop(lifecycle.await.expect("lifecycle writer"));
|
||||
};
|
||||
timeout(Duration::from_secs(5), async {
|
||||
let (put, (), ()) = tokio::join!(put, release_metadata, release_lifecycle);
|
||||
put.expect("owner task").expect("commit");
|
||||
})
|
||||
.await
|
||||
.expect("PUT and queued writers finish");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_put_cannot_publish_into_a_recreated_bucket() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "deferred-put-generation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let (mut sender, inner) = tokio::io::duplex(16);
|
||||
let polled = Arc::new(tokio::sync::Notify::new());
|
||||
let body = ObservedPutBody {
|
||||
inner,
|
||||
polled: polled.clone(),
|
||||
};
|
||||
let put_store = store.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
|
||||
put_store
|
||||
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
|
||||
store
|
||||
.delete_bucket(
|
||||
bucket,
|
||||
&DeleteBucketOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove old bucket while PUT stalls");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
sender.write_all(b"body").await.expect("finish body");
|
||||
sender.shutdown().await.expect("EOF");
|
||||
let error = timeout(Duration::from_secs(5), put)
|
||||
.await
|
||||
.expect("commit gate")
|
||||
.expect("owner")
|
||||
.expect_err("old request generation must not publish");
|
||||
assert!(
|
||||
error.to_string().contains("generation") || matches!(error, Error::BucketNotFound(..)),
|
||||
"{error:?}"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_object_info(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_put_rechecks_current_object_lock_at_commit() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "deferred-put-retention";
|
||||
store
|
||||
.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("locked bucket");
|
||||
let mut old = PutObjReader::from_vec(b"old".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut old, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("unlocked predecessor");
|
||||
let (mut sender, inner) = tokio::io::duplex(16);
|
||||
let polled = Arc::new(tokio::sync::Notify::new());
|
||||
let body = ObservedPutBody {
|
||||
inner,
|
||||
polled: polled.clone(),
|
||||
};
|
||||
let put_store = store.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::new(HashReader::from_stream(body, 4, 4, None, None, false).expect("body"));
|
||||
put_store
|
||||
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(5), polled.notified()).await.expect("body paused");
|
||||
let mut protected = PutObjReader::from_vec(b"protected".to_vec());
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
"object",
|
||||
&mut protected,
|
||||
&ObjectOptions {
|
||||
user_defined: std::collections::HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("install a protected predecessor while the client stalls");
|
||||
sender.write_all(b"body").await.expect("finish stale body");
|
||||
sender.shutdown().await.expect("EOF");
|
||||
let error = timeout(Duration::from_secs(5), put)
|
||||
.await
|
||||
.expect("commit gate")
|
||||
.expect("task")
|
||||
.expect_err("pre-body state cannot admit an overwrite of the newly protected version");
|
||||
assert!(matches!(error, Error::PrefixAccessDenied(..)), "{error:?}");
|
||||
let info = store
|
||||
.get_object_info(bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("protected object remains");
|
||||
assert_eq!(info.size, 9);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
|
||||
replica_write_with_waiting_metadata_writer(false).await;
|
||||
|
||||
@@ -9,12 +9,14 @@ use rustfs_storage_api as storage_contracts;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) mod metadata_lock {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
BucketOperations, CompletePart, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
BucketOperations, CompletePart, DeleteBucketOptions, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO,
|
||||
ObjectOperations,
|
||||
};
|
||||
pub(crate) use super::{Error, ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rio::HashReader;
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
}
|
||||
|
||||
|
||||
@@ -3930,6 +3930,34 @@ mod test {
|
||||
assert!(versions_with_free.versions.iter().all(|version| version.num_versions == 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_version_rejects_mod_times_that_cannot_round_trip() {
|
||||
for nanos in [0, i128::from(i64::MAX) + 1, i128::from(i64::MIN) - 1, 1_i128 << 64] {
|
||||
for deleted in [false, true] {
|
||||
let mut meta = FileMeta::new();
|
||||
let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
|
||||
info.deleted = deleted;
|
||||
info.mod_time = Some(OffsetDateTime::from_unix_timestamp_nanos(nanos).expect("time within supported date range"));
|
||||
assert!(
|
||||
meta.add_version(info).is_err(),
|
||||
"unrepresentable time {nanos} must be rejected before publication"
|
||||
);
|
||||
}
|
||||
}
|
||||
for nanos in [-1, 1, i64::MIN, i64::MAX] {
|
||||
let mut meta = FileMeta::new();
|
||||
let mut info = crate::fileinfo::FileInfo::new("object", 2, 1);
|
||||
let time = OffsetDateTime::from_unix_timestamp_nanos(i128::from(nanos)).expect("valid codec time");
|
||||
info.mod_time = Some(time);
|
||||
meta.add_version(info).expect("representable times remain supported");
|
||||
let reopened = FileMeta::load(&meta.marshal_msg().expect("serialize version")).expect("reopen version");
|
||||
let versions = reopened
|
||||
.get_file_info_versions("bucket", "object", false)
|
||||
.expect("read version history");
|
||||
assert_eq!(versions.versions[0].mod_time, Some(time));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_integrity_validation() {
|
||||
// Test data integrity checks
|
||||
|
||||
@@ -858,13 +858,24 @@ impl FileMetaVersion {
|
||||
VersionType::Object => self
|
||||
.object
|
||||
.as_ref()
|
||||
.map(|v| v.erasure_algorithm.valid() && v.bitrot_checksum_algo.valid() && v.mod_time.is_some())
|
||||
.map(|v| {
|
||||
v.erasure_algorithm.valid()
|
||||
&& v.bitrot_checksum_algo.valid()
|
||||
&& v.mod_time.is_some_and(|time| {
|
||||
let nanos = time.unix_timestamp_nanos();
|
||||
nanos != 0 && i64::try_from(nanos).is_ok()
|
||||
})
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
VersionType::Legacy => self.legacy_object.as_ref().map(MetaObjectV1::valid).unwrap_or_default(),
|
||||
VersionType::Delete => self
|
||||
.delete_marker
|
||||
.as_ref()
|
||||
.map(|v| v.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) > OffsetDateTime::UNIX_EPOCH)
|
||||
.map(|v| {
|
||||
v.mod_time.is_some_and(|time| {
|
||||
time > OffsetDateTime::UNIX_EPOCH && i64::try_from(time.unix_timestamp_nanos()).is_ok()
|
||||
})
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
_ => false,
|
||||
}
|
||||
|
||||
@@ -77,6 +77,9 @@ pub enum Error {
|
||||
#[error("Replacement target is not ready: {0}")]
|
||||
ReplacementTargetNotReady(String),
|
||||
|
||||
#[error("Replacement bucket namespace changed; a fresh identity-bound pass is required")]
|
||||
ReplacementNamespaceChanged,
|
||||
|
||||
#[error("replacement recovery retry budget exhausted")]
|
||||
ReplacementRetryBudgetExhausted,
|
||||
|
||||
@@ -132,7 +135,10 @@ impl Error {
|
||||
/// Callers must cut their own concurrency instead of retrying at full speed.
|
||||
pub(crate) fn is_slow_down(&self) -> bool {
|
||||
match self {
|
||||
Error::Storage(error) => matches!(error, EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles),
|
||||
Error::Storage(error) => matches!(
|
||||
error,
|
||||
EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles | EcstoreError::DeleteRecoveryWorkLimit
|
||||
),
|
||||
Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
|
||||
Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
|
||||
message.contains("Please reduce your request rate")
|
||||
@@ -229,9 +235,10 @@ pub(crate) fn classify_heal_error(error: &Error) -> HealErrorClass {
|
||||
Error::DanglingDeleteDeferred { retry_not_before } => HealErrorClass::DanglingGrace {
|
||||
retry_not_before: Some(*retry_not_before),
|
||||
},
|
||||
Error::ReplacementTargetNotReady(_) | Error::TransientSkip { .. } | Error::BucketDiscoveryIncomplete => {
|
||||
HealErrorClass::Transient
|
||||
}
|
||||
Error::ReplacementTargetNotReady(_)
|
||||
| Error::ReplacementNamespaceChanged
|
||||
| Error::TransientSkip { .. }
|
||||
| Error::BucketDiscoveryIncomplete => HealErrorClass::Transient,
|
||||
Error::LegacyMrfUnverified { .. } => HealErrorClass::Permanent,
|
||||
Error::Storage(error) => classify_storage_error(error),
|
||||
Error::Disk(error) => classify_disk_error(error),
|
||||
@@ -285,6 +292,7 @@ pub(crate) fn classify_storage_error(error: &EcstoreError) -> HealErrorClass {
|
||||
| EcstoreError::FaultyDisk
|
||||
| EcstoreError::FaultyRemoteDisk
|
||||
| EcstoreError::SlowDown
|
||||
| EcstoreError::DeleteRecoveryWorkLimit
|
||||
| EcstoreError::OperationCanceled
|
||||
| EcstoreError::RemoteClientUnavailable(_) => HealErrorClass::Transient,
|
||||
_ if error.is_quorum_error() => HealErrorClass::Transient,
|
||||
|
||||
@@ -468,7 +468,16 @@ impl ErasureSetHealer {
|
||||
let task_id = self.get_or_create_task_id(set_disk_id).await?;
|
||||
|
||||
// 2. initialize or resume resume state
|
||||
let (resume_manager, checkpoint_manager) = self.initialize_resume_state(&task_id, set_disk_id, buckets).await?;
|
||||
let (resume_manager, mut checkpoint_manager) = self.initialize_resume_state(&task_id, set_disk_id, buckets).await?;
|
||||
let replacement_buckets = if self.replacement_task_id.is_some() {
|
||||
let current = resume_manager.prepare_replacement_bucket_pass(self.storage.as_ref()).await?;
|
||||
// Preparation may have durably rewound the companion checkpoint.
|
||||
checkpoint_manager = CheckpointManager::load_from_disk(self.disk.clone(), &task_id).await?;
|
||||
Some(current)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let buckets = replacement_buckets.as_deref().unwrap_or(buckets);
|
||||
|
||||
let admin_buckets = if self.admin_task.is_some() {
|
||||
Some(
|
||||
@@ -489,6 +498,9 @@ impl ErasureSetHealer {
|
||||
.execute_heal_with_resume(buckets, set_disk_id, &resume_manager, &checkpoint_manager)
|
||||
.await;
|
||||
|
||||
if self.replacement_task_id.is_some() {
|
||||
resume_manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
|
||||
}
|
||||
result?;
|
||||
self.verify_replacement_identity_fence("completion").await?;
|
||||
|
||||
@@ -967,9 +979,16 @@ impl ErasureSetHealer {
|
||||
// Recheck even resumed buckets: their user-object cursor does not
|
||||
// prove that the replacement holds the internal bucket records.
|
||||
for bucket in buckets {
|
||||
self.storage
|
||||
.heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
|
||||
.await?;
|
||||
if self.replacement_task_id.is_some() {
|
||||
let expected = resume_manager.replacement_bucket_incarnation(bucket).await?;
|
||||
self.storage
|
||||
.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, &self.heal_opts, targets)
|
||||
.await?;
|
||||
} else {
|
||||
self.storage
|
||||
.heal_replacement_bucket_metadata(bucket, &self.heal_opts, targets)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.heal_replacement_pool_metadata(
|
||||
@@ -1019,6 +1038,7 @@ impl ErasureSetHealer {
|
||||
// never hot-retried in place here.
|
||||
if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
|
||||
if self.replacement_task_id.is_some() {
|
||||
resume_manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
|
||||
let state = resume_manager.get_state().await;
|
||||
let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
|
||||
if state.can_retry()
|
||||
@@ -1452,7 +1472,11 @@ impl ErasureSetHealer {
|
||||
"Erasure set bucket started"
|
||||
);
|
||||
|
||||
let incarnation = self.admin_bucket_incarnation(checkpoint_manager, bucket).await?;
|
||||
let incarnation = if self.replacement_task_id.is_some() {
|
||||
Some(resume_manager.replacement_bucket_incarnation(bucket).await?)
|
||||
} else {
|
||||
self.admin_bucket_incarnation(checkpoint_manager, bucket).await?
|
||||
};
|
||||
let unresolved_delete_recovery = self.heal_admin_bucket_delete_recovery(bucket, incarnation).await?;
|
||||
if unresolved_delete_recovery && !self.heal_opts.dry_run {
|
||||
resume_manager.set_delete_recovery_rescan_pending(true).await?;
|
||||
@@ -1744,15 +1768,20 @@ impl ErasureSetHealer {
|
||||
(0, Err(Error::TaskCancelled))
|
||||
} else {
|
||||
let result = if replacement_object {
|
||||
storage
|
||||
.heal_object_with_task_cancel(
|
||||
&bucket_name,
|
||||
&object_name,
|
||||
version_id.as_deref(),
|
||||
&heal_opts,
|
||||
&cancel_token,
|
||||
)
|
||||
.await
|
||||
match incarnation {
|
||||
Some(expected) => storage
|
||||
.heal_replacement_object_at_incarnation(
|
||||
&bucket_name,
|
||||
&object_name,
|
||||
version_id.as_deref(),
|
||||
expected,
|
||||
&heal_opts,
|
||||
&cancel_token,
|
||||
)
|
||||
.await
|
||||
.map(|result| (result.item, result.error)),
|
||||
None => Err(Error::ReplacementNamespaceChanged),
|
||||
}
|
||||
} else {
|
||||
storage
|
||||
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
|
||||
@@ -2638,6 +2667,7 @@ mod resume_loop_tests {
|
||||
struct FakeStorage {
|
||||
admin_disk: Mutex<Option<DiskStore>>,
|
||||
bucket_discovery_incomplete: AtomicBool,
|
||||
bucket_discovery_error: Mutex<Option<EcstoreError>>,
|
||||
bucket_discovery_scopes: Mutex<Vec<Option<crate::heal::storage_api::storage::DiskSetSelector>>>,
|
||||
bucket_pages: Mutex<HashMap<String, Page>>,
|
||||
incarnations: Mutex<HashMap<String, uuid::Uuid>>,
|
||||
@@ -2729,6 +2759,9 @@ mod resume_loop_tests {
|
||||
if self.bucket_discovery_incomplete.load(Ordering::SeqCst) {
|
||||
return Err(Error::BucketDiscoveryIncomplete);
|
||||
}
|
||||
if let Some(error) = self.bucket_discovery_error.lock().expect("discovery failure").clone() {
|
||||
return Err(Error::Storage(error));
|
||||
}
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
@@ -2905,6 +2938,16 @@ mod resume_loop_tests {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn heal_replacement_bucket_metadata_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected: uuid::Uuid,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
self.validate_bucket_incarnation(bucket, Some(expected)).await?;
|
||||
self.heal_replacement_bucket_metadata(bucket, opts, targets).await
|
||||
}
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -3021,6 +3064,16 @@ mod resume_loop_tests {
|
||||
let temp = TempDir::new().unwrap();
|
||||
let disk = make_disk(&temp).await;
|
||||
let storage = Arc::new(FakeStorage::default());
|
||||
if !target_endpoints.is_empty() {
|
||||
storage.bucket_pages.lock().expect("replacement inventory").insert(
|
||||
"b".to_string(),
|
||||
Page {
|
||||
items: Vec::new(),
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
}
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let healer = ErasureSetHealer::new(
|
||||
storage.clone(),
|
||||
@@ -3040,7 +3093,17 @@ mod resume_loop_tests {
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let checkpoint = CheckpointManager::new(disk, task_id.clone()).await.unwrap();
|
||||
if !healer.target_endpoints.is_empty() {
|
||||
resume
|
||||
.prepare_replacement_bucket_pass(storage.as_ref())
|
||||
.await
|
||||
.expect("replacement fixture identity admission");
|
||||
}
|
||||
let checkpoint = if CheckpointManager::has_checkpoint(&disk, &task_id).await {
|
||||
CheckpointManager::load_from_disk(disk, &task_id).await.unwrap()
|
||||
} else {
|
||||
CheckpointManager::new(disk, task_id.clone()).await.unwrap()
|
||||
};
|
||||
Env {
|
||||
healer,
|
||||
storage,
|
||||
@@ -3080,6 +3143,60 @@ mod resume_loop_tests {
|
||||
(processed, successful, failed, skipped, result.map(|_| ()))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_bucket_pass_rejects_incomplete_inventory_without_reusing_or_clearing_progress() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
env.resume
|
||||
.set_resume_cursor(Some("held-cursor".to_string()))
|
||||
.await
|
||||
.expect("cursor before discovery failure");
|
||||
env.checkpoint
|
||||
.add_processed_object("held-key".to_string())
|
||||
.await
|
||||
.expect("dedup before discovery failure");
|
||||
// Model a page whose dedup identities must survive until cursor
|
||||
// publication. complete_page deliberately prunes that identity set.
|
||||
env.checkpoint
|
||||
.advance_page(0, 7)
|
||||
.await
|
||||
.expect("persist the retained page identities");
|
||||
let before = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
|
||||
.await
|
||||
.expect("durable checkpoint before discovery failure")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(before.current_object_index, 7);
|
||||
assert!(before.processed_objects.contains("held-key"));
|
||||
let state = env.resume.get_state().await;
|
||||
env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst);
|
||||
assert!(matches!(
|
||||
env.resume.prepare_replacement_bucket_pass(env.storage.as_ref()).await,
|
||||
Err(Error::BucketDiscoveryIncomplete)
|
||||
));
|
||||
assert_eq!(env.resume.get_state().await, state, "partial discovery must never publish a fresh plan");
|
||||
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
|
||||
.await
|
||||
.expect("durable checkpoint")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(checkpoint.current_object_index, 7);
|
||||
assert!(checkpoint.processed_objects.contains("held-key"));
|
||||
env.storage.bucket_discovery_incomplete.store(false, Ordering::SeqCst);
|
||||
*env.storage.bucket_discovery_error.lock().expect("inject quorum failure") =
|
||||
Some(EcstoreError::InsufficientReadQuorum("bucket".to_string(), "inventory".to_string()));
|
||||
assert!(matches!(
|
||||
env.resume.prepare_replacement_bucket_pass(env.storage.as_ref()).await,
|
||||
Err(Error::Storage(EcstoreError::InsufficientReadQuorum(..)))
|
||||
));
|
||||
assert_eq!(env.resume.get_state().await, state, "quorum failure is not an empty namespace");
|
||||
*env.storage.bucket_discovery_error.lock().expect("restore inventory") = None;
|
||||
env.resume
|
||||
.prepare_replacement_bucket_pass(env.storage.as_ref())
|
||||
.await
|
||||
.expect("unchanged complete inventory resumes existing pass");
|
||||
assert_eq!(env.resume.get_state().await.resume_cursor.as_deref(), Some("held-cursor"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_empty_bucket_no_panic() {
|
||||
let env = make_env().await;
|
||||
@@ -4055,7 +4172,7 @@ mod resume_loop_tests {
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env().await;
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
@@ -4110,7 +4227,7 @@ mod resume_loop_tests {
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env().await;
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer.heal_opts.no_lock = true;
|
||||
env.healer = env
|
||||
@@ -4167,7 +4284,7 @@ mod resume_loop_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn auto_replacement_retries_a_transient_object_before_rewinding_the_scan() {
|
||||
let mut env = make_env().await;
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
@@ -4199,7 +4316,7 @@ mod resume_loop_tests {
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("1")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env().await;
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
@@ -4231,7 +4348,7 @@ mod resume_loop_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn auto_replacement_cancellation_interrupts_object_retry() {
|
||||
let mut env = make_env().await;
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
@@ -4247,14 +4364,18 @@ mod resume_loop_tests {
|
||||
env.storage.set_outcome("retry", None, HealOutcome::Transient);
|
||||
let cancel = env.healer.cancel_token.clone();
|
||||
let observe = async {
|
||||
loop {
|
||||
let notified = env.storage.heal_started.notified();
|
||||
if !env.storage.calls().is_empty() {
|
||||
break;
|
||||
let started = tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
let notified = env.storage.heal_started.notified();
|
||||
if !env.storage.calls().is_empty() {
|
||||
break;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
cancel.cancel();
|
||||
started.expect("replacement object retry must start");
|
||||
};
|
||||
|
||||
let ((_, successful, _, _, result), ()) = tokio::join!(run(&env), observe);
|
||||
@@ -4272,7 +4393,7 @@ mod resume_loop_tests {
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env().await;
|
||||
let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
let execution = crate::heal::storage::ReplacementExecution::for_test(Vec::new(), Vec::new());
|
||||
env.healer = env
|
||||
|
||||
@@ -51,6 +51,25 @@ async fn replacement_env() -> Env {
|
||||
)
|
||||
.await
|
||||
.expect("replacement intent should persist before scanning");
|
||||
env.resume
|
||||
.prepare_replacement_bucket_pass(env.storage.as_ref())
|
||||
.await
|
||||
.expect("the new replacement generation must admit its bucket identities before direct execution");
|
||||
// Admission durably resets the companion checkpoint. Reload its current
|
||||
// revision instead of retaining the previous generation's CAS instance.
|
||||
env.checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
|
||||
.await
|
||||
.expect("reload the admitted replacement pass checkpoint");
|
||||
assert_eq!(
|
||||
env.resume
|
||||
.replacement_bucket_incarnation("b")
|
||||
.await
|
||||
.expect("the fixture bucket must belong to the replacement pass"),
|
||||
env.storage
|
||||
.admit_bucket_incarnation("b")
|
||||
.await
|
||||
.expect("fixture bucket identity")
|
||||
);
|
||||
env.healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
|
||||
@@ -881,6 +881,13 @@ async fn auto_format_fixture(
|
||||
// Preserve the constructor's explicit open/closed state before any fixture await.
|
||||
let node_limit = node.as_ref().map(|node| node.snapshot().limit);
|
||||
let fixture = resume_admission_fixture(node.clone()).await;
|
||||
// The endpoint monitor independently probes format.json after five seconds.
|
||||
// Drain it before aging the cache so the path-only pause observes this scanner.
|
||||
for pool in &fixture._store.ecstore.pools {
|
||||
tokio::time::timeout(Duration::from_secs(30), pool.stop_endpoint_monitor_for_test())
|
||||
.await
|
||||
.expect("endpoint reconnection must stop before the scanner owns format probes");
|
||||
}
|
||||
let disks = crate::heal::local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
|
||||
@@ -27,6 +27,7 @@ use super::{
|
||||
BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET, storage_api::owner::EcstoreDiskBytes,
|
||||
};
|
||||
|
||||
mod bucket_pass;
|
||||
mod checkpoint;
|
||||
mod gc;
|
||||
mod handoff;
|
||||
@@ -345,6 +346,16 @@ pub struct ResumeState {
|
||||
/// input after restart.
|
||||
#[serde(default)]
|
||||
pub replacement_buckets: Vec<String>,
|
||||
/// Current namespace pass, separate from the immutable generation plan.
|
||||
/// Legacy snapshots have no binding and must discard every old cursor.
|
||||
#[serde(default)]
|
||||
pub replacement_bucket_incarnations: Option<std::collections::BTreeMap<String, Uuid>>,
|
||||
/// Last proven retired identity per name, retained for returning stale volumes.
|
||||
#[serde(default)]
|
||||
pub replacement_retired_bucket_incarnations: std::collections::BTreeMap<String, Uuid>,
|
||||
/// Reapply the checkpoint rewind after any interruption between the two writes.
|
||||
#[serde(default)]
|
||||
pub replacement_bucket_pass_reset_pending: bool,
|
||||
/// A task-scoped generation assigned before formatting an automatic
|
||||
/// replacement. A new unformatted replacement receives a new generation
|
||||
/// and therefore cannot reuse an older disk's cursor or checkpoint.
|
||||
@@ -454,6 +465,9 @@ impl ResumeState {
|
||||
replacement_targets: Vec::new(),
|
||||
replacement_target_identities: Vec::new(),
|
||||
replacement_buckets: Vec::new(),
|
||||
replacement_bucket_incarnations: None,
|
||||
replacement_retired_bucket_incarnations: std::collections::BTreeMap::new(),
|
||||
replacement_bucket_pass_reset_pending: false,
|
||||
replacement_generation: None,
|
||||
replacement_phase: ReplacementPhase::None,
|
||||
replacement_execution_protocol: 0,
|
||||
@@ -985,7 +999,16 @@ impl ResumeManager {
|
||||
return Err(replacement_recovery_conflict("invalid retired legacy generation"));
|
||||
}
|
||||
}
|
||||
if state.replacement_execution_protocol != 1
|
||||
let valid_execution_protocol = match state.replacement_execution_protocol {
|
||||
1 => state.replacement_bucket_incarnations.is_none(),
|
||||
2 => state
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.is_some_and(|identities| identities.values().all(|incarnation| !incarnation.is_nil())),
|
||||
_ => false,
|
||||
};
|
||||
if !valid_execution_protocol
|
||||
|| state.replacement_retired_bucket_incarnations.values().any(Uuid::is_nil)
|
||||
|| state.replacement_predecessor.as_deref()
|
||||
!= state.replacement_lineage.last().map(|link| link.predecessor.as_str())
|
||||
{
|
||||
|
||||
@@ -0,0 +1,411 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::heal::storage::HealStorageAPI;
|
||||
use crate::heal::storage_api::storage::DiskSetSelector;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
async fn namespace_snapshot(storage: &dyn HealStorageAPI, state: &ResumeState) -> Result<BTreeMap<String, Uuid>> {
|
||||
let (pool, set) = crate::heal::utils::parse_set_disk_id(&state.set_disk_id)?;
|
||||
// This API rejects partial topology and propagates read-quorum failures.
|
||||
let buckets = storage
|
||||
.list_buckets_for_admin_heal(Some(DiskSetSelector::new(pool, set)))
|
||||
.await?;
|
||||
let mut identities = BTreeMap::new();
|
||||
for bucket in buckets {
|
||||
let Some(incarnation) = storage.replacement_bucket_incarnation(&bucket.name).await? else {
|
||||
// A returning disk may still list a retired volume. Only the admitted
|
||||
// old UUID plus its durable receipt can discharge that obligation.
|
||||
if let Some(expected) = state
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.and_then(|map| map.get(&bucket.name))
|
||||
.or_else(|| state.replacement_retired_bucket_incarnations.get(&bucket.name))
|
||||
&& storage.mrf_bucket_incarnation_retired(&bucket.name, *expected).await?
|
||||
{
|
||||
continue;
|
||||
}
|
||||
return Err(Error::ReplacementNamespaceChanged);
|
||||
};
|
||||
if incarnation.is_nil() {
|
||||
return Err(Error::InvalidBucketIncarnation);
|
||||
}
|
||||
identities.insert(bucket.name, incarnation);
|
||||
}
|
||||
Ok(identities)
|
||||
}
|
||||
|
||||
impl ResumeManager {
|
||||
pub(crate) async fn replacement_bucket_incarnation(&self, bucket: &str) -> Result<Uuid> {
|
||||
self.state
|
||||
.read()
|
||||
.await
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.and_then(|identities| identities.get(bucket).copied())
|
||||
.ok_or(Error::ReplacementNamespaceChanged)
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_replacement_bucket_pass(&self, storage: &dyn HealStorageAPI) -> Result<Vec<String>> {
|
||||
let previous = self.get_state().await;
|
||||
let current = namespace_snapshot(storage, &previous).await?;
|
||||
if let Some(identities) = previous.replacement_bucket_incarnations.as_ref() {
|
||||
for (bucket, expected) in identities {
|
||||
if current.get(bucket) != Some(expected) && !storage.mrf_bucket_incarnation_retired(bucket, *expected).await? {
|
||||
// Absence or a different current identity alone is not a deletion receipt.
|
||||
return Err(Error::ReplacementNamespaceChanged);
|
||||
}
|
||||
}
|
||||
}
|
||||
if previous.replacement_bucket_incarnations.as_ref() != Some(¤t) {
|
||||
let mut state = self.state.write().await;
|
||||
if let Some(identities) = previous.replacement_bucket_incarnations.as_ref() {
|
||||
for (bucket, expected) in identities {
|
||||
if current.get(bucket) != Some(expected) {
|
||||
state
|
||||
.replacement_retired_bucket_incarnations
|
||||
.insert(bucket.clone(), *expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
state.replacement_bucket_incarnations = Some(current.clone());
|
||||
if state.replacement_generation.is_some() {
|
||||
// Protocol-1 readers reject this before interpreting a new pass cursor.
|
||||
state.replacement_execution_protocol = 2;
|
||||
}
|
||||
state.replacement_bucket_pass_reset_pending = true;
|
||||
state.reset_for_retry();
|
||||
state.pending_buckets = current.keys().cloned().collect();
|
||||
state.total_objects = 0;
|
||||
state.total_bytes = 0;
|
||||
state.baseline_generation = None;
|
||||
state.baseline_known = false;
|
||||
drop(state);
|
||||
self.save_state_strict().await?;
|
||||
}
|
||||
if self.get_state().await.replacement_bucket_pass_reset_pending {
|
||||
let checkpoint = if CheckpointManager::has_checkpoint(&self.disk, &previous.task_id).await {
|
||||
CheckpointManager::load_from_disk(self.disk.clone(), &previous.task_id).await?
|
||||
} else {
|
||||
CheckpointManager::new(self.disk.clone(), previous.task_id.clone()).await?
|
||||
};
|
||||
// Unlike the throttled retry writer this is durable before clearing the intent.
|
||||
checkpoint.reset_after_delete_recovery().await?;
|
||||
self.state.write().await.replacement_bucket_pass_reset_pending = false;
|
||||
self.save_state_strict().await?;
|
||||
}
|
||||
Ok(current.into_keys().collect())
|
||||
}
|
||||
|
||||
pub(crate) async fn verify_replacement_bucket_pass(&self, storage: &dyn HealStorageAPI) -> Result<()> {
|
||||
let state = self.get_state().await;
|
||||
let current = namespace_snapshot(storage, &state).await?;
|
||||
if state.replacement_bucket_incarnations.as_ref() != Some(¤t) {
|
||||
return Err(Error::ReplacementNamespaceChanged);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::heal::storage::{ECStoreHealStorage, HealStorageAPI};
|
||||
use crate::heal::storage_api::{
|
||||
storage::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _},
|
||||
test_support::{
|
||||
ObjectOptions, PutObjReader, delete_config, init_bucket_metadata_sys, isolated_store_over_temp_disks, read_config,
|
||||
save_config,
|
||||
},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn replacement_bucket_pass_retires_deleted_identity_and_restarts_recreated_bucket() {
|
||||
// Keep the concrete storage futures off the test executor stack.
|
||||
let (directories, store) = Box::pin(isolated_store_over_temp_disks()).await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "replacement-pass-bucket";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original generation");
|
||||
let old = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("original identity");
|
||||
let storage = ECStoreHealStorage::new(store.clone());
|
||||
let (_anchor_dir, disk) = super::super::tests::schema_test_disk().await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = ResumeManager::new_replacement_intent(
|
||||
disk.clone(),
|
||||
task_id.clone(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec![bucket.to_owned()],
|
||||
vec!["replacement-a".to_string()],
|
||||
vec![ReplacementTargetIdentity {
|
||||
endpoint: "replacement-a".to_string(),
|
||||
canonical_path: "/mnt/replacement-a".to_string(),
|
||||
physical_device_ids: vec!["device-a".to_string()],
|
||||
filesystem_identity: "1:2:3".to_string(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("persist generation");
|
||||
// A legacy name-only cursor cannot be attributed to the currently live bucket.
|
||||
manager
|
||||
.set_resume_cursor(Some("unbound-legacy-cursor".to_string()))
|
||||
.await
|
||||
.expect("legacy cursor");
|
||||
let legacy_checkpoint = CheckpointManager::new(disk.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("legacy checkpoint");
|
||||
legacy_checkpoint
|
||||
.add_processed_object("legacy-key".to_string())
|
||||
.await
|
||||
.expect("legacy dedup");
|
||||
legacy_checkpoint.complete_page(0, 99).await.expect("persist legacy cursor");
|
||||
assert_eq!(
|
||||
manager
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("admit original pass"),
|
||||
[bucket]
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_state()
|
||||
.await
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.and_then(|m| m.get(bucket))
|
||||
.copied(),
|
||||
Some(old)
|
||||
);
|
||||
assert!(
|
||||
manager.get_state().await.resume_cursor.is_none(),
|
||||
"legacy cursor must not enter the fresh pass"
|
||||
);
|
||||
let fresh_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("fresh checkpoint")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(fresh_checkpoint.current_object_index, 0);
|
||||
assert!(fresh_checkpoint.processed_objects.is_empty());
|
||||
manager
|
||||
.set_resume_cursor(Some("old-generation-cursor".to_string()))
|
||||
.await
|
||||
.expect("persist old cursor");
|
||||
let checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("checkpoint");
|
||||
checkpoint
|
||||
.add_processed_object("same-key".to_string())
|
||||
.await
|
||||
.expect("old dedup entry");
|
||||
checkpoint.complete_page(0, 7).await.expect("persist old position and dedup");
|
||||
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original generation");
|
||||
assert!(matches!(
|
||||
manager.verify_replacement_bucket_pass(&storage).await,
|
||||
Err(Error::ReplacementNamespaceChanged)
|
||||
));
|
||||
manager
|
||||
.record_replacement_failure(&Error::ReplacementNamespaceChanged, 99)
|
||||
.await
|
||||
.expect("defer without exhausting replacement");
|
||||
assert_eq!(manager.get_state().await.retry_count, 0);
|
||||
let resumed = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("restart replacement manager");
|
||||
let retirement_path = format!("bucket-retirements/{bucket}/{old}.json");
|
||||
let retirement = Box::pin(read_config(store.clone(), &retirement_path))
|
||||
.await
|
||||
.expect("authoritative retirement");
|
||||
Box::pin(delete_config(store.clone(), &retirement_path))
|
||||
.await
|
||||
.expect("simulate missing retirement evidence");
|
||||
assert!(
|
||||
matches!(
|
||||
resumed.prepare_replacement_bucket_pass(&storage).await,
|
||||
Err(Error::ReplacementNamespaceChanged)
|
||||
),
|
||||
"physical absence alone cannot retire an admitted identity"
|
||||
);
|
||||
Box::pin(save_config(store.clone(), &retirement_path, b"corrupt retirement".to_vec()))
|
||||
.await
|
||||
.expect("inject unreadable proof");
|
||||
assert!(
|
||||
matches!(resumed.prepare_replacement_bucket_pass(&storage).await, Err(Error::Storage(_))),
|
||||
"a corrupt proof cannot be treated as an absent bucket"
|
||||
);
|
||||
assert_eq!(resumed.get_state().await.resume_cursor.as_deref(), Some("old-generation-cursor"));
|
||||
Box::pin(save_config(store.clone(), &retirement_path, retirement))
|
||||
.await
|
||||
.expect("restore proof");
|
||||
assert!(
|
||||
resumed
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("retirement proves deleted plan item")
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
// A disk returning after DELETE may expose the retired physical volume
|
||||
// again; the persisted receipt identity still discharges that old name.
|
||||
tokio::fs::create_dir(directories[0].path().join(bucket))
|
||||
.await
|
||||
.expect("return stale retired volume");
|
||||
let returned = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("restart after retirement");
|
||||
assert!(
|
||||
returned
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("retired stale volume is not a new bucket")
|
||||
.is_empty()
|
||||
);
|
||||
tokio::fs::remove_dir(directories[0].path().join(bucket))
|
||||
.await
|
||||
.expect("remove stale test volume");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate same name");
|
||||
let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
|
||||
assert_ne!(new, old);
|
||||
Box::pin(store.put_object(
|
||||
bucket,
|
||||
"same-key",
|
||||
&mut PutObjReader::from_vec(b"new payload".to_vec()),
|
||||
&ObjectOptions::default(),
|
||||
))
|
||||
.await
|
||||
.expect("new generation payload");
|
||||
assert_eq!(
|
||||
resumed
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("new generation gets independent pass"),
|
||||
[bucket]
|
||||
);
|
||||
let state = resumed.get_state().await;
|
||||
assert_eq!(
|
||||
state.replacement_execution_protocol, 2,
|
||||
"older binaries must reject identity-bound cursors"
|
||||
);
|
||||
assert_eq!(
|
||||
state
|
||||
.replacement_bucket_incarnations
|
||||
.as_ref()
|
||||
.and_then(|m| m.get(bucket))
|
||||
.copied(),
|
||||
Some(new)
|
||||
);
|
||||
assert!(state.resume_cursor.is_none());
|
||||
let checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("restarted checkpoint")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!((checkpoint.current_bucket_index, checkpoint.current_object_index), (0, 0));
|
||||
assert!(checkpoint.processed_objects.is_empty());
|
||||
|
||||
let opts = HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
storage
|
||||
.heal_replacement_object_at_incarnation(bucket, "same-key", None, old, &opts, &CancellationToken::new())
|
||||
.await
|
||||
.is_err(),
|
||||
"a stale worker cannot mutate a recreated bucket"
|
||||
);
|
||||
let targets = directories
|
||||
.iter()
|
||||
.map(|dir| dir.path().to_string_lossy().into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(
|
||||
storage
|
||||
.heal_replacement_bucket_metadata_at_incarnation(bucket, old, &opts, &targets)
|
||||
.await
|
||||
.is_err(),
|
||||
"a stale metadata pass cannot certify the new incarnation"
|
||||
);
|
||||
storage
|
||||
.heal_replacement_bucket_metadata_at_incarnation(bucket, new, &opts, &targets)
|
||||
.await
|
||||
.expect("current incarnation can repair metadata");
|
||||
let mut reader =
|
||||
Box::pin(store.get_object_reader(bucket, "same-key", None, Default::default(), &ObjectOptions::default()))
|
||||
.await
|
||||
.expect("open new generation payload");
|
||||
assert_eq!(reader.read_all().await.expect("read new payload"), b"new payload");
|
||||
drop(reader);
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("new identity remains"),
|
||||
new
|
||||
);
|
||||
// Simulate interruption after publishing the reset obligation, before
|
||||
// replacing the old checkpoint.
|
||||
let stale_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("checkpoint before interruption");
|
||||
stale_checkpoint
|
||||
.add_processed_object("stale-before-reset".to_string())
|
||||
.await
|
||||
.expect("stale dedup");
|
||||
stale_checkpoint
|
||||
.complete_page(0, 42)
|
||||
.await
|
||||
.expect("persist interrupted checkpoint");
|
||||
resumed.state.write().await.replacement_bucket_pass_reset_pending = true;
|
||||
resumed.save_state_strict().await.expect("persist reset obligation");
|
||||
let interrupted = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("restart interrupted pass");
|
||||
interrupted
|
||||
.prepare_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("finish interrupted checkpoint rewind");
|
||||
let repaired_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("repaired checkpoint")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(repaired_checkpoint.current_object_index, 0);
|
||||
assert!(repaired_checkpoint.processed_objects.is_empty());
|
||||
assert!(!interrupted.get_state().await.replacement_bucket_pass_reset_pending);
|
||||
let loaded = ResumeManager::load_replacement_intent(disk, &task_id)
|
||||
.await
|
||||
.expect("new pass survives restart");
|
||||
loaded
|
||||
.verify_replacement_bucket_pass(&storage)
|
||||
.await
|
||||
.expect("new pass matches current namespace");
|
||||
}
|
||||
}
|
||||
@@ -277,7 +277,7 @@ impl ResumeManager {
|
||||
}
|
||||
return self.ensure_handoff_successor().await;
|
||||
}
|
||||
if state.replacement_execution_protocol != 1
|
||||
if !matches!(state.replacement_execution_protocol, 1 | 2)
|
||||
|| !matches!(
|
||||
state.replacement_phase,
|
||||
ReplacementPhase::Intent
|
||||
@@ -458,6 +458,12 @@ impl ResumeManager {
|
||||
predecessor.replacement_targets.clone(),
|
||||
link.targets.clone(),
|
||||
);
|
||||
if predecessor.replacement_execution_protocol == 2 {
|
||||
state.replacement_execution_protocol = 2;
|
||||
state.replacement_bucket_incarnations = predecessor.replacement_bucket_incarnations.clone();
|
||||
state.replacement_retired_bucket_incarnations = predecessor.replacement_retired_bucket_incarnations.clone();
|
||||
state.replacement_bucket_pass_reset_pending = true;
|
||||
}
|
||||
state.replacement_phase = ReplacementPhase::OwnershipPending;
|
||||
// The constructor marks this compatibility grant consumed for every
|
||||
// successor, including descendants of a recovered older generation.
|
||||
@@ -580,7 +586,7 @@ fn preformat_handoff_retry_candidate(state: &ResumeState) -> bool {
|
||||
&& state.replacement_generation.as_deref() == Some(state.task_id.as_str())
|
||||
&& !state.replacement_preformat_handoff_retry_done
|
||||
&& state.replacement_phase == ReplacementPhase::Intent
|
||||
&& state.replacement_execution_protocol == 1
|
||||
&& matches!(state.replacement_execution_protocol, 1 | 2)
|
||||
&& state.max_retries >= 2
|
||||
&& state.replacement_predecessor.is_none()
|
||||
&& state.replacement_handoff.is_none()
|
||||
|
||||
@@ -918,3 +918,33 @@ async fn committed_predecessor_is_collectible_once_its_successor_is_verified() {
|
||||
"the verified successor keeps its own record until cleanup"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handoff_preserves_bucket_pass_protocol_and_rewinds_successor_progress() {
|
||||
let (_dirs, parent, execution) = fixture(1).await;
|
||||
let identities = [("bucket-a".to_string(), Uuid::new_v4())].into_iter().collect();
|
||||
{
|
||||
let mut state = parent.state.write().await;
|
||||
state.replacement_execution_protocol = 2;
|
||||
state.replacement_bucket_incarnations = Some(identities);
|
||||
state.resume_cursor = Some("old-pass-cursor".to_string());
|
||||
}
|
||||
parent.save_state_strict().await.expect("persist identity-bound predecessor");
|
||||
let child = parent
|
||||
.prepare_replacement_handoff(&execution, Vec::new())
|
||||
.await
|
||||
.expect("handoff protocol two");
|
||||
let state = child.get_state().await;
|
||||
assert_eq!(state.replacement_execution_protocol, 2);
|
||||
assert_eq!(
|
||||
state.replacement_bucket_incarnations,
|
||||
parent.get_state().await.replacement_bucket_incarnations
|
||||
);
|
||||
assert!(state.replacement_bucket_pass_reset_pending);
|
||||
assert!(state.resume_cursor.is_none());
|
||||
assert!(state.completed_buckets.is_empty());
|
||||
let recovered = ResumeManager::load_replacement_intent(child.disk.clone(), &state.task_id)
|
||||
.await
|
||||
.expect("reload protocol-two successor");
|
||||
assert_eq!(recovered.get_state().await, state);
|
||||
}
|
||||
|
||||
@@ -345,6 +345,16 @@ pub(crate) fn merge_replacement_recovery_candidate(
|
||||
|
||||
let candidate_revision = candidate.state.replacement_revision;
|
||||
let current_revision = current.state.replacement_revision;
|
||||
let candidate_protocol = candidate.state.replacement_execution_protocol;
|
||||
let current_protocol = current.state.replacement_execution_protocol;
|
||||
if (candidate_protocol > current_protocol && candidate_revision <= current_revision)
|
||||
|| (candidate_protocol < current_protocol && candidate_revision >= current_revision)
|
||||
{
|
||||
return Err(Error::ReplacementGenerationConflict {
|
||||
task_id: candidate.state.task_id,
|
||||
reason: "replacement execution protocol regresses with durable revision".to_string(),
|
||||
});
|
||||
}
|
||||
if candidate_revision == current_revision && candidate.state != current.state {
|
||||
return Err(Error::ReplacementGenerationConflict {
|
||||
task_id: candidate.state.task_id,
|
||||
@@ -374,7 +384,10 @@ fn same_replacement_generation_binding(left: &ResumeState, right: &ResumeState)
|
||||
&& left.replacement_target_identities == right.replacement_target_identities
|
||||
&& left.replacement_buckets == right.replacement_buckets
|
||||
&& left.replacement_generation == right.replacement_generation
|
||||
&& left.replacement_execution_protocol == right.replacement_execution_protocol
|
||||
// A revision-ordered protocol-1 -> protocol-2 pass admission preserves
|
||||
// the generation's target ownership and immutable lineage.
|
||||
&& matches!(left.replacement_execution_protocol, 1 | 2)
|
||||
&& matches!(right.replacement_execution_protocol, 1 | 2)
|
||||
&& left.replacement_predecessor == right.replacement_predecessor
|
||||
&& left.replacement_lineage == right.replacement_lineage
|
||||
&& left.replacement_legacy_import == right.replacement_legacy_import
|
||||
@@ -592,6 +605,12 @@ impl ResumeManager {
|
||||
if state.replacement_generation.as_deref() != Some(state.task_id.as_str()) {
|
||||
return Err(replacement_recovery_conflict("replacement failure has no matching generation"));
|
||||
}
|
||||
if matches!(error, Error::ReplacementNamespaceChanged | Error::BucketDiscoveryIncomplete) {
|
||||
// Namespace churn is not evidence that the replacement is irreparable.
|
||||
state.error_message = Some(error.to_string());
|
||||
drop(state);
|
||||
return self.save_state_strict().await;
|
||||
}
|
||||
if let Error::DanglingDeleteDeferred { retry_not_before } = error {
|
||||
if !super::dangling_delete_deadline_is_representable(*retry_not_before) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
|
||||
@@ -3629,3 +3629,129 @@ async fn blocked_checkpoint_is_not_revalidated_before_its_schedule() {
|
||||
assert_eq!(serde_json::from_slice::<serde_json::Value>(&retained).expect("marker JSON"), marker);
|
||||
temp_dir.close().expect("remove scheduled revalidation fixture");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_candidate_merge_rejects_protocol_downgrade_in_either_read_order() {
|
||||
for downgrade in [false, true] {
|
||||
for reverse in [false, true] {
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let mut protocol_one = replacement_candidate_state(&task_id);
|
||||
protocol_one.replacement_revision = if downgrade { 11 } else { 9 };
|
||||
let mut protocol_two = replacement_candidate_state(&task_id);
|
||||
protocol_two.replacement_execution_protocol = 2;
|
||||
protocol_two.replacement_bucket_incarnations = Some(Default::default());
|
||||
protocol_two.replacement_revision = 10;
|
||||
let (first, second) = if reverse {
|
||||
(protocol_two, protocol_one)
|
||||
} else {
|
||||
(protocol_one, protocol_two)
|
||||
};
|
||||
let mut selected = None;
|
||||
merge_replacement_recovery_candidate(
|
||||
&mut selected,
|
||||
ReplacementRecoveryCandidate::new(first, "anchor-a", false).expect("first durable copy"),
|
||||
)
|
||||
.expect("initial observation");
|
||||
let result = merge_replacement_recovery_candidate(
|
||||
&mut selected,
|
||||
ReplacementRecoveryCandidate::new(second, "anchor-b", true).expect("second durable copy"),
|
||||
);
|
||||
if downgrade {
|
||||
assert!(
|
||||
matches!(result, Err(Error::ReplacementGenerationConflict { .. })),
|
||||
"protocol 1 at a newer revision must not erase protocol 2, reverse={reverse}"
|
||||
);
|
||||
} else {
|
||||
result.expect("protocol 1 at an older revision may advance to protocol 2");
|
||||
let state = selected.expect("selected upgraded generation").state;
|
||||
assert_eq!(state.replacement_execution_protocol, 2);
|
||||
assert_eq!(state.replacement_revision, 10);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn replacement_recovery_survives_same_named_bucket_deletion_and_recreation() {
|
||||
use crate::heal::storage_api::{
|
||||
storage::{BucketOperations as _, DeleteBucketOptions, DiskSetSelector, MakeBucketOptions, StorageAdminApi as _},
|
||||
test_support::{init_bucket_metadata_sys, isolated_store_over_temp_disks},
|
||||
};
|
||||
|
||||
let (_directories, store) = Box::pin(isolated_store_over_temp_disks()).await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "ahm-replacement";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create legal same-name bucket");
|
||||
let disk = store
|
||||
.disk_set_inventory(DiskSetSelector::new(0, 0))
|
||||
.await
|
||||
.expect("survivor inventory")
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.next()
|
||||
.expect("survivor disk");
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let manager = ResumeManager::new_replacement_intent(
|
||||
disk.clone(),
|
||||
task_id.clone(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["other-bucket".to_string()],
|
||||
vec!["replacement-a".to_string()],
|
||||
vec![ReplacementTargetIdentity {
|
||||
endpoint: "replacement-a".to_string(),
|
||||
canonical_path: "/mnt/replacement-a".to_string(),
|
||||
physical_device_ids: vec!["device-a".to_string()],
|
||||
filesystem_identity: "1:2:3".to_string(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("persist sealed replacement intent");
|
||||
let expected = manager.get_state().await;
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete same-name user bucket");
|
||||
let resumed = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("sealed intent remains recoverable");
|
||||
assert_eq!(resumed.get_state().await, expected);
|
||||
assert_eq!(
|
||||
ResumeUtils::get_replacement_intent_tasks(&disk)
|
||||
.await
|
||||
.expect("scanner recovery enumeration"),
|
||||
std::slice::from_ref(&task_id)
|
||||
);
|
||||
resumed
|
||||
.mark_replacement_completed_and_verified()
|
||||
.await
|
||||
.expect("replacement can finish after deletion");
|
||||
resumed
|
||||
.ensure_replacement_completion_proof()
|
||||
.await
|
||||
.expect("persist completion proof");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate same-name user bucket");
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete recreated bucket");
|
||||
let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), &task_id)
|
||||
.await
|
||||
.expect("completion proof survives recreation");
|
||||
assert_eq!(proof.task_id, task_id);
|
||||
assert_eq!(
|
||||
ResumeManager::load_replacement_intent(disk, &task_id)
|
||||
.await
|
||||
.expect("verified intent survives")
|
||||
.get_state()
|
||||
.await
|
||||
.replacement_phase,
|
||||
ReplacementPhase::Verified
|
||||
);
|
||||
}
|
||||
|
||||
@@ -476,6 +476,15 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Published identity for a replacement pass; None proves missing metadata only.
|
||||
async fn replacement_bucket_incarnation(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||
match self.admit_bucket_incarnation(bucket).await {
|
||||
Ok(id) => Ok(Some(id)),
|
||||
Err(Error::StaleBucketIncarnation { .. }) => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the exact old bucket generation has a durable deletion record.
|
||||
/// Alternate backends must fail closed unless they can validate the same
|
||||
/// deployment, bucket and incarnation identity.
|
||||
@@ -598,6 +607,23 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
self.heal_object(bucket, object, version_id, opts).await
|
||||
}
|
||||
|
||||
/// Replacement writes retain both cancellation ownership and the admitted bucket fence.
|
||||
async fn heal_replacement_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
task_cancel: &CancellationToken,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
if task_cancel.is_cancelled() {
|
||||
return Err(Error::TaskCancelled);
|
||||
}
|
||||
self.heal_object_at_incarnation(bucket, object, version_id, expected, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_object_with_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -663,6 +689,17 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Err(Error::Storage(StorageError::PreconditionFailed))
|
||||
}
|
||||
|
||||
async fn heal_replacement_bucket_metadata_at_incarnation(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_expected: Uuid,
|
||||
_opts: &HealOpts,
|
||||
_targets: &[String],
|
||||
) -> Result<()> {
|
||||
// An unfenced fallback could certify a different same-name bucket.
|
||||
Err(Error::Storage(StorageError::PreconditionFailed))
|
||||
}
|
||||
|
||||
/// Read target-specific physical evidence for one replacement version.
|
||||
///
|
||||
/// This is only used by automatic replacement healing after the normal
|
||||
@@ -1316,6 +1353,35 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_replacement_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
task_cancel: &CancellationToken,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
run_background_object_inner(
|
||||
self.admission.as_ref(),
|
||||
|| self.ecstore.physical_heal_plan(object, opts).map_err(Error::Storage),
|
||||
opts,
|
||||
Some(task_cancel),
|
||||
|result: &HealStorageObjectResult| result.error.is_none(),
|
||||
boxed_object_operation(|| async {
|
||||
let result = self
|
||||
.ecstore
|
||||
.heal_object_at_incarnation(bucket, object, version_id.unwrap_or_default(), expected, opts)
|
||||
.await
|
||||
.map_err(|error| incarnation_storage_error(bucket, expected, error))?;
|
||||
Ok(self
|
||||
.object_result_with_receipt(bucket, object, version_id, opts, result, Some(expected))
|
||||
.await)
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn heal_mrf_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1520,6 +1586,13 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn replacement_bucket_incarnation(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||
self.ecstore
|
||||
.replacement_bucket_incarnation_from_disk(bucket)
|
||||
.await
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn mrf_bucket_incarnation_retired(&self, bucket: &str, expected: Uuid) -> Result<bool> {
|
||||
self.ecstore
|
||||
.is_bucket_incarnation_retired(bucket, expected)
|
||||
@@ -1959,6 +2032,19 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn heal_replacement_bucket_metadata_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected: Uuid,
|
||||
opts: &HealOpts,
|
||||
targets: &[String],
|
||||
) -> Result<()> {
|
||||
self.ecstore
|
||||
.heal_replacement_bucket_metadata_at_incarnation(bucket, expected, opts, targets)
|
||||
.await
|
||||
.map_err(|error| incarnation_storage_error(bucket, expected, error))
|
||||
}
|
||||
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -98,10 +98,19 @@ pub(crate) mod storage {
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::storage_contracts::DeleteBucketOptions;
|
||||
pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod status {
|
||||
pub(crate) use super::storage_contracts::BucketInfo;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::config::com::{delete_config, read_config, save_config};
|
||||
pub(crate) use rustfs_ecstore::api::object::{ObjectOptions, PutObjReader};
|
||||
}
|
||||
|
||||
@@ -1310,7 +1310,22 @@ impl HealTask {
|
||||
.load(Ordering::Acquire)
|
||||
.max(self.retry_attempts)
|
||||
.saturating_add(1);
|
||||
manager.record_replacement_failure(&failure, attempt).await
|
||||
// A deletion can race any stage, including hidden-transaction recovery.
|
||||
// Only a successful complete inventory may reclassify that failure;
|
||||
// lookup/quorum failures remain the original storage failure.
|
||||
if manager.get_state().await.replacement_bucket_incarnations.is_some()
|
||||
&& !matches!(failure, Error::TaskCancelled | Error::TaskTimeout)
|
||||
&& matches!(
|
||||
manager.verify_replacement_bucket_pass(self.storage.as_ref()).await,
|
||||
Err(Error::ReplacementNamespaceChanged)
|
||||
)
|
||||
{
|
||||
manager
|
||||
.record_replacement_failure(&Error::ReplacementNamespaceChanged, attempt)
|
||||
.await
|
||||
} else {
|
||||
manager.record_replacement_failure(&failure, attempt).await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
match persisted {
|
||||
|
||||
@@ -247,7 +247,12 @@ impl HealTask {
|
||||
*self.replacement_resume_disk.write().await = Some(disk.clone());
|
||||
let state = manager.get_state().await;
|
||||
self.replacement_start_retry_count.store(state.retry_count, Ordering::Release);
|
||||
buckets = state.replacement_buckets;
|
||||
buckets = if state.completed {
|
||||
state.replacement_buckets
|
||||
} else {
|
||||
self.await_with_control(manager.prepare_replacement_bucket_pass(self.storage.as_ref()))
|
||||
.await?
|
||||
};
|
||||
Some((disk, manager, identities))
|
||||
} else {
|
||||
None
|
||||
@@ -478,14 +483,27 @@ impl HealTask {
|
||||
self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "bucket prepass")
|
||||
.await?;
|
||||
}
|
||||
let expected = if let Some((_, manager, _)) = replacement_resume.as_ref() {
|
||||
Some(manager.replacement_bucket_incarnation(bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let heal_result = self
|
||||
.await_with_control(self.storage.heal_bucket(bucket, &bucket_heal_opts))
|
||||
.await_with_control(async {
|
||||
match expected {
|
||||
Some(id) => self.storage.heal_bucket_at_incarnation(bucket, id, &bucket_heal_opts).await,
|
||||
None => self.storage.heal_bucket(bucket, &bucket_heal_opts).await,
|
||||
}
|
||||
})
|
||||
.await;
|
||||
match heal_result {
|
||||
Ok(result) => {
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
Err(err) => {
|
||||
if let Some((_, manager, _)) = replacement_resume.as_ref() {
|
||||
manager.verify_replacement_bucket_pass(self.storage.as_ref()).await?;
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
@@ -505,7 +523,7 @@ impl HealTask {
|
||||
if let Some(result) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.recover_delete_transactions(bucket, None, None, &bucket_heal_opts),
|
||||
.recover_delete_transactions(bucket, None, expected, &bucket_heal_opts),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
||||
@@ -1056,6 +1056,7 @@ async fn automatic_replacement_uses_target_scoped_format() {
|
||||
let temp = TempDir::new().expect("temporary resume disk directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
resume_disk: Mutex::new(Some(disk.clone())),
|
||||
..Default::default()
|
||||
@@ -1264,6 +1265,7 @@ async fn automatic_replacement_rejects_a_new_identity_after_format() {
|
||||
let first_identity = replacement_identity("replacement-a", "device-a", "filesystem-a");
|
||||
let second_identity = replacement_identity("replacement-a", "device-b", "filesystem-b");
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([
|
||||
vec![first_identity.clone()],
|
||||
@@ -1330,6 +1332,7 @@ async fn automatic_replacement_reuses_an_existing_non_target_resume_anchor() {
|
||||
.await
|
||||
.expect("existing intent should be stored on the non-target anchor");
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_resume_disk: Mutex::new(Some(anchor.clone())),
|
||||
..Default::default()
|
||||
@@ -2998,6 +3001,7 @@ async fn replacement_dropped_waiter_keeps_executor_until_format_and_failure_pers
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![identity.clone()], vec![identity]])),
|
||||
replacement_execution_fixture: Mutex::new(Some(execution)),
|
||||
@@ -3012,7 +3016,9 @@ async fn replacement_dropped_waiter_keeps_executor_until_format_and_failure_pers
|
||||
let task = task.clone();
|
||||
async move { task.execute().await }
|
||||
});
|
||||
entered.notified().await;
|
||||
tokio::time::timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("owned format I/O must start");
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("waiter aborted").is_cancelled());
|
||||
task.cancel_token.cancel();
|
||||
@@ -3122,6 +3128,7 @@ async fn replacement_format_failures_keep_the_durable_budget_across_new_executor
|
||||
request.heal_endpoints = vec!["replacement-a".to_string()];
|
||||
for _attempt in 1..=3 {
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
resume_disk: Mutex::new(Some(disk.clone())),
|
||||
format_error: Mutex::new(Some(Error::other("injected format failure"))),
|
||||
@@ -5683,6 +5690,7 @@ async fn replacement_owned_executor_keeps_manager_drain_pending_through_failure_
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![identity.clone()], vec![identity]])),
|
||||
replacement_execution_fixture: Mutex::new(Some(execution)),
|
||||
@@ -5699,7 +5707,9 @@ async fn replacement_owned_executor_keeps_manager_drain_pending_through_failure_
|
||||
let task = task.clone();
|
||||
async move { task.execute_owned(Some(parent)).await }
|
||||
});
|
||||
entered.notified().await;
|
||||
tokio::time::timeout(Duration::from_secs(5), entered.notified())
|
||||
.await
|
||||
.expect("owned format I/O must start");
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("waiter aborted").is_cancelled());
|
||||
task.cancel_token.cancel();
|
||||
|
||||
@@ -45,6 +45,7 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-io-metrics/hotpath-cpu"
|
||||
hotpath.workspace = true
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
rustfs-common = { workspace = true }
|
||||
async-trait.workspace = true
|
||||
futures.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -694,6 +694,132 @@ impl DistributedLock {
|
||||
self.acquire_guard(&req).await
|
||||
}
|
||||
|
||||
/// Acquire exclusive locks for many independent resources in one batched
|
||||
/// round: one [`LockClient::acquire_locks_batch`] RPC per client instead of
|
||||
/// one quorum round-trip per key. Returns one guard or error per resource.
|
||||
///
|
||||
/// Bulk operations (S3 `DeleteObjects`) are the only multi-key write-lock
|
||||
/// holders, and their keys are independent. Per-key outcomes are
|
||||
/// deadlock-free without any serial fallback: the wire call is a single
|
||||
/// round per client and every lock server orders concurrent batches by the
|
||||
/// same global (shard, key) order, so no acquirer can hold one key while
|
||||
/// waiting for another. A key that fails to reach quorum releases its
|
||||
/// partial grants (awaited) before its error is returned.
|
||||
///
|
||||
/// Guards keep the same TTL heartbeat and lock-lost semantics as
|
||||
/// [`Self::acquire_guard`]. Contention is waited out server-side up to
|
||||
/// `timeout` (each request carries it), matching the single-key path.
|
||||
pub async fn acquire_guard_batch(
|
||||
&self,
|
||||
resources: &[ObjectKey],
|
||||
owner: &str,
|
||||
timeout: Duration,
|
||||
ttl: Duration,
|
||||
) -> Vec<Result<DistributedLockGuard>> {
|
||||
if resources.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
if self.clients.is_empty() {
|
||||
return resources
|
||||
.iter()
|
||||
.map(|_| Err(LockError::internal("No lock clients available")))
|
||||
.collect();
|
||||
}
|
||||
|
||||
let required_quorum = self.required_quorum(LockType::Exclusive);
|
||||
let attempt_started = Instant::now();
|
||||
let requests: Vec<LockRequest> = resources
|
||||
.iter()
|
||||
.map(|resource| {
|
||||
LockRequest::new(resource.clone(), LockType::Exclusive, owner)
|
||||
.with_acquire_timeout(timeout)
|
||||
.with_ttl(ttl)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// One batched round per client; a client-level failure is a per-key
|
||||
// failure for every key that still needed that client's vote.
|
||||
let rounds = join_all(self.clients.iter().map(|client| {
|
||||
let requests = requests.clone();
|
||||
async move { client.acquire_locks_batch(&requests).await }
|
||||
}))
|
||||
.await;
|
||||
|
||||
let mut results: Vec<Result<DistributedLockGuard>> = Vec::with_capacity(resources.len());
|
||||
let mut orphaned_entries: Vec<(LockId, Arc<dyn LockClient>)> = Vec::new();
|
||||
for (key_idx, resource) in resources.iter().enumerate() {
|
||||
let mut entries: Vec<HeldLockEntry> = Vec::new();
|
||||
let mut failure: Option<String> = None;
|
||||
for (client_idx, client) in self.clients.iter().enumerate() {
|
||||
match &rounds[client_idx] {
|
||||
Ok(responses) => match responses.get(key_idx) {
|
||||
Some(response) if response.success => {
|
||||
let lock_id = response
|
||||
.lock_info
|
||||
.as_ref()
|
||||
.map(|info| info.id.clone())
|
||||
.unwrap_or_else(|| LockId::new_unique(resource));
|
||||
entries.push(HeldLockEntry {
|
||||
lock_id,
|
||||
client: client.clone(),
|
||||
valid_until: attempt_started.checked_add(ttl).unwrap_or(attempt_started),
|
||||
});
|
||||
}
|
||||
Some(response) => {
|
||||
failure.get_or_insert_with(|| {
|
||||
response
|
||||
.error
|
||||
.clone()
|
||||
.unwrap_or_else(|| "distributed lock acquisition failed".to_string())
|
||||
});
|
||||
}
|
||||
None => {
|
||||
failure.get_or_insert_with(|| "client returned incomplete batch lock response".to_string());
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
failure.get_or_insert_with(|| err.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if entries.len() >= required_quorum {
|
||||
let heartbeat = LockHeartbeatConfig {
|
||||
ttl,
|
||||
interval: derive_refresh_interval(entries.len(), ttl, None),
|
||||
refresh_quorum: required_quorum,
|
||||
owner: owner.to_string(),
|
||||
resource: resource.clone(),
|
||||
};
|
||||
results.push(Ok(DistributedLockGuard::new(
|
||||
LockId::new_unique(resource),
|
||||
entries,
|
||||
LockType::Exclusive,
|
||||
heartbeat,
|
||||
)));
|
||||
} else {
|
||||
let achieved = entries.len();
|
||||
orphaned_entries.extend(entries.iter().map(HeldLockEntry::release_entry));
|
||||
if let Some(error) = failure {
|
||||
tracing::debug!(resource = %resource, owner, required_quorum, achieved, "batch lock acquire failed: {error}");
|
||||
}
|
||||
results.push(Err(LockError::QuorumNotReached {
|
||||
required: required_quorum,
|
||||
achieved,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// One bounded synchronous pass so the caller cannot race these
|
||||
// unlocks on an immediate retry; anything a slow peer still holds
|
||||
// keeps retrying in the background instead of parking the request
|
||||
// on the deferred unlock schedule.
|
||||
if !orphaned_entries.is_empty() {
|
||||
Self::release_entries_bounded(orphaned_entries, "distributed_lock_batch_quorum_rollback").await;
|
||||
}
|
||||
results
|
||||
}
|
||||
|
||||
fn spawn_lock_requests_until(
|
||||
&self,
|
||||
pending: &mut JoinSet<LockAcquireTaskResult>,
|
||||
@@ -755,6 +881,19 @@ impl DistributedLock {
|
||||
Self::release_entries_deferred(pending, context).await;
|
||||
}
|
||||
|
||||
/// Release entries with one synchronous pass, then continue the retry
|
||||
/// schedule in the background.
|
||||
///
|
||||
/// The batch-acquire rollback path runs on the request: it must not park
|
||||
/// the caller on the deferred unlock schedule when a peer is unreachable,
|
||||
/// while still confirming the healthy quorum is free before returning.
|
||||
async fn release_entries_bounded(entries: Vec<(LockId, Arc<dyn LockClient>)>, context: &'static str) {
|
||||
let pending = Self::release_pending_once(entries, 1, context).await;
|
||||
if !pending.is_empty() {
|
||||
Self::spawn_release_cleanup(pending, context);
|
||||
}
|
||||
}
|
||||
|
||||
async fn release_pending_once(
|
||||
pending: Vec<(LockId, Arc<dyn LockClient>)>,
|
||||
attempt: usize,
|
||||
@@ -2856,4 +2995,120 @@ mod tests {
|
||||
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
/// Per-object scripted client: grants only the named objects and records
|
||||
/// every released lock's resource.
|
||||
#[derive(Debug)]
|
||||
struct ScriptedBatchClient {
|
||||
grants: Vec<String>,
|
||||
releases: Arc<Mutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
impl ScriptedBatchClient {
|
||||
fn build(grants: &[&str]) -> (Arc<dyn LockClient>, Arc<Mutex<Vec<String>>>) {
|
||||
let releases = Arc::new(Mutex::new(Vec::new()));
|
||||
let client = Arc::new(Self {
|
||||
grants: grants.iter().map(|object| (*object).to_string()).collect(),
|
||||
releases: Arc::clone(&releases),
|
||||
}) as Arc<dyn LockClient>;
|
||||
(client, releases)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LockClient for ScriptedBatchClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
|
||||
if self.grants.iter().any(|object| object == request.resource.object.as_ref()) {
|
||||
let info = LockInfo {
|
||||
id: request.lock_id.clone(),
|
||||
resource: request.resource.clone(),
|
||||
lock_type: request.lock_type,
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: request.owner.clone(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + request.ttl,
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: request.metadata.clone(),
|
||||
priority: request.priority,
|
||||
wait_start_time: None,
|
||||
};
|
||||
Ok(LockResponse::success(info, Duration::ZERO))
|
||||
} else {
|
||||
Ok(LockResponse::failure("denied by test script", Duration::ZERO))
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.releases
|
||||
.lock()
|
||||
.expect("release log")
|
||||
.push(lock_id.resource.object.to_string());
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn refresh(&self, _lock_id: &LockId) -> crate::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.release(lock_id).await
|
||||
}
|
||||
|
||||
async fn check_status(&self, _lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn get_stats(&self) -> crate::Result<LockStats> {
|
||||
Ok(LockStats::default())
|
||||
}
|
||||
|
||||
async fn close(&self) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn is_online(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn is_local(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// A batch key granted by fewer than quorum clients fails alone, and its
|
||||
/// partial grants are released synchronously before the call returns
|
||||
/// (PR9155 review coverage for the distributed rollback path).
|
||||
#[tokio::test]
|
||||
async fn acquire_guard_batch_releases_partial_grants_below_quorum() {
|
||||
let (c0, c0_releases) = ScriptedBatchClient::build(&["kept", "lost"]);
|
||||
let (c1, _c1_releases) = ScriptedBatchClient::build(&["kept"]);
|
||||
let (c2, _c2_releases) = ScriptedBatchClient::build(&[]);
|
||||
let lock = DistributedLock::new("batch-test".to_string(), vec![c0, c1, c2], 2);
|
||||
|
||||
let resources = vec![ObjectKey::new("bucket", "kept"), ObjectKey::new("bucket", "lost")];
|
||||
let results = lock
|
||||
.acquire_guard_batch(&resources, "owner", Duration::from_millis(200), Duration::from_secs(30))
|
||||
.await;
|
||||
|
||||
assert!(results[0].is_ok(), "kept reached quorum: {:?}", results[0].as_ref().err());
|
||||
let error = results[1].as_ref().expect_err("lost must fail below quorum");
|
||||
assert!(
|
||||
matches!(
|
||||
error,
|
||||
LockError::QuorumNotReached {
|
||||
required: 2,
|
||||
achieved: 1
|
||||
}
|
||||
),
|
||||
"lost must report its partial vote: {error:?}"
|
||||
);
|
||||
// Exactly the orphaned grant is released before the call returns,
|
||||
// while "kept" stays held by its live guard.
|
||||
assert_eq!(
|
||||
c0_releases.lock().expect("release log").clone(),
|
||||
vec!["lost".to_string()],
|
||||
"only the below-quorum key's partial grant is released synchronously"
|
||||
);
|
||||
drop(results);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,10 +31,13 @@ pub mod manager_trait;
|
||||
pub mod metrics;
|
||||
pub mod object_pool;
|
||||
pub mod optimized_notify;
|
||||
mod publication;
|
||||
pub mod shard;
|
||||
pub mod state;
|
||||
pub mod types;
|
||||
|
||||
#[cfg(test)]
|
||||
mod publication_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
|
||||
@@ -95,6 +95,8 @@ impl ObjectLockState {
|
||||
// Reset atomic state
|
||||
self.atomic_state = crate::fast_lock::state::AtomicLockState::new();
|
||||
|
||||
self.publication = std::sync::OnceLock::new();
|
||||
|
||||
// Clear owners
|
||||
*self.current_owner.write() = None;
|
||||
self.shared_owners.write().clear();
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! FIFO admission only for the bucket publication fence. Consecutive readers
|
||||
//! form a group; a later writer cannot overtake them, nor can later readers
|
||||
//! overtake a queued writer. This does not limit a current holder's I/O time.
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use std::{collections::VecDeque, sync::Arc};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use super::{
|
||||
state::ObjectLockState,
|
||||
types::{LockMode, ObjectKey, ObjectLockRequest},
|
||||
};
|
||||
|
||||
pub(super) fn applies(key: &ObjectKey) -> bool {
|
||||
key.version.is_none() && key.object.as_ref() == rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct PublicationQueue {
|
||||
entries: Mutex<VecDeque<(Arc<()>, LockMode)>>,
|
||||
pub(super) changed: Notify,
|
||||
}
|
||||
|
||||
pub(super) struct PublicationTicket {
|
||||
pub(super) state: Arc<ObjectLockState>,
|
||||
pub(super) queue: Arc<PublicationQueue>,
|
||||
token: Arc<()>,
|
||||
mode: LockMode,
|
||||
}
|
||||
|
||||
impl PublicationTicket {
|
||||
// Called while the shard map is locked. The waiter counter pins the map
|
||||
// entry until ticket destruction, including cancellation between polls.
|
||||
pub(super) fn new(state: Arc<ObjectLockState>, mode: LockMode) -> Option<Self> {
|
||||
let counted = match mode {
|
||||
LockMode::Shared => state.atomic_state.inc_readers_waiting(),
|
||||
LockMode::Exclusive => state.atomic_state.inc_writers_waiting(),
|
||||
};
|
||||
if !counted {
|
||||
return None;
|
||||
}
|
||||
let queue = state
|
||||
.publication
|
||||
.get_or_init(|| Arc::new(PublicationQueue::default()))
|
||||
.clone();
|
||||
let token = Arc::new(());
|
||||
queue.entries.lock().push_back((token.clone(), mode));
|
||||
Some(Self {
|
||||
state,
|
||||
queue,
|
||||
token,
|
||||
mode,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn try_acquire(&self, request: &ObjectLockRequest) -> bool {
|
||||
// Lock order: shard map -> queue -> owner metadata. Release never
|
||||
// locks the queue; ticket destruction only locks the queue.
|
||||
let entries = self.queue.entries.lock();
|
||||
let mut found = false;
|
||||
for (token, mode) in entries.iter() {
|
||||
if Arc::ptr_eq(token, &self.token) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
if self.mode == LockMode::Exclusive || *mode == LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return false;
|
||||
}
|
||||
match self.mode {
|
||||
LockMode::Shared => self.state.try_acquire_admitted_shared(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => self.state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PublicationTicket {
|
||||
fn drop(&mut self) {
|
||||
{
|
||||
let mut entries = self.queue.entries.lock();
|
||||
if let Some(index) = entries.iter().position(|(token, _)| Arc::ptr_eq(token, &self.token)) {
|
||||
entries.remove(index);
|
||||
}
|
||||
}
|
||||
match self.mode {
|
||||
LockMode::Shared => self.state.atomic_state.dec_readers_waiting(),
|
||||
LockMode::Exclusive => self.state.atomic_state.dec_writers_waiting(),
|
||||
}
|
||||
self.queue.changed.notify_waiters();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use tokio::{task::JoinHandle, time::timeout};
|
||||
|
||||
fn manager() -> Arc<FastObjectLockManager> {
|
||||
Arc::new(FastObjectLockManager::with_config(LockConfig {
|
||||
shard_count: 1,
|
||||
max_idle_time: Duration::ZERO,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
fn key() -> ObjectKey {
|
||||
ObjectKey::new("publication-bucket", rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
}
|
||||
|
||||
fn queued(manager: &Arc<FastObjectLockManager>, owner: &str, mode: LockMode) -> JoinHandle<FastLockGuard> {
|
||||
let manager = manager.clone();
|
||||
let owner = owner.to_owned();
|
||||
tokio::spawn(async move {
|
||||
let mut request = ObjectLockRequest::new_read(key(), owner);
|
||||
request.mode = mode;
|
||||
manager
|
||||
.acquire_lock(request.with_acquire_timeout(Duration::from_secs(5)))
|
||||
.await
|
||||
.expect("queued publication lock")
|
||||
})
|
||||
}
|
||||
|
||||
async fn waiters(manager: &FastObjectLockManager, count: u64) {
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while manager.current_lock_counts().1 != count {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("waiter admission must be observable");
|
||||
}
|
||||
|
||||
async fn acquired(task: JoinHandle<FastLockGuard>) -> FastLockGuard {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("queue must progress")
|
||||
.expect("acquire task")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_serves_read_groups_and_writers_in_arrival_order() {
|
||||
let manager = manager();
|
||||
let holder = manager.acquire_write_lock(key(), "holder").await.expect("initial holder");
|
||||
let w1 = queued(&manager, "w1", LockMode::Exclusive);
|
||||
waiters(&manager, 1).await;
|
||||
let r1 = queued(&manager, "r1", LockMode::Shared);
|
||||
waiters(&manager, 2).await;
|
||||
let r2 = queued(&manager, "r2", LockMode::Shared);
|
||||
waiters(&manager, 3).await;
|
||||
let w2 = queued(&manager, "w2", LockMode::Exclusive);
|
||||
waiters(&manager, 4).await;
|
||||
let r3 = queued(&manager, "r3", LockMode::Shared);
|
||||
waiters(&manager, 5).await;
|
||||
assert_eq!(manager.cleanup_expired_traditional().await, 0, "queued state must stay mapped");
|
||||
drop(holder);
|
||||
assert_eq!(
|
||||
manager.cleanup_expired_traditional().await,
|
||||
0,
|
||||
"an unlocked state with queued tickets must stay mapped"
|
||||
);
|
||||
let w1 = acquired(w1).await;
|
||||
assert!(!r1.is_finished() && !r2.is_finished() && !w2.is_finished() && !r3.is_finished());
|
||||
drop(w1);
|
||||
let r1 = acquired(r1).await;
|
||||
let r2 = acquired(r2).await;
|
||||
assert!(!w2.is_finished() && !r3.is_finished(), "later requests cannot pass the reader group");
|
||||
drop(r1);
|
||||
assert!(!w2.is_finished(), "the second reader still holds the fence");
|
||||
drop(r2);
|
||||
let w2 = acquired(w2).await;
|
||||
assert!(!r3.is_finished(), "a later reader cannot starve the queued writer");
|
||||
drop(w2);
|
||||
drop(acquired(r3).await);
|
||||
assert_eq!(manager.current_lock_counts(), (0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_cancellation_and_timeout_remove_their_turns() {
|
||||
let manager = manager();
|
||||
let holder = manager.acquire_write_lock(key(), "holder").await.expect("initial holder");
|
||||
let cancelled = queued(&manager, "cancelled", LockMode::Exclusive);
|
||||
waiters(&manager, 1).await;
|
||||
let expiring = tokio::spawn({
|
||||
let manager = manager.clone();
|
||||
async move {
|
||||
manager
|
||||
.acquire_lock(ObjectLockRequest::new_write(key(), "expires").with_acquire_timeout(Duration::from_millis(500)))
|
||||
.await
|
||||
}
|
||||
});
|
||||
waiters(&manager, 2).await;
|
||||
let reader = queued(&manager, "survivor", LockMode::Shared);
|
||||
waiters(&manager, 3).await;
|
||||
cancelled.abort();
|
||||
assert!(cancelled.await.expect_err("cancel waiter").is_cancelled());
|
||||
assert!(matches!(expiring.await.expect("timeout task"), Err(LockResult::Timeout)));
|
||||
waiters(&manager, 1).await;
|
||||
assert_eq!(manager.cleanup_expired_traditional().await, 0);
|
||||
drop(holder);
|
||||
drop(acquired(reader).await);
|
||||
assert_eq!(manager.current_lock_counts(), (0, 0));
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while manager.cleanup_expired_traditional().await == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("an empty queue must allow map cleanup");
|
||||
// An empty queue neither keeps a stale turn nor prevents subsequent writers.
|
||||
drop(manager.acquire_write_lock(key(), "next").await.expect("queue recovered"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_batch_fast_path_cannot_bypass_a_reader() {
|
||||
for all_or_nothing in [false, true] {
|
||||
let manager = manager();
|
||||
let holder = manager.acquire_write_lock(key(), "holder").await.expect("initial holder");
|
||||
let reader = queued(&manager, "queued-reader", LockMode::Shared);
|
||||
waiters(&manager, 1).await;
|
||||
drop(holder);
|
||||
// This future can try the batch fast path before the reader runs again.
|
||||
let batch = manager.acquire_locks_batch(
|
||||
BatchLockRequest::new("batch")
|
||||
.add_write_lock(key())
|
||||
.with_all_or_nothing(all_or_nothing),
|
||||
);
|
||||
tokio::pin!(batch);
|
||||
let reader = tokio::select! {
|
||||
biased;
|
||||
_ = &mut batch => panic!("batch overtook an already queued reader"),
|
||||
reader = acquired(reader) => reader,
|
||||
};
|
||||
drop(reader);
|
||||
let batch = timeout(Duration::from_secs(3), batch)
|
||||
.await
|
||||
.expect("batch progresses after reader");
|
||||
assert!(batch.all_acquired);
|
||||
drop(batch);
|
||||
assert_eq!(manager.current_lock_counts(), (0, 0));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_queue_is_scoped_to_the_unversioned_bucket_fence() {
|
||||
assert!(publication::applies(&key()));
|
||||
assert!(!publication::applies(&ObjectKey::new("publication-bucket", "plain-object")));
|
||||
assert!(!publication::applies(&ObjectKey::with_version(
|
||||
"publication-bucket",
|
||||
rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH,
|
||||
"version"
|
||||
)));
|
||||
}
|
||||
|
||||
async fn cluster_waiters(managers: &[Arc<FastObjectLockManager>], minimum: u64) {
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while !managers.iter().all(|manager| manager.current_lock_counts().1 >= minimum) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all real local peers registered the waiter");
|
||||
}
|
||||
|
||||
fn cluster_queued(lock: &Arc<crate::NamespaceLock>, owner: &str, kind: crate::LockType) -> JoinHandle<crate::NamespaceLockGuard> {
|
||||
let lock = lock.clone();
|
||||
let owner = owner.to_owned();
|
||||
tokio::spawn(async move {
|
||||
lock.acquire_guard(
|
||||
&crate::LockRequest::new(key(), kind, owner)
|
||||
.with_acquire_timeout(Duration::from_secs(5))
|
||||
.with_ttl(Duration::from_secs(30)),
|
||||
)
|
||||
.await
|
||||
.expect("distributed acquire")
|
||||
.expect("distributed quorum")
|
||||
})
|
||||
}
|
||||
|
||||
async fn cluster_acquired(task: JoinHandle<crate::NamespaceLockGuard>) -> crate::NamespaceLockGuard {
|
||||
timeout(Duration::from_secs(3), task)
|
||||
.await
|
||||
.expect("quorum must progress")
|
||||
.expect("cluster task")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_distributed_quorum_preserves_bidirectional_progress() {
|
||||
let managers = (0..3).map(|_| manager()).collect::<Vec<_>>();
|
||||
let clients = managers
|
||||
.iter()
|
||||
.map(|manager| {
|
||||
Arc::new(crate::LocalClient::with_manager(Arc::new(crate::GlobalLockManager::Enabled(
|
||||
manager.clone(),
|
||||
)))) as Arc<dyn crate::LockClient>
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let lock = Arc::new(crate::NamespaceLock::with_clients("publication-fairness".to_owned(), clients));
|
||||
// Hold every voter while establishing a deterministic arrival order.
|
||||
// A distributed holder may return at quorum and release its spare vote.
|
||||
let mut holders = Vec::new();
|
||||
for manager in &managers {
|
||||
holders.push(manager.acquire_write_lock(key(), "holder").await.expect("hold voter"));
|
||||
}
|
||||
let w1 = cluster_queued(&lock, "w1", crate::LockType::Exclusive);
|
||||
cluster_waiters(&managers, 1).await;
|
||||
let r1 = cluster_queued(&lock, "r1", crate::LockType::Shared);
|
||||
cluster_waiters(&managers, 2).await;
|
||||
let w2 = cluster_queued(&lock, "w2", crate::LockType::Exclusive);
|
||||
cluster_waiters(&managers, 3).await;
|
||||
let r2 = cluster_queued(&lock, "r2", crate::LockType::Shared);
|
||||
cluster_waiters(&managers, 4).await;
|
||||
drop(holders);
|
||||
let w1 = cluster_acquired(w1).await;
|
||||
assert!(!r1.is_finished() && !w2.is_finished() && !r2.is_finished());
|
||||
drop(w1);
|
||||
let r1 = cluster_acquired(r1).await;
|
||||
assert!(!w2.is_finished() && !r2.is_finished());
|
||||
drop(r1);
|
||||
let w2 = cluster_acquired(w2).await;
|
||||
assert!(!r2.is_finished());
|
||||
drop(w2);
|
||||
drop(cluster_acquired(r2).await);
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while managers.iter().any(|manager| manager.current_lock_counts() != (0, 0)) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all quorum guards and pending cleanup must release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_distributed_cancel_and_timeout_leave_no_orphan_turns() {
|
||||
let managers = (0..3).map(|_| manager()).collect::<Vec<_>>();
|
||||
let clients = managers
|
||||
.iter()
|
||||
.map(|manager| {
|
||||
Arc::new(crate::LocalClient::with_manager(Arc::new(crate::GlobalLockManager::Enabled(
|
||||
manager.clone(),
|
||||
)))) as Arc<dyn crate::LockClient>
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let lock = Arc::new(crate::NamespaceLock::with_clients("publication-cancellation".to_owned(), clients));
|
||||
let mut holders = Vec::new();
|
||||
for manager in &managers {
|
||||
holders.push(manager.acquire_write_lock(key(), "holder").await.expect("hold voter"));
|
||||
}
|
||||
let cancelled = cluster_queued(&lock, "cancelled", crate::LockType::Exclusive);
|
||||
cluster_waiters(&managers, 1).await;
|
||||
cancelled.abort();
|
||||
assert!(cancelled.await.expect_err("cancel distributed waiter").is_cancelled());
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while managers.iter().any(|manager| manager.current_lock_counts().1 != 0) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("aborted peer attempts must remove their tickets");
|
||||
let timed_out = lock
|
||||
.acquire_guard(
|
||||
&crate::LockRequest::new(key(), crate::LockType::Exclusive, "timeout")
|
||||
.with_acquire_timeout(Duration::from_millis(100)),
|
||||
)
|
||||
.await
|
||||
.expect("contention is a failed acquire");
|
||||
assert!(timed_out.is_none());
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while managers.iter().any(|manager| manager.current_lock_counts().1 != 0) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("timed out peer attempts must remove their tickets");
|
||||
let reader = cluster_queued(&lock, "next-reader", crate::LockType::Shared);
|
||||
cluster_waiters(&managers, 1).await;
|
||||
let writer = cluster_queued(&lock, "next-writer", crate::LockType::Exclusive);
|
||||
cluster_waiters(&managers, 2).await;
|
||||
drop(holders);
|
||||
let reader = cluster_acquired(reader).await;
|
||||
assert!(!writer.is_finished());
|
||||
drop(reader);
|
||||
drop(cluster_acquired(writer).await);
|
||||
timeout(Duration::from_secs(3), async {
|
||||
while managers.iter().any(|manager| manager.current_lock_counts() != (0, 0)) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("distributed holders and queues must drain");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_queue_does_not_change_ordinary_key_writer_preference() {
|
||||
let manager = manager();
|
||||
let ordinary = ObjectKey::new("publication-bucket", "ordinary-key");
|
||||
let holder = manager
|
||||
.acquire_write_lock(ordinary.clone(), "holder")
|
||||
.await
|
||||
.expect("ordinary holder");
|
||||
let reader = tokio::spawn({
|
||||
let manager = manager.clone();
|
||||
let ordinary = ordinary.clone();
|
||||
async move { manager.acquire_read_lock(ordinary, "reader").await.expect("ordinary reader") }
|
||||
});
|
||||
waiters(&manager, 1).await;
|
||||
let writer = tokio::spawn({
|
||||
let manager = manager.clone();
|
||||
async move { manager.acquire_write_lock(ordinary, "writer").await.expect("ordinary writer") }
|
||||
});
|
||||
waiters(&manager, 2).await;
|
||||
drop(holder);
|
||||
let writer = acquired(writer).await;
|
||||
assert!(!reader.is_finished(), "ordinary keys retain writer preference");
|
||||
drop(writer);
|
||||
drop(acquired(reader).await);
|
||||
assert_eq!(manager.current_lock_counts(), (0, 0));
|
||||
}
|
||||
@@ -21,6 +21,7 @@ use tokio::time::timeout;
|
||||
use crate::fast_lock::{
|
||||
metrics::ShardMetrics,
|
||||
object_pool::ObjectStatePool,
|
||||
publication::{self, PublicationTicket},
|
||||
state::ObjectLockState,
|
||||
types::{LockMode, LockResult, ObjectKey, ObjectLockRequest},
|
||||
};
|
||||
@@ -134,6 +135,9 @@ impl LockShard {
|
||||
/// Acquire lock with fast path optimization
|
||||
pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> {
|
||||
let start_time = Instant::now();
|
||||
if publication::applies(&request.key) {
|
||||
return self.acquire_publication(request, start_time).await;
|
||||
}
|
||||
|
||||
// Try fast path first
|
||||
if let Some(_state) = self.try_fast_path(request) {
|
||||
@@ -145,15 +149,65 @@ impl LockShard {
|
||||
self.acquire_lock_slow_path(request, start_time).await
|
||||
}
|
||||
|
||||
fn publication_ticket(&self, request: &ObjectLockRequest) -> Option<PublicationTicket> {
|
||||
let mut objects = self.objects.write();
|
||||
let state = objects
|
||||
.entry(request.key.clone())
|
||||
.or_insert_with(|| Arc::new(*self.object_pool.acquire()))
|
||||
.clone();
|
||||
PublicationTicket::new(state, request.mode)
|
||||
}
|
||||
|
||||
async fn acquire_publication(&self, request: &ObjectLockRequest, started: Instant) -> Result<(), LockResult> {
|
||||
let Some(ticket) = self.publication_ticket(request) else {
|
||||
return Err(LockResult::Timeout);
|
||||
};
|
||||
let mut first_attempt = true;
|
||||
loop {
|
||||
// Register before checking so a release/cancellation cannot be lost
|
||||
// between the eligibility check and the notification wait.
|
||||
let notified = ticket.queue.changed.notified();
|
||||
let remaining = request.acquire_timeout.saturating_sub(started.elapsed());
|
||||
if !first_attempt && remaining.is_zero() {
|
||||
self.metrics.record_timeout();
|
||||
return Err(LockResult::Timeout);
|
||||
}
|
||||
if ticket.try_acquire(request) {
|
||||
self.metrics.record_slow_path_success();
|
||||
return Ok(());
|
||||
}
|
||||
first_attempt = false;
|
||||
if remaining.is_zero() || timeout(remaining, notified).await.is_err() {
|
||||
self.metrics.record_timeout();
|
||||
return Err(LockResult::Timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try fast path only (without fallback to slow path)
|
||||
pub fn try_fast_path_only(&self, request: &ObjectLockRequest) -> bool {
|
||||
if publication::applies(&request.key) {
|
||||
// Batch acquisition must join the same queue, even for a no-wait
|
||||
// attempt. Dropping a failed ticket does not consume a queue turn.
|
||||
return self
|
||||
.publication_ticket(request)
|
||||
.is_some_and(|ticket| ticket.try_acquire(request));
|
||||
}
|
||||
// Early check to avoid unnecessary lock contention
|
||||
if let Some(state) = self.objects.read().get(&request.key)
|
||||
&& !state.atomic_state.is_fast_path_available(request.mode)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
self.try_fast_path(request).is_some()
|
||||
// Count the acquisition like `acquire_lock` does: batch callers use
|
||||
// this path for every uncontended key, and the metric promises the
|
||||
// total of successful acquisitions.
|
||||
if self.try_fast_path(request).is_some() {
|
||||
self.metrics.record_fast_path_success();
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Try fast path lock acquisition (lock-free when possible)
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
@@ -78,13 +78,21 @@ impl AtomicLockState {
|
||||
|
||||
/// Try to acquire shared lock (fast path)
|
||||
pub fn try_acquire_shared(&self) -> bool {
|
||||
self.try_acquire_shared_with_mask(NO_WRITER_AND_NO_WAITING_WRITERS)
|
||||
}
|
||||
|
||||
// Publication admission already orders readers against waiting writers.
|
||||
fn try_acquire_admitted_shared(&self) -> bool {
|
||||
self.try_acquire_shared_with_mask(WRITER_FLAG_MASK)
|
||||
}
|
||||
|
||||
fn try_acquire_shared_with_mask(&self, blocked_mask: u64) -> bool {
|
||||
self.update_access_time();
|
||||
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
|
||||
// Fast path check - cannot acquire if there's a writer or writers waiting
|
||||
if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 {
|
||||
if (current & blocked_mask) != 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -334,6 +342,7 @@ pub struct ObjectLockState {
|
||||
// First cache line: Most frequently accessed data
|
||||
/// Atomic state for fast operations
|
||||
pub atomic_state: AtomicLockState,
|
||||
pub(super) publication: OnceLock<Arc<super::publication::PublicationQueue>>,
|
||||
|
||||
// Second cache line: Notification mechanisms
|
||||
/// Notification for readers (traditional)
|
||||
@@ -377,6 +386,7 @@ impl ObjectLockState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
atomic_state: AtomicLockState::new(),
|
||||
publication: OnceLock::new(),
|
||||
read_notify: Notify::new(),
|
||||
write_notify: Notify::new(),
|
||||
optimized_notify: OptimizedNotify::new(),
|
||||
@@ -392,6 +402,19 @@ impl ObjectLockState {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.record_shared_owner(owner, lock_timeout);
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn try_acquire_admitted_shared(&self, owner: &Arc<str>, lock_timeout: Duration) -> bool {
|
||||
if !self.atomic_state.try_acquire_admitted_shared() {
|
||||
return false;
|
||||
}
|
||||
self.record_shared_owner(owner, lock_timeout);
|
||||
true
|
||||
}
|
||||
|
||||
fn record_shared_owner(&self, owner: &Arc<str>, lock_timeout: Duration) {
|
||||
self.atomic_state.update_access_time();
|
||||
let mut shared = self.shared_owners.write();
|
||||
if let Some(entry) = shared.iter_mut().find(|entry| entry.owner.as_ref() == owner.as_ref()) {
|
||||
@@ -406,7 +429,6 @@ impl ObjectLockState {
|
||||
lock_timeout,
|
||||
});
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Try fast path exclusive lock acquisition
|
||||
@@ -437,6 +459,9 @@ impl ObjectLockState {
|
||||
Some(shared.remove(pos))
|
||||
};
|
||||
if self.atomic_state.release_shared() {
|
||||
if let Some(queue) = self.publication.get() {
|
||||
queue.changed.notify_waiters();
|
||||
}
|
||||
if shared.is_empty() {
|
||||
drop(shared);
|
||||
self.optimized_notify.notify_writer();
|
||||
@@ -480,6 +505,9 @@ impl ObjectLockState {
|
||||
if self.atomic_state.release_exclusive() {
|
||||
*current = None;
|
||||
drop(current);
|
||||
if let Some(queue) = self.publication.get() {
|
||||
queue.changed.notify_waiters();
|
||||
}
|
||||
// Notify waiters using optimized system - prefer writers over readers
|
||||
if self
|
||||
.atomic_state
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::{
|
||||
GlobalLockManager, ObjectKey,
|
||||
error::Result,
|
||||
fast_lock::{FastLockGuard, LockManager, LockMode, ObjectLockRequest},
|
||||
fast_lock::{FastLockGuard, LockManager, LockMode, LockResult, ObjectLockRequest},
|
||||
types::{LockPriority, LockRequest, LockType},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
@@ -112,4 +112,61 @@ impl LocalLock {
|
||||
.with_ttl(ttl);
|
||||
self.acquire_guard(&req).await
|
||||
}
|
||||
|
||||
/// Acquire exclusive locks for many independent resources in one batched
|
||||
/// pass; see [`crate::distributed_lock::DistributedLock::acquire_guard_batch`].
|
||||
/// The manager's batch acquire sorts every request into one global
|
||||
/// (shard, key) order, so concurrent batches cannot deadlock. Returns one
|
||||
/// guard or error per resource, in `resources` order.
|
||||
pub async fn acquire_guard_batch(
|
||||
&self,
|
||||
resources: &[ObjectKey],
|
||||
owner: &str,
|
||||
timeout: Duration,
|
||||
ttl: Duration,
|
||||
) -> Vec<Result<FastLockGuard>> {
|
||||
if resources.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let batch = crate::fast_lock::BatchLockRequest {
|
||||
requests: resources
|
||||
.iter()
|
||||
.map(|resource| {
|
||||
ObjectLockRequest::new_write(resource.clone(), owner)
|
||||
.with_acquire_timeout(timeout)
|
||||
.with_lock_timeout(ttl)
|
||||
})
|
||||
.collect(),
|
||||
owner: owner.into(),
|
||||
all_or_nothing: false,
|
||||
};
|
||||
let outcome = self.manager.acquire_locks_batch(batch).await;
|
||||
let mut guards = outcome
|
||||
.successful_locks
|
||||
.into_iter()
|
||||
.zip(outcome.guards)
|
||||
.collect::<std::collections::HashMap<ObjectKey, FastLockGuard>>();
|
||||
let mut failures = outcome
|
||||
.failed_locks
|
||||
.into_iter()
|
||||
.collect::<std::collections::HashMap<ObjectKey, LockResult>>();
|
||||
resources
|
||||
.iter()
|
||||
.map(|resource| {
|
||||
if let Some(guard) = guards.remove(resource) {
|
||||
return Ok(guard);
|
||||
}
|
||||
match failures.remove(resource) {
|
||||
Some(LockResult::Timeout) => Err(crate::error::LockError::timeout(resource.to_string(), timeout)),
|
||||
Some(LockResult::Conflict { current_owner, .. }) => Err(crate::error::LockError::AlreadyLocked {
|
||||
resource: resource.to_string(),
|
||||
owner: current_owner.to_string(),
|
||||
}),
|
||||
Some(LockResult::Acquired) | None => {
|
||||
Err(crate::error::LockError::internal("batch lock acquire returned no outcome for key"))
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,6 +88,17 @@ impl NamespaceLockWrapper {
|
||||
pub async fn get_read_lock(&self, timeout: Duration) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
|
||||
self.lock.get_read_lock(self.resource.clone(), &self.owner, timeout).await
|
||||
}
|
||||
|
||||
/// Acquire write locks for `resources` in one batched round using this
|
||||
/// wrapper's owner; see [`NamespaceLock::acquire_write_locks_batch`].
|
||||
/// One guard or error per resource, in `resources` order.
|
||||
pub async fn get_write_locks_batch(
|
||||
&self,
|
||||
resources: &[ObjectKey],
|
||||
timeout: Duration,
|
||||
) -> Vec<std::result::Result<NamespaceLockGuard, crate::error::LockError>> {
|
||||
self.lock.acquire_write_locks_batch(resources, &self.owner, timeout).await
|
||||
}
|
||||
}
|
||||
|
||||
impl NamespaceLockGuard {
|
||||
@@ -349,6 +360,38 @@ impl NamespaceLock {
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquire exclusive locks for many independent resources in one batched
|
||||
/// round; see [`crate::distributed_lock::DistributedLock::acquire_guard_batch`].
|
||||
/// Returns one guard or error per resource, in `resources` order.
|
||||
///
|
||||
/// This is the lock primitive bulk mutations (S3 `DeleteObjects`) should
|
||||
/// use: per-key quorum round-trips serialize a 1000-key batch, while the
|
||||
/// batched form is one wire round per lock client and stays deadlock-free
|
||||
/// without any canonical serial fallback (every lock server orders
|
||||
/// concurrent batches by the same global key order).
|
||||
pub async fn acquire_write_locks_batch(
|
||||
&self,
|
||||
resources: &[ObjectKey],
|
||||
owner: &str,
|
||||
timeout: Duration,
|
||||
) -> Vec<std::result::Result<NamespaceLockGuard, crate::error::LockError>> {
|
||||
let ttl = crate::fast_lock::DEFAULT_LOCK_TIMEOUT;
|
||||
match self {
|
||||
Self::Distributed(lock) => lock
|
||||
.acquire_guard_batch(resources, owner, timeout, ttl)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|result| result.map(NamespaceLockGuard::Standard))
|
||||
.collect(),
|
||||
Self::Local(lock) => lock
|
||||
.acquire_guard_batch(resources, owner, timeout, ttl)
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|result| result.map(NamespaceLockGuard::Fast))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get health information
|
||||
pub async fn get_health(&self) -> crate::types::HealthInfo {
|
||||
let lock_stats = self.get_stats().await;
|
||||
|
||||
@@ -881,7 +881,7 @@ pub fn parse_replicate_decision(_bucket: &str, s: &str) -> std::io::Result<Repli
|
||||
/// writers may use different textual representations. `invalid` preserves the
|
||||
/// distinction between truly absent legacy metadata and corrupt/conflicting
|
||||
/// compatibility aliases.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ReplicationGenerationSnapshot {
|
||||
pub timestamp: Option<String>,
|
||||
pub mutation_id: Option<String>,
|
||||
|
||||
@@ -424,7 +424,7 @@ impl EcObjectStore {
|
||||
|
||||
async fn object_reader(&self, range: Option<HTTPRangeSpec>) -> Result<SelectGetObjectReader> {
|
||||
#[cfg(test)]
|
||||
self.reader_open_count.fetch_add(1, Ordering::Relaxed);
|
||||
test::pause_before_reader_open(&self.input.bucket, self.reader_open_count.fetch_add(1, Ordering::Relaxed) + 1).await;
|
||||
self.snapshot(None)
|
||||
.await?
|
||||
.open_reader(range)
|
||||
@@ -1416,7 +1416,6 @@ mod test {
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
use http::HeaderMap;
|
||||
use parking_lot::Mutex;
|
||||
use rustfs_test_utils::PutObjectCommitBarrier;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::{
|
||||
CSVInput, CSVOutput, CompressionType, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, JSONOutput,
|
||||
@@ -1433,6 +1432,54 @@ mod test {
|
||||
|
||||
use tokio::{io::AsyncReadExt, sync::Semaphore};
|
||||
|
||||
struct ReaderOpenPause {
|
||||
bucket: String,
|
||||
ordinal: usize,
|
||||
reached: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
static READER_OPEN_PAUSE: Mutex<Option<Arc<ReaderOpenPause>>> = Mutex::new(None);
|
||||
|
||||
struct ReaderOpenPauseGuard(Arc<ReaderOpenPause>);
|
||||
|
||||
impl ReaderOpenPauseGuard {
|
||||
fn install(bucket: &str, ordinal: usize) -> Self {
|
||||
let hook = Arc::new(ReaderOpenPause {
|
||||
bucket: bucket.to_owned(),
|
||||
ordinal,
|
||||
reached: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
*READER_OPEN_PAUSE.lock() = Some(Arc::clone(&hook));
|
||||
Self(hook)
|
||||
}
|
||||
async fn wait(&self) {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), self.0.reached.notified())
|
||||
.await
|
||||
.expect("query reached next range");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReaderOpenPauseGuard {
|
||||
fn drop(&mut self) {
|
||||
*READER_OPEN_PAUSE.lock() = None;
|
||||
self.0.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pause_before_reader_open(bucket: &str, ordinal: usize) {
|
||||
let hook = READER_OPEN_PAUSE
|
||||
.lock()
|
||||
.as_ref()
|
||||
.filter(|hook| hook.bucket == bucket && hook.ordinal == ordinal)
|
||||
.cloned();
|
||||
if let Some(hook) = hook {
|
||||
hook.reached.notify_one();
|
||||
hook.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
const JSON_DOCUMENT_TEST_MEMORY_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -1705,7 +1752,7 @@ mod test {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn legacy_constructor_reuses_head_snapshot_for_body() {
|
||||
async fn legacy_constructor_revalidates_head_snapshot_before_body() {
|
||||
const BUCKET: &str = "s3select-lazy-snapshot-head-body";
|
||||
const OBJECT: &str = "input.csv";
|
||||
const OLD_DATA: &[u8] = b"id,name\n1,old\n";
|
||||
@@ -1727,34 +1774,15 @@ mod test {
|
||||
.expect("HEAD should lazily prepare the snapshot");
|
||||
assert_eq!(head.meta.size, u64::try_from(OLD_DATA.len()).expect("fixture length should fit in u64"));
|
||||
|
||||
let commit_barrier = PutObjectCommitBarrier::before_namespace(BUCKET, OBJECT);
|
||||
let writer = tokio::spawn(async move {
|
||||
env.put_object_bytes(BUCKET, OBJECT, NEW_DATA.to_vec()).await;
|
||||
});
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release_and_wait_until_namespace_pending().await;
|
||||
assert!(!writer.is_finished(), "overwrite must wait for the lazy snapshot");
|
||||
|
||||
let result = store
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), env.put_object_bytes(BUCKET, OBJECT, NEW_DATA.to_vec()))
|
||||
.await
|
||||
.expect("HEAD snapshot must not block overwrite");
|
||||
let error = store
|
||||
.get_opts(&Path::from(OBJECT), GetOptions::default())
|
||||
.await
|
||||
.expect("body should reuse the HEAD snapshot");
|
||||
let GetResultPayload::Stream(stream) = result.payload else {
|
||||
panic!("expected streaming snapshot body");
|
||||
};
|
||||
let bytes = stream
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.expect("collect lazy snapshot body")
|
||||
.concat();
|
||||
assert_eq!(bytes, OLD_DATA);
|
||||
.expect_err("first body open must revalidate HEAD generation");
|
||||
assert!(error.to_string().contains("object read semantics changed"));
|
||||
assert_eq!(store.reader_open_count.load(Ordering::Relaxed), 1);
|
||||
|
||||
drop(store);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), writer)
|
||||
.await
|
||||
.expect("overwrite should finish after the lazy snapshot is released")
|
||||
.expect("overwrite task should join");
|
||||
assert_eq!(read_current_object(BUCKET, OBJECT).await, NEW_DATA);
|
||||
}
|
||||
|
||||
@@ -2089,39 +2117,23 @@ mod test {
|
||||
selected_start,
|
||||
selected_start,
|
||||
);
|
||||
let commit_barrier = PutObjectCommitBarrier::before_namespace(BUCKET, OBJECT);
|
||||
let writer = tokio::spawn(async move {
|
||||
env.put_object_bytes(BUCKET, OBJECT, NEW_DATA.to_vec()).await;
|
||||
let store = Arc::new(store);
|
||||
let pause = ReaderOpenPauseGuard::install(BUCKET, 2);
|
||||
let reading = tokio::spawn({
|
||||
let store = Arc::clone(&store);
|
||||
async move { store.get_opts(&Path::from(OBJECT), GetOptions::default()).await }
|
||||
});
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release_and_wait_until_namespace_pending().await;
|
||||
assert!(
|
||||
!writer.is_finished(),
|
||||
"overwrite must remain blocked while the SelectObjectContent snapshot is alive"
|
||||
);
|
||||
|
||||
let result = store
|
||||
.get_opts(&Path::from(OBJECT), GetOptions::default())
|
||||
pause.wait().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), env.put_object_bytes(BUCKET, OBJECT, NEW_DATA.to_vec()))
|
||||
.await
|
||||
.expect("read CSV header and body from one snapshot");
|
||||
let GetResultPayload::Stream(stream) = result.payload else {
|
||||
panic!("expected streaming CSV header/body payload");
|
||||
.expect("writer must complete between Select range opens");
|
||||
drop(pause);
|
||||
let error = match reading.await.expect("query task") {
|
||||
Ok(_) => panic!("query must not combine old context with replacement body"),
|
||||
Err(error) => error,
|
||||
};
|
||||
let bytes = stream
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.expect("collect CSV header/body snapshot")
|
||||
.concat();
|
||||
|
||||
assert_eq!(bytes, b"old_header,value\nold_body,1\n");
|
||||
assert!(error.to_string().contains("object read semantics changed"));
|
||||
assert_eq!(store.reader_open_count.load(Ordering::Relaxed), 2);
|
||||
assert!(!writer.is_finished(), "overwrite must remain blocked after both snapshot readers finish");
|
||||
|
||||
drop(store);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), writer)
|
||||
.await
|
||||
.expect("overwrite should finish after the snapshot is released")
|
||||
.expect("overwrite task should join");
|
||||
assert_eq!(read_current_object(BUCKET, OBJECT).await, NEW_DATA);
|
||||
}
|
||||
|
||||
@@ -2143,42 +2155,23 @@ mod test {
|
||||
|
||||
let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
|
||||
let store = scan_range_csv_store(BUCKET, OBJECT, snapshot, "aa", None, 4, 5);
|
||||
let commit_barrier = PutObjectCommitBarrier::before_namespace(BUCKET, OBJECT);
|
||||
let writer = tokio::spawn(async move {
|
||||
env.put_object_bytes(BUCKET, OBJECT, NEW_DATA.to_vec()).await;
|
||||
let store = Arc::new(store);
|
||||
let pause = ReaderOpenPauseGuard::install(BUCKET, 2);
|
||||
let reading = tokio::spawn({
|
||||
let store = Arc::clone(&store);
|
||||
async move { store.get_opts(&Path::from(OBJECT), GetOptions::default()).await }
|
||||
});
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release_and_wait_until_namespace_pending().await;
|
||||
assert!(
|
||||
!writer.is_finished(),
|
||||
"overwrite must remain blocked while the SelectObjectContent snapshot is alive"
|
||||
);
|
||||
|
||||
let result = store
|
||||
.get_opts(&Path::from(OBJECT), GetOptions::default())
|
||||
pause.wait().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), env.put_object_bytes(BUCKET, OBJECT, NEW_DATA.to_vec()))
|
||||
.await
|
||||
.expect("read ScanRange context and main body from one snapshot");
|
||||
let GetResultPayload::Stream(stream) = result.payload else {
|
||||
panic!("expected streaming ScanRange context payload");
|
||||
.expect("writer must complete between Select range opens");
|
||||
drop(pause);
|
||||
let error = match reading.await.expect("query task") {
|
||||
Ok(_) => panic!("query must not combine old context with replacement body"),
|
||||
Err(error) => error,
|
||||
};
|
||||
let bytes = stream
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.expect("collect ScanRange context snapshot")
|
||||
.concat();
|
||||
|
||||
assert_eq!(bytes, b"a222\r\n");
|
||||
assert!(error.to_string().contains("object read semantics changed"));
|
||||
assert_eq!(store.reader_open_count.load(Ordering::Relaxed), 2);
|
||||
assert!(
|
||||
!writer.is_finished(),
|
||||
"overwrite must remain blocked after context and main readers finish"
|
||||
);
|
||||
|
||||
drop(store);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), writer)
|
||||
.await
|
||||
.expect("overwrite should finish after the snapshot is released")
|
||||
.expect("overwrite task should join");
|
||||
assert_eq!(read_current_object(BUCKET, OBJECT).await, NEW_DATA);
|
||||
}
|
||||
|
||||
@@ -2468,6 +2461,117 @@ mod test {
|
||||
assert!(err.to_string().contains("ScanRange"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn parquet_snapshot_rejects_overwrite_between_footer_and_data_ranges() {
|
||||
use datafusion::arrow::{
|
||||
array::StringArray,
|
||||
datatypes::{DataType, Field, Schema},
|
||||
record_batch::RecordBatch,
|
||||
};
|
||||
use datafusion::object_store::ObjectStoreExt as _;
|
||||
use datafusion::parquet::{
|
||||
arrow::{
|
||||
ArrowWriter,
|
||||
arrow_reader::ArrowReaderOptions,
|
||||
async_reader::{AsyncFileReader, ParquetRecordBatchStreamBuilder},
|
||||
},
|
||||
basic::Compression,
|
||||
errors::{ParquetError, Result as ParquetResult},
|
||||
file::{
|
||||
metadata::{ParquetMetaData, ParquetMetaDataReader},
|
||||
properties::WriterProperties,
|
||||
},
|
||||
};
|
||||
use futures::future::BoxFuture;
|
||||
|
||||
struct SnapshotParquetReader {
|
||||
store: Arc<EcObjectStore>,
|
||||
path: Path,
|
||||
file_size: u64,
|
||||
}
|
||||
|
||||
impl AsyncFileReader for SnapshotParquetReader {
|
||||
fn get_bytes(&mut self, range: std::ops::Range<u64>) -> BoxFuture<'_, ParquetResult<Bytes>> {
|
||||
Box::pin(async move {
|
||||
self.store
|
||||
.get_range(&self.path, range)
|
||||
.await
|
||||
.map_err(|error| ParquetError::External(Box::new(error)))
|
||||
})
|
||||
}
|
||||
|
||||
fn get_metadata<'a>(
|
||||
&'a mut self,
|
||||
options: Option<&'a ArrowReaderOptions>,
|
||||
) -> BoxFuture<'a, ParquetResult<Arc<ParquetMetaData>>> {
|
||||
Box::pin(async move {
|
||||
let file_size = self.file_size;
|
||||
let metadata = ParquetMetaDataReader::new()
|
||||
.with_arrow_reader_options(options)
|
||||
.load_and_finish(self, file_size)
|
||||
.await?;
|
||||
Ok(Arc::new(metadata))
|
||||
})
|
||||
}
|
||||
}
|
||||
const BUCKET: &str = "select-parquet-range-generation";
|
||||
const OBJECT: &str = "input.parquet";
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Utf8, false)]));
|
||||
let values = (0..6000)
|
||||
.map(|index| format!("{index:08}{}", "x".repeat(248)))
|
||||
.collect::<Vec<_>>();
|
||||
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(StringArray::from(values))]).expect("Parquet batch");
|
||||
let props = WriterProperties::builder()
|
||||
.set_dictionary_enabled(false)
|
||||
.set_compression(Compression::UNCOMPRESSED)
|
||||
.build();
|
||||
let mut bytes = Vec::new();
|
||||
let mut writer = ArrowWriter::try_new(&mut bytes, schema, Some(props)).expect("Parquet writer");
|
||||
writer.write(&batch).expect("Parquet rows");
|
||||
writer.close().expect("Parquet footer");
|
||||
assert!(bytes.len() > 1024 * 1024, "fixture requires data outside the footer prefetch");
|
||||
let env = crate::storage_api::select_test_ecstore_env().await;
|
||||
env.make_bucket(BUCKET, false).await;
|
||||
env.put_object_bytes(BUCKET, OBJECT, bytes.clone()).await;
|
||||
let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await;
|
||||
let mut input = (*csv_input(BUCKET, OBJECT)).clone();
|
||||
input.request.input_serialization.csv = None;
|
||||
input.request.input_serialization.parquet = Some(s3s::dto::ParquetInput {});
|
||||
let store = Arc::new(EcObjectStore::new_with_snapshot(Arc::new(input), Arc::clone(&snapshot)).expect("Parquet store"));
|
||||
let (footer_ready, footer_loaded) = tokio::sync::oneshot::channel();
|
||||
let (resume_data, data_ready) = tokio::sync::oneshot::channel();
|
||||
let file_size = snapshot.logical_size();
|
||||
let query_store = store.clone();
|
||||
let query = tokio::spawn(async move {
|
||||
let reader = SnapshotParquetReader {
|
||||
store: query_store,
|
||||
path: Path::from(OBJECT),
|
||||
file_size,
|
||||
};
|
||||
let builder = ParquetRecordBatchStreamBuilder::new(reader).await?;
|
||||
footer_ready.send(()).expect("signal fully decoded footer");
|
||||
data_ready.await.expect("resume data ranges after overwrite");
|
||||
let stream = builder.build()?;
|
||||
stream.try_collect::<Vec<_>>().await
|
||||
});
|
||||
footer_loaded
|
||||
.await
|
||||
.expect("read and decode the original footer before overwriting");
|
||||
// Same bytes, size and ETag still constitute a new generation (new data_dir).
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), env.put_object_bytes(BUCKET, OBJECT, bytes))
|
||||
.await
|
||||
.expect("overwrite cannot wait for Parquet's next range");
|
||||
resume_data.send(()).expect("resume the original query");
|
||||
let error = query
|
||||
.await
|
||||
.expect("Parquet task")
|
||||
.expect_err("footer and row data must not span generations");
|
||||
assert!(error.to_string().contains("object read semantics changed"));
|
||||
assert!(snapshot.ensure_valid().is_err());
|
||||
assert!(store.reader_open_count.load(Ordering::Relaxed) >= 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_range_conversion_for_parquet_bounded_ranges() {
|
||||
let range = http_range_spec_from_get_range(&GetRange::Bounded(10..20));
|
||||
|
||||
@@ -194,7 +194,9 @@ impl QueryExecutionTracker {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn query_guard(&self) -> Option<QueryExecutionGuard> {
|
||||
/// Retains admission for queued or running work that can outlive its request.
|
||||
/// Returns `None` once the execution has finished or reached its deadline.
|
||||
pub fn query_guard(&self) -> Option<QueryExecutionGuard> {
|
||||
let query_guard = self.inner.query_guard.lock();
|
||||
if Instant::now() >= self.inner.deadline || self.status() != QueryExecutionStatus::Active {
|
||||
return None;
|
||||
@@ -297,6 +299,9 @@ pub struct SessionCtxFactory {
|
||||
|
||||
pub const DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// Maximum inferred fields and explicit projection expressions for S3 Select.
|
||||
pub const MAX_SELECT_COLUMNS: usize = 1024;
|
||||
|
||||
impl Default for SessionCtxFactory {
|
||||
fn default() -> Self {
|
||||
Self::new(false)
|
||||
@@ -436,6 +441,26 @@ impl SessionCtxFactory {
|
||||
};
|
||||
let rt = RuntimeEnvBuilder::new().with_memory_limit(memory_limit_bytes, 1.0).build()?;
|
||||
let config = SessionConfig::new().with_target_partitions(self.target_partitions);
|
||||
// Arrow CSV reserves two usize offsets for every cell outside the
|
||||
// DataFusion memory pool. Bound that reservation to at most 4 MiB and
|
||||
// one sixteenth of the configured query budget, including narrow input.
|
||||
let config = if context.input.request.input_serialization.csv.is_some() {
|
||||
let batch_rows = (memory_limit_bytes / 16 / (MAX_SELECT_COLUMNS * 2 * size_of::<usize>())).clamp(1, 256);
|
||||
config.with_batch_size(batch_rows)
|
||||
} else {
|
||||
config
|
||||
};
|
||||
// Parquet's dictionary views can expand during an explicit SQL cast.
|
||||
// Keep every downstream operator (including FilterExec's internal
|
||||
// coalescer) at one row, and disable the separate coalesce optimizer.
|
||||
let config = if context.input.request.input_serialization.parquet.is_some() {
|
||||
let mut config = config.with_batch_size(1).with_target_partitions(1);
|
||||
config.options_mut().execution.coalesce_batches = false;
|
||||
config.options_mut().optimizer.repartition_file_scans = false;
|
||||
config
|
||||
} else {
|
||||
config
|
||||
};
|
||||
let custom_two_byte_record_delimiter = context
|
||||
.input
|
||||
.request
|
||||
@@ -810,7 +835,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_scan_range_keeps_file_repartitioning() {
|
||||
async fn parquet_scan_range_keeps_single_partition_decode() {
|
||||
let mut context = test_context();
|
||||
let request = &mut Arc::make_mut(&mut context.input).request;
|
||||
request.scan_range = Some(ScanRange {
|
||||
@@ -826,7 +851,9 @@ mod tests {
|
||||
.await
|
||||
.expect("Parquet ScanRange session should be created");
|
||||
|
||||
assert!(session.inner().config().options().optimizer.repartition_file_scans);
|
||||
assert!(!session.inner().config().options().optimizer.repartition_file_scans);
|
||||
assert_eq!(session.inner().config().batch_size(), 1);
|
||||
assert_eq!(session.inner().config().target_partitions(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1006,6 +1033,39 @@ mod tests {
|
||||
assert_eq!(session.inner().config().target_partitions(), SessionConfig::new().target_partitions());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decoder_batch_budgets_keep_parquet_downstream_execution_single_row() {
|
||||
let factory = SessionCtxFactory::new(true);
|
||||
for (budget, expected) in [(64 * 1024 * 1024, 256), (8 * 1024 * 1024, 32), (1024, 1)] {
|
||||
let session = factory
|
||||
.create_session_ctx_inner(&test_context(), None, JsonSource::default(), None, None, budget)
|
||||
.await
|
||||
.expect("CSV session");
|
||||
assert_eq!(session.inner().config().batch_size(), expected);
|
||||
}
|
||||
for parquet in [false, true] {
|
||||
let mut context = test_context();
|
||||
let input = Arc::make_mut(&mut context.input);
|
||||
input.request.input_serialization.csv = None;
|
||||
if parquet {
|
||||
input.request.input_serialization.parquet = Some(ParquetInput::default());
|
||||
} else {
|
||||
input.request.input_serialization.json = Some(JSONInput {
|
||||
type_: Some(JSONType::from_static(JSONType::LINES)),
|
||||
});
|
||||
}
|
||||
let session = factory.create_session_ctx(&context).await.expect("non-CSV session");
|
||||
assert_eq!(
|
||||
session.inner().config().batch_size(),
|
||||
if parquet { 1 } else { SessionConfig::new().batch_size() }
|
||||
);
|
||||
if parquet {
|
||||
assert!(!session.inner().config().options().execution.coalesce_batches);
|
||||
assert_eq!(session.inner().config().target_partitions(), 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_factory_applies_memory_limit() {
|
||||
let factory = SessionCtxFactory::new(true);
|
||||
|
||||
@@ -45,6 +45,7 @@ hotpath.workspace = true
|
||||
rustfs-s3select-api = { workspace = true }
|
||||
async-trait.workspace = true
|
||||
bytes.workspace = true
|
||||
csv-core.workspace = true
|
||||
datafusion = { workspace = true, default-features = false, features = ["parquet", "recursive_protection", "sql"] }
|
||||
derive_builder = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
@@ -56,6 +57,7 @@ tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"]
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
flate2.workspace = true
|
||||
rustfs-test-utils = { workspace = true, features = ["put-object-commit-barrier"] }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -32,6 +32,16 @@
|
||||
|
||||
For comprehensive documentation, examples, and usage guides, please visit the main [RustFS repository](https://github.com/rustfs/rustfs).
|
||||
|
||||
## Resource limits
|
||||
|
||||
JSON LINES schema inference reads the first 1,000 nonblank records, preserving Arrow's type inference and source field order. It buffers at most 4 MiB of decompressed input. Reaching that byte limit before the inference sample or end of the object is an error; RustFS does not truncate a record or silently shorten the sample. CSV inference reads only the first non-comment record (the header when enabled), with the same 4 MiB prefix limit. Quoting, escaping, comments, header modes and record delimiters still apply.
|
||||
|
||||
CSV schemas have at most 1,024 columns. JSON LINES schemas have at most 1,024 distinct fields across all nested objects in the inference sample. SQL expressions have at most 256 KiB and 1,024 projection expressions; wildcard expansion must also fit 1,024 output columns. Inputs exceeding these limits now return explicit errors, including inputs older versions attempted to process without these bounds. These changes do not alter Parquet schema inference or JSON DOCUMENT input handling.
|
||||
|
||||
CSV decoding uses at most 256 rows per batch, reduced further for smaller configured query memory limits. At 1,024 columns, Arrow's two-offset-per-cell reservation is at most 4 MiB on 64-bit systems and at most one sixteenth of the configured query budget (with a minimum one-row batch). Schema prefix storage and temporary inference work are reserved against the existing query memory pool. The configured memory limit can reject a sample before the 4 MiB hard prefix limit.
|
||||
|
||||
SQL parsing, schema inference for CSV and JSON LINES, and SQL planning run on blocking workers after query admission. Queued and running workers retain that admission until they finish, even if the HTTP request is canceled or its query deadline expires. Request cancellation stops waiting immediately; it cannot forcibly interrupt synchronous third-party code or admit replacement CPU work before that code exits.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -17,8 +17,9 @@ use datafusion::arrow::{
|
||||
error::ArrowError,
|
||||
json::reader::{ValueIter, infer_json_schema_from_iterator},
|
||||
};
|
||||
use rustfs_s3select_api::query::session::MAX_SELECT_COLUMNS;
|
||||
use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor};
|
||||
use std::{collections::HashMap, fmt, sync::Arc};
|
||||
use std::{cell::Cell, collections::HashMap, fmt, rc::Rc, sync::Arc};
|
||||
|
||||
/// Infers a JSON LINES schema with Arrow's types and the source's first-seen
|
||||
/// key order.
|
||||
@@ -27,10 +28,9 @@ use std::{collections::HashMap, fmt, sync::Arc};
|
||||
/// process-wide `serde_json/preserve_order` feature is enabled, so `SELECT *`
|
||||
/// would otherwise reorder fields and CSV output columns.
|
||||
pub(super) fn infer_schema(bytes: &[u8], max_records: usize) -> Result<Schema, ArrowError> {
|
||||
let schema = infer_json_schema_from_iterator(ValueIter::new(bytes, Some(max_records)))?;
|
||||
let mut order = KeyOrder::default();
|
||||
// Same records as `ValueIter`: one value per line, blank lines skipped.
|
||||
// Arrow has already rejected invalid UTF-8 and JSON within this prefix.
|
||||
// Check width before Arrow merges records; retain its type inference.
|
||||
let records = bytes
|
||||
.split(|byte| *byte == b'\n')
|
||||
.filter_map(|line| std::str::from_utf8(line).ok())
|
||||
@@ -43,6 +43,7 @@ pub(super) fn infer_schema(bytes: &[u8], max_records: usize) -> Result<Schema, A
|
||||
.deserialize(&mut deserializer)
|
||||
.map_err(|error| ArrowError::JsonError(format!("Not valid JSON: {error}")))?;
|
||||
}
|
||||
let schema = infer_json_schema_from_iterator(ValueIter::new(bytes, Some(max_records)))?;
|
||||
Ok(Schema::new_with_metadata(order.reorder(schema.fields()), schema.metadata().clone()))
|
||||
}
|
||||
|
||||
@@ -52,16 +53,25 @@ pub(super) fn infer_schema(bytes: &[u8], max_records: usize) -> Result<Schema, A
|
||||
struct KeyOrder {
|
||||
positions: HashMap<String, usize>,
|
||||
children: Vec<KeyOrder>,
|
||||
field_count: Rc<Cell<usize>>,
|
||||
}
|
||||
|
||||
impl KeyOrder {
|
||||
fn child(&mut self, key: String) -> &mut KeyOrder {
|
||||
fn child<E: de::Error>(&mut self, key: String) -> Result<&mut KeyOrder, E> {
|
||||
let next = self.children.len();
|
||||
let position = *self.positions.entry(key).or_insert(next);
|
||||
if position == next {
|
||||
self.children.push(KeyOrder::default());
|
||||
let count = self.field_count.get();
|
||||
if count >= MAX_SELECT_COLUMNS {
|
||||
return Err(E::custom("S3 Select JSON schema exceeds 1024 fields"));
|
||||
}
|
||||
self.field_count.set(count + 1);
|
||||
self.children.push(KeyOrder {
|
||||
field_count: Rc::clone(&self.field_count),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
&mut self.children[position]
|
||||
Ok(&mut self.children[position])
|
||||
}
|
||||
|
||||
fn reorder(&self, fields: &Fields) -> Fields {
|
||||
@@ -107,7 +117,7 @@ impl<'de> Visitor<'de> for &mut KeyOrder {
|
||||
|
||||
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<(), A::Error> {
|
||||
while let Some(key) = map.next_key::<String>()? {
|
||||
map.next_value_seed(self.child(key))?;
|
||||
map.next_value_seed(self.child::<A::Error>(key)?)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -177,6 +187,22 @@ mod tests {
|
||||
assert_eq!(names(schema.fields()), ["b", "a"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_json_schema_counts_nested_fields_across_the_sample() {
|
||||
let rows = (0..1023)
|
||||
.map(|index| format!("{{\"outer\":{{\"f{index}\":1}}}}\n"))
|
||||
.collect::<String>();
|
||||
let schema = infer_schema(rows.as_bytes(), 2000).expect("1024 total fields");
|
||||
assert_eq!(schema.fields().len(), 1);
|
||||
let overflow = format!("{rows}{{\"outer\":{{\"extra\":1}}}}\n");
|
||||
assert!(
|
||||
infer_schema(overflow.as_bytes(), 2000)
|
||||
.expect_err("nested width limit")
|
||||
.to_string()
|
||||
.contains("1024 fields")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_arrow_errors_for_invalid_records() {
|
||||
assert!(infer_schema(b"{\"a\":1}\n[1]\n", 1000).is_err());
|
||||
|
||||
@@ -31,7 +31,7 @@ use datafusion::{
|
||||
file_format::{DEFAULT_SCHEMA_INFER_MAX_RECORD, csv::CsvFormat, json::JsonFormat},
|
||||
listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},
|
||||
},
|
||||
error::Result as DFResult,
|
||||
error::{DataFusionError, Result as DFResult},
|
||||
execution::{RecordBatchStream, SendableRecordBatchStream},
|
||||
object_store::ObjectStoreExt as _,
|
||||
sql::sqlparser::parser::ParserError,
|
||||
@@ -65,11 +65,13 @@ use crate::{
|
||||
dispatcher::{json_document_table::JsonDocumentTable, json_lines_schema, parquet_table::ParquetSelectTable},
|
||||
execution::factory::QueryExecutionFactoryRef,
|
||||
instance::{DEFAULT_MAX_CONCURRENT_QUERIES, DEFAULT_QUERY_TIMEOUT_SECS},
|
||||
metadata::{ContextProviderExtension, MetadataProvider, TableHandleProviderRef, base_table::BaseTableProvider},
|
||||
metadata::{MetadataProvider, TableHandleProviderRef, base_table::BaseTableProvider},
|
||||
sql::logical::planner::DefaultLogicalPlanner,
|
||||
sql::planner::prepare_s3_select_statement,
|
||||
sql::planner::{prepare_s3_select_statement, validate_expanded_projection_width},
|
||||
};
|
||||
|
||||
use super::schema_inference::{PrefixBoundary, infer_csv_schema, read_schema_prefix};
|
||||
|
||||
static IGNORE: LazyLock<FileHeaderInfo> = LazyLock::new(|| FileHeaderInfo::from_static(FileHeaderInfo::IGNORE));
|
||||
static NONE: LazyLock<FileHeaderInfo> = LazyLock::new(|| FileHeaderInfo::from_static(FileHeaderInfo::NONE));
|
||||
static USE: LazyLock<FileHeaderInfo> = LazyLock::new(|| FileHeaderInfo::from_static(FileHeaderInfo::USE));
|
||||
@@ -166,20 +168,27 @@ impl QueryDispatcher for SimpleQueryDispatcher {
|
||||
let stmt = match query_state_machine.prepared_statement() {
|
||||
Some(statement) => statement.clone(),
|
||||
None => {
|
||||
let (statement, source) = self.prepare_query_statement(query.content())?;
|
||||
let (statement, source) = self
|
||||
.prepare_query_statement_in_worker(query.content(), &query_tracker)
|
||||
.await?;
|
||||
if source_path_requires_expansion(source.path()) {
|
||||
return Err(SelectError::DataSourcePathUnsupported.into());
|
||||
}
|
||||
statement
|
||||
}
|
||||
};
|
||||
let scheme_provider = self.build_scheme_provider(session).await?;
|
||||
let logical_planner = DefaultLogicalPlanner::new(&scheme_provider);
|
||||
|
||||
let logical_plan = self
|
||||
.statement_to_logical_plan(stmt, &logical_planner, Arc::clone(&query_state_machine))
|
||||
.await?;
|
||||
Ok(logical_plan)
|
||||
let scheme_provider = self.build_scheme_provider(session, &query_tracker).await?;
|
||||
let machine = Arc::clone(&query_state_machine);
|
||||
let runtime = tokio::runtime::Handle::current();
|
||||
run_planning_worker(&query_tracker, "SQL plan", move || {
|
||||
validate_expanded_projection_width(&stmt, scheme_provider.source_column_count())?;
|
||||
let planner = DefaultLogicalPlanner::new(&scheme_provider);
|
||||
machine.begin_analyze();
|
||||
let plan = runtime.block_on(planner.prepared_statement_to_plan(stmt, &machine.session))?;
|
||||
machine.end_analyze();
|
||||
Ok(plan)
|
||||
})
|
||||
.await
|
||||
})
|
||||
.await?;
|
||||
if !is_json_document_input(&self.input) {
|
||||
@@ -270,7 +279,10 @@ impl SimpleQueryDispatcher {
|
||||
// Keep parser and analyzer errors in the planning phase. Successful
|
||||
// preparation is cached here because the source path configures the
|
||||
// object store before schema inference starts.
|
||||
let (prepared_statement, source) = match self.prepare_query_statement(query.content()) {
|
||||
let (prepared_statement, source) = match self
|
||||
.run_with_query_deadline(&query_tracker, self.prepare_query_statement_in_worker(query.content(), &query_tracker))
|
||||
.await
|
||||
{
|
||||
Ok((statement, source)) => (Some(statement), source),
|
||||
Err(_) => (None, JsonSource::default()),
|
||||
};
|
||||
@@ -298,6 +310,20 @@ impl SimpleQueryDispatcher {
|
||||
Ok(Arc::new(state_machine))
|
||||
}
|
||||
|
||||
async fn prepare_query_statement_in_worker(
|
||||
&self,
|
||||
sql: &str,
|
||||
tracker: &QueryExecutionTracker,
|
||||
) -> QueryResult<(ExtStatement, JsonSource)> {
|
||||
// Reject before cloning the request into an owned blocking job.
|
||||
if sql.len() > crate::sql::parser::MAX_SQL_EXPRESSION_BYTES {
|
||||
return Err(ParserError::ParserError("S3 Select expression exceeds 256 KiB".to_owned()).into());
|
||||
}
|
||||
let dispatcher = self.clone();
|
||||
let sql = sql.to_owned();
|
||||
run_planning_worker(tracker, "SQL parse", move || dispatcher.prepare_query_statement(&sql)).await
|
||||
}
|
||||
|
||||
fn prepare_query_statement(&self, sql: &str) -> QueryResult<(ExtStatement, JsonSource)> {
|
||||
let mut statements = self.parser.parse(sql)?;
|
||||
if statements.len() > 1 {
|
||||
@@ -383,22 +409,6 @@ impl SimpleQueryDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
async fn statement_to_logical_plan<S: ContextProviderExtension + Send + Sync>(
|
||||
&self,
|
||||
stmt: ExtStatement,
|
||||
logical_planner: &DefaultLogicalPlanner<'_, S>,
|
||||
query_state_machine: Arc<QueryStateMachine>,
|
||||
) -> QueryResult<Plan> {
|
||||
// begin analyze
|
||||
query_state_machine.begin_analyze();
|
||||
let logical_plan = logical_planner
|
||||
.prepared_statement_to_plan(stmt, &query_state_machine.session)
|
||||
.await?;
|
||||
query_state_machine.end_analyze();
|
||||
|
||||
Ok(logical_plan)
|
||||
}
|
||||
|
||||
async fn start_logical_plan(&self, logical_plan: Plan, query_state_machine: Arc<QueryStateMachine>) -> QueryResult<Output> {
|
||||
let execution = self
|
||||
.query_execution_factory
|
||||
@@ -408,7 +418,11 @@ impl SimpleQueryDispatcher {
|
||||
execution.start().await
|
||||
}
|
||||
|
||||
async fn build_scheme_provider(&self, session: &SessionCtx) -> QueryResult<MetadataProvider> {
|
||||
async fn build_scheme_provider(
|
||||
&self,
|
||||
session: &SessionCtx,
|
||||
tracker: &QueryExecutionTracker,
|
||||
) -> QueryResult<MetadataProvider> {
|
||||
if self.input.request.input_serialization.parquet.is_some() {
|
||||
let provider = ParquetSelectTable::try_new(session.inner(), self.input.as_ref()).await?;
|
||||
let current_session_table_provider = self.build_table_handle_provider()?;
|
||||
@@ -431,6 +445,7 @@ impl SimpleQueryDispatcher {
|
||||
|
||||
let path = format!("s3://{}/{}", self.input.bucket, self.input.key);
|
||||
let table_path = ListingTableUrl::parse(path)?;
|
||||
let mut csv_format = None;
|
||||
let (listing_options, need_rename_volume_name, need_ignore_volume_name) =
|
||||
if let Some(csv) = self.input.request.input_serialization.csv.as_ref() {
|
||||
let mut need_rename_volume_name = false;
|
||||
@@ -493,6 +508,7 @@ impl SimpleQueryDispatcher {
|
||||
.with_comment(None)
|
||||
.with_newlines_in_values(true);
|
||||
}
|
||||
csv_format = Some(CsvFormat::default().with_options(file_format.options().clone()));
|
||||
(
|
||||
ListingOptions::new(Arc::new(file_format)).with_file_extension(EXACT_OBJECT_FILE_EXTENSION),
|
||||
need_rename_volume_name,
|
||||
@@ -509,11 +525,39 @@ impl SimpleQueryDispatcher {
|
||||
return Err(SelectError::InvalidDataSource.into());
|
||||
};
|
||||
|
||||
let resolve_schema = if self.input.request.input_serialization.json.is_some() {
|
||||
infer_json_lines_schema(session, &table_path).await?
|
||||
} else {
|
||||
listing_options.infer_schema(session.inner(), &table_path).await?
|
||||
};
|
||||
let store = session.inner().runtime_env().object_store(table_path.object_store())?;
|
||||
let boundary = csv_format
|
||||
.as_ref()
|
||||
.map(PrefixBoundary::csv)
|
||||
.unwrap_or_else(PrefixBoundary::json_lines);
|
||||
let prefix = read_schema_prefix(
|
||||
store
|
||||
.get(table_path.prefix())
|
||||
.await
|
||||
.map_err(DataFusionError::from)?
|
||||
.into_stream(),
|
||||
&session.inner().runtime_env().memory_pool,
|
||||
boundary,
|
||||
)
|
||||
.await?;
|
||||
// Account for decoded values, temporary field maps and Arrow schema
|
||||
// construction as well as the retained byte prefix.
|
||||
prefix
|
||||
.reservation
|
||||
.try_resize(prefix.bytes.capacity() + prefix.bytes.len().saturating_mul(16))?;
|
||||
let resolve_schema = run_planning_worker(tracker, "schema inference", move || {
|
||||
let schema = if let Some(format) = csv_format {
|
||||
infer_csv_schema(&prefix.bytes, &format)?
|
||||
} else {
|
||||
Arc::new(
|
||||
json_lines_schema::infer_schema(&prefix.bytes, DEFAULT_SCHEMA_INFER_MAX_RECORD)
|
||||
.map_err(DataFusionError::from)?,
|
||||
)
|
||||
};
|
||||
drop(prefix);
|
||||
Ok(schema)
|
||||
})
|
||||
.await?;
|
||||
let config = if need_rename_volume_name {
|
||||
let mut new_fields = Vec::new();
|
||||
for (i, field) in resolve_schema.fields().iter().enumerate() {
|
||||
@@ -554,12 +598,48 @@ impl SimpleQueryDispatcher {
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the object like DataFusion's `JsonFormat::infer_schema` does, but
|
||||
/// keeps the source key order (see `json_lines_schema::infer_schema`).
|
||||
async fn infer_json_lines_schema(session: &SessionCtx, table_path: &ListingTableUrl) -> DFResult<SchemaRef> {
|
||||
let store = session.inner().runtime_env().object_store(table_path.object_store())?;
|
||||
let bytes = store.get(table_path.prefix()).await?.bytes().await?;
|
||||
Ok(Arc::new(json_lines_schema::infer_schema(&bytes, DEFAULT_SCHEMA_INFER_MAX_RECORD)?))
|
||||
#[cfg(test)]
|
||||
#[derive(Clone)]
|
||||
struct PlanningWorkerProbe {
|
||||
phase: &'static str,
|
||||
entered: Arc<tokio::sync::Notify>,
|
||||
release: Arc<Mutex<std::sync::mpsc::Receiver<()>>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static PLANNING_WORKER_PROBE: PlanningWorkerProbe;
|
||||
}
|
||||
|
||||
/// Admission is acquired before queueing and is owned by the worker, not its
|
||||
/// JoinHandle. Request cancellation cannot admit replacement CPU work while a
|
||||
/// detached parser, schema inference or planner is still queued/running.
|
||||
async fn run_planning_worker<T: Send + 'static>(
|
||||
tracker: &QueryExecutionTracker,
|
||||
phase: &'static str,
|
||||
work: impl FnOnce() -> QueryResult<T> + Send + 'static,
|
||||
) -> QueryResult<T> {
|
||||
let guard = tracker.query_guard().ok_or(QueryError::Cancel)?;
|
||||
#[cfg(test)]
|
||||
let probe = PLANNING_WORKER_PROBE
|
||||
.try_with(|probe| probe.clone())
|
||||
.ok()
|
||||
.filter(|probe| probe.phase == phase);
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let _guard = guard;
|
||||
#[cfg(test)]
|
||||
if let Some(probe) = probe {
|
||||
probe.entered.notify_one();
|
||||
probe
|
||||
.release
|
||||
.lock()
|
||||
.recv_timeout(Duration::from_secs(10))
|
||||
.expect("release planning worker");
|
||||
}
|
||||
work()
|
||||
})
|
||||
.await
|
||||
.map_err(|error| DataFusionError::Execution(format!("S3 Select {phase} worker failed: {error}")))?
|
||||
}
|
||||
|
||||
fn is_json_document_input(input: &SelectObjectContentInput) -> bool {
|
||||
@@ -894,7 +974,7 @@ mod tests {
|
||||
},
|
||||
},
|
||||
};
|
||||
use rustfs_test_utils::{PutObjectCommitBarrier, TestECStoreEnv};
|
||||
use rustfs_test_utils::TestECStoreEnv;
|
||||
use s3s::dto::{
|
||||
CSVInput, CSVOutput, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, JSONOutput, JSONType,
|
||||
OutputSerialization, ParquetInput, SelectObjectContentInput, SelectObjectContentRequest,
|
||||
@@ -1168,6 +1248,24 @@ mod tests {
|
||||
.expect("production query dispatcher should build")
|
||||
}
|
||||
|
||||
fn snapshot_marker_values(batch: &datafusion::arrow::record_batch::RecordBatch) -> Vec<String> {
|
||||
use datafusion::arrow::array::cast::AsArray;
|
||||
let column = batch.column(0);
|
||||
match column.data_type() {
|
||||
DataType::Utf8 => column
|
||||
.as_string::<i32>()
|
||||
.iter()
|
||||
.map(|value| value.expect("marker").to_owned())
|
||||
.collect(),
|
||||
DataType::Utf8View => column
|
||||
.as_string_view()
|
||||
.iter()
|
||||
.map(|value| value.expect("marker").to_owned())
|
||||
.collect(),
|
||||
other => panic!("unexpected marker type {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_utf8_output(output: Output) -> Vec<String> {
|
||||
let Output::StreamData(stream) = output else {
|
||||
panic!("snapshot query should return rows");
|
||||
@@ -1177,16 +1275,7 @@ mod tests {
|
||||
.await
|
||||
.expect("collect snapshot query output")
|
||||
.iter()
|
||||
.flat_map(|batch| {
|
||||
batch
|
||||
.column(0)
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.expect("snapshot marker column should be Utf8")
|
||||
.iter()
|
||||
.map(|value| value.expect("snapshot marker should not be null").to_string())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.flat_map(snapshot_marker_values)
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -1219,32 +1308,24 @@ mod tests {
|
||||
.expect("infer old-generation schema")
|
||||
.expect("SELECT should produce a logical plan");
|
||||
|
||||
let commit_barrier = PutObjectCommitBarrier::before_namespace(&input.bucket, &input.key);
|
||||
let writer_env = env;
|
||||
let writer_bucket = input.bucket.clone();
|
||||
let writer_object = input.key.clone();
|
||||
let writer = tokio::spawn(async move {
|
||||
writer_env
|
||||
.put_object_bytes(&writer_bucket, &writer_object, new_generation)
|
||||
.await;
|
||||
});
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release_and_wait_until_namespace_pending().await;
|
||||
assert!(!writer.is_finished(), "overwrite must wait for the SelectObjectContent snapshot");
|
||||
|
||||
let output = dispatcher
|
||||
.execute_logical_plan(logical_plan, query_state_machine)
|
||||
tokio::time::timeout(Duration::from_secs(5), env.put_object_bytes(&input.bucket, &input.key, new_generation))
|
||||
.await
|
||||
.expect("scan old-generation rows");
|
||||
let values = collect_utf8_output(output).await;
|
||||
assert_eq!(values, expected_old_markers);
|
||||
assert!(!writer.is_finished(), "overwrite must remain blocked while Query owns the snapshot");
|
||||
.expect("overwrite must finish while the planned Query still owns its snapshot");
|
||||
|
||||
// A reader already pinned during planning may finish with old bytes.
|
||||
// Any subsequent range must reject the replacement generation.
|
||||
match dispatcher.execute_logical_plan(logical_plan, query_state_machine).await {
|
||||
Ok(Output::StreamData(stream)) => match stream.try_collect::<Vec<_>>().await {
|
||||
Ok(batches) => {
|
||||
let values: Vec<String> = batches.iter().flat_map(snapshot_marker_values).collect();
|
||||
assert_eq!(values, expected_old_markers);
|
||||
}
|
||||
Err(error) => assert!(error.to_string().contains("object read semantics changed"), "{error}"),
|
||||
},
|
||||
Err(error) => assert!(error.to_string().contains("object read semantics changed"), "{error}"),
|
||||
Ok(_) => panic!("snapshot query must return rows or a consistency error"),
|
||||
}
|
||||
drop(query);
|
||||
tokio::time::timeout(Duration::from_secs(5), writer)
|
||||
.await
|
||||
.expect("overwrite should finish after snapshot release")
|
||||
.expect("overwrite task should join");
|
||||
}
|
||||
|
||||
fn json_snapshot_input() -> Arc<SelectObjectContentInput> {
|
||||
@@ -1387,6 +1468,42 @@ mod tests {
|
||||
bytes
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn production_parquet_dictionary_select_keeps_views_and_single_row_filter_casts() {
|
||||
use datafusion::arrow::array::cast::AsArray;
|
||||
let mut input = parquet_snapshot_input();
|
||||
let request = Arc::make_mut(&mut input);
|
||||
request.bucket = "select-dictionary-budget".to_owned();
|
||||
request.request.expression = "SELECT CAST(v AS VARCHAR) FROM S3Object WHERE length(v) > 0 LIMIT 2".to_owned();
|
||||
let env = snapshot_test_env().await;
|
||||
env.make_bucket(&input.bucket, false).await;
|
||||
let bytes = crate::dispatcher::parquet_table::dictionary_parquet_for_test(1024 * 1024, 8192);
|
||||
assert!(bytes.len() < 2 * 1024 * 1024);
|
||||
env.put_object_bytes(&input.bucket, &input.key, bytes).await;
|
||||
let snapshot = env.prepare_select_object_snapshot(&input.bucket, &input.key).await;
|
||||
let query = Query::new_with_snapshot(
|
||||
QueryContext {
|
||||
input: Arc::clone(&input),
|
||||
},
|
||||
input.request.expression.clone(),
|
||||
snapshot,
|
||||
);
|
||||
let dispatcher = production_dispatcher(input);
|
||||
let output = dispatcher.execute_query(&query).await.expect("production dictionary SELECT");
|
||||
let mut stream = output.into_record_batch_stream().expect("record stream");
|
||||
for _ in 0..2 {
|
||||
let batch = stream.try_next().await.expect("budgeted decode").expect("row");
|
||||
assert_eq!(batch.num_rows(), 1);
|
||||
assert_eq!(batch.column(0).data_type(), &DataType::Utf8View);
|
||||
assert_eq!(batch.column(0).as_string_view().value(0).len(), 1024 * 1024);
|
||||
assert!(
|
||||
batch.get_array_memory_size() < 2 * 1024 * 1024,
|
||||
"dictionary references must not materialize 8 GiB"
|
||||
);
|
||||
}
|
||||
assert!(stream.try_next().await.expect("EOF").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn json_schema_inference_and_scan_use_one_snapshot_generation() {
|
||||
run_snapshot_generation_race(
|
||||
@@ -1410,7 +1527,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn planner_failure_drops_snapshot_and_unblocks_overwrite() {
|
||||
async fn planner_failure_does_not_block_overwrite_while_query_is_alive() {
|
||||
let mut input = json_snapshot_input();
|
||||
let input_mut = Arc::make_mut(&mut input);
|
||||
input_mut.bucket = "s3select-planner-failure-snapshot".to_string();
|
||||
@@ -1434,34 +1551,19 @@ mod tests {
|
||||
.await
|
||||
.expect("build production snapshot session");
|
||||
|
||||
let commit_barrier = PutObjectCommitBarrier::before_namespace(&input.bucket, &input.key);
|
||||
let writer_env = env;
|
||||
let writer_bucket = input.bucket.clone();
|
||||
let writer_object = input.key.clone();
|
||||
let writer = tokio::spawn(async move {
|
||||
writer_env
|
||||
.put_object_bytes(&writer_bucket, &writer_object, b"{\"new_schema_poison\":1629}\n".to_vec())
|
||||
.await;
|
||||
});
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release_and_wait_until_namespace_pending().await;
|
||||
assert!(!writer.is_finished(), "overwrite must wait for the planner's snapshot");
|
||||
|
||||
let Err(error) = dispatcher.build_logical_plan(Arc::clone(&query_state_machine)).await else {
|
||||
panic!("missing schema binding must fail planning");
|
||||
};
|
||||
assert!(error.to_string().contains("missing_binding"));
|
||||
assert!(
|
||||
!writer.is_finished(),
|
||||
"planner failure must not release snapshots still owned by Query/qsm"
|
||||
);
|
||||
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
env.put_object_bytes(&input.bucket, &input.key, b"{\"new_schema_poison\":1629}\n".to_vec()),
|
||||
)
|
||||
.await
|
||||
.expect("overwrite must finish even while failed planner and Query retain their snapshots");
|
||||
drop(query_state_machine);
|
||||
drop(query);
|
||||
tokio::time::timeout(Duration::from_secs(5), writer)
|
||||
.await
|
||||
.expect("overwrite should finish after failed-plan snapshot release")
|
||||
.expect("overwrite task should join");
|
||||
}
|
||||
|
||||
fn test_query_tracker(
|
||||
@@ -1741,6 +1843,296 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn bounded_inference_machine(
|
||||
input: SelectObjectContentInput,
|
||||
data: Vec<u8>,
|
||||
) -> (Arc<SimpleQueryDispatcher>, Arc<QueryStateMachine>) {
|
||||
let input = Arc::new(input);
|
||||
let optimizer = Arc::new(CascadeOptimizerBuilder::default().build());
|
||||
let dispatcher = test_dispatcher_for_input(
|
||||
Arc::clone(&input),
|
||||
Arc::new(Semaphore::new(1)),
|
||||
Duration::from_secs(30),
|
||||
Arc::new(SqlQueryExecutionFactory::new(optimizer, Arc::new(LocalScheduler {}))),
|
||||
);
|
||||
let query = Query::new(
|
||||
QueryContext {
|
||||
input: Arc::clone(&input),
|
||||
},
|
||||
input.request.expression.clone(),
|
||||
);
|
||||
let machine = dispatcher.build_query_state_machine(query).await.expect("build query");
|
||||
let url = ObjectStoreUrl::parse(format!("s3://{}", input.bucket)).expect("store URL");
|
||||
let store = machine.session.inner().runtime_env().object_store(&url).expect("store");
|
||||
store
|
||||
.put(&Path::from(input.key.as_str()), data.into())
|
||||
.await
|
||||
.expect("put object");
|
||||
(dispatcher, machine)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_schema_inference_rejects_wide_csv_and_json_before_planning() {
|
||||
for json in [false, true] {
|
||||
let mut input = test_input();
|
||||
input.request.expression = "SELECT * FROM S3Object".to_owned();
|
||||
let data = if json {
|
||||
input.request.input_serialization.csv = None;
|
||||
input.request.input_serialization.json = Some(JSONInput {
|
||||
type_: Some(JSONType::from_static(JSONType::LINES)),
|
||||
});
|
||||
format!("{{{}}}\n", (0..1025).map(|index| format!("\"c{index}\":1")).collect::<Vec<_>>().join(","))
|
||||
} else {
|
||||
input.request.input_serialization.csv.as_mut().expect("CSV").file_header_info =
|
||||
Some(FileHeaderInfo::from_static(FileHeaderInfo::USE));
|
||||
format!("{}\n", (0..1025).map(|index| format!("c{index}")).collect::<Vec<_>>().join(","))
|
||||
};
|
||||
let (dispatcher, machine) = bounded_inference_machine(input, data.into_bytes()).await;
|
||||
let error = dispatcher
|
||||
.build_logical_plan(machine)
|
||||
.await
|
||||
.err()
|
||||
.expect("wide schema rejected");
|
||||
assert!(error.to_string().contains("1024"), "{error}");
|
||||
assert_eq!(dispatcher.query_admission.available_permits(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_json_lines_prefix_rejects_overflow_without_truncating_records() {
|
||||
let mut input = test_input();
|
||||
input.request.expression = "SELECT * FROM S3Object".to_owned();
|
||||
input.request.input_serialization.csv = None;
|
||||
input.request.input_serialization.json = Some(JSONInput {
|
||||
type_: Some(JSONType::from_static(JSONType::LINES)),
|
||||
});
|
||||
// Individually valid records remain below the existing 1 MiB record
|
||||
// limit. The 1000-record sample, however, cannot fit its byte budget.
|
||||
let data = format!("{{\"payload\":\"{}\"}}\n", "x".repeat(8192)).repeat(600);
|
||||
let (dispatcher, machine) = bounded_inference_machine(input, data.into_bytes()).await;
|
||||
let error = dispatcher.build_logical_plan(machine).await.err().expect("prefix limit");
|
||||
assert!(error.to_string().contains("4 MiB prefix"), "{error}");
|
||||
assert_eq!(dispatcher.query_admission.available_permits(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_csv_decoder_executes_maximum_width_in_small_batches() {
|
||||
let mut input = test_input();
|
||||
input.request.expression = "SELECT * FROM S3Object".to_owned();
|
||||
input.request.input_serialization.csv.as_mut().expect("CSV").file_header_info =
|
||||
Some(FileHeaderInfo::from_static(FileHeaderInfo::USE));
|
||||
let header = (0..1024).map(|index| format!("c{index}")).collect::<Vec<_>>().join(",");
|
||||
let row = format!("{}\n", vec!["value"; 1024].join(","));
|
||||
let data = format!("{header}\n{}", row.repeat(600));
|
||||
let (dispatcher, machine) = bounded_inference_machine(input, data.into_bytes()).await;
|
||||
assert_eq!(machine.session.inner().config().batch_size(), 256);
|
||||
let plan = dispatcher
|
||||
.build_logical_plan(Arc::clone(&machine))
|
||||
.await
|
||||
.expect("plan")
|
||||
.expect("SELECT plan");
|
||||
let output = dispatcher.execute_logical_plan(plan, machine).await.expect("execute");
|
||||
let mut stream = output.into_record_batch_stream().expect("batches");
|
||||
let mut rows = 0;
|
||||
while let Some(batch) = stream.next().await {
|
||||
let batch = batch.expect("decode bounded CSV");
|
||||
assert_eq!(batch.num_columns(), 1024);
|
||||
assert!(batch.num_rows() <= 256);
|
||||
assert_eq!(array_value_to_string(batch.column(1023), 0).expect("value"), "value");
|
||||
rows += batch.num_rows();
|
||||
}
|
||||
assert_eq!(rows, 600);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn bounded_json_lines_gzip_inference_does_not_read_the_object_tail() {
|
||||
use std::io::{Read, Write};
|
||||
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
|
||||
for _ in 0..500_000 {
|
||||
encoder.write_all(b"{\"z\":1,\"a\":\"value\"}\n").expect("gzip input");
|
||||
}
|
||||
let mut compressed = encoder.finish().expect("gzip");
|
||||
// The original whole-object .bytes() path must encounter this invalid
|
||||
// trailer; prefix inference and LIMIT 1 must not reach it.
|
||||
let trailer = compressed.len() - 8;
|
||||
compressed[trailer] ^= 1;
|
||||
assert!(
|
||||
flate2::read::GzDecoder::new(compressed.as_slice())
|
||||
.read_to_end(&mut Vec::new())
|
||||
.is_err()
|
||||
);
|
||||
let env = snapshot_test_env().await;
|
||||
let mut input = (*json_snapshot_input()).clone();
|
||||
input.bucket = "bounded-lines-gzip".to_owned();
|
||||
input.key = "records.gz".to_owned();
|
||||
input.request.expression = "SELECT * FROM S3Object LIMIT 1".to_owned();
|
||||
input.request.input_serialization.compression_type =
|
||||
Some(s3s::dto::CompressionType::from_static(s3s::dto::CompressionType::GZIP));
|
||||
env.make_bucket(&input.bucket, false).await;
|
||||
env.put_object_bytes(&input.bucket, &input.key, compressed).await;
|
||||
let snapshot = env.prepare_select_object_snapshot(&input.bucket, &input.key).await;
|
||||
let input = Arc::new(input);
|
||||
let dispatcher = production_dispatcher(Arc::clone(&input));
|
||||
let query = Query::new_with_snapshot(
|
||||
QueryContext {
|
||||
input: Arc::clone(&input),
|
||||
},
|
||||
input.request.expression.clone(),
|
||||
snapshot,
|
||||
);
|
||||
let machine = dispatcher.build_query_state_machine(query).await.expect("machine");
|
||||
let plan = dispatcher
|
||||
.build_logical_plan(Arc::clone(&machine))
|
||||
.await
|
||||
.expect("bounded inference succeeds")
|
||||
.expect("plan");
|
||||
let output = dispatcher.execute_logical_plan(plan, machine).await.expect("execute");
|
||||
let batches: Vec<_> = output
|
||||
.into_record_batch_stream()
|
||||
.expect("stream")
|
||||
.try_collect()
|
||||
.await
|
||||
.expect("one row");
|
||||
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 1);
|
||||
assert_eq!(field_names(batches[0].schema().fields()), ["z", "a"]);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn blocking_select_planning_keeps_runtime_responsive_and_admission_after_cancel() {
|
||||
for phase in ["SQL parse", "schema inference", "SQL plan"] {
|
||||
let (release, receiver) = std::sync::mpsc::channel();
|
||||
let probe = super::PlanningWorkerProbe {
|
||||
phase,
|
||||
entered: Arc::new(tokio::sync::Notify::new()),
|
||||
release: Arc::new(parking_lot::Mutex::new(receiver)),
|
||||
};
|
||||
let entered = Arc::clone(&probe.entered);
|
||||
let mut input = test_input();
|
||||
input.request.expression = "SELECT * FROM S3Object".to_owned();
|
||||
let (dispatcher, machine) = bounded_inference_machine(input, b"name\nvalue\n".to_vec()).await;
|
||||
let admission = Arc::clone(&dispatcher.query_admission);
|
||||
let task = if phase == "SQL parse" {
|
||||
let query = machine.query.clone();
|
||||
machine
|
||||
.query_tracker()
|
||||
.expect("tracker")
|
||||
.finish(&dispatcher.query_execution_owner);
|
||||
tokio::spawn(super::PLANNING_WORKER_PROBE.scope(probe, async move {
|
||||
dispatcher.build_query_state_machine(query).await?;
|
||||
Ok::<_, QueryError>(())
|
||||
}))
|
||||
} else {
|
||||
tokio::spawn(super::PLANNING_WORKER_PROBE.scope(probe, async move {
|
||||
dispatcher.build_logical_plan(machine).await?;
|
||||
Ok::<_, QueryError>(())
|
||||
}))
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), entered.notified())
|
||||
.await
|
||||
.expect("worker entered without blocking Tokio");
|
||||
// The single-thread executor stays responsive while the worker is held.
|
||||
tokio::time::timeout(Duration::from_secs(1), tokio::task::yield_now())
|
||||
.await
|
||||
.expect("runtime heartbeat");
|
||||
assert_eq!(admission.available_permits(), 0, "{phase}");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("canceled request").is_cancelled());
|
||||
assert_eq!(
|
||||
admission.available_permits(),
|
||||
0,
|
||||
"cancellation must retain the {phase} worker's admission"
|
||||
);
|
||||
release.send(()).expect("release worker");
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while admission.available_permits() != 1 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("worker completion releases admission");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocking_select_parser_retains_admission_while_queued_after_cancel() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.max_blocking_threads(1)
|
||||
.build()
|
||||
.expect("runtime");
|
||||
runtime.block_on(async {
|
||||
let (release, receiver) = std::sync::mpsc::channel();
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let signal = Arc::clone(&entered);
|
||||
let blocker = tokio::task::spawn_blocking(move || {
|
||||
signal.notify_one();
|
||||
receiver
|
||||
.recv_timeout(Duration::from_secs(10))
|
||||
.expect("release occupied worker");
|
||||
});
|
||||
entered.notified().await;
|
||||
let admission = Arc::new(Semaphore::new(1));
|
||||
let (dispatcher, input) = test_dispatcher(Arc::clone(&admission), Duration::from_secs(30));
|
||||
let query = Query::new(QueryContext { input }, "SELECT * FROM S3Object".to_owned());
|
||||
let request = tokio::spawn(async move { dispatcher.build_query_state_machine(query).await.map(|_| ()) });
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while admission.available_permits() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("admission before worker queue");
|
||||
request.abort();
|
||||
assert!(request.await.expect_err("canceled request").is_cancelled());
|
||||
assert_eq!(admission.available_permits(), 0, "queued worker must retain admission");
|
||||
release.send(()).expect("unblock worker pool");
|
||||
blocker.await.expect("blocking slot released");
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while admission.available_permits() != 1 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("queued parser finished");
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_select_projection_checks_wildcard_expansion_before_planner() {
|
||||
let mut input = test_input();
|
||||
input.request.expression = "SELECT *, * FROM S3Object".to_owned();
|
||||
input.request.input_serialization.csv.as_mut().expect("CSV").file_header_info =
|
||||
Some(FileHeaderInfo::from_static(FileHeaderInfo::USE));
|
||||
let data = format!("{}\n", (0..600).map(|index| format!("c{index}")).collect::<Vec<_>>().join(","));
|
||||
let (dispatcher, machine) = bounded_inference_machine(input, data.into_bytes()).await;
|
||||
let error = dispatcher
|
||||
.build_logical_plan(machine)
|
||||
.await
|
||||
.err()
|
||||
.expect("expanded projection rejected");
|
||||
assert!(error.to_string().contains("expanded projection exceeds 1024"), "{error}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_select_projection_is_rejected_before_object_schema_access() {
|
||||
let (dispatcher, input) = test_dispatcher(Arc::new(Semaphore::new(1)), Duration::from_secs(30));
|
||||
let sql = format!(
|
||||
"SELECT {} FROM S3Object",
|
||||
(0..30_000).map(|index| format!("{index}")).collect::<Vec<_>>().join(",")
|
||||
);
|
||||
let machine = dispatcher
|
||||
.build_query_state_machine(Query::new(QueryContext { input }, sql))
|
||||
.await
|
||||
.expect("deferred parser error");
|
||||
// No object is present: a storage lookup would produce the wrong error.
|
||||
let error = dispatcher
|
||||
.build_logical_plan(machine)
|
||||
.await
|
||||
.err()
|
||||
.expect("wide projection rejected");
|
||||
assert!(error.to_string().contains("1024 columns"), "{error}");
|
||||
}
|
||||
|
||||
async fn json_lines_plan_schema(data: &'static [u8], sql: &str) -> SchemaRef {
|
||||
let mut input = test_input();
|
||||
input.key = "records.json".to_owned();
|
||||
|
||||
@@ -17,3 +17,4 @@ mod json_document_table;
|
||||
mod json_lines_schema;
|
||||
pub mod manager;
|
||||
mod parquet_table;
|
||||
mod schema_inference;
|
||||
|
||||
@@ -17,16 +17,20 @@ use std::{fmt, ops::Range, sync::Arc};
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use datafusion::{
|
||||
arrow::datatypes::SchemaRef,
|
||||
arrow::{
|
||||
array::{Array, cast::AsArray, downcast_dictionary_array},
|
||||
datatypes::{DataType, FieldRef, Schema, SchemaRef},
|
||||
record_batch::RecordBatch,
|
||||
},
|
||||
catalog::Session,
|
||||
common::Result as DFResult,
|
||||
common::{DataFusionError, Result as DFResult},
|
||||
datasource::{
|
||||
TableProvider,
|
||||
listing::{ListingTableUrl, PartitionedFile},
|
||||
physical_plan::{FileScanConfigBuilder, ParquetSource, parquet::ParquetAccessPlan},
|
||||
source::DataSourceExec,
|
||||
},
|
||||
execution::object_store::ObjectStoreUrl,
|
||||
execution::{memory_pool::MemoryLimit, object_store::ObjectStoreUrl},
|
||||
logical_expr::{Expr, TableProviderFilterPushDown, TableType},
|
||||
object_store::{Error as ObjectStoreError, ObjectStore, ObjectStoreExt, path::Path},
|
||||
parquet::{
|
||||
@@ -34,9 +38,13 @@ use datafusion::{
|
||||
errors::{ParquetError, Result as ParquetResult},
|
||||
file::metadata::{ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData},
|
||||
},
|
||||
physical_plan::ExecutionPlan,
|
||||
physical_plan::{
|
||||
ExecutionPlan, SendableRecordBatchStream,
|
||||
stream::RecordBatchStreamAdapter,
|
||||
streaming::{PartitionStream, StreamingTableExec},
|
||||
},
|
||||
};
|
||||
use futures::{FutureExt, TryFutureExt, future::BoxFuture};
|
||||
use futures::{FutureExt, TryFutureExt, TryStreamExt, future::BoxFuture, stream};
|
||||
use rustfs_s3select_api::{
|
||||
QueryResult,
|
||||
object_store::{SelectScanRange, scan_range_from_bounds},
|
||||
@@ -50,6 +58,7 @@ pub struct ParquetSelectTable {
|
||||
object_path: String,
|
||||
object_size: u64,
|
||||
access_plan: Option<Arc<ParquetAccessPlan>>,
|
||||
expanded_row_budget: usize,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ParquetSelectTable {
|
||||
@@ -67,14 +76,24 @@ struct ObjectStoreParquetReader {
|
||||
store: Arc<dyn ObjectStore>,
|
||||
path: Path,
|
||||
file_size: u64,
|
||||
metadata_read_limit: usize,
|
||||
}
|
||||
|
||||
impl AsyncFileReader for ObjectStoreParquetReader {
|
||||
fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, ParquetResult<Bytes>> {
|
||||
if range.end.saturating_sub(range.start) > self.metadata_read_limit as u64 {
|
||||
return async { Err(ParquetError::External(Box::new(parquet_budget_error()))) }.boxed();
|
||||
}
|
||||
self.store.get_range(&self.path, range).map_err(parquet_store_error).boxed()
|
||||
}
|
||||
|
||||
fn get_byte_ranges(&mut self, ranges: Vec<Range<u64>>) -> BoxFuture<'_, ParquetResult<Vec<Bytes>>> {
|
||||
let total = ranges
|
||||
.iter()
|
||||
.try_fold(0u64, |sum, range| sum.checked_add(range.end.saturating_sub(range.start)));
|
||||
if total.is_none_or(|total| total > self.metadata_read_limit as u64) {
|
||||
return async { Err(ParquetError::External(Box::new(parquet_budget_error()))) }.boxed();
|
||||
}
|
||||
async move { self.store.get_ranges(&self.path, &ranges).await.map_err(parquet_store_error) }.boxed()
|
||||
}
|
||||
|
||||
@@ -111,17 +130,34 @@ impl ParquetSelectTable {
|
||||
.await
|
||||
.map_err(datafusion::common::DataFusionError::from)?;
|
||||
|
||||
let memory_budget = match state.runtime_env().memory_pool.memory_limit() {
|
||||
MemoryLimit::Finite(bytes) => bytes,
|
||||
_ => rustfs_s3select_api::query::session::DEFAULT_S3SELECT_MEMORY_LIMIT_BYTES,
|
||||
};
|
||||
let reader = ObjectStoreParquetReader {
|
||||
store: Arc::clone(&store),
|
||||
path: object_location,
|
||||
file_size: object_meta.size,
|
||||
metadata_read_limit: (memory_budget / 16).max(1),
|
||||
};
|
||||
let builder = ParquetRecordBatchStreamBuilder::new(reader)
|
||||
.await
|
||||
.map_err(datafusion::common::DataFusionError::from)?;
|
||||
let schema = Arc::clone(builder.schema());
|
||||
let schema = Arc::new(Schema::new_with_metadata(
|
||||
builder
|
||||
.schema()
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|field| parquet_view_field(field, 0))
|
||||
.collect::<DFResult<Vec<_>>>()?,
|
||||
builder.schema().metadata().clone(),
|
||||
));
|
||||
let metadata = Arc::clone(builder.metadata());
|
||||
// This is admission for valid Parquet declarations, not a replacement
|
||||
// for limits inside the footer parser or native compression codecs.
|
||||
let expanded_row_budget = (memory_budget / 16).max(1);
|
||||
let access_plan = parquet_access_plan(input, object_meta.size, metadata.as_ref())?;
|
||||
admit_parquet_decode(metadata.as_ref(), access_plan.as_deref(), memory_budget / 4)?;
|
||||
|
||||
Ok(Arc::new(Self {
|
||||
schema,
|
||||
@@ -129,11 +165,12 @@ impl ParquetSelectTable {
|
||||
object_path: input.key.clone(),
|
||||
object_size: object_meta.size,
|
||||
access_plan,
|
||||
expanded_row_budget,
|
||||
}))
|
||||
}
|
||||
|
||||
fn partitioned_file(&self) -> PartitionedFile {
|
||||
let file = PartitionedFile::new(self.object_path.clone(), self.object_size);
|
||||
let file = PartitionedFile::new(self.object_path.clone(), self.object_size).with_arrow_schema(Arc::clone(&self.schema));
|
||||
if let Some(access_plan) = self.access_plan.as_ref() {
|
||||
file.with_extension(access_plan.as_ref().clone())
|
||||
} else {
|
||||
@@ -154,20 +191,41 @@ impl TableProvider for ParquetSelectTable {
|
||||
|
||||
async fn scan(
|
||||
&self,
|
||||
_state: &dyn Session,
|
||||
state: &dyn Session,
|
||||
projection: Option<&Vec<usize>>,
|
||||
filters: &[Expr],
|
||||
limit: Option<usize>,
|
||||
) -> DFResult<Arc<dyn ExecutionPlan>> {
|
||||
if state.config_options().execution.batch_size.get() != 1 || state.config_options().execution.coalesce_batches {
|
||||
return Err(DataFusionError::Plan(
|
||||
"S3 Select Parquet requires single-row execution without batch coalescing".into(),
|
||||
));
|
||||
}
|
||||
let scan_limit = if filters.is_empty() { limit } else { None };
|
||||
let file_source = Arc::new(ParquetSource::new(Arc::clone(&self.schema)));
|
||||
let config = FileScanConfigBuilder::new(self.object_store_url.clone(), file_source)
|
||||
.with_file(self.partitioned_file())
|
||||
.with_projection_indices(projection.cloned())?
|
||||
.with_limit(scan_limit)
|
||||
.with_batch_size(Some(1))
|
||||
.build();
|
||||
let plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(config);
|
||||
Ok(plan)
|
||||
let input: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(config);
|
||||
let schema = input.schema();
|
||||
let partition: Arc<dyn PartitionStream> = Arc::new(ParquetBudgetPartition {
|
||||
input,
|
||||
schema: Arc::clone(&schema),
|
||||
expanded_row_budget: self.expanded_row_budget,
|
||||
});
|
||||
// An opaque source boundary keeps expression/cast pushdown outside the
|
||||
// expansion check. One file group is read serially, without repartitioning.
|
||||
Ok(Arc::new(StreamingTableExec::try_new(
|
||||
schema,
|
||||
vec![partition],
|
||||
None,
|
||||
[],
|
||||
false,
|
||||
scan_limit,
|
||||
)?))
|
||||
}
|
||||
|
||||
fn supports_filters_pushdown(&self, filters: &[&Expr]) -> DFResult<Vec<TableProviderFilterPushDown>> {
|
||||
@@ -175,6 +233,161 @@ impl TableProvider for ParquetSelectTable {
|
||||
}
|
||||
}
|
||||
|
||||
const PARQUET_MAX_TYPE_DEPTH: usize = 64;
|
||||
|
||||
fn parquet_budget_error() -> DataFusionError {
|
||||
DataFusionError::ResourcesExhausted("S3 Select Parquet decoded data exceeds the query memory budget".into())
|
||||
}
|
||||
|
||||
fn parquet_view_field(field: &FieldRef, depth: usize) -> DFResult<FieldRef> {
|
||||
if depth >= PARQUET_MAX_TYPE_DEPTH {
|
||||
return Err(parquet_budget_error());
|
||||
}
|
||||
let recurse = |child: &FieldRef| parquet_view_field(child, depth + 1);
|
||||
let data_type = match field.data_type() {
|
||||
DataType::Utf8 | DataType::LargeUtf8 => DataType::Utf8View,
|
||||
DataType::Binary | DataType::LargeBinary => DataType::BinaryView,
|
||||
DataType::List(child) => DataType::List(recurse(child)?),
|
||||
DataType::LargeList(child) => DataType::LargeList(recurse(child)?),
|
||||
DataType::ListView(child) => DataType::ListView(recurse(child)?),
|
||||
DataType::LargeListView(child) => DataType::LargeListView(recurse(child)?),
|
||||
DataType::FixedSizeList(child, size) => DataType::FixedSizeList(recurse(child)?, *size),
|
||||
DataType::Map(child, sorted) => DataType::Map(recurse(child)?, *sorted),
|
||||
DataType::Struct(fields) => DataType::Struct(fields.iter().map(recurse).collect::<DFResult<Vec<_>>>()?.into()),
|
||||
// Preserve the dictionary: its indices already share the value buffer.
|
||||
// A later cast is still checked by the actual per-row expansion budget.
|
||||
other => other.clone(),
|
||||
};
|
||||
Ok(Arc::new(field.as_ref().clone().with_data_type(data_type)))
|
||||
}
|
||||
|
||||
fn admit_parquet_decode(metadata: &ParquetMetaData, access_plan: Option<&ParquetAccessPlan>, limit: usize) -> DFResult<()> {
|
||||
// The pinned DataFusion reader drains one active row group before fetching
|
||||
// the next. Parquet's DataRequest hands its ranges to that group and clears
|
||||
// them from PushBuffers. Do not reject a large file of many small groups.
|
||||
for (index, group) in metadata.row_groups().iter().enumerate() {
|
||||
if access_plan.is_some_and(|plan| !plan.should_scan(index)) {
|
||||
continue;
|
||||
}
|
||||
let mut resident = 0usize;
|
||||
for column in group.columns() {
|
||||
let decoded = usize::try_from(column.uncompressed_size()).map_err(|_| parquet_budget_error())?;
|
||||
let compressed = usize::try_from(column.compressed_size()).map_err(|_| parquet_budget_error())?;
|
||||
let values = usize::try_from(column.num_values()).map_err(|_| parquet_budget_error())?;
|
||||
// Include a conservative view/offset/validity allowance, including
|
||||
// repeated nested values that can all belong to a single row.
|
||||
resident = resident
|
||||
.checked_add(compressed)
|
||||
.and_then(|sum| sum.checked_add(decoded))
|
||||
.and_then(|sum| values.checked_mul(32).and_then(|views| sum.checked_add(views)))
|
||||
.ok_or_else(parquet_budget_error)?;
|
||||
if resident > limit {
|
||||
return Err(parquet_budget_error());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ParquetBudgetPartition {
|
||||
input: Arc<dyn ExecutionPlan>,
|
||||
schema: SchemaRef,
|
||||
expanded_row_budget: usize,
|
||||
}
|
||||
|
||||
impl PartitionStream for ParquetBudgetPartition {
|
||||
fn schema(&self) -> &SchemaRef {
|
||||
&self.schema
|
||||
}
|
||||
|
||||
fn execute(&self, context: Arc<datafusion::execution::TaskContext>) -> SendableRecordBatchStream {
|
||||
let schema = Arc::clone(&self.schema);
|
||||
let input = match self.input.execute(0, context) {
|
||||
Ok(input) => input,
|
||||
Err(error) => return Box::pin(RecordBatchStreamAdapter::new(schema, stream::once(async move { Err(error) }))),
|
||||
};
|
||||
let budget = self.expanded_row_budget;
|
||||
let rows = stream::try_unfold((input, None::<RecordBatch>, 0usize), move |(mut input, mut batch, mut row)| async move {
|
||||
loop {
|
||||
if let Some(current) = batch.as_ref()
|
||||
&& row < current.num_rows()
|
||||
{
|
||||
let mut remaining = budget;
|
||||
for column in current.columns() {
|
||||
charge_parquet_value(column.as_ref(), row, 0, &mut remaining)?;
|
||||
}
|
||||
let output = current.slice(row, 1);
|
||||
row += 1;
|
||||
return Ok(Some((output, (input, batch, row))));
|
||||
}
|
||||
batch = input.try_next().await?;
|
||||
row = 0;
|
||||
if batch.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
});
|
||||
Box::pin(RecordBatchStreamAdapter::new(schema, rows))
|
||||
}
|
||||
}
|
||||
|
||||
fn charge_parquet_value(array: &dyn Array, row: usize, depth: usize, remaining: &mut usize) -> DFResult<()> {
|
||||
if depth >= PARQUET_MAX_TYPE_DEPTH {
|
||||
return Err(parquet_budget_error());
|
||||
}
|
||||
// Offset, validity and view bookkeeping still costs memory for empty cells.
|
||||
*remaining = remaining.checked_sub(32).ok_or_else(parquet_budget_error)?;
|
||||
if array.is_null(row) {
|
||||
return Ok(());
|
||||
}
|
||||
let bytes = match array.data_type() {
|
||||
DataType::Utf8View => array.as_string_view().value(row).len(),
|
||||
DataType::BinaryView => array.as_binary_view().value(row).len(),
|
||||
DataType::Utf8 => array.as_string::<i32>().value(row).len(),
|
||||
DataType::LargeUtf8 => array.as_string::<i64>().value(row).len(),
|
||||
DataType::Binary => array.as_binary::<i32>().value(row).len(),
|
||||
DataType::LargeBinary => array.as_binary::<i64>().value(row).len(),
|
||||
DataType::FixedSizeBinary(size) => usize::try_from(*size).map_err(|_| parquet_budget_error())?,
|
||||
DataType::Struct(_) => {
|
||||
for child in array.as_struct().columns() {
|
||||
charge_parquet_value(child.as_ref(), row, depth + 1, remaining)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
DataType::Dictionary(_, _) => {
|
||||
return downcast_dictionary_array!(array => {
|
||||
let key = array.key(row).ok_or_else(parquet_budget_error)?;
|
||||
charge_parquet_value(array.values().as_ref(), key, depth + 1, remaining)
|
||||
}, _ => Err(parquet_budget_error()));
|
||||
}
|
||||
DataType::List(_)
|
||||
| DataType::LargeList(_)
|
||||
| DataType::ListView(_)
|
||||
| DataType::LargeListView(_)
|
||||
| DataType::FixedSizeList(_, _)
|
||||
| DataType::Map(_, _) => {
|
||||
let values: Arc<dyn Array> = match array.data_type() {
|
||||
DataType::List(_) => array.as_list::<i32>().value(row),
|
||||
DataType::LargeList(_) => array.as_list::<i64>().value(row),
|
||||
DataType::ListView(_) => array.as_list_view::<i32>().value(row),
|
||||
DataType::LargeListView(_) => array.as_list_view::<i64>().value(row),
|
||||
DataType::FixedSizeList(_, _) => array.as_fixed_size_list().value(row),
|
||||
DataType::Map(_, _) => Arc::new(array.as_map().value(row)),
|
||||
_ => return Err(parquet_budget_error()),
|
||||
};
|
||||
for child_row in 0..values.len() {
|
||||
charge_parquet_value(values.as_ref(), child_row, depth + 1, remaining)?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
DataType::Boolean | DataType::Null => 1,
|
||||
other => other.primitive_width().ok_or_else(parquet_budget_error)?,
|
||||
};
|
||||
*remaining = remaining.checked_sub(bytes).ok_or_else(parquet_budget_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parquet_access_plan(
|
||||
input: &SelectObjectContentInput,
|
||||
object_size: u64,
|
||||
@@ -218,6 +431,11 @@ fn parquet_store_error(err: ObjectStoreError) -> ParquetError {
|
||||
ParquetError::External(Box::new(err))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn dictionary_parquet_for_test(value_bytes: usize, rows: usize) -> Vec<u8> {
|
||||
tests::dictionary_file(value_bytes, rows, false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -328,8 +546,347 @@ mod tests {
|
||||
assert_eq!(error.select_error(), SelectError::ParquetParsingError);
|
||||
}
|
||||
|
||||
pub(super) fn dictionary_file(value_bytes: usize, rows: usize, compressed: bool) -> Vec<u8> {
|
||||
use datafusion::arrow::array::{DictionaryArray, StringArray};
|
||||
use datafusion::arrow::datatypes::Int32Type;
|
||||
use datafusion::parquet::{
|
||||
arrow::arrow_writer::ArrowWriterOptions, basic::Compression, file::properties::WriterProperties,
|
||||
};
|
||||
let values = Arc::new(StringArray::from(vec!["x".repeat(value_bytes)]));
|
||||
let keys = Int32Array::from(vec![0; rows]);
|
||||
let column = DictionaryArray::<Int32Type>::try_new(keys, values).expect("dictionary");
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("v", column.data_type().clone(), true)]));
|
||||
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(column)]).expect("dictionary batch");
|
||||
let properties = WriterProperties::builder()
|
||||
.set_dictionary_enabled(true)
|
||||
.set_dictionary_page_size_limit(value_bytes * 2)
|
||||
.set_compression(if compressed {
|
||||
Compression::SNAPPY
|
||||
} else {
|
||||
Compression::UNCOMPRESSED
|
||||
})
|
||||
.build();
|
||||
let options = ArrowWriterOptions::new()
|
||||
.with_properties(properties)
|
||||
.with_skip_arrow_metadata(true);
|
||||
let mut bytes = Vec::new();
|
||||
let mut writer = ArrowWriter::try_new_with_options(&mut bytes, schema, options).expect("writer");
|
||||
writer.write(&batch).expect("dictionary data");
|
||||
writer.close().expect("footer");
|
||||
bytes
|
||||
}
|
||||
|
||||
async fn register_parquet_fixture(context: &SessionContext, store: &Arc<InMemory>, bytes: Vec<u8>) {
|
||||
store
|
||||
.put(&Path::from("dictionary.parquet"), Bytes::from(bytes).into())
|
||||
.await
|
||||
.expect("fixture");
|
||||
let table = ParquetSelectTable::try_new(&context.state(), &parquet_input("dictionary.parquet"))
|
||||
.await
|
||||
.expect("valid dictionary metadata");
|
||||
context.register_table("source", table).expect("table");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dictionary_byte_array_uses_shared_views_and_single_row_casts_after_filter() {
|
||||
let store = Arc::new(InMemory::new());
|
||||
let context = parquet_session(store.clone());
|
||||
let bytes = dictionary_file(1024 * 1024, 8192, false);
|
||||
assert!(bytes.len() < 2 * 1024 * 1024, "legal small dictionary file");
|
||||
register_parquet_fixture(&context, &store, bytes).await;
|
||||
let mut rows = context
|
||||
.sql("SELECT v FROM source LIMIT 2")
|
||||
.await
|
||||
.expect("plan")
|
||||
.execute_stream()
|
||||
.await
|
||||
.expect("stream");
|
||||
let first = rows.try_next().await.expect("decode").expect("first row");
|
||||
let second = rows.try_next().await.expect("decode").expect("second row");
|
||||
assert_eq!(first.num_rows(), 1);
|
||||
assert_eq!(second.num_rows(), 1);
|
||||
assert_eq!(first.column(0).data_type(), &DataType::Utf8View);
|
||||
assert!(first.get_array_memory_size() < 2 * 1024 * 1024, "no 8192-fold dictionary expansion");
|
||||
let first_views = first.column(0).as_string_view();
|
||||
let second_views = second.column(0).as_string_view();
|
||||
assert!(
|
||||
first_views.data_buffers()[0].ptr_eq(&second_views.data_buffers()[0]),
|
||||
"rows share the dictionary bytes"
|
||||
);
|
||||
drop(rows);
|
||||
|
||||
// BYTEA requests a materializing Binary cast. FilterExec must not
|
||||
// rebuild an 8192-row batch before evaluating that cast.
|
||||
let frame = context
|
||||
.sql("SELECT CAST(v AS BYTEA) FROM source WHERE length(v) > 1 LIMIT 2")
|
||||
.await
|
||||
.expect("cast plan");
|
||||
let physical = frame.clone().create_physical_plan().await.expect("physical plan");
|
||||
let display = datafusion::physical_plan::displayable(physical.as_ref())
|
||||
.indent(false)
|
||||
.to_string();
|
||||
assert!(display.contains("FilterExec"), "{display}");
|
||||
assert!(!display.contains("CoalesceBatchesExec"), "{display}");
|
||||
let mut rows = frame.execute_stream().await.expect("cast stream");
|
||||
for _ in 0..2 {
|
||||
let batch = rows.try_next().await.expect("bounded cast").expect("cast row");
|
||||
assert_eq!(batch.num_rows(), 1);
|
||||
assert_eq!(batch.column(0).data_type(), &DataType::Binary);
|
||||
assert_eq!(batch.column(0).as_binary::<i32>().value(0).len(), 1024 * 1024);
|
||||
assert!(batch.get_array_memory_size() < 2 * 1024 * 1024);
|
||||
}
|
||||
let frame = context
|
||||
.table("source")
|
||||
.await
|
||||
.expect("table")
|
||||
.filter(datafusion::prelude::col("v").is_not_null())
|
||||
.expect("filter")
|
||||
.select(vec![datafusion::logical_expr::cast(
|
||||
datafusion::prelude::col("v"),
|
||||
DataType::LargeUtf8,
|
||||
)])
|
||||
.expect("large string cast")
|
||||
.limit(0, Some(2))
|
||||
.expect("limit");
|
||||
let mut rows = frame.execute_stream().await.expect("LargeUtf8 stream");
|
||||
for _ in 0..2 {
|
||||
let batch = rows.try_next().await.expect("bounded large string cast").expect("row");
|
||||
assert_eq!(batch.num_rows(), 1);
|
||||
assert_eq!(batch.column(0).data_type(), &DataType::LargeUtf8);
|
||||
assert!(batch.get_array_memory_size() < 2 * 1024 * 1024);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_actual_expanded_single_row_is_rejected_before_cast() {
|
||||
let store = Arc::new(InMemory::new());
|
||||
let context = parquet_session(store.clone());
|
||||
// Compression keeps the stored input small. Valid footer declarations
|
||||
// fit the decode allowance, but this one expanded row exceeds 4 MiB.
|
||||
register_parquet_fixture(&context, &store, dictionary_file(5 * 1024 * 1024, 2, true)).await;
|
||||
let mut stream = context
|
||||
.sql("SELECT CAST(v AS BYTEA) FROM source")
|
||||
.await
|
||||
.expect("plan")
|
||||
.execute_stream()
|
||||
.await
|
||||
.expect("stream");
|
||||
let error = stream.try_next().await.expect_err("reject before materializing the cast");
|
||||
assert!(error.to_string().contains("query memory budget"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_many_small_row_groups_can_exceed_the_query_budget_in_total() {
|
||||
use datafusion::arrow::array::StringArray;
|
||||
use datafusion::parquet::{
|
||||
arrow::arrow_writer::ArrowWriterOptions, basic::Compression, file::properties::WriterProperties,
|
||||
};
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Utf8, false)]));
|
||||
let column = Arc::new(StringArray::from(vec!["x".repeat(64 * 1024)]));
|
||||
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![column]).expect("small row group");
|
||||
let properties = WriterProperties::builder().set_compression(Compression::UNCOMPRESSED).build();
|
||||
let options = ArrowWriterOptions::new()
|
||||
.with_properties(properties)
|
||||
.with_skip_arrow_metadata(true);
|
||||
let mut bytes = Vec::new();
|
||||
let mut writer = ArrowWriter::try_new_with_options(&mut bytes, schema, options).expect("writer");
|
||||
for _ in 0..80 {
|
||||
writer.write(&batch).expect("row");
|
||||
writer.flush().expect("separate row group");
|
||||
}
|
||||
writer.close().expect("footer");
|
||||
assert!(bytes.len() > 4 * 1024 * 1024);
|
||||
let store = Arc::new(InMemory::new());
|
||||
let context = parquet_session_with_memory(store.clone(), 4 * 1024 * 1024);
|
||||
register_parquet_fixture(&context, &store, bytes).await;
|
||||
let mut rows = context
|
||||
.sql("SELECT length(v) FROM source")
|
||||
.await
|
||||
.expect("plan")
|
||||
.execute_stream()
|
||||
.await
|
||||
.expect("stream");
|
||||
let mut count = 0;
|
||||
while let Some(batch) = rows.try_next().await.expect("serial row group decode") {
|
||||
assert_eq!(batch.num_rows(), 1);
|
||||
count += batch.num_rows();
|
||||
}
|
||||
assert_eq!(count, 80);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_scan_range_admits_only_the_selected_row_groups() {
|
||||
use datafusion::arrow::array::StringArray;
|
||||
use datafusion::parquet::{
|
||||
arrow::arrow_writer::ArrowWriterOptions, basic::Compression, file::properties::WriterProperties,
|
||||
};
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Utf8, false)]));
|
||||
let properties = WriterProperties::builder().set_compression(Compression::SNAPPY).build();
|
||||
let options = ArrowWriterOptions::new()
|
||||
.with_properties(properties)
|
||||
.with_skip_arrow_metadata(true);
|
||||
let mut bytes = Vec::new();
|
||||
let mut writer = ArrowWriter::try_new_with_options(&mut bytes, Arc::clone(&schema), options).expect("writer");
|
||||
for value in ["selected".to_string(), "x".repeat(2 * 1024 * 1024)] {
|
||||
let column = Arc::new(StringArray::from(vec![value]));
|
||||
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![column]).expect("row");
|
||||
writer.write(&batch).expect("write row group");
|
||||
writer.flush().expect("separate row groups");
|
||||
}
|
||||
writer.close().expect("footer");
|
||||
let bytes = Bytes::from(bytes);
|
||||
let builder = ParquetRecordBatchReaderBuilder::try_new(bytes.clone()).expect("metadata");
|
||||
let starts = builder
|
||||
.metadata()
|
||||
.row_groups()
|
||||
.iter()
|
||||
.map(|group| i64::try_from(row_group_start_offset(group).expect("row group start")).expect("offset"))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(starts.len(), 2);
|
||||
assert_ne!(starts[0], starts[1]);
|
||||
let store = Arc::new(InMemory::new());
|
||||
store.put(&Path::from("range.parquet"), bytes.into()).await.expect("fixture");
|
||||
let context = parquet_session_with_memory(store, 4 * 1024 * 1024);
|
||||
let mut input = parquet_input("range.parquet");
|
||||
input.request.scan_range = Some(ScanRange {
|
||||
start: Some(starts[0]),
|
||||
end: Some(starts[0]),
|
||||
});
|
||||
let table = ParquetSelectTable::try_new(&context.state(), &input)
|
||||
.await
|
||||
.expect("an unselected large row group must not reject the small range");
|
||||
context.register_table("source", table).expect("table");
|
||||
let rows = context
|
||||
.sql("SELECT v FROM source")
|
||||
.await
|
||||
.expect("query")
|
||||
.collect()
|
||||
.await
|
||||
.expect("decode only the selected row group");
|
||||
assert_eq!(rows.len(), 1);
|
||||
assert_eq!(rows[0].num_rows(), 1);
|
||||
assert_eq!(rows[0].column(0).as_string_view().value(0), "selected");
|
||||
input.request.scan_range = Some(ScanRange {
|
||||
start: Some(starts[1]),
|
||||
end: Some(starts[1]),
|
||||
});
|
||||
let error = match ParquetSelectTable::try_new(&context.state(), &input).await {
|
||||
Ok(_) => panic!("the selected oversized row group must still be rejected before decoding"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.to_string().contains("query memory budget"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_actual_row_allowance_tracks_the_configured_query_budget() {
|
||||
let store = Arc::new(InMemory::new());
|
||||
let context = parquet_session_with_memory(store.clone(), 4 * 1024 * 1024);
|
||||
register_parquet_fixture(&context, &store, dictionary_file(512 * 1024, 2, true)).await;
|
||||
let mut rows = context
|
||||
.sql("SELECT v FROM source")
|
||||
.await
|
||||
.expect("plan")
|
||||
.execute_stream()
|
||||
.await
|
||||
.expect("stream");
|
||||
let error = rows
|
||||
.try_next()
|
||||
.await
|
||||
.expect_err("512 KiB row exceeds the configured 256 KiB allowance");
|
||||
assert!(error.to_string().contains("query memory budget"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_valid_compressed_dictionary_is_rejected_before_decoder_allocation() {
|
||||
let store = Arc::new(InMemory::new());
|
||||
let context = parquet_session(store.clone());
|
||||
let bytes = dictionary_file(20 * 1024 * 1024, 1, true);
|
||||
assert!(bytes.len() < 2 * 1024 * 1024);
|
||||
store
|
||||
.put(&Path::from("large.parquet"), Bytes::from(bytes).into())
|
||||
.await
|
||||
.expect("fixture");
|
||||
let error = match ParquetSelectTable::try_new(&context.state(), &parquet_input("large.parquet")).await {
|
||||
Ok(_) => panic!("valid declared decoded size must be admitted before page decode"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.to_string().contains("query memory budget"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parquet_nested_views_decode_through_the_actual_file_schema() {
|
||||
use datafusion::arrow::array::{ListArray, StringArray};
|
||||
use datafusion::arrow::buffer::OffsetBuffer;
|
||||
use datafusion::parquet::arrow::arrow_writer::ArrowWriterOptions;
|
||||
let store = Arc::new(InMemory::new());
|
||||
let context = parquet_session(store.clone());
|
||||
let child = Arc::new(Field::new("item", DataType::Utf8, true));
|
||||
let values = Arc::new(StringArray::from(vec![Some("first value beyond inline size"), None, Some("last")]));
|
||||
let list = Arc::new(ListArray::new(child, OffsetBuffer::new(vec![0, 2, 3].into()), values, None));
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("nested", list.data_type().clone(), false)]));
|
||||
let batch = RecordBatch::try_new(Arc::clone(&schema), vec![list]).expect("nested rows");
|
||||
let mut bytes = Vec::new();
|
||||
let mut writer =
|
||||
ArrowWriter::try_new_with_options(&mut bytes, schema, ArrowWriterOptions::new().with_skip_arrow_metadata(true))
|
||||
.expect("nested writer");
|
||||
writer.write(&batch).expect("rows");
|
||||
writer.close().expect("footer");
|
||||
register_parquet_fixture(&context, &store, bytes).await;
|
||||
let mut rows = context
|
||||
.sql("SELECT nested FROM source")
|
||||
.await
|
||||
.expect("plan")
|
||||
.execute_stream()
|
||||
.await
|
||||
.expect("stream");
|
||||
let first = rows.try_next().await.expect("decode").expect("first row");
|
||||
let values = first.column(0).as_list::<i32>().value(0);
|
||||
assert_eq!(values.data_type(), &DataType::Utf8View);
|
||||
assert_eq!(values.as_string_view().value(0), "first value beyond inline size");
|
||||
assert!(values.is_null(1));
|
||||
let second = rows.try_next().await.expect("decode").expect("second row");
|
||||
assert_eq!(second.column(0).as_list::<i32>().value(0).as_string_view().value(0), "last");
|
||||
assert!(rows.try_next().await.expect("EOF").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parquet_nested_view_schema_and_expansion_follow_logical_references() {
|
||||
use datafusion::arrow::array::{ListArray, StringViewArray};
|
||||
use datafusion::arrow::buffer::OffsetBuffer;
|
||||
let child = Arc::new(
|
||||
Field::new("item", DataType::Utf8, true)
|
||||
.with_metadata(std::collections::HashMap::from([("custom".to_owned(), "kept".to_owned())])),
|
||||
);
|
||||
let field = Arc::new(Field::new("nested", DataType::List(child), false));
|
||||
let converted = parquet_view_field(&field, 0).expect("recursive views");
|
||||
let DataType::List(child) = converted.data_type() else { panic!("list preserved") };
|
||||
assert_eq!(child.data_type(), &DataType::Utf8View);
|
||||
assert_eq!(child.metadata().get("custom").map(String::as_str), Some("kept"));
|
||||
let values = Arc::new(StringViewArray::from(vec!["long repeated dictionary value"; 4]));
|
||||
let list = ListArray::new(Arc::clone(child), OffsetBuffer::new(vec![0, 4].into()), values, None);
|
||||
let mut budget = 64;
|
||||
assert!(
|
||||
charge_parquet_value(&list, 0, 0, &mut budget).is_err(),
|
||||
"nested references count separately"
|
||||
);
|
||||
let mut budget = 1024;
|
||||
charge_parquet_value(&list, 0, 0, &mut budget).expect("small nested row");
|
||||
}
|
||||
|
||||
fn parquet_session(store: Arc<dyn ObjectStore>) -> SessionContext {
|
||||
let context = SessionContext::new();
|
||||
parquet_session_with_memory(store, 64 * 1024 * 1024)
|
||||
}
|
||||
|
||||
fn parquet_session_with_memory(store: Arc<dyn ObjectStore>, budget: usize) -> SessionContext {
|
||||
let mut config = datafusion::prelude::SessionConfig::new()
|
||||
.with_batch_size(1)
|
||||
.with_target_partitions(1);
|
||||
config.options_mut().execution.coalesce_batches = false;
|
||||
let runtime = datafusion::execution::runtime_env::RuntimeEnvBuilder::new()
|
||||
.with_memory_limit(budget, 1.0)
|
||||
.build()
|
||||
.expect("query memory pool");
|
||||
let context = SessionContext::new_with_config_rt(config, Arc::new(runtime));
|
||||
let store_url = ObjectStoreUrl::parse("s3://test-bucket").expect("valid test object store URL");
|
||||
context.register_object_store(store_url.as_ref(), store);
|
||||
context
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use bytes::Bytes;
|
||||
use csv_core::{ReadRecordResult, Reader, ReaderBuilder, Terminator};
|
||||
use datafusion::{
|
||||
arrow::{
|
||||
csv::reader::Format,
|
||||
datatypes::{DataType, Schema, SchemaRef},
|
||||
},
|
||||
datasource::file_format::{DEFAULT_SCHEMA_INFER_MAX_RECORD, csv::CsvFormat},
|
||||
error::{DataFusionError, Result},
|
||||
execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation},
|
||||
object_store::Error as ObjectStoreError,
|
||||
};
|
||||
use futures::{StreamExt, stream::BoxStream};
|
||||
use rustfs_s3select_api::query::session::MAX_SELECT_COLUMNS;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) const MAX_SCHEMA_PREFIX_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
pub(super) struct SchemaPrefix {
|
||||
pub bytes: Vec<u8>,
|
||||
pub reservation: MemoryReservation,
|
||||
}
|
||||
|
||||
pub(super) enum PrefixBoundary {
|
||||
JsonLines { records: usize, line_start: usize },
|
||||
Csv { reader: Box<Reader>, fields: usize },
|
||||
}
|
||||
|
||||
impl PrefixBoundary {
|
||||
pub fn json_lines() -> Self {
|
||||
Self::JsonLines {
|
||||
records: 0,
|
||||
line_start: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn csv(format: &CsvFormat) -> Self {
|
||||
let options = format.options();
|
||||
let mut builder = ReaderBuilder::new();
|
||||
builder
|
||||
.delimiter(options.delimiter)
|
||||
.quote(options.quote)
|
||||
.escape(options.escape)
|
||||
.comment(options.comment);
|
||||
if let Some(terminator) = options.terminator {
|
||||
builder.terminator(Terminator::Any(terminator));
|
||||
}
|
||||
Self::Csv {
|
||||
reader: Box::new(builder.build()),
|
||||
fields: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn consume(&mut self, input: &[u8], prefix: &[u8]) -> Result<(usize, bool)> {
|
||||
match self {
|
||||
Self::JsonLines { records, line_start } => {
|
||||
for (index, byte) in input.iter().enumerate() {
|
||||
if *byte != b'\n' {
|
||||
continue;
|
||||
}
|
||||
let end = prefix.len() - input.len() + index + 1;
|
||||
// The complete line is available even when a UTF-8 space
|
||||
// straddles input chunks. Match Arrow's blank-line rule.
|
||||
let nonblank = !std::str::from_utf8(&prefix[*line_start..end])
|
||||
.unwrap_or("invalid UTF-8")
|
||||
.trim()
|
||||
.is_empty();
|
||||
*line_start = end;
|
||||
if nonblank {
|
||||
*records += 1;
|
||||
}
|
||||
if *records == DEFAULT_SCHEMA_INFER_MAX_RECORD {
|
||||
return Ok((index + 1, true));
|
||||
}
|
||||
}
|
||||
Ok((input.len(), false))
|
||||
}
|
||||
Self::Csv { reader, fields } => {
|
||||
let mut consumed = 0;
|
||||
let mut output = [0u8; 8192];
|
||||
let mut ends = [0usize; MAX_SELECT_COLUMNS + 1];
|
||||
loop {
|
||||
let (result, read, _, count) =
|
||||
reader.read_record(&input[consumed..], &mut output, &mut ends[..MAX_SELECT_COLUMNS + 1 - *fields]);
|
||||
consumed += read;
|
||||
*fields += count;
|
||||
if *fields > MAX_SELECT_COLUMNS || result == ReadRecordResult::OutputEndsFull {
|
||||
return Err(DataFusionError::ResourcesExhausted(
|
||||
"S3 Select CSV schema exceeds 1024 columns".to_owned(),
|
||||
));
|
||||
}
|
||||
match result {
|
||||
ReadRecordResult::Record | ReadRecordResult::End => return Ok((consumed, true)),
|
||||
ReadRecordResult::InputEmpty => return Ok((consumed, false)),
|
||||
ReadRecordResult::OutputFull if consumed == input.len() => return Ok((consumed, false)),
|
||||
ReadRecordResult::OutputFull => {}
|
||||
ReadRecordResult::OutputEndsFull => {
|
||||
return Err(DataFusionError::ResourcesExhausted(
|
||||
"S3 Select CSV schema exceeds 1024 columns".to_owned(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads only complete inference records. Hitting the byte bound is an error,
|
||||
/// never a shorter successful sample that can silently omit later fields.
|
||||
pub(super) async fn read_schema_prefix(
|
||||
mut stream: BoxStream<'static, std::result::Result<Bytes, ObjectStoreError>>,
|
||||
memory_pool: &Arc<dyn MemoryPool>,
|
||||
mut boundary: PrefixBoundary,
|
||||
) -> Result<SchemaPrefix> {
|
||||
let reservation = MemoryConsumer::new("S3 Select schema prefix").register(memory_pool);
|
||||
let mut bytes = Vec::new();
|
||||
let mut parsed = 0;
|
||||
while let Some(chunk) = stream.next().await.transpose()? {
|
||||
let mut offset = 0;
|
||||
while offset < chunk.len() {
|
||||
let remaining = MAX_SCHEMA_PREFIX_BYTES - bytes.len();
|
||||
if remaining == 0 {
|
||||
return Err(prefix_limit_error());
|
||||
}
|
||||
let end = chunk.len().min(offset + remaining);
|
||||
let required = bytes.len() + end - offset;
|
||||
if required > bytes.capacity() {
|
||||
let capacity = required.next_power_of_two();
|
||||
reservation.try_resize(capacity)?;
|
||||
bytes
|
||||
.try_reserve_exact(capacity - bytes.len())
|
||||
.map_err(|_| prefix_limit_error())?;
|
||||
reservation.try_resize(bytes.capacity())?;
|
||||
}
|
||||
bytes.extend_from_slice(&chunk[offset..end]);
|
||||
// csv-core recognizes a BOM only when its three bytes arrive
|
||||
// together, but treats a BOM with no following bytes as EOF after
|
||||
// stripping it. Wait for data or the actual end of the stream.
|
||||
if parsed == 0 && (bytes.len() < 3 || bytes.as_slice() == b"\xef\xbb\xbf") {
|
||||
offset = end;
|
||||
continue;
|
||||
}
|
||||
let (consumed, done) = boundary.consume(&bytes[parsed..], &bytes)?;
|
||||
parsed += consumed;
|
||||
bytes.truncate(parsed);
|
||||
offset = end;
|
||||
if done {
|
||||
return Ok(SchemaPrefix { bytes, reservation });
|
||||
}
|
||||
}
|
||||
}
|
||||
if parsed < bytes.len() {
|
||||
boundary.consume(&bytes[parsed..], &bytes)?;
|
||||
}
|
||||
boundary.consume(&[], &bytes)?;
|
||||
Ok(SchemaPrefix { bytes, reservation })
|
||||
}
|
||||
|
||||
fn prefix_limit_error() -> DataFusionError {
|
||||
DataFusionError::ResourcesExhausted("S3 Select schema inference exceeds the 4 MiB prefix limit".to_owned())
|
||||
}
|
||||
|
||||
pub(super) fn infer_csv_schema(bytes: &[u8], format: &CsvFormat) -> Result<SchemaRef> {
|
||||
let options = format.options();
|
||||
let mut arrow = Format::default()
|
||||
.with_header(options.has_header.unwrap_or(false))
|
||||
.with_delimiter(options.delimiter)
|
||||
.with_quote(options.quote);
|
||||
if let Some(escape) = options.escape {
|
||||
arrow = arrow.with_escape(escape);
|
||||
}
|
||||
if let Some(comment) = options.comment {
|
||||
arrow = arrow.with_comment(comment);
|
||||
}
|
||||
if let Some(terminator) = options.terminator {
|
||||
arrow = arrow.with_terminator(terminator);
|
||||
}
|
||||
let (schema, _) = arrow.infer_schema(bytes, Some(0))?;
|
||||
// Preserve CsvFormat's zero-record Utf8 inference and duplicate-header
|
||||
// merge semantics. Width has already been checked before Arrow allocation.
|
||||
let fields = schema
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|field| field.as_ref().clone().with_data_type(DataType::Utf8))
|
||||
.collect::<Vec<_>>();
|
||||
Ok(Arc::new(Schema::try_merge([Schema::new(fields)])?))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use datafusion::execution::memory_pool::GreedyMemoryPool;
|
||||
use futures::stream;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
fn pool() -> Arc<dyn MemoryPool> {
|
||||
Arc::new(GreedyMemoryPool::new(64 * 1024 * 1024))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_schema_prefix_stops_polling_after_complete_sample() {
|
||||
let polls = Arc::new(AtomicUsize::new(0));
|
||||
let counter = Arc::clone(&polls);
|
||||
let chunks = stream::iter([
|
||||
Ok(Bytes::from("{\"a\":1}\n".repeat(DEFAULT_SCHEMA_INFER_MAX_RECORD))),
|
||||
Err(ObjectStoreError::Generic {
|
||||
store: "unread tail",
|
||||
source: Box::new(std::io::Error::other("tail must not be read")),
|
||||
}),
|
||||
])
|
||||
.inspect(move |_| {
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
})
|
||||
.boxed();
|
||||
let prefix = read_schema_prefix(chunks, &pool(), PrefixBoundary::json_lines())
|
||||
.await
|
||||
.expect("bounded prefix");
|
||||
assert_eq!(polls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
super::super::json_lines_schema::infer_schema(&prefix.bytes, DEFAULT_SCHEMA_INFER_MAX_RECORD)
|
||||
.expect("schema")
|
||||
.fields()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_schema_prefix_preserves_csv_quotes_bom_and_chunk_boundaries() {
|
||||
let format = CsvFormat::default().with_has_header(true).with_comment(Some(b'#'));
|
||||
// All the commas below belong to one quoted header, not extra columns.
|
||||
let input = format!("\u{feff}#comment\n\"{}\",second\nvalue,2\n", "comma,".repeat(1025));
|
||||
let chunks = input
|
||||
.into_bytes()
|
||||
.into_iter()
|
||||
.map(|byte| Ok(Bytes::from(vec![byte])))
|
||||
.collect::<Vec<_>>();
|
||||
let prefix = read_schema_prefix(stream::iter(chunks).boxed(), &pool(), PrefixBoundary::csv(&format))
|
||||
.await
|
||||
.expect("quoted header");
|
||||
let schema = infer_csv_schema(&prefix.bytes, &format).expect("schema");
|
||||
assert_eq!(schema.fields().len(), 2);
|
||||
assert_eq!(schema.field(1).name(), "second");
|
||||
assert!(!prefix.bytes.ends_with(b"value,2\n"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_csv_bom_prefix_waits_for_data_or_transport_eof() {
|
||||
let format = CsvFormat::default().with_has_header(true);
|
||||
let chunks = stream::iter([
|
||||
Ok(Bytes::from_static(b"\xef\xbb\xbf")),
|
||||
Ok(Bytes::new()),
|
||||
Ok(Bytes::from_static(b"first,second\n")),
|
||||
Err(ObjectStoreError::Generic {
|
||||
store: "unread tail",
|
||||
source: Box::new(std::io::Error::other("tail must not be read")),
|
||||
}),
|
||||
])
|
||||
.boxed();
|
||||
let prefix = read_schema_prefix(chunks, &pool(), PrefixBoundary::csv(&format))
|
||||
.await
|
||||
.expect("BOM-only chunks must not signal EOF");
|
||||
assert_eq!(prefix.bytes, b"\xef\xbb\xbffirst,second\n");
|
||||
let schema = infer_csv_schema(&prefix.bytes, &format).expect("header schema");
|
||||
assert_eq!(schema.fields().len(), 2);
|
||||
assert_eq!(schema.field(0).name(), "first");
|
||||
assert_eq!(schema.field(1).name(), "second");
|
||||
|
||||
let prefix = read_schema_prefix(
|
||||
stream::iter([Ok(Bytes::from_static(b"\xef\xbb\xbf"))]).boxed(),
|
||||
&pool(),
|
||||
PrefixBoundary::csv(&format),
|
||||
)
|
||||
.await
|
||||
.expect("a BOM-only object ends at real transport EOF");
|
||||
assert_eq!(prefix.bytes, b"\xef\xbb\xbf");
|
||||
assert!(
|
||||
infer_csv_schema(&prefix.bytes, &format)
|
||||
.expect("empty schema")
|
||||
.fields()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_schema_prefix_counts_nonblank_utf8_lines_across_chunks() {
|
||||
let input = format!("\u{2003}\n{}", "{\"a\":1}\n".repeat(DEFAULT_SCHEMA_INFER_MAX_RECORD));
|
||||
let expected = input.len();
|
||||
let mut chunks = input
|
||||
.into_bytes()
|
||||
.into_iter()
|
||||
.map(|byte| Ok(Bytes::from(vec![byte])))
|
||||
.collect::<Vec<_>>();
|
||||
chunks.push(Err(ObjectStoreError::Generic {
|
||||
store: "unread tail",
|
||||
source: Box::new(std::io::Error::other("tail")),
|
||||
}));
|
||||
let prefix = read_schema_prefix(stream::iter(chunks).boxed(), &pool(), PrefixBoundary::json_lines())
|
||||
.await
|
||||
.expect("complete records");
|
||||
assert_eq!(prefix.bytes.len(), expected);
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,10 @@ impl MetadataProvider {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn source_column_count(&self) -> usize {
|
||||
self.provider.schema().fields().len()
|
||||
}
|
||||
|
||||
fn build_table_handle(&self) -> datafusion::common::Result<TableHandle> {
|
||||
self.current_session_table_provider.build_table_handle(self.provider.clone())
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@ use rustfs_s3select_api::{
|
||||
|
||||
use super::dialect::RustFsDialect;
|
||||
|
||||
pub(crate) const MAX_SQL_EXPRESSION_BYTES: usize = 256 * 1024;
|
||||
|
||||
pub type Result<T, E = ParserError> = std::result::Result<T, E>;
|
||||
|
||||
// Use `Parser::expected` instead, if possible
|
||||
@@ -54,7 +56,12 @@ impl<'a> ExtParser<'a> {
|
||||
/// Parse the specified tokens with dialect
|
||||
fn new_with_dialect(sql: &str, dialect: &'a dyn Dialect) -> Result<Self> {
|
||||
let mut tokenizer = Tokenizer::new(dialect, sql);
|
||||
// Bound token allocation before the full recursive SQL parser runs.
|
||||
if sql.len() > MAX_SQL_EXPRESSION_BYTES {
|
||||
return parser_err!("S3 Select expression exceeds 256 KiB");
|
||||
}
|
||||
let mut tokens = tokenizer.tokenize()?;
|
||||
validate_projection_width(&tokens)?;
|
||||
rewrite_source_object_wildcards(&mut tokens);
|
||||
Ok(ExtParser {
|
||||
parser: Parser::new(dialect).with_tokens(tokens),
|
||||
@@ -106,6 +113,41 @@ impl<'a> ExtParser<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_projection_width(tokens: &[Token]) -> Result<()> {
|
||||
let mut depth = 0usize;
|
||||
let mut projections: Vec<(usize, usize)> = Vec::new();
|
||||
for token in tokens {
|
||||
match token {
|
||||
Token::Word(word) if word.keyword == Keyword::SELECT => projections.push((depth, 1)),
|
||||
Token::Word(word)
|
||||
if word.keyword == Keyword::FROM && projections.last().is_some_and(|(level, _)| *level == depth) =>
|
||||
{
|
||||
projections.pop();
|
||||
}
|
||||
Token::LParen | Token::LBracket => depth += 1,
|
||||
Token::RParen | Token::RBracket => {
|
||||
depth = depth.saturating_sub(1);
|
||||
while projections.last().is_some_and(|(level, _)| *level > depth) {
|
||||
projections.pop();
|
||||
}
|
||||
}
|
||||
Token::Comma => {
|
||||
if let Some((level, columns)) = projections.last_mut()
|
||||
&& *level == depth
|
||||
{
|
||||
*columns += 1;
|
||||
if *columns > rustfs_s3select_api::query::session::MAX_SELECT_COLUMNS {
|
||||
return parser_err!("S3 Select projection exceeds 1024 columns");
|
||||
}
|
||||
}
|
||||
}
|
||||
Token::SemiColon => projections.clear(),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn rewrite_source_object_wildcards(tokens: &mut [Token]) {
|
||||
let mut paren_depth = 0usize;
|
||||
let mut in_from = false;
|
||||
@@ -444,6 +486,31 @@ mod tests {
|
||||
assert!(memory_size < 10000, "Parsed statements should not use excessive memory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_projection_preflight_respects_quoted_commas_and_nested_expressions() {
|
||||
let projection = (0..1024)
|
||||
.map(|index| format!("COALESCE('a,b', 'c') AS c{index}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
assert!(ExtParser::parse_sql(&format!("SELECT {projection} FROM S3Object")).is_ok());
|
||||
let error = ExtParser::parse_sql(&format!("SELECT {projection}, 1 FROM S3Object")).expect_err("column limit");
|
||||
assert!(error.to_string().contains("1024 columns"));
|
||||
let nested = format!("SELECT (SELECT 1), {projection} FROM S3Object");
|
||||
assert!(
|
||||
ExtParser::parse_sql(&nested)
|
||||
.expect_err("nested SELECT cannot reset outer width")
|
||||
.to_string()
|
||||
.contains("1024 columns")
|
||||
);
|
||||
let long_literal = "x".repeat(256 * 1024);
|
||||
assert!(
|
||||
ExtParser::parse_sql(&format!("SELECT '{long_literal}' FROM S3Object"))
|
||||
.expect_err("byte limit")
|
||||
.to_string()
|
||||
.contains("256 KiB")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ext_parser_large_query() {
|
||||
// Test with a reasonably large query
|
||||
|
||||
@@ -90,6 +90,28 @@ impl<'a, S: ContextProviderExtension + Send + Sync + 'a> SqlPlanner<'a, S> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_expanded_projection_width(statement: &ExtStatement, source_columns: usize) -> QueryResult<()> {
|
||||
let ExtStatement::SqlStatement(statement) = statement;
|
||||
let Statement::Query(query) = statement.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
let SetExpr::Select(select) = query.body.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
let width = select.projection.iter().try_fold(0usize, |width, item| {
|
||||
let columns = if matches!(item, SelectItem::Wildcard(_) | SelectItem::QualifiedWildcard(..)) {
|
||||
source_columns
|
||||
} else {
|
||||
1
|
||||
};
|
||||
width.checked_add(columns)
|
||||
});
|
||||
if width.is_none_or(|width| width > rustfs_s3select_api::query::session::MAX_SELECT_COLUMNS) {
|
||||
return Err(unsupported_structure("S3 Select expanded projection exceeds 1024 columns"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn classify_planner_error(error: datafusion::common::DataFusionError) -> QueryError {
|
||||
if matches!(
|
||||
&error,
|
||||
@@ -414,6 +436,9 @@ fn implicit_source_alias(source_path: &[JsonPathSegment]) -> Ident {
|
||||
}
|
||||
|
||||
fn prepare_select(select: &mut Select) -> QueryResult<JsonSource> {
|
||||
if select.projection.len() > rustfs_s3select_api::query::session::MAX_SELECT_COLUMNS {
|
||||
return Err(unsupported_structure("S3 Select projection exceeds 1024 columns"));
|
||||
}
|
||||
if !select.optimizer_hints.is_empty()
|
||||
|| select.distinct.is_some()
|
||||
|| select.select_modifiers.is_some()
|
||||
|
||||
@@ -3424,6 +3424,7 @@ where
|
||||
runtime_config_generation_seen = scanner_runtime_config_generation();
|
||||
}
|
||||
|
||||
let mut maintenance_wait_deadline: Option<Instant> = None;
|
||||
loop {
|
||||
if ctx.is_cancelled() {
|
||||
break;
|
||||
@@ -3486,6 +3487,10 @@ where
|
||||
convergence_retry_interval = Some(pause_backlog_delay.max(Duration::from_secs(1)));
|
||||
}
|
||||
wait_plan = usage_bootstrap_rebuild.wait_plan(wait_plan, convergence_retry_interval);
|
||||
let wait_started = Instant::now();
|
||||
let wait_deadline = maintenance_wait_deadline
|
||||
.map_or(wait_started + wait_plan.delay, |deadline| deadline.min(wait_started + wait_plan.delay));
|
||||
wait_plan.delay = wait_deadline.saturating_duration_since(wait_started);
|
||||
let dirty_generation_before_wait = dirty_usage_generation();
|
||||
let dirty_usage_pending_before_wait = dirty_usage_buckets_pending();
|
||||
let maintenance_generation_before_wait = scanner_maintenance_generation();
|
||||
@@ -3559,6 +3564,7 @@ where
|
||||
break;
|
||||
}
|
||||
ScannerCycleWakeReason::RuntimeConfig => {
|
||||
maintenance_wait_deadline = None;
|
||||
runtime_config_generation_seen = scanner_runtime_config_generation();
|
||||
maintenance_generation_seen = None;
|
||||
scanner_activity_seen = None;
|
||||
@@ -3566,6 +3572,7 @@ where
|
||||
continue;
|
||||
}
|
||||
ScannerCycleWakeReason::MaintenanceConfig => {
|
||||
maintenance_wait_deadline = Some(wait_deadline);
|
||||
maintenance_generation_seen = None;
|
||||
scanner_activity_seen = None;
|
||||
clean_idle_backoff.reset();
|
||||
@@ -3584,6 +3591,7 @@ where
|
||||
| ScannerCycleWakeReason::ClusterActivityUnavailable
|
||||
| ScannerCycleWakeReason::Recovery => {}
|
||||
}
|
||||
maintenance_wait_deadline = None;
|
||||
|
||||
if wake_reason == ScannerCycleWakeReason::DirtyUsage {
|
||||
debug!(
|
||||
|
||||
@@ -354,6 +354,60 @@ async fn restarted_main_loop_completes_durable_pause_backlog_catch_up() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn running_main_loop_starts_next_cycle_despite_maintenance_config_churn() {
|
||||
temp_env::async_with_vars([(ENV_SCANNER_CYCLE, Some("2")), (ENV_SCANNER_START_DELAY_SECS, Some("0"))], async {
|
||||
crate::runtime_config::refresh_scanner_runtime_config_for_tests();
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
global_metrics().set_cycle(None).await;
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
let ctx = CancellationToken::new();
|
||||
let ready = ScannerRuntimeObservedProbe::install(&store, false);
|
||||
let scanner_ctx = ctx.clone();
|
||||
let scanner_store = store.clone();
|
||||
let scanner_task = tokio::spawn(async move { run_data_scanner(scanner_ctx, scanner_store).await });
|
||||
ready.wait().await;
|
||||
let ScannerCycleStateStartup::Ready { cycle: initial, .. } = load_scanner_cycle_state_for_startup(store.clone()).await
|
||||
else {
|
||||
panic!("initial scanner cycle must persist its state");
|
||||
};
|
||||
drop(ready);
|
||||
let churn_ctx = ctx.clone();
|
||||
let churn = tokio::spawn(async move {
|
||||
loop {
|
||||
crate::scanner_io::record_scanner_maintenance_change("churning-lifecycle");
|
||||
tokio::select! {
|
||||
_ = churn_ctx.cancelled() => break,
|
||||
_ = tokio::time::sleep(Duration::from_millis(100)) => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
let progressed = tokio::time::timeout(Duration::from_secs(8), async {
|
||||
loop {
|
||||
if let ScannerCycleStateStartup::Ready { cycle, .. } = load_scanner_cycle_state_for_startup(store.clone()).await
|
||||
&& cycle.next > initial.next
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
ctx.cancel();
|
||||
churn.await.expect("churn task");
|
||||
tokio::time::timeout(Duration::from_secs(5), scanner_task)
|
||||
.await
|
||||
.expect("scanner cancellation")
|
||||
.expect("scanner task")
|
||||
.expect("scanner exit");
|
||||
assert!(progressed.is_ok(), "maintenance wakes must not indefinitely move the next cycle deadline");
|
||||
global_metrics().set_cycle(None).await;
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() {
|
||||
|
||||
@@ -110,6 +110,7 @@ pub enum StorageErrorCode {
|
||||
ConditionalDeleteConflict,
|
||||
GuardedDeleteUnsupported,
|
||||
DeleteRecoveryLegacyAmbiguous,
|
||||
DeleteRecoveryWorkLimit,
|
||||
}
|
||||
|
||||
impl StorageErrorCode {
|
||||
@@ -202,6 +203,7 @@ impl StorageErrorCode {
|
||||
Self::ConditionalDeleteConflict => 0x57,
|
||||
Self::GuardedDeleteUnsupported => 0x58,
|
||||
Self::DeleteRecoveryLegacyAmbiguous => 0x59,
|
||||
Self::DeleteRecoveryWorkLimit => 0x5A,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -294,6 +296,7 @@ impl StorageErrorCode {
|
||||
0x57 => Some(Self::ConditionalDeleteConflict),
|
||||
0x58 => Some(Self::GuardedDeleteUnsupported),
|
||||
0x59 => Some(Self::DeleteRecoveryLegacyAmbiguous),
|
||||
0x5A => Some(Self::DeleteRecoveryWorkLimit),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -394,6 +397,7 @@ mod tests {
|
||||
(StorageErrorCode::ConditionalDeleteConflict, 0x57),
|
||||
(StorageErrorCode::GuardedDeleteUnsupported, 0x58),
|
||||
(StorageErrorCode::DeleteRecoveryLegacyAmbiguous, 0x59),
|
||||
(StorageErrorCode::DeleteRecoveryWorkLimit, 0x5A),
|
||||
];
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -90,8 +90,8 @@ pub use bucket::BUCKET_LIFECYCLE_LOCK_OBJECT;
|
||||
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
|
||||
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::MultipartUploadInspection;
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use multipart::{MultipartCompletionAuthorization, MultipartUploadInspection};
|
||||
pub use object::AuthoritativeListOperations;
|
||||
pub use object::DeleteAccounting;
|
||||
pub use object::ObjectLockDeleteOptions;
|
||||
|
||||
@@ -15,6 +15,16 @@
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Request authorization against the source selected under completion locks.
|
||||
/// Implementations must not mutate storage or acquire object/upload locks again.
|
||||
/// Called before comparing client checksums/parts or acknowledging a replay.
|
||||
#[async_trait::async_trait]
|
||||
pub trait MultipartCompletionAuthorization: Send + Sync {
|
||||
type Error: std::error::Error + Send + Sync + 'static;
|
||||
|
||||
async fn authorize(&self, metadata: &HashMap<String, String>) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
/// Checks upload absence before removing a bucket's data protections.
|
||||
#[async_trait::async_trait]
|
||||
pub trait MultipartUploadInspection: Send + Sync {
|
||||
|
||||
@@ -574,6 +574,23 @@ pub trait ListOperations: Send + Sync + fmt::Debug {
|
||||
/// this capability does not provide a snapshot or retain a mutation lock.
|
||||
#[async_trait::async_trait]
|
||||
pub trait AuthoritativeListOperations: ListOperations {
|
||||
/// A bounded, authoritative page of an unversioned internal namespace.
|
||||
/// Unlike ordinary S3 listing, uncertainty or a versioned/deleted entry is
|
||||
/// an error. A nontruncated result proves EOF; truncated results resume at
|
||||
/// their last returned key. This does not weaken full-walk absence checks.
|
||||
/// Cancellation drops the scan workers and publishes no persistent cursor.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error for an invalid page size, unreadable metadata, insufficient
|
||||
/// read quorum, or a producer cut that has no safe continuation key.
|
||||
async fn list_unversioned_objects_authoritative(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
marker: Option<String>,
|
||||
max_keys: i32,
|
||||
) -> Result<Self::ListObjectVersionsInfo, Self::Error>;
|
||||
|
||||
/// Resolves every set at its version read quorum, including reserved prefixes.
|
||||
///
|
||||
/// # Errors
|
||||
|
||||
@@ -105,6 +105,18 @@ pub const AMZ_RUSTFS_SNOWBALL_PREFIX: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix"
|
||||
pub const AMZ_RUSTFS_SNOWBALL_IGNORE_DIRS: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Dirs";
|
||||
pub const AMZ_RUSTFS_SNOWBALL_IGNORE_ERRORS: &str = "X-Amz-Meta-Rustfs-Snowball-Ignore-Errors";
|
||||
|
||||
/// Transport controls must not become commands when stored metadata is sent to a tier.
|
||||
/// Accepts both persisted metadata names and their HTTP header spellings.
|
||||
pub fn is_snowball_transport_key(key: &str) -> bool {
|
||||
let key = key.to_ascii_lowercase();
|
||||
let name = key.strip_prefix("x-amz-meta-").unwrap_or(&key);
|
||||
name == "snowball-auto-extract"
|
||||
|| key == "x-amz-snowball-auto-extract"
|
||||
|| ["snowball-prefix", "snowball-ignore-dirs", "snowball-ignore-errors"]
|
||||
.iter()
|
||||
.any(|suffix| name.ends_with(suffix))
|
||||
}
|
||||
|
||||
// Object lock enabled
|
||||
pub const AMZ_OBJECT_LOCK_ENABLED: &str = "x-amz-bucket-object-lock-enabled";
|
||||
|
||||
@@ -284,9 +296,33 @@ pub fn is_minio_header(header_key: &str) -> bool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::is_sse_header;
|
||||
use super::{is_snowball_transport_key, is_sse_header};
|
||||
use proptest::prelude::*;
|
||||
|
||||
#[test]
|
||||
fn snowball_transport_keys_cover_header_and_stored_aliases() {
|
||||
for key in [
|
||||
"snowball-auto-extract",
|
||||
"X-Amz-Meta-Snowball-Auto-Extract",
|
||||
"X-Amz-Snowball-Auto-Extract",
|
||||
"snowball-prefix",
|
||||
"minio-snowball-ignore-dirs",
|
||||
"rustfs-snowball-ignore-errors",
|
||||
"x-amz-meta-other-vendor-snowball-prefix",
|
||||
"other-vendor-snowball-ignore-errors",
|
||||
] {
|
||||
assert!(is_snowball_transport_key(key), "{key}");
|
||||
}
|
||||
for key in [
|
||||
"owner",
|
||||
"snowball-auto-extract-extra",
|
||||
"snowball-prefix-extra",
|
||||
"content-type",
|
||||
] {
|
||||
assert!(!is_snowball_transport_key(key), "{key}");
|
||||
}
|
||||
}
|
||||
|
||||
const STORED_SSE_HEADERS: &[&str] = &[
|
||||
"x-amz-server-side-encryption",
|
||||
"x-amz-server-side-encryption-aws-kms-key-id",
|
||||
|
||||
+749
-116
File diff suppressed because it is too large
Load Diff
@@ -19,6 +19,11 @@ use s3s::header::CONTENT_TYPE;
|
||||
use s3s::{Body, S3Error, S3ErrorCode, S3Response, S3Result, s3_error};
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
// Each compatible request can run two 64 MiB derivations. Keep both CPU and
|
||||
// memory bounded without occupying runtime workers or queueing unbounded jobs.
|
||||
static ADMIN_DECRYPT_PERMITS: Semaphore = Semaphore::const_new(2);
|
||||
|
||||
/// Returns `true` if `s` contains any whitespace character.
|
||||
///
|
||||
@@ -45,9 +50,20 @@ pub(crate) async fn read_compatible_admin_body(
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
|
||||
|
||||
if is_compat_admin_request(path) {
|
||||
decrypt_stream_io(secret_key.as_bytes(), body.as_ref())
|
||||
.or_else(|_| decrypt_data(secret_key.as_bytes(), body.as_ref()))
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to decrypt MinIO admin payload: {}", e))
|
||||
let permit = ADMIN_DECRYPT_PERMITS
|
||||
.try_acquire()
|
||||
.map_err(|_| s3_error!(SlowDown, "Admin payload decryption is busy"))?;
|
||||
let secret_key = secret_key.to_owned();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// Cancellation of the request must not release admission while the
|
||||
// non-cancellable derivation is still running.
|
||||
let _permit = permit;
|
||||
decrypt_stream_io(secret_key.as_bytes(), body.as_ref())
|
||||
.or_else(|_| decrypt_data(secret_key.as_bytes(), body.as_ref()))
|
||||
.map_err(|e| s3_error!(InvalidRequest, "failed to decrypt MinIO admin payload: {}", e))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| s3_error!(InternalError, "Admin payload decryption failed"))?
|
||||
} else {
|
||||
Ok(body.to_vec())
|
||||
}
|
||||
@@ -155,6 +171,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(admin_decrypt)]
|
||||
async fn reads_legacy_compat_payload_as_fallback() {
|
||||
let payload = b"{\"ok\":true}".to_vec();
|
||||
let encrypted = encrypt_data(b"secret", &payload).expect("encrypt payload");
|
||||
@@ -166,6 +183,71 @@ mod tests {
|
||||
assert_eq!(decoded, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(admin_decrypt)]
|
||||
async fn compatible_admin_decryption_rejects_excess_work_without_blocking_native_requests() {
|
||||
let permits = ADMIN_DECRYPT_PERMITS
|
||||
.acquire_many(2)
|
||||
.await
|
||||
.expect("reserve decryption capacity");
|
||||
let error = read_compatible_admin_body(Body::from(vec![0; 64]), 1024, "/minio/admin/v3/add-user", "secret")
|
||||
.await
|
||||
.expect_err("excess KDF work must not queue");
|
||||
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
|
||||
let plain = read_compatible_admin_body(Body::from(String::from("{}")), 1024, "/rustfs/admin/v3/add-user", "secret")
|
||||
.await
|
||||
.expect("native admin does not use KDF capacity");
|
||||
assert_eq!(plain, b"{}");
|
||||
drop(permits);
|
||||
let encrypted = encrypt_stream_io(b"secret", b"{}").expect("encrypt compatible request");
|
||||
assert_eq!(
|
||||
read_compatible_admin_body(Body::from(encrypted), 1024, "/minio/admin/v3/add-user", "secret")
|
||||
.await
|
||||
.expect("capacity returns after work finishes"),
|
||||
b"{}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(admin_decrypt)]
|
||||
fn cancelled_admin_decryption_keeps_capacity_until_blocking_work_finishes() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.max_blocking_threads(1)
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build single-worker runtime");
|
||||
runtime.block_on(async {
|
||||
let (release, wait) = std::sync::mpsc::channel();
|
||||
let blocker = tokio::task::spawn_blocking(move || wait.recv().expect("release blocker"));
|
||||
let request = tokio::spawn(read_compatible_admin_body(
|
||||
Body::from(vec![0; 64]),
|
||||
1024,
|
||||
"/minio/admin/v3/add-user",
|
||||
"secret",
|
||||
));
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
ADMIN_DECRYPT_PERMITS.available_permits(),
|
||||
1,
|
||||
"decryption was queued off the runtime worker"
|
||||
);
|
||||
request.abort();
|
||||
assert!(request.await.expect_err("request cancelled").is_cancelled());
|
||||
assert_eq!(
|
||||
ADMIN_DECRYPT_PERMITS.available_permits(),
|
||||
1,
|
||||
"cancellation must not release queued work admission"
|
||||
);
|
||||
release.send(()).expect("unblock worker");
|
||||
blocker.await.expect("blocking task finished");
|
||||
let all = ADMIN_DECRYPT_PERMITS
|
||||
.acquire_many(2)
|
||||
.await
|
||||
.expect("decryption finished and released capacity");
|
||||
drop(all);
|
||||
});
|
||||
}
|
||||
|
||||
async fn body_bytes(mut body: Body) -> Vec<u8> {
|
||||
body.store_all_limited(64 * 1024).await.expect("body should read").to_vec()
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user