diff --git a/.config/nextest.toml b/.config/nextest.toml index 8eb9284ba..0ef3e198c 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -60,6 +60,16 @@ test-group = 'ecstore-serial-flaky' filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)' test-group = 'ecstore-serial-flaky' +# Serialize the bucket-incarnation / lifecycle-fence tests. They drive +# init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global +# OnceLock state that serial_test's #[serial] cannot protect across nextest's +# process boundary, and they delete+recreate buckets — the same shape that +# raced into InsufficientWriteQuorum in backlog#937. Preventive only, no +# retries. The matching ci-profile override is after [profile.ci]. +[[profile.default.overrides]] +filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))' +test-group = 'ecstore-serial-flaky' + # Serialize the 4-disk reliability / degraded-read e2e tests (see the # e2e-reliability test-group note above). The matching ci-profile override is at # the end of the file, after [profile.ci] is declared. @@ -139,6 +149,12 @@ test-group = 'ecstore-serial-flaky' filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)' test-group = 'ecstore-serial-flaky' +# Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile +# too (see the matching default-profile override near the top). No retries. +[[profile.ci.overrides]] +filter = 'package(rustfs-ecstore) & (test(bucket_delete_waits_for_config_mutation_fence) | test(stale_config_request_cannot_mutate_a_recreated_bucket) | test(disk_incarnation_read_detects_stale_cache_until_peer_reload) | test(lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata) | test(expiry_configs_are_resolved_from_the_owning_store))' +test-group = 'ecstore-serial-flaky' + # --------------------------------------------------------------------------- # e2e-smoke profile — PR smoke subset of the e2e_test crate (backlog#1149 ci-4) # --------------------------------------------------------------------------- diff --git a/crates/e2e_test/src/inline_fast_path_cluster_test.rs b/crates/e2e_test/src/inline_fast_path_cluster_test.rs index d0585b307..085a10a4d 100644 --- a/crates/e2e_test/src/inline_fast_path_cluster_test.rs +++ b/crates/e2e_test/src/inline_fast_path_cluster_test.rs @@ -1687,6 +1687,44 @@ async fn four_node_inline_storage_and_get_boundaries() -> TestResult { Ok(()) } +#[tokio::test] +#[serial] +async fn four_node_empty_legacy_volumes_start_as_fresh() -> TestResult { + init_logging(); + + let mut cluster = RustFSTestClusterEnvironment::new(4).await?; + for data_dir in cluster.nodes.iter().flat_map(|node| &node.data_dirs) { + tokio::fs::create_dir_all(Path::new(data_dir).join(".minio.sys")).await?; + } + + cluster.start().await?; + + // Starting is not the assertion. The regression is that an empty legacy + // `.minio.sys` must be classified as a *fresh* volume, not as an existing + // MinIO deployment to adopt or migrate. Pin what that classification leaves + // on disk and in the namespace. + let buckets = cluster.create_s3_client(0)?.list_buckets().send().await?; + assert!( + buckets.buckets().is_empty(), + "a fresh classification must not adopt buckets from the pre-existing directories, got {:?}", + buckets.buckets().iter().filter_map(|b| b.name()).collect::>() + ); + + for data_dir in cluster.nodes.iter().flat_map(|node| &node.data_dirs) { + assert!( + Path::new(data_dir).join(".rustfs.sys").join("format.json").is_file(), + "each drive must be formatted as fresh: {data_dir} has no .rustfs.sys/format.json" + ); + let mut legacy = tokio::fs::read_dir(Path::new(data_dir).join(".minio.sys")).await?; + assert!( + legacy.next_entry().await?.is_none(), + "the empty legacy directory must be left untouched, not migrated into: {data_dir}" + ); + } + + Ok(()) +} + #[tokio::test] #[serial] async fn four_node_inline_fallback_controls() -> TestResult { diff --git a/crates/e2e_test/src/multipart_auth_test.rs b/crates/e2e_test/src/multipart_auth_test.rs index 941e8d075..4ae950e76 100644 --- a/crates/e2e_test/src/multipart_auth_test.rs +++ b/crates/e2e_test/src/multipart_auth_test.rs @@ -62,6 +62,33 @@ fn md5_hex(input: impl AsRef<[u8]>) -> String { hex::encode(hasher.finalize()) } +async fn create_restricted_user( + env: &RustFSTestEnvironment, + username: &str, + secret_key: &str, +) -> Result<(), Box> { + let url = format!("{}/rustfs/admin/v3/add-user?accessKey={username}", env.url); + let body = serde_json::json!({ + "secretKey": secret_key, + "status": "enabled" + }) + .to_string(); + crate::common::awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?; + Ok(()) +} + +fn restricted_user_client(env: &RustFSTestEnvironment, username: &str, secret_key: &str) -> aws_sdk_s3::Client { + let credentials = aws_sdk_s3::config::Credentials::new(username, secret_key, None, None, "snowball-pax-auth-test"); + let config = aws_sdk_s3::Config::builder() + .credentials_provider(credentials) + .region(aws_sdk_s3::config::Region::new("us-east-1")) + .endpoint_url(&env.url) + .force_path_style(true) + .behavior_version_latest() + .build(); + aws_sdk_s3::Client::from_conf(config) +} + /// Env var consumed by the local SSE-S3 DEK provider when KMS is not configured. /// /// Since rustfs#3564 the server fails closed on managed SSE (SSE-S3 or @@ -5658,6 +5685,70 @@ async fn test_signed_put_object_extract_preserves_object_lock_retention() -> Res Ok(()) } +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_pax_retention_overrides_request_retention() +-> Result<(), Box> { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-pax-retention-precedence"; + let archive_key = "retention.tar"; + let extracted_key = "alpha.txt"; + let request_retain_until = aws_sdk_s3::primitives::DateTime::from_secs(2_114_380_800); + let pax_retain_until = "2040-01-01T00:00:00Z"; + + let client = env.create_s3_client(); + client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + + let pax = HashMap::from([ + ("minio.metadata.x-amz-object-lock-mode", "COMPLIANCE".to_string()), + ("minio.metadata.x-amz-object-lock-retain-until-date", pax_retain_until.to_string()), + ]); + let archive = make_tar_with_pax_entry(extracted_key, b"alpha-body", None, &pax).await; + + client + .put_object() + .bucket(bucket) + .key(archive_key) + .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance) + .object_lock_retain_until_date(request_retain_until) + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await?; + + let retention = client + .get_object_retention() + .bucket(bucket) + .key(extracted_key) + .send() + .await? + .retention() + .expect("retention should be present") + .clone(); + assert_eq!(retention.mode().map(|value| value.as_str()), Some("COMPLIANCE")); + assert_eq!( + retention + .retain_until_date() + .expect("retain_until_date should be present") + .fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)?, + pax_retain_until + ); + + Ok(()) +} + #[tokio::test] #[serial] async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box> { @@ -5782,6 +5873,282 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id() Ok(()) } +#[tokio::test] +#[serial] +async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions() +-> Result<(), Box> { + init_logging(); + if !crate::common::awscurl_available() { + return Ok(()); + } + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let bucket = "signed-extract-pax-auth"; + let put_only_user = "snowball-put-only"; + let put_only_secret = "snowball-put-only-secret"; + let conditional_user = "snowball-retention-condition"; + let conditional_secret = "snowball-retention-condition-secret"; + let wrong_action_user = "snowball-wrong-action"; + let wrong_action_secret = "snowball-wrong-action-secret"; + let version_condition_user = "snowball-version-condition"; + let version_condition_secret = "snowball-version-condition-secret"; + let pax_context_user = "snowball-pax-context"; + let pax_context_secret = "snowball-pax-context-secret"; + let conditional_version_id = Uuid::new_v4().to_string(); + let admin_client = env.create_s3_client(); + admin_client + .create_bucket() + .bucket(bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + create_restricted_user(&env, put_only_user, put_only_secret).await?; + create_restricted_user(&env, conditional_user, conditional_secret).await?; + create_restricted_user(&env, wrong_action_user, wrong_action_secret).await?; + create_restricted_user(&env, version_condition_user, version_condition_secret).await?; + create_restricted_user(&env, pax_context_user, pax_context_secret).await?; + + let object_resource = format!("arn:aws:s3:::{bucket}/*"); + let context_archive_resources = [ + format!("arn:aws:s3:::{bucket}/tag-context.tar"), + format!("arn:aws:s3:::{bucket}/lock-context.tar"), + ]; + let tag_entry_resource = format!("arn:aws:s3:::{bucket}/tag-context-entry.txt"); + let lock_entry_resource = format!("arn:aws:s3:::{bucket}/lock-context-entry.txt"); + let policy = serde_json::json!({ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "PutOnly", + "Effect": "Allow", + "Principal": { "AWS": [put_only_user] }, + "Action": ["s3:PutObject"], + "Resource": [object_resource.clone()] + }, + { + "Sid": "RetentionWithLimit", + "Effect": "Allow", + "Principal": { "AWS": [conditional_user] }, + "Action": ["s3:PutObject", "s3:PutObjectRetention"], + "Resource": [object_resource.clone()] + }, + { + "Sid": "DenyRetentionBeyondCutoff", + "Effect": "Deny", + "Principal": { "AWS": [conditional_user] }, + "Action": ["s3:PutObject"], + "Resource": [object_resource.clone()], + "Condition": { + "DateGreaterThan": { + "s3:object-lock-retain-until-date": "2030-01-01T00:00:00Z" + } + } + }, + { + "Sid": "WrongAdditionalAction", + "Effect": "Allow", + "Principal": { "AWS": [wrong_action_user] }, + "Action": ["s3:PutObject", "s3:PutObjectLegalHold"], + "Resource": [object_resource.clone()] + }, + { + "Sid": "VersionConditionPut", + "Effect": "Allow", + "Principal": { "AWS": [version_condition_user] }, + "Action": ["s3:PutObject"], + "Resource": [object_resource.clone()] + }, + { + "Sid": "VersionConditionReplicate", + "Effect": "Allow", + "Principal": { "AWS": [version_condition_user] }, + "Action": ["s3:ReplicateObject"], + "Resource": [object_resource], + "Condition": { + "StringEquals": { + "s3:VersionId": conditional_version_id.clone() + } + } + }, + { + "Sid": "PaxContextArchives", + "Effect": "Allow", + "Principal": { "AWS": [pax_context_user] }, + "Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging"], + "Resource": context_archive_resources + }, + { + "Sid": "PaxTagContextPut", + "Effect": "Allow", + "Principal": { "AWS": [pax_context_user] }, + "Action": ["s3:PutObject"], + "Resource": [tag_entry_resource.clone()], + "Condition": { + "StringEquals": { + "s3:RequestObjectTag/classification": "public" + } + } + }, + { + "Sid": "PaxTagContextAction", + "Effect": "Allow", + "Principal": { "AWS": [pax_context_user] }, + "Action": ["s3:PutObjectTagging"], + "Resource": [tag_entry_resource] + }, + { + "Sid": "PaxLockContextPut", + "Effect": "Allow", + "Principal": { "AWS": [pax_context_user] }, + "Action": ["s3:PutObject"], + "Resource": [lock_entry_resource.clone()], + "Condition": { + "StringEquals": { + "s3:object-lock-mode": "COMPLIANCE" + } + } + }, + { + "Sid": "PaxLockContextAction", + "Effect": "Allow", + "Principal": { "AWS": [pax_context_user] }, + "Action": ["s3:PutObjectRetention"], + "Resource": [lock_entry_resource] + } + ] + }) + .to_string(); + admin_client.put_bucket_policy().bucket(bucket).policy(policy).send().await?; + + let put_only_client = restricted_user_client(&env, put_only_user, put_only_secret); + let conditional_client = restricted_user_client(&env, conditional_user, conditional_secret); + let wrong_action_client = restricted_user_client(&env, wrong_action_user, wrong_action_secret); + let cases = [ + ( + "legal-hold.tar", + put_only_client, + HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]), + ), + ( + "retention-condition.tar", + conditional_client, + HashMap::from([ + ("minio.metadata.x-amz-object-lock-mode", "COMPLIANCE".to_string()), + ("minio.metadata.x-amz-object-lock-retain-until-date", "2099-01-01T00:00:00Z".to_string()), + ]), + ), + ( + "version-id.tar", + wrong_action_client, + HashMap::from([("minio.versionId", Uuid::new_v4().to_string())]), + ), + ]; + + for (archive_key, client, pax) in cases { + let archive = make_tar_with_pax_entry("entry.txt", b"must-not-write", None, &pax).await; + let err = client + .put_object() + .bucket(bucket) + .key(archive_key) + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await + .expect_err("missing, conditional, or wrong PAX privilege must be rejected"); + assert_eq!( + err.as_service_error().and_then(|error| error.meta().code()), + Some("AccessDenied"), + "{archive_key}" + ); + } + + let version_condition_client = restricted_user_client(&env, version_condition_user, version_condition_secret); + let matching_version_pax = HashMap::from([("minio.versionId", conditional_version_id)]); + let archive = make_tar_with_pax_entry("condition-entry.txt", b"condition-body", None, &matching_version_pax).await; + version_condition_client + .put_object() + .bucket(bucket) + .key("version-condition.tar") + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await?; + + let pax_context_client = restricted_user_client(&env, pax_context_user, pax_context_secret); + let tag_pax = HashMap::from([("minio.metadata.x-amz-tagging", "classification=public".to_string())]); + let archive = make_tar_with_pax_entry("tag-context-entry.txt", b"tag-context-body", None, &tag_pax).await; + pax_context_client + .put_object() + .bucket(bucket) + .key("tag-context.tar") + .tagging("classification=restricted") + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await?; + let tags = admin_client + .get_object_tagging() + .bucket(bucket) + .key("tag-context-entry.txt") + .send() + .await?; + assert!( + tags.tag_set() + .iter() + .any(|tag| tag.key() == "classification" && tag.value() == "public") + ); + + let pax_retain_until = "2040-01-01T00:00:00Z"; + let lock_pax = HashMap::from([ + ("minio.metadata.x-amz-object-lock-mode", "COMPLIANCE".to_string()), + ("minio.metadata.x-amz-object-lock-retain-until-date", pax_retain_until.to_string()), + ]); + let archive = make_tar_with_pax_entry("lock-context-entry.txt", b"lock-context-body", None, &lock_pax).await; + pax_context_client + .put_object() + .bucket(bucket) + .key("lock-context.tar") + .object_lock_mode(aws_sdk_s3::types::ObjectLockMode::Governance) + .object_lock_retain_until_date(aws_sdk_s3::primitives::DateTime::from_secs(2_114_380_800)) + .body(ByteStream::from(archive)) + .customize() + .mutate_request(|req| { + req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true"); + }) + .send() + .await?; + let retention = admin_client + .get_object_retention() + .bucket(bucket) + .key("lock-context-entry.txt") + .send() + .await? + .retention() + .expect("PAX retention should be present") + .clone(); + assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("COMPLIANCE")); + assert_eq!( + retention + .retain_until_date() + .expect("PAX retain-until should be present") + .fmt(aws_sdk_s3::primitives::DateTimeFormat::DateTime)?, + pax_retain_until + ); + + Ok(()) +} + #[tokio::test] #[serial] async fn test_signed_put_object_extract_accepts_compat_header() -> Result<(), Box> { diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e67cffadc..9a5a01a77 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -130,13 +130,15 @@ pub mod bucket { pub mod metadata_sys { pub use crate::bucket::metadata_sys::{ - BucketMetadataSys, acquire_bucket_metadata_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy, + BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, + capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config, - get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config, - get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config, - init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, - update, update_bucket_targets_under_transaction_lock, update_config_with, update_under_transaction_lock, + get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config, + get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, + get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, + set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, + update_under_transaction_lock, }; } @@ -403,11 +405,11 @@ pub mod notification { pub mod object { pub use crate::object_api::{ BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, - GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo, - ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, - ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, - register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook, - unregister_object_mutation_hook, + GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver, + ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, + ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, + lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook, + unregister_get_object_body_cache_hook, unregister_object_mutation_hook, }; pub use crate::store::PreparedGetObjectReader; } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index bd48d3ff1..59d063daa 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -42,6 +42,7 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{ use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats}; use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity}; use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop; +use crate::bucket::object_lock::ObjectLockApi; use crate::bucket::versioning::VersioningApi as _; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::client::object_api_utils::new_getobjectreader; @@ -90,8 +91,8 @@ use rustfs_utils::{ string::{parse_bool, strings_has_prefix_fold}, }; use s3s::dto::{ - BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, - RestoreRequestType, RestoreStatus, Timestamp, + BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus, + Timestamp, }; use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION}; use sha2::{Digest, Sha256}; @@ -309,6 +310,7 @@ struct ExpiryTask { obj_info: ObjectInfo, event: lifecycle::Event, src: LcEventSrc, + bucket_incarnation_id: Uuid, } impl ExpiryOp for ExpiryTask { @@ -587,6 +589,7 @@ struct NewerNoncurrentTask { versions: Vec, event: lifecycle::Event, src: LcEventSrc, + bucket_incarnation_id: Uuid, } impl ExpiryOp for NewerNoncurrentTask { @@ -697,13 +700,20 @@ impl ExpiryState { queued } - pub fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) -> bool { + pub fn enqueue_by_days( + &mut self, + oi: &ObjectInfo, + event: &lifecycle::Event, + src: &LcEventSrc, + bucket_incarnation_id: Uuid, + ) -> bool { let trace = LifecycleExpiryTrace::for_object(oi, event, src, 1); trace.emit(EVENT_LIFECYCLE_EXPIRED_DETECTED, "detected", None); let task = ExpiryTask { obj_info: oi.clone(), event: event.clone(), src: src.clone(), + bucket_incarnation_id, }; let wrkr = self.get_worker_ch(task.op_hash()); if wrkr.is_none() { @@ -730,6 +740,7 @@ impl ExpiryState { versions: Vec, lc_event: lifecycle::Event, src: &LcEventSrc, + bucket_incarnation_id: Uuid, ) -> bool { if versions.is_empty() { return true; @@ -743,6 +754,7 @@ impl ExpiryState { versions, event: lc_event.clone(), src: src.clone(), + bucket_incarnation_id, }; let wrkr = self.get_worker_ch(task.op_hash()); if wrkr.is_none() { @@ -855,9 +867,23 @@ impl ExpiryState { let trace = LifecycleExpiryTrace::for_object(&v.obj_info, &v.event, &v.src, 1); trace.emit(EVENT_LIFECYCLE_DELETE_DISPATCHED, "delete_dispatched", None); let deleted = if !v.obj_info.transitioned_object.status.is_empty() { - apply_expiry_on_transitioned_object(api.clone(), &v.obj_info, &v.event, &v.src).await + apply_expiry_on_transitioned_object( + api.clone(), + &v.obj_info, + &v.event, + &v.src, + v.bucket_incarnation_id, + ) + .await } else { - apply_expiry_on_non_transitioned_objects(api.clone(), &v.obj_info, &v.event, &v.src).await + apply_expiry_on_non_transitioned_objects( + api.clone(), + &v.obj_info, + &v.event, + &v.src, + v.bucket_incarnation_id, + ) + .await }; if deleted { trace.emit(EVENT_LIFECYCLE_DELETE_COMPLETED, "delete_completed", None); @@ -875,7 +901,14 @@ impl ExpiryState { let version_count = u64::try_from(v.versions.len()).unwrap_or(u64::MAX); let trace = LifecycleExpiryTrace::for_batch(&v.bucket, &v.event, &v.src, version_count); trace.emit(EVENT_LIFECYCLE_DELETE_DISPATCHED, "delete_dispatched", None); - crate::client::object_handlers_common::delete_object_versions(&api, &v.bucket, &v.versions, v.event.clone()).await; + crate::client::object_handlers_common::delete_object_versions( + &api, + &v.bucket, + &v.versions, + v.event.clone(), + v.bucket_incarnation_id, + ) + .await; trace.emit(EVENT_LIFECYCLE_DELETE_COMPLETED, "delete_completed", None); } else if v.as_any().is::() { @@ -2886,7 +2919,7 @@ async fn read_stale_multipart_candidate( ) { Ok(file_info) => (Some(file_info.metadata), file_info.mod_time), Err(err) => { - warn!( + debug!( event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_LIFECYCLE, @@ -3265,10 +3298,27 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) { } pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) { - let Some(lifecycle) = runtime_sources::bucket_lifecycle_config(&oi.bucket).await else { + let Some(api) = runtime_sources::object_store_handle() else { return; }; - let Some(api) = runtime_sources::object_store_handle() else { + let configs = match metadata_boundary::get_expiry_configs(&api, &oi.bucket).await { + Ok(configs) => configs, + Err(err) => { + observe_lifecycle_observability_event(EVENT_LIFECYCLE_EVALUATION_FAILED, "failed", Some("metadata_unavailable")); + warn!( + event = EVENT_LIFECYCLE_EVALUATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + bucket = %oi.bucket, + object = %oi.name, + error = %err, + reason = "metadata_unavailable", + "Failed to load authoritative lifecycle metadata" + ); + return; + } + }; + let Some(lifecycle) = configs.lifecycle else { return; }; @@ -3312,16 +3362,13 @@ pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) { object_infos.push(oi.clone()); } - let lock_config = match metadata_boundary::get_object_lock_config(&oi.bucket).await { - Ok((cfg, _)) => Some(Arc::new(cfg)), - Err(_) => None, - }; let object_opts = object_infos .iter() .map(lifecycle::object_opts_from_object_info) .collect::>(); - let events = match Evaluator::new(Arc::new(lifecycle)) - .with_lock_retention(lock_config) + let lock_config = configs.object_lock; + let events = match Evaluator::new(lifecycle) + .with_lock_retention(lock_config.clone()) .eval(&object_opts) .await { @@ -3350,6 +3397,19 @@ pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) { if event.due != Some(OffsetDateTime::UNIX_EPOCH) { continue; } + if matches!( + event.action, + IlmAction::DeleteAction + | IlmAction::DeleteVersionAction + | IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction + ) && !matches!( + object_lock_boundary::check_object_lock_for_deletion_with_config(lock_config.as_deref(), object, false), + Ok(None) + ) { + record_scanner_lifecycle_expiry_blocked(&src, 1); + continue; + } match event.action { IlmAction::DeleteAction @@ -3357,7 +3417,7 @@ pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) { | IlmAction::DeleteRestoredVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => { - apply_expiry_rule(event, &src, object).await; + enqueue_expiry_rule_with_incarnation(event, &src, object, configs.bucket_incarnation_id).await; } IlmAction::DeleteVersionAction => { to_delete_objs.push(ObjectToDelete { @@ -3377,10 +3437,13 @@ pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) { && let Some(event) = noncurrent_event { let expiry_state = runtime_sources::expiry_state_handle(); - expiry_state - .write() - .await - .enqueue_by_newer_noncurrent(&oi.bucket, to_delete_objs, event, &src); + expiry_state.write().await.enqueue_by_newer_noncurrent( + &oi.bucket, + to_delete_objs, + event, + &src, + configs.bucket_incarnation_id, + ); } } @@ -3757,11 +3820,17 @@ fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfigu }) } -async fn apply_existing_object_expiry(api: Arc, object: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { +async fn apply_existing_object_expiry( + api: Arc, + object: &ObjectInfo, + event: &lifecycle::Event, + src: &LcEventSrc, + bucket_incarnation_id: Uuid, +) { if object.is_remote() { - apply_expiry_on_transitioned_object(api, object, event, src).await; + apply_expiry_on_transitioned_object(api, object, event, src, bucket_incarnation_id).await; } else { - apply_expiry_on_non_transitioned_objects(api, object, event, src).await; + apply_expiry_on_non_transitioned_objects(api, object, event, src, bucket_incarnation_id).await; } } @@ -3770,6 +3839,7 @@ struct ExistingObjectExpiryContext<'a> { bucket: &'a str, lc: Arc, lock_config: Option>, + bucket_incarnation_id: Uuid, src: &'a LcEventSrc, defer_date_expiry_once: bool, } @@ -3821,6 +3891,17 @@ async fn enqueue_expiry_for_existing_object_group( | IlmAction::DeleteRestoredVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => { + if !event.action.delete_restored() { + let object_lock_result = object_lock_boundary::check_object_lock_for_deletion_with_config( + context.lock_config.as_deref(), + object, + false, + ); + if !matches!(object_lock_result, Ok(None)) { + record_scanner_lifecycle_expiry_blocked(context.src, 1); + continue; + } + } let now = OffsetDateTime::now_utc(); if event.due.is_some_and(|due| due.unix_timestamp() <= now.unix_timestamp()) { if context.defer_date_expiry_once @@ -3864,10 +3945,17 @@ async fn enqueue_expiry_for_existing_object_group( record_scanner_lifecycle_expiry_blocked(context.src, 1); continue; } - apply_existing_object_expiry(context.api.clone(), object, event, context.src).await; + apply_existing_object_expiry( + context.api.clone(), + object, + event, + context.src, + context.bucket_incarnation_id, + ) + .await; } } else { - apply_expiry_rule(event, context.src, object).await; + enqueue_expiry_rule_with_incarnation(event, context.src, object, context.bucket_incarnation_id).await; } } _ => {} @@ -3878,22 +3966,22 @@ async fn enqueue_expiry_for_existing_object_group( && let Some(event) = noncurrent_event { let expiry_state = runtime_sources::expiry_state_handle(); - expiry_state - .write() - .await - .enqueue_by_newer_noncurrent(context.bucket, to_delete_objs, event, context.src); + expiry_state.write().await.enqueue_by_newer_noncurrent( + context.bucket, + to_delete_objs, + event, + context.src, + context.bucket_incarnation_id, + ); } } pub async fn enqueue_expiry_for_existing_objects(api: Arc, bucket: &str) -> Result<(), Error> { - let Ok((lc, _)) = metadata_boundary::get_lifecycle_config(bucket).await else { + let configs = metadata_boundary::get_expiry_configs(&api, bucket).await?; + let Some(lc) = configs.lifecycle else { return Ok(()); }; - let lc = Arc::new(lc); - let lock_config = metadata_boundary::get_object_lock_config(bucket) - .await - .ok() - .map(|(cfg, _)| Arc::new(cfg)); + let lock_config = configs.object_lock; let mut marker = None; let mut version_marker = None; let src = LcEventSrc::Scanner; @@ -3903,6 +3991,7 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc, bucket: &str bucket, lc: lc.clone(), lock_config: lock_config.clone(), + bucket_incarnation_id: configs.bucket_incarnation_id, src: &src, defer_date_expiry_once, }; @@ -4057,20 +4146,45 @@ fn transitioned_object_delete_opts( action: IlmAction, versioned: bool, version_suspended: bool, -) -> ObjectOptions { + bucket_incarnation_id: Uuid, +) -> crate::error::Result { let mut opts = ObjectOptions { versioned, version_suspended, expiration: ExpirationOptions { expire: true }, + expected_bucket_incarnation_id: Some(bucket_incarnation_id), ..Default::default() }; if action.delete_versioned() { opts.version_id = oi.version_id.map(|id| id.to_string()); } if action.delete_restored() { + let etag = oi + .etag + .as_deref() + .filter(|etag| !etag.is_empty()) + .ok_or_else(|| Error::other("restored-copy expiry requires an object etag"))?; + let data_dir = oi + .data_dir + .ok_or_else(|| Error::other("restored-copy expiry requires a local data directory"))?; + let restore_expiry = oi + .restore_expires + .ok_or_else(|| Error::other("restored-copy expiry requires a restore expiry"))?; opts.transition.expire_restored = true; + opts.transition.status.clone_from(&oi.transitioned_object.status); + opts.transition.tier.clone_from(&oi.transitioned_object.tier); + opts.transition.etag = etag.to_string(); + opts.transition.expected_data_dir = Some(data_dir); + opts.transition.expected_remote_name.clone_from(&oi.transitioned_object.name); + opts.transition + .expected_remote_version_id + .clone_from(&oi.transitioned_object.version_id); + opts.transition.restore_expiry = restore_expiry; + if let Some(version_id) = oi.version_id { + opts.version_id = Some(version_id.to_string()); + } } - opts + Ok(opts) } pub async fn expire_transitioned_object( @@ -4078,12 +4192,14 @@ pub async fn expire_transitioned_object( oi: &ObjectInfo, lc_event: &lifecycle::Event, _src: &LcEventSrc, + bucket_incarnation_id: Uuid, ) -> Result { let snapshot = lifecycle_delete_config_snapshot(&api, oi) .await .map_err(std::io::Error::other)?; let (versioned, version_suspended) = snapshot.versioning_config().delete_state(&oi.name); - let mut opts = transitioned_object_delete_opts(oi, lc_event.action, versioned, version_suspended); + let mut opts = transitioned_object_delete_opts(oi, lc_event.action, versioned, version_suspended, bucket_incarnation_id) + .map_err(std::io::Error::other)?; opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)); //let tags = LcAuditEvent::new(src, lcEvent).Tags(); if lc_event.action.delete_restored() { @@ -4509,7 +4625,7 @@ const _MAX_RESTORE_OBJECT_REQUEST_SIZE: i64 = 2 << 20; pub async fn eval_action_from_lifecycle( lc: &BucketLifecycleConfiguration, - lr: Option, + lock_config: Option<&ObjectLockConfiguration>, oi: &ObjectInfo, ) -> lifecycle::Event { let event = lc.eval(&oi.to_lifecycle_opts()).await; @@ -4522,7 +4638,7 @@ pub async fn eval_action_from_lifecycle( "Evaluated lifecycle action during secondary scan" ); - let lock_enabled = if let Some(lr) = lr { lr.mode.is_some() } else { false }; + let lock_enabled = lock_config.is_some_and(ObjectLockApi::enabled); let object_locked = object_lock_boundary::is_object_locked_by_metadata(&oi.user_defined, oi.delete_marker); match event.action { @@ -4538,12 +4654,14 @@ pub async fn eval_action_from_lifecycle( { return lifecycle::Event::default(); } - // Lifecycle operations should never bypass governance retention - if object_locked - || (lock_enabled - && object_lock_boundary::check_object_lock_for_deletion(&oi.bucket, oi, false) - .await - .is_some()) + // Destructive expiry never bypasses retention. Restore expiry only + // removes the local copy; the retained logical version remains. + if !event.action.delete_restored() + && (object_locked + || !matches!( + object_lock_boundary::check_object_lock_for_deletion_with_config(lock_config, oi, false), + Ok(None) + )) { //if serverDebugLog { if oi.version_id.is_some() { @@ -4676,9 +4794,10 @@ pub async fn apply_expiry_on_transitioned_object( oi: &ObjectInfo, lc_event: &lifecycle::Event, src: &LcEventSrc, + bucket_incarnation_id: Uuid, ) -> bool { let time_ilm = Metrics::time_ilm(lc_event.action); - if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src).await { + if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src, bucket_incarnation_id).await { return false; } time_ilm(1)(); @@ -4691,6 +4810,7 @@ pub async fn apply_expiry_on_non_transitioned_objects( oi: &ObjectInfo, lc_event: &lifecycle::Event, _src: &LcEventSrc, + bucket_incarnation_id: Uuid, ) -> bool { let snapshot = match lifecycle_delete_config_snapshot(&api, oi).await { Ok(snapshot) => snapshot, @@ -4714,6 +4834,7 @@ pub async fn apply_expiry_on_non_transitioned_objects( version_suspended, expiration: ExpirationOptions { expire: true }, delete_replication_config_snapshot: Some(Arc::new(snapshot)), + expected_bucket_incarnation_id: Some(bucket_incarnation_id), ..Default::default() }; @@ -4775,10 +4896,61 @@ pub async fn apply_expiry_on_non_transitioned_objects( true } -pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { +async fn enqueue_expiry_rule_with_incarnation( + event: &lifecycle::Event, + src: &LcEventSrc, + oi: &ObjectInfo, + bucket_incarnation_id: Uuid, +) -> bool { let expiry_state = runtime_sources::expiry_state_handle(); let mut expiry_state = expiry_state.write().await; - expiry_state.enqueue_by_days(oi, event, src) + expiry_state.enqueue_by_days(oi, event, src, bucket_incarnation_id) +} + +pub(crate) async fn apply_expiry_rule_in(api: Arc, event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + let Ok(_lifecycle_guard) = api.acquire_bucket_lifecycle_read_lock(&oi.bucket).await else { + return false; + }; + let Ok(bucket_incarnation_id) = api.bucket_incarnation_id_from_disk(&oi.bucket).await else { + return false; + }; + let current = match api + .get_object_info( + &oi.bucket, + &oi.name, + &ObjectOptions { + version_id: oi.version_id.map(|version_id| version_id.to_string()), + versioned: oi.version_id.is_some(), + expected_bucket_incarnation_id: Some(bucket_incarnation_id), + ..Default::default() + }, + ) + .await + { + Ok(current) => current, + Err(_) => return false, + }; + if current.version_id != oi.version_id + || current.data_dir != oi.data_dir + || current.mod_time != oi.mod_time + || current.etag != oi.etag + || current.delete_marker != oi.delete_marker + || current.transitioned_object.name != oi.transitioned_object.name + || current.transitioned_object.version_id != oi.transitioned_object.version_id + || current.transitioned_object.tier != oi.transitioned_object.tier + || current.transitioned_object.status != oi.transitioned_object.status + || current.restore_expires != oi.restore_expires + { + return false; + } + enqueue_expiry_rule_with_incarnation(event, src, oi, bucket_incarnation_id).await +} + +pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { + let Some(api) = runtime_sources::object_store_handle() else { + return false; + }; + apply_expiry_rule_in(api, event, src, oi).await } fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> DeletedObject { @@ -4853,11 +5025,11 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, mod tests { use super::{ DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX, - DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EXPIRED_DETECTED, EVENT_LIFECYCLE_NOT_ENQUEUED, ExpiryState, - FreeVersionTask, ManualTransitionJobRecoveryOutcome, ManualTransitionQueueSnapshot, ManualTransitionRunOptions, - ManualTransitionRunReport, StaleMultipartUploadCandidate, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL, - TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL, TRANSITION_COMPLETE, TierFreeVersionRecoverySchedule, - TransitionEnqueueOutcome, TransitionState, TransitionedObject, VersionReplicationScan, + DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EVALUATION_FAILED, EVENT_LIFECYCLE_EXPIRED_DETECTED, + EVENT_LIFECYCLE_NOT_ENQUEUED, ExpiryState, ExpiryTask, FreeVersionTask, ManualTransitionJobRecoveryOutcome, + ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, StaleMultipartUploadCandidate, + TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL, TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL, TRANSITION_COMPLETE, + TierFreeVersionRecoverySchedule, TransitionEnqueueOutcome, TransitionState, TransitionedObject, VersionReplicationScan, cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at, enqueue_recovered_free_version_with_state, enqueue_transition_for_existing_objects_scoped, enqueue_transition_with_lifecycle, enqueue_transition_with_lifecycle_report, eval_action_from_lifecycle, @@ -4937,8 +5109,9 @@ mod tests { use rustfs_data_usage::TierStats; use rustfs_filemeta::{FileInfo, FileMeta}; use s3s::dto::{ - BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry, OutputLocation, - RestoreRequest, RestoreRequestType, S3Location, Timestamp, Transition, TransitionStorageClass, + BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry, + ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, OutputLocation, RestoreRequest, + RestoreRequestType, S3Location, Timestamp, Transition, TransitionStorageClass, }; use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE}; use serial_test::serial; @@ -5854,29 +6027,44 @@ mod tests { bucket: "bucket".to_string(), name: "object".to_string(), version_id: Some(vid), + data_dir: Some(Uuid::new_v4()), + etag: Some("etag".to_string()), + restore_expires: Some(OffsetDateTime::now_utc() - StdDuration::from_secs(1)), + transitioned_object: TransitionedObject { + name: "remote-object".to_string(), + tier: "tier".to_string(), + status: TRANSITION_COMPLETE.to_string(), + ..Default::default() + }, ..Default::default() }; // Plain version expiry: exact version, real delete. - let opts = transitioned_object_delete_opts(&oi, IlmAction::DeleteVersionAction, true, false); + let incarnation = Uuid::new_v4(); + let opts = transitioned_object_delete_opts(&oi, IlmAction::DeleteVersionAction, true, false, incarnation) + .expect("build version expiry options"); assert_eq!(opts.version_id.as_deref(), Some(vid_str.as_str())); + assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation)); assert!(!opts.transition.expire_restored); assert!(opts.expiration.expire); // Restore-expiry of the latest version: restored-copy cleanup only. - let opts = transitioned_object_delete_opts(&oi, IlmAction::DeleteRestoredAction, true, false); - assert!(opts.version_id.is_none()); + let opts = transitioned_object_delete_opts(&oi, IlmAction::DeleteRestoredAction, true, false, incarnation) + .expect("build restored expiry options"); + assert_eq!(opts.version_id.as_deref(), Some(vid_str.as_str())); assert!(opts.transition.expire_restored); // Restore-expiry of a noncurrent version: restored-copy cleanup of the // exact version. Routing this through the full transitioned-object // delete instead would remove the remote tier data. - let opts = transitioned_object_delete_opts(&oi, IlmAction::DeleteRestoredVersionAction, true, false); + let opts = transitioned_object_delete_opts(&oi, IlmAction::DeleteRestoredVersionAction, true, false, incarnation) + .expect("build restored-version expiry options"); assert_eq!(opts.version_id.as_deref(), Some(vid_str.as_str())); assert!(opts.transition.expire_restored); // Whole-object expiry stays a real delete. - let opts = transitioned_object_delete_opts(&oi, IlmAction::DeleteAction, false, false); + let opts = transitioned_object_delete_opts(&oi, IlmAction::DeleteAction, false, false, incarnation) + .expect("build object expiry options"); assert!(opts.version_id.is_none()); assert!(!opts.transition.expire_restored); } @@ -5905,7 +6093,7 @@ mod tests { ..Default::default() }; - let queued = state.enqueue_by_days(&object, &event, &LcEventSrc::Scanner); + let queued = state.enqueue_by_days(&object, &event, &LcEventSrc::Scanner, Uuid::new_v4()); assert!(!queued); assert_eq!(state.stats.missed_tasks(), 1); @@ -6011,8 +6199,9 @@ mod tests { ..Default::default() }; - let first = state.enqueue_by_days(&object, &event, &LcEventSrc::Scanner); - let second = state.enqueue_by_days(&object, &event, &LcEventSrc::Scanner); + let incarnation = Uuid::new_v4(); + let first = state.enqueue_by_days(&object, &event, &LcEventSrc::Scanner, incarnation); + let second = state.enqueue_by_days(&object, &event, &LcEventSrc::Scanner, incarnation); assert!(first); assert!(!second); @@ -6031,6 +6220,39 @@ mod tests { assert!(observed.contains(&(EVENT_LIFECYCLE_NOT_ENQUEUED, "not_enqueued", Some("queue_full")))); } + #[tokio::test] + async fn expiry_task_retains_enqueue_time_bucket_incarnation() { + let state = ExpiryState::new_with_unconsumed_worker_channel(1); + let incarnation = Uuid::new_v4(); + let object = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + ..Default::default() + }; + let event = crate::bucket::lifecycle::lifecycle::Event { + action: IlmAction::DeleteAction, + ..Default::default() + }; + { + let mut state = state.write().await; + assert!(state.enqueue_by_days(&object, &event, &LcEventSrc::Scanner, incarnation)); + } + + let receiver = state.read().await.tasks_rx[0].clone(); + let task = receiver + .lock() + .await + .recv() + .await + .expect("expiry task should be queued") + .expect("expiry task payload should be present"); + let task = task + .as_any() + .downcast_ref::() + .expect("queued payload should be an expiry task"); + assert_eq!(task.bucket_incarnation_id, incarnation); + } + #[tokio::test] async fn enqueue_tier_journal_entry_reports_error_when_worker_queue_full() { let state = ExpiryState::new_with_unconsumed_worker_channel(1); @@ -7741,6 +7963,48 @@ mod tests { } } + fn all_versions_expiration_lifecycle() -> BucketLifecycleConfiguration { + BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(1), + expired_object_all_versions: Some(true), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("delete-all".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + } + } + + fn lock_enabled_without_default_retention() -> ObjectLockConfiguration { + ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + } + } + + fn lock_enabled_with_default_retention() -> ObjectLockConfiguration { + ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: Some(ObjectLockRule { + default_retention: Some(DefaultRetention { + days: Some(30), + mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), + years: None, + }), + }), + } + } + fn latest_transition_lifecycle() -> BucketLifecycleConfiguration { BucketLifecycleConfiguration { expiry_updated_at: None, @@ -9937,6 +10201,124 @@ mod tests { assert_eq!(event.action, IlmAction::DeleteAction); } + #[tokio::test] + async fn existing_object_lifecycle_skips_current_expiration_for_bucket_default_retention() { + let lc = latest_expiration_lifecycle(); + let mut object = current_object(ReplicationStatusType::Completed); + object.mod_time = Some(OffsetDateTime::now_utc()); + let lock_config = lock_enabled_with_default_retention(); + + let event = eval_action_from_lifecycle(&lc, Some(&lock_config), &object).await; + + assert_eq!(event.action, IlmAction::NoneAction); + } + + #[tokio::test] + async fn existing_object_lifecycle_skips_delete_all_when_lock_enabled_without_default_retention() { + let lc = all_versions_expiration_lifecycle(); + let object = current_object(ReplicationStatusType::Completed); + let lock_config = lock_enabled_without_default_retention(); + + let event = eval_action_from_lifecycle(&lc, Some(&lock_config), &object).await; + + assert_eq!(event.action, IlmAction::NoneAction); + } + + #[tokio::test] + #[serial] + async fn lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata() { + let (_disk_paths, ecstore) = setup_test_env().await; + let bucket = format!("lifecycle-lock-metadata-error-{}", Uuid::new_v4().simple()); + let object = "due/object"; + create_test_bucket(&ecstore, &bucket).await; + + let mut reader = PutObjReader::from_vec(b"must survive lifecycle metadata failure".to_vec()); + let object_info = ecstore + .put_object( + &bucket, + object, + &mut reader, + &ObjectOptions { + mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(2)), + ..Default::default() + }, + ) + .await + .expect("due object should be created"); + + let lifecycle = latest_expiration_lifecycle(); + let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system should be initialized"); + let sys = sys.read().await.clone(); + let mut metadata = (*sys.get(&bucket).await.expect("bucket metadata should exist")).clone(); + metadata.lifecycle_config_xml = crate::bucket::utils::serialize(&lifecycle).unwrap(); + metadata.lifecycle_config = Some(lifecycle); + metadata.object_lock_config_xml = b"".to_vec(); + metadata.object_lock_config = None; + sys.persist_and_set(metadata) + .await + .expect("corrupt Object Lock payload should be persisted for the read-boundary test"); + sys.reload_from_store(&bucket) + .await + .expect("peer-style reload should publish the malformed persisted snapshot"); + + let exact_error = super::metadata_boundary::get_expiry_configs(&ecstore, &bucket) + .await + .expect_err("malformed Object Lock metadata must reject lifecycle config resolution"); + assert!( + exact_error + .to_string() + .contains("persisted bucket Object Lock configuration is invalid") + ); + + let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await; + let observed = Arc::new(StdMutex::new(Vec::new())); + let observed_events = Arc::clone(&observed); + let _observer = set_lifecycle_observability_observer(move |event, state, reason| { + observed_events + .lock() + .expect("lifecycle metadata error observer should not poison") + .push((event, state, reason)); + }); + + super::enqueue_immediate_expiry(&object_info, LcEventSrc::S3PutObject).await; + + assert!( + observed.lock().expect("observed events should not poison").contains(&( + EVENT_LIFECYCLE_EVALUATION_FAILED, + "failed", + Some("metadata_unavailable") + )), + "immediate expiry must expose the authoritative metadata failure" + ); + { + let state = runtime_state.read().await; + assert_eq!(state.stats.pending_tasks(), 0, "immediate expiry must not enqueue a delete"); + } + assert!( + ecstore + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .is_ok(), + "immediate expiry must leave the due object intact" + ); + + let scanner_error = super::enqueue_expiry_for_existing_objects(ecstore.clone(), &bucket) + .await + .expect_err("scanner must propagate the authoritative Object Lock metadata error"); + assert_eq!(scanner_error.to_string(), exact_error.to_string()); + { + let state = runtime_state.read().await; + assert_eq!(state.stats.pending_tasks(), 0, "scanner must not enqueue a delete"); + } + assert!( + ecstore + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .is_ok(), + "scanner must leave the due object intact" + ); + } + #[tokio::test] async fn existing_object_lifecycle_skips_current_expiration_for_explicit_legal_hold() { let lc = latest_expiration_lifecycle(); @@ -9972,6 +10354,37 @@ mod tests { assert_eq!(event.action, IlmAction::NoneAction); } + #[tokio::test] + async fn restored_copy_expiry_is_not_blocked_by_retention() { + let lifecycle = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: Vec::new(), + }; + let retain_until = (OffsetDateTime::now_utc() + time::Duration::days(30)) + .format(&time::format_description::well_known::Rfc3339) + .unwrap(); + let mut object = current_object_with_metadata( + ReplicationStatusType::Completed, + HashMap::from([ + ( + X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), + ObjectLockRetentionMode::COMPLIANCE.to_string(), + ), + (X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until), + ]), + ); + object.transitioned_object.status = TRANSITION_COMPLETE.to_string(); + object.restore_expires = Some(OffsetDateTime::now_utc() - time::Duration::hours(1)); + + let current = eval_action_from_lifecycle(&lifecycle, None, &object).await; + assert_eq!(current.action, IlmAction::DeleteRestoredAction); + + object.is_latest = false; + object.successor_mod_time = Some(OffsetDateTime::now_utc()); + let noncurrent = eval_action_from_lifecycle(&lifecycle, None, &object).await; + assert_eq!(noncurrent.action, IlmAction::DeleteRestoredVersionAction); + } + #[tokio::test] async fn existing_object_lifecycle_skips_transition_while_replication_pending() { let lc = latest_transition_lifecycle(); diff --git a/crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs b/crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs index 6867726aa..22e3b52a0 100644 --- a/crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs +++ b/crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs @@ -12,20 +12,127 @@ // See the License for the specific language governing permissions and // limitations under the License. -use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration}; -use time::OffsetDateTime; +use std::sync::Arc; -use crate::bucket::metadata_sys; -use crate::error::Result; +use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::bucket::metadata_sys::{self, ObjectLockConfigState}; +use crate::error::{Error, Result}; + +#[derive(Debug)] +pub(crate) struct LifecycleExpiryConfigs { + pub(crate) lifecycle: Option>, + pub(crate) object_lock: Option>, + pub(crate) bucket_incarnation_id: Uuid, +} + +pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result { + let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?; + let sys = metadata_sys::bucket_metadata_sys_of(&api.ctx)?; + let sys = sys.read().await.clone(); + let metadata = sys.get_authoritative_metadata(bucket).await?; + if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id != bucket_incarnation_id { + return Err(Error::other(format!("bucket lifecycle metadata is not authoritative: {bucket}"))); + } + + let lifecycle = if metadata.lifecycle_config.is_none() && !metadata.lifecycle_config_xml.is_empty() { + return Err(Error::other("persisted bucket lifecycle configuration is invalid")); + } else { + metadata + .lifecycle_config + .clone() + .filter(|config| !config.rules.is_empty()) + .map(Arc::new) + }; + if lifecycle.is_none() { + return Ok(LifecycleExpiryConfigs { + lifecycle: None, + object_lock: None, + bucket_incarnation_id, + }); + } + let object_lock = match metadata_sys::object_lock_config_state_from_authoritative_metadata(&metadata)? { + ObjectLockConfigState::Configured { config, .. } => Some(Arc::new(config)), + ObjectLockConfigState::ConfirmedAbsent => None, + ObjectLockConfigState::Fabricated => { + return Err(Error::other(format!("bucket Object Lock metadata is not authoritative: {bucket}"))); + } + }; + + Ok(LifecycleExpiryConfigs { + lifecycle, + object_lock, + bucket_incarnation_id, + }) +} pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> { metadata_sys::get_lifecycle_config(bucket).await } -pub(crate) async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> { - metadata_sys::get_object_lock_config(bucket).await -} +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::metadata::BucketMetadata; + use crate::bucket::metadata_sys::{self, test_support::isolated_store_over_temp_disks}; + use crate::storage_api_contracts::bucket::MakeBucketOptions; + use s3s::dto::{ExpirationStatus, LifecycleExpiration, LifecycleRule}; + use serial_test::serial; -pub(crate) async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { - metadata_sys::get_replication_config(bucket).await + fn lifecycle_config() -> BucketLifecycleConfiguration { + BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(1), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("expire".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + } + } + + #[tokio::test] + #[serial] + async fn expiry_configs_are_resolved_from_the_owning_store() { + let (_dirs_a, store_a) = isolated_store_over_temp_disks().await; + let (_dirs_b, store_b) = isolated_store_over_temp_disks().await; + let bucket = "same-name-expiry-config"; + store_a + .peer_sys + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .unwrap(); + store_b + .peer_sys + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .unwrap(); + metadata_sys::init_bucket_metadata_sys(store_a.clone(), vec![bucket.to_string()]).await; + metadata_sys::init_bucket_metadata_sys(store_b.clone(), vec![bucket.to_string()]).await; + + let mut metadata = BucketMetadata::new(bucket); + let lifecycle = lifecycle_config(); + metadata.lifecycle_config_xml = crate::bucket::utils::serialize(&lifecycle).unwrap(); + metadata.lifecycle_config = Some(lifecycle); + metadata_sys::set_new_bucket_metadata_in(&store_a.ctx, metadata) + .await + .unwrap(); + metadata_sys::set_new_bucket_metadata_in(&store_b.ctx, BucketMetadata::new(bucket)) + .await + .unwrap(); + + assert!(get_expiry_configs(&store_a, bucket).await.unwrap().lifecycle.is_some()); + assert!(get_expiry_configs(&store_b, bucket).await.unwrap().lifecycle.is_none()); + } } diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index a84973d28..823263072 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -19,6 +19,7 @@ pub mod core; pub mod evaluator; pub mod manual_transition_job; mod metadata_boundary; +pub(crate) use metadata_boundary::get_expiry_configs; mod object_lock_boundary; pub use self::core as lifecycle; mod replication_sink; diff --git a/crates/ecstore/src/bucket/lifecycle/object_lock_boundary.rs b/crates/ecstore/src/bucket/lifecycle/object_lock_boundary.rs index 166371607..79ec3c7a1 100644 --- a/crates/ecstore/src/bucket/lifecycle/object_lock_boundary.rs +++ b/crates/ecstore/src/bucket/lifecycle/object_lock_boundary.rs @@ -21,12 +21,12 @@ pub(crate) fn is_object_locked_by_metadata(user_defined: &HashMap, obj_info: &ObjectInfo, bypass_governance: bool, -) -> Option { - objectlock_sys::check_object_lock_for_deletion(bucket, obj_info, bypass_governance).await +) -> crate::error::Result> { + objectlock_sys::check_object_lock_for_deletion_with_config(config, obj_info, bypass_governance) } #[cfg(test)] diff --git a/crates/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs index b0ed82eec..2b02231fd 100644 --- a/crates/ecstore/src/bucket/metadata.rs +++ b/crates/ecstore/src/bucket/metadata.rs @@ -18,7 +18,7 @@ use super::versioning::VersioningApi; use super::{quota::BucketQuota, target::BucketTargets}; use crate::bucket::replication::invalid_replication_config_status_field; use crate::bucket::utils::deserialize; -use crate::config::com::{read_config, save_config}; +use crate::config::com::{read_config, read_config_preserve_empty, save_config}; use crate::disk::BUCKET_META_PREFIX; use crate::error::{Error, Result}; use crate::runtime::sources as runtime_sources; @@ -37,6 +37,7 @@ use std::io::{Read, Write}; use std::sync::Arc; use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset}; use tracing::error; +use uuid::Uuid; fn read_msgp_str(rd: &mut R) -> Result { let len = rmp::decode::read_str_len(rd)? as usize; @@ -226,6 +227,7 @@ fn write_bin_field(wr: &mut W, key: &str, val: &[u8]) -> Result<()> { } pub const BUCKET_METADATA_FILE: &str = ".metadata.bin"; +pub const BUCKET_INCARNATION_FILE: &str = ".bucket-incarnation"; pub const BUCKET_METADATA_FORMAT: u16 = 1; pub const BUCKET_METADATA_VERSION: u16 = 1; @@ -277,6 +279,8 @@ pub struct BucketMetadata { pub name: String, pub created: OffsetDateTime, pub lock_enabled: bool, // While marked as unused, it may need to be retained + pub bucket_incarnation_id: Uuid, + pub(crate) bucket_incarnation_sidecar: bool, pub policy_config_json: Vec, pub notification_config_xml: Vec, pub lifecycle_config_xml: Vec, @@ -347,6 +351,8 @@ impl Default for BucketMetadata { name: Default::default(), created: OffsetDateTime::UNIX_EPOCH, lock_enabled: Default::default(), + bucket_incarnation_id: Uuid::nil(), + bucket_incarnation_sidecar: false, policy_config_json: Default::default(), notification_config_xml: Default::default(), lifecycle_config_xml: Default::default(), @@ -414,6 +420,7 @@ impl BucketMetadata { pub fn new(name: &str) -> Self { BucketMetadata { name: name.to_string(), + bucket_incarnation_id: Uuid::new_v4(), ..Default::default() } } @@ -479,6 +486,11 @@ impl BucketMetadata { "Name" => self.name = read_msgp_str(rd)?, "Created" => self.created = read_msgp_time_value(rd)?, "LockEnabled" => self.lock_enabled = read_msgp_bool(rd)?, + "BucketIncarnationID" => { + let bytes = read_msgp_bin(rd)?; + self.bucket_incarnation_id = + Uuid::from_slice(&bytes).map_err(|err| Error::other(format!("invalid BucketIncarnationID: {err}")))?; + } "PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?, "NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?, "LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?, @@ -535,8 +547,8 @@ impl BucketMetadata { /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility. pub fn encode_to(&self, wr: &mut W) -> Result<()> { - // Map size: MinIO fields (25) + RustFS extensions (18) - let map_len: u32 = 43; + // Map size: MinIO fields (25) + RustFS extensions (19) + let map_len: u32 = 44; rmp::encode::write_map_len(wr, map_len)?; // MinIO field order (same as Go struct) @@ -549,6 +561,8 @@ impl BucketMetadata { rmp::encode::write_str(wr, "LockEnabled")?; rmp::encode::write_bool(wr, self.lock_enabled)?; + write_bin_field(wr, "BucketIncarnationID", self.bucket_incarnation_id.as_bytes())?; + write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?; write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?; write_bin_field(wr, "LifecycleConfigXML", &self.lifecycle_config_xml)?; @@ -748,6 +762,10 @@ impl BucketMetadata { self.quota_config_updated_at = updated; } OBJECT_LOCK_CONFIG => { + self.object_lock_config = None; + if !data.is_empty() { + self.lock_enabled = true; + } self.object_lock_config_xml = data; self.object_lock_config_updated_at = updated; } @@ -1115,6 +1133,29 @@ impl BucketMetadata { } } +pub(crate) async fn load_bucket_incarnation(api: Arc, bucket: &str) -> Result> { + let path = format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"); + let data = match read_config_preserve_empty(api, &path).await { + Ok(data) => data, + Err(Error::ConfigNotFound) => return Ok(None), + Err(err) => return Err(err), + }; + let incarnation = + Uuid::from_slice(&data).map_err(|err| Error::other(format!("persisted bucket incarnation is invalid: {err}")))?; + if incarnation.is_nil() { + return Err(Error::other("persisted bucket incarnation is nil")); + } + Ok(Some(incarnation)) +} + +pub(crate) async fn save_bucket_incarnation(api: Arc, bucket: &str, incarnation: Uuid) -> Result<()> { + if incarnation.is_nil() { + return Err(Error::other("cannot persist a nil bucket incarnation")); + } + let path = format!("{BUCKET_META_PREFIX}/{bucket}/{BUCKET_INCARNATION_FILE}"); + save_config(api, &path, incarnation.as_bytes().to_vec()).await +} + pub async fn load_bucket_metadata(api: Arc, bucket: &str) -> Result { load_bucket_metadata_parse(api, bucket, true).await } @@ -1142,6 +1183,23 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence( } }; + let incarnation = load_bucket_incarnation(api, bucket).await?; + if persisted { + if let Some(incarnation) = incarnation { + if !bm.bucket_incarnation_id.is_nil() && bm.bucket_incarnation_id != incarnation { + return Err(Error::other("bucket incarnation sidecar does not match bucket metadata")); + } + bm.bucket_incarnation_id = incarnation; + bm.bucket_incarnation_sidecar = true; + } else if !bm.bucket_incarnation_id.is_nil() { + return Err(Error::other(format!( + "bucket incarnation sidecar is missing for new-format metadata: {bucket}" + ))); + } + } else if incarnation.is_some() { + return Err(Error::other("bucket incarnation sidecar exists without bucket metadata")); + } + bm.default_timestamps(); if parse { @@ -1209,6 +1267,10 @@ mod test { // Same 4-byte format|version header (1|1) and msgpack layout as MinIO. BucketMetadata::check_header(&blob).expect("valid .metadata.bin header"); let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata"); + assert!( + bm.bucket_incarnation_id.is_nil(), + "legacy MinIO metadata has no RustFS bucket incarnation field" + ); // Raw config fields survive the msgpack decode (PascalCase MinIO field names). assert_eq!(bm.name, "interop"); @@ -1291,6 +1353,42 @@ mod test { let new = BucketMetadata::unmarshal(&buf).unwrap(); assert_eq!(bm.name, new.name); + assert!(!bm.bucket_incarnation_id.is_nil()); + assert_eq!(bm.bucket_incarnation_id, new.bucket_incarnation_id); + } + + #[test] + fn bucket_incarnation_msgpack_rejects_invalid_binary_length() { + let mut fixture = Vec::new(); + rmp::encode::write_map_len(&mut fixture, 1).unwrap(); + rmp::encode::write_str(&mut fixture, "BucketIncarnationID").unwrap(); + rmp::encode::write_bin(&mut fixture, &[0_u8; 15]).unwrap(); + + let err = BucketMetadata::unmarshal(&fixture).expect_err("non-UUID incarnation bytes must fail closed"); + assert!(err.to_string().contains("invalid BucketIncarnationID")); + } + + #[test] + fn same_name_bucket_metadata_gets_a_new_incarnation() { + let old = BucketMetadata::new("recreated"); + let new = BucketMetadata::new("recreated"); + + assert!(!old.bucket_incarnation_id.is_nil()); + assert!(!new.bucket_incarnation_id.is_nil()); + assert_ne!(old.bucket_incarnation_id, new.bucket_incarnation_id); + } + + #[test] + fn site_replication_config_updates_cannot_replace_bucket_incarnation() { + let mut metadata = BucketMetadata::new("site-replication-update"); + let incarnation = metadata.bucket_incarnation_id; + + metadata + .update_config(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec()) + .unwrap(); + metadata.update_config(OBJECT_LOCK_CONFIG, Vec::new()).unwrap(); + + assert_eq!(metadata.bucket_incarnation_id, incarnation); } #[test] diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index b765dd199..442a5d55d 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::metadata::{BUCKET_TARGETS_FILE, BucketMetadata, load_bucket_metadata}; +use super::metadata::{ + BUCKET_TARGETS_FILE, BucketMetadata, load_bucket_incarnation, load_bucket_metadata, save_bucket_incarnation, +}; use super::quota::BucketQuota; use super::target::BucketTargets; use crate::bucket::bucket_target_sys::BucketTargetSys; @@ -30,8 +32,8 @@ use rustfs_policy::policy::BucketPolicy; use s3s::dto::ReplicationConfiguration; use s3s::dto::{ AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration, - ObjectLockConfiguration, PublicAccessBlockConfiguration, RequestPaymentConfiguration, ServerSideEncryptionConfiguration, - Tagging, VersioningConfiguration, WebsiteConfiguration, + ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, PublicAccessBlockConfiguration, + RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration, }; use std::collections::HashSet; use std::time::Duration; @@ -44,6 +46,7 @@ use tokio::sync::{Mutex, RwLock}; use tokio::time::sleep; use tokio_util::sync::CancellationToken; use tracing::{error, warn}; +use uuid::Uuid; const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60); @@ -53,6 +56,73 @@ enum MetadataLoadMode { Refresh, } +#[derive(Debug, Clone)] +pub enum ObjectLockConfigState { + Configured { + config: ObjectLockConfiguration, + updated_at: OffsetDateTime, + }, + ConfirmedAbsent, + Fabricated, +} + +enum BucketMetadataAuthority { + Authoritative(Arc), + Fabricated, + MissingBucket, +} + +pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMetadata) -> Result { + if bm.object_lock_config.is_none() && !bm.object_lock_config_xml.is_empty() { + return Err(Error::other("persisted bucket Object Lock configuration is invalid")); + } + + if let Some(config) = bm.object_lock_config.clone() { + validate_authoritative_object_lock_config(&config)?; + return Ok(ObjectLockConfigState::Configured { + config, + updated_at: bm.object_lock_config_updated_at, + }); + } + + if bm.lock_enabled { + return Ok(ObjectLockConfigState::Configured { + config: ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }, + updated_at: bm.object_lock_config_updated_at, + }); + } + + Ok(ObjectLockConfigState::ConfirmedAbsent) +} + +fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -> Result<()> { + if config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str) != Some(ObjectLockEnabled::ENABLED) { + return Err(Error::other("persisted bucket Object Lock enabled state is invalid")); + } + + let Some(rule) = config.rule.as_ref() else { + return Ok(()); + }; + let Some(retention) = rule.default_retention.as_ref() else { + return Err(Error::other("persisted bucket Object Lock rule has no default retention")); + }; + if !retention + .mode + .as_ref() + .is_some_and(|mode| matches!(mode.as_str(), ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE)) + { + return Err(Error::other("persisted bucket Object Lock retention mode is invalid")); + } + match (retention.days, retention.years) { + (Some(days), None) if (1..=36_500).contains(&days) => Ok(()), + (None, Some(years)) if (1..=100).contains(&years) => Ok(()), + _ => Err(Error::other("persisted bucket Object Lock retention period is invalid")), + } +} + pub async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { // The metadata system is inherently per-store (it holds the store handle // and that store's bucket cache), so it lives on the store's own instance @@ -103,8 +173,8 @@ pub async fn reload_bucket_metadata(api: Arc, bucket: &str) -> Result<( .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) .await?; let sys = bucket_metadata_sys_of(&api.ctx)?; - let lock = sys.read().await; - lock.reload_from_store_under_namespace(bucket, &namespace_guard).await + let sys = sys.read().await.clone(); + sys.reload_from_store_under_namespace(bucket, &namespace_guard).await } /// Drop a bucket's cached metadata from the in-memory map. @@ -218,6 +288,11 @@ pub(crate) fn bucket_metadata_sys_of(ctx: &crate::runtime::instance::InstanceCon get_bucket_metadata_sys() } +pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result> { + let sys = bucket_metadata_sys_of(ctx)?; + Ok(sys.read().await.api.clone()) +} + pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result> { let sys = bucket_metadata_sys_of(ctx)?; let lock = sys.read().await; @@ -230,30 +305,116 @@ pub(crate) async fn created_at_in(ctx: &crate::runtime::instance::InstanceContex lock.created_at(bucket).await } +pub(crate) async fn get_config_from_disk_with_presence_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result<(BucketMetadata, bool)> { + let sys = bucket_metadata_sys_of(ctx)?; + let api = sys.read().await.api.clone(); + load_bucket_metadata_parse_with_presence(api, bucket, true).await +} + +pub(crate) async fn get_bucket_incarnation_id_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result { + let sys = bucket_metadata_sys_of(ctx)?; + let sys = sys.read().await.clone(); + sys.get_bucket_incarnation_id_from_disk(bucket).await +} + +pub(crate) async fn get_cached_bucket_incarnation_id_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result { + let sys = bucket_metadata_sys_of(ctx)?; + let sys = sys.read().await.clone(); + sys.get_bucket_incarnation_id(bucket).await +} + pub(crate) async fn set_bucket_metadata_in(ctx: &crate::runtime::instance::InstanceContext, bm: BucketMetadata) -> Result<()> { let sys = bucket_metadata_sys_of(ctx)?; let lock = sys.read().await; lock.persist_and_set(bm).await } +pub(crate) async fn set_new_bucket_metadata_in( + ctx: &crate::runtime::instance::InstanceContext, + bm: BucketMetadata, +) -> Result<()> { + let sys = bucket_metadata_sys_of(ctx)?; + let lock = sys.read().await; + lock.persist_new_and_set(bm).await +} + +pub(crate) async fn cache_bucket_metadata_in(ctx: &crate::runtime::instance::InstanceContext, bm: BucketMetadata) -> Result<()> { + let sys = bucket_metadata_sys_of(ctx)?; + let lock = sys.read().await; + lock.set(bm.name.clone(), Arc::new(bm)).await; + Ok(()) +} + pub(crate) async fn remove_bucket_metadata_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result { let sys = bucket_metadata_sys_of(ctx)?; let lock = sys.read().await; Ok(lock.remove(bucket).await) } +#[cfg(test)] +pub(crate) async fn inject_object_lock_disk_read_error_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result<()> { + let sys = bucket_metadata_sys_of(ctx)?; + let sys = sys.read().await.clone(); + sys.object_lock_disk_read_errors.write().await.insert(bucket.to_string()); + Ok(()) +} + /// Rewrite one config file of a bucket's metadata, serialized cluster-wide. /// /// See [`acquire_bucket_metadata_transaction_lock`] for why every config /// write — not just the replication-targets one — has to hold that lock. pub async fn update(bucket: &str, config_file: &str, data: Vec) -> Result { - update_with_sys(get_bucket_metadata_sys()?, bucket, config_file, data).await + Box::pin(update_with_sys(get_bucket_metadata_sys()?, bucket, config_file, data)).await } pub async fn delete(bucket: &str, config_file: &str) -> Result { delete_with_sys(get_bucket_metadata_sys()?, bucket, config_file).await } +pub async fn update_if_incarnation( + bucket: &str, + config_file: &str, + data: Vec, + expected_incarnation_id: Uuid, +) -> Result { + // Boxed for the same reason as [`update`]: this is the path an authorized + // bucket-config mutation actually takes once it carries an incarnation, so + // leaving it inlined puts the whole resolve/load/save chain back on the + // caller's stack (rustfs#5648). + Box::pin(update_with_sys_expected( + get_bucket_metadata_sys()?, + bucket, + config_file, + data, + Some(expected_incarnation_id), + )) + .await +} + +pub async fn delete_if_incarnation(bucket: &str, config_file: &str, expected_incarnation_id: Uuid) -> Result { + Box::pin(delete_with_sys_expected( + get_bucket_metadata_sys()?, + bucket, + config_file, + Some(expected_incarnation_id), + )) + .await +} + +pub async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result { + let guard = acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await?; + Ok(guard.incarnation_id) +} + /// [`update`] against an explicitly supplied metadata system. /// /// The free functions resolve the instance's own system; this variant takes @@ -266,31 +427,136 @@ async fn update_with_sys( config_file: &str, data: Vec, ) -> Result { - let (_transaction_guard, mut sys) = acquire_config_write_guards(sys, bucket).await?; - sys.update(bucket, config_file, data).await + update_with_sys_expected(sys, bucket, config_file, data, None).await +} + +async fn update_with_sys_expected( + sys: Arc>, + bucket: &str, + config_file: &str, + data: Vec, + expected_incarnation_id: Option, +) -> Result { + let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?; + update_under_config_write_guard(sys, &guard, config_file, data).await } /// [`delete`] against an explicitly supplied metadata system. See /// [`update_with_sys`]. async fn delete_with_sys(sys: Arc>, bucket: &str, config_file: &str) -> Result { - let (_transaction_guard, mut sys) = acquire_config_write_guards(sys, bucket).await?; - sys.delete(bucket, config_file).await + delete_with_sys_expected(sys, bucket, config_file, None).await } -/// Take, in the one order every config write uses, the two guards a -/// read-modify-write of a bucket's metadata needs: the cluster-wide -/// transaction lock first, then this process's metadata-system write guard. -/// -/// The order is load-bearing. Acquiring the process-local guard first would -/// park every local reader and writer of *every* bucket behind a lock whose -/// holder may be another node, turning remote contention into a local stall. -async fn acquire_config_write_guards( +async fn delete_with_sys_expected( sys: Arc>, bucket: &str, -) -> Result<(rustfs_lock::NamespaceLockGuard, tokio::sync::OwnedRwLockWriteGuard)> { - let transaction_guard = acquire_transaction_lock_with_sys(&sys, bucket).await?; - let sys_guard = sys.write_owned().await; - Ok((transaction_guard, sys_guard)) + config_file: &str, + expected_incarnation_id: Option, +) -> Result { + let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?; + delete_under_config_write_guard(sys, &guard, config_file).await +} + +/// Owns the complete bucket-config mutation fence. +/// +/// Lock order: bucket lifecycle sentinel (read), then metadata transaction +/// (write). The immutable incarnation is checked before any persisted +/// metadata can be rewritten. +pub struct BucketMetadataMutationGuard { + bucket: String, + incarnation_id: Uuid, + lifecycle_guard: rustfs_lock::NamespaceLockGuard, + transaction_guard: rustfs_lock::NamespaceLockGuard, +} + +impl BucketMetadataMutationGuard { + fn ensure_valid(&self, bucket: &str) -> Result<()> { + if self.bucket != bucket { + return Err(Error::other("bucket metadata mutation guard does not match bucket")); + } + if self.lifecycle_guard.is_lock_lost() || self.transaction_guard.is_lock_lost() { + return Err(Error::other(format!("bucket metadata mutation lock was lost: {bucket}"))); + } + Ok(()) + } +} + +async fn acquire_config_write_guard(sys: Arc>, bucket: &str) -> Result { + acquire_config_write_guard_for_incarnation(sys, bucket, None).await +} + +async fn acquire_config_write_guard_for_incarnation( + sys: Arc>, + bucket: &str, + expected_incarnation_id: Option, +) -> Result { + let metadata_sys = sys.read().await.clone(); + let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?; + + // Legacy buckets are migrated while the lifecycle fence prevents a + // same-name replacement. The second read under the write transaction is + // the CAS source of truth for the actual rewrite. + await_bucket_namespace_operation( + Some(&lifecycle_guard), + bucket, + "bucket config incarnation migration", + metadata_sys.get_bucket_incarnation_id(bucket), + ) + .await?; + let transaction_guard = await_bucket_namespace_operation( + Some(&lifecycle_guard), + bucket, + "bucket config transaction lock acquisition", + acquire_transaction_lock_with_sys(&sys, bucket), + ) + .await?; + await_bucket_namespace_operation( + Some(&lifecycle_guard), + bucket, + "bucket config existence validation", + await_bucket_namespace_operation( + Some(&transaction_guard), + bucket, + "bucket config existence transaction validation", + async { + match metadata_sys + .api + .peer_sys + .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) + .await + { + Ok(_) => Ok(()), + Err(crate::disk::error::Error::VolumeNotFound) => Err(Error::BucketNotFound(bucket.to_string())), + Err(err) => Err(err.into()), + } + }, + ), + ) + .await?; + let current_incarnation_id = await_bucket_namespace_operation( + Some(&lifecycle_guard), + bucket, + "bucket config incarnation validation", + await_bucket_namespace_operation( + Some(&transaction_guard), + bucket, + "bucket config incarnation transaction validation", + load_bucket_incarnation(metadata_sys.api.clone(), bucket), + ), + ) + .await? + .filter(|incarnation_id| !incarnation_id.is_nil()) + .ok_or_else(|| Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))?; + if expected_incarnation_id.is_some_and(|expected| expected != current_incarnation_id) { + return Err(Error::BucketNotFound(bucket.to_string())); + } + + Ok(BucketMetadataMutationGuard { + bucket: bucket.to_string(), + incarnation_id: current_incarnation_id, + lifecycle_guard, + transaction_guard, + }) } /// Rewrite one config file while the caller already holds this bucket's @@ -298,14 +564,69 @@ async fn acquire_config_write_guards( /// /// [`update`] would deadlock here: the lock is not reentrant, so a holder /// that called it would block until its own guard timed out. -pub async fn update_under_transaction_lock(bucket: &str, config_file: &str, data: Vec) -> Result { - let bucket_meta_sys_lock = get_bucket_metadata_sys()?; - let mut bucket_meta_sys = bucket_meta_sys_lock.write().await; - bucket_meta_sys.update(bucket, config_file, data).await +pub async fn update_under_transaction_lock( + guard: &BucketMetadataMutationGuard, + bucket: &str, + config_file: &str, + data: Vec, +) -> Result { + guard.ensure_valid(bucket)?; + update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await } -pub async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Vec) -> Result { - update_under_transaction_lock(bucket, BUCKET_TARGETS_FILE, data).await +pub async fn update_bucket_targets_under_transaction_lock( + guard: &BucketMetadataMutationGuard, + bucket: &str, + data: Vec, +) -> Result { + update_under_transaction_lock(guard, bucket, BUCKET_TARGETS_FILE, data).await +} + +async fn update_under_config_write_guard( + sys: Arc>, + guard: &BucketMetadataMutationGuard, + config_file: &str, + data: Vec, +) -> Result { + guard.ensure_valid(&guard.bucket)?; + let metadata_sys = sys.read().await.clone(); + let updated = await_bucket_namespace_operation( + Some(&guard.lifecycle_guard), + &guard.bucket, + "bucket config mutation", + await_bucket_namespace_operation( + Some(&guard.transaction_guard), + &guard.bucket, + "bucket config transaction", + metadata_sys.update_checked(&guard.bucket, config_file, data, true, guard.incarnation_id), + ), + ) + .await?; + guard.ensure_valid(&guard.bucket)?; + Ok(updated) +} + +async fn delete_under_config_write_guard( + sys: Arc>, + guard: &BucketMetadataMutationGuard, + config_file: &str, +) -> Result { + guard.ensure_valid(&guard.bucket)?; + let metadata_sys = sys.read().await.clone(); + let updated = await_bucket_namespace_operation( + Some(&guard.lifecycle_guard), + &guard.bucket, + "bucket config deletion", + await_bucket_namespace_operation( + Some(&guard.transaction_guard), + &guard.bucket, + "bucket config deletion transaction", + metadata_sys.update_checked(&guard.bucket, config_file, Vec::new(), false, guard.incarnation_id), + ), + ) + .await?; + guard.ensure_valid(&guard.bucket)?; + Ok(updated) } /// Read-modify-write one bucket config file under both guards a config @@ -327,8 +648,24 @@ pub async fn update_config_with(bucket: &str, config_file: &str, mutate: F) - where F: FnOnce(&BucketMetadata) -> Result> + Send, { - let (_transaction_guard, mut sys) = acquire_config_write_guards(get_bucket_metadata_sys()?, bucket).await?; - sys.update_config_with(bucket, config_file, mutate).await + let sys = get_bucket_metadata_sys()?; + let guard = Box::pin(acquire_config_write_guard(sys.clone(), bucket)).await?; + guard.ensure_valid(bucket)?; + let metadata_sys = sys.read().await.clone(); + let updated = await_bucket_namespace_operation( + Some(&guard.lifecycle_guard), + bucket, + "bucket config read-modify-write", + await_bucket_namespace_operation( + Some(&guard.transaction_guard), + bucket, + "bucket config read-modify-write transaction", + Box::pin(metadata_sys.update_config_with_checked(bucket, config_file, mutate, guard.incarnation_id)), + ), + ) + .await?; + guard.ensure_valid(bucket)?; + Ok(updated) } /// Acquire a bucket's metadata transaction lock, held across a whole @@ -347,8 +684,27 @@ where /// /// Callers that hold this guard must use [`update_under_transaction_lock`] /// rather than [`update`] — see that function. -pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result { - acquire_transaction_lock_with_sys(&get_bucket_metadata_sys()?, bucket).await +pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result { + acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await +} + +pub(crate) async fn acquire_bucket_metadata_transaction_lock_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result { + acquire_transaction_lock_with_sys(&bucket_metadata_sys_of(ctx)?, bucket).await +} + +pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result { + let sys = bucket_metadata_sys_of(ctx)?; + let api = sys.read().await.object_store(); + let lock = api + .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket)) + .await?; + Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?) } async fn acquire_transaction_lock_with_sys( @@ -376,7 +732,7 @@ fn bucket_metadata_transaction_lock_key(bucket: &str) -> String { pub async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; - let bucket_meta_sys = bucket_meta_sys_lock.read().await; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); bucket_meta_sys.get_bucket_policy(bucket).await } @@ -385,7 +741,7 @@ pub async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDate /// This preserves the exact format of the policy document as it was PUT. pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; - let bucket_meta_sys = bucket_meta_sys_lock.read().await; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); bucket_meta_sys.get_bucket_policy_raw(bucket).await } @@ -462,11 +818,61 @@ pub async fn get_sse_config(bucket: &str) -> Result<(ServerSideEncryptionConfigu pub async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; - let bucket_meta_sys = bucket_meta_sys_lock.read().await; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); bucket_meta_sys.get_object_lock_config(bucket).await } +pub async fn get_object_lock_config_state(bucket: &str) -> Result { + let bucket_meta_sys_lock = get_bucket_metadata_sys()?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); + + bucket_meta_sys.get_object_lock_config_state(bucket).await +} + +pub(crate) async fn get_object_lock_config_state_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result { + let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); + + bucket_meta_sys.get_object_lock_config_state(bucket).await +} + +/// Re-read the Object Lock state and bucket incarnation from the same +/// authoritative metadata blob while the caller holds the transaction lock. +pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in( + ctx: &crate::runtime::instance::InstanceContext, + bucket: &str, +) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> { + let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?; + let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone(); + + match bucket_meta_sys + .read_authoritative_metadata_from_disk_under_transaction_lock(bucket) + .await? + { + BucketMetadataAuthority::Authoritative(metadata) + if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() => + { + let object_lock_config_updated_at = metadata.object_lock_config_updated_at; + Ok(( + object_lock_config_state_from_authoritative_metadata(&metadata)?, + metadata.bucket_incarnation_id, + object_lock_config_updated_at, + )) + } + BucketMetadataAuthority::Authoritative(_) => { + Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}"))) + } + BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())), + BucketMetadataAuthority::Fabricated => { + Err(Error::other(format!("bucket Object Lock metadata is not authoritative: {bucket}"))) + } + } +} + pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> { let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys = bucket_meta_sys_lock.read().await; @@ -537,12 +943,11 @@ pub async fn list_bucket_targets(bucket: &str) -> Result { bucket_meta_sys.get_bucket_targets_config(bucket).await } -/// Bound and lifetime of the negative cache for buckets with no persisted -/// metadata. Entries are invalidated the moment real metadata is cached, so -/// the TTL only bounds staleness for out-of-band creations whose reload -/// notification was lost; the capacity bounds memory under bogus-name floods. -const ABSENT_BUCKET_METADATA_TTL: Duration = Duration::from_secs(30); -const ABSENT_BUCKET_METADATA_MAX_ENTRIES: u64 = 10_000; +/// Bound and lifetime of the negative cache for physically missing buckets. +/// Existing legacy buckets without metadata use a separate lifetime marker so +/// auth checks do not repeat erasure reads every TTL. +const MISSING_BUCKET_TTL: Duration = Duration::from_secs(30); +const MISSING_BUCKET_MAX_ENTRIES: u64 = 10_000; const PEER_METADATA_NOT_PERSISTED: &str = "no persisted bucket metadata readable; peer cache left unchanged"; #[derive(Debug)] struct MetadataPublishLockRegistry { @@ -573,9 +978,9 @@ struct MetadataPublishGuard { _guard: tokio::sync::OwnedMutexGuard, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct BucketMetadataSys { - metadata_map: RwLock>>, + metadata_map: Arc>>>, /// Serializes metadata-map commits and their derived cache updates for one /// bucket. Namespace locks, when present, are acquired before this lock. metadata_publish_locks: Arc, @@ -594,26 +999,32 @@ pub struct BucketMetadataSys { /// it cannot invert against a path that already holds one of the others. lazy_load_locks: Arc, #[cfg(test)] - lazy_load_lock_probe: std::sync::atomic::AtomicBool, + lazy_load_lock_probe: Arc, /// Counts disk loads taken by the lazy `get_config` path, so a test can /// prove concurrent misses collapse into one. #[cfg(test)] - lazy_disk_loads: std::sync::atomic::AtomicUsize, - /// Buckets recently observed to have no persisted metadata. Serving the - /// fabricated default from here (instead of re-reading disk) keeps the - /// per-request cost of repeated lookups for such names bounded — without - /// this, every request naming a nonexistent bucket pays a namespace-lock - /// acquisition plus a full erasure-set metadata fanout (reachable - /// pre-auth via CORS preflight, and per-key in DeleteObjects). - absent_metadata: moka::future::Cache, + lazy_disk_loads: Arc, + #[cfg(test)] + legacy_migration_write_error: Arc, + #[cfg(test)] + legacy_migration_lock_probe: Arc, + #[cfg(test)] + object_lock_disk_read_errors: Arc>>, + /// Existing buckets confirmed to have no persisted metadata. These entries + /// are non-authoritative and never enter `metadata_map`; config publish, + /// peer reload, refresh, and bucket deletion fence their invalidation. + fabricated_metadata: Arc>>, + /// Physically missing names are TTL-bounded to limit memory under bogus + /// name floods while avoiding repeated namespace and erasure reads. + missing_buckets: moka::future::Cache, api: Arc, - initialized: RwLock, + initialized: Arc>, } impl BucketMetadataSys { pub fn new(api: Arc) -> Self { Self { - metadata_map: RwLock::new(HashMap::new()), + metadata_map: Arc::new(RwLock::new(HashMap::new())), metadata_publish_locks: Arc::new(MetadataPublishLockRegistry { locks: StdMutex::new(HashMap::new()), }), @@ -621,15 +1032,22 @@ impl BucketMetadataSys { locks: StdMutex::new(HashMap::new()), }), #[cfg(test)] - lazy_load_lock_probe: std::sync::atomic::AtomicBool::new(false), + lazy_load_lock_probe: Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(test)] - lazy_disk_loads: std::sync::atomic::AtomicUsize::new(0), - absent_metadata: moka::future::Cache::builder() - .max_capacity(ABSENT_BUCKET_METADATA_MAX_ENTRIES) - .time_to_live(ABSENT_BUCKET_METADATA_TTL) + lazy_disk_loads: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + #[cfg(test)] + legacy_migration_write_error: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + legacy_migration_lock_probe: Arc::new(std::sync::atomic::AtomicBool::new(false)), + #[cfg(test)] + object_lock_disk_read_errors: Arc::new(RwLock::new(HashSet::new())), + fabricated_metadata: Arc::new(RwLock::new(HashSet::new())), + missing_buckets: moka::future::Cache::builder() + .max_capacity(MISSING_BUCKET_MAX_ENTRIES) + .time_to_live(MISSING_BUCKET_TTL) .build(), api, - initialized: RwLock::new(false), + initialized: Arc::new(RwLock::new(false)), } } @@ -822,6 +1240,8 @@ impl BucketMetadataSys { .lock_metadata_publish(bucket, namespace_guard, "stale bucket metadata removal") .await?; let removed = self.metadata_map.write().await.remove(bucket).is_some(); + self.fabricated_metadata.write().await.remove(bucket); + self.missing_buckets.insert(bucket.to_string(), ()).await; if removed { BucketTargetSys::get().delete(bucket).await; clear_bucket_durability(bucket); @@ -844,19 +1264,19 @@ impl BucketMetadataSys { .lock_metadata_publish(bucket, namespace_guard, "initial bucket metadata publish") .await?; self.metadata_map.write().await.insert(bucket.to_string(), Arc::clone(&bm)); - self.absent_metadata.invalidate(bucket).await; + self.fabricated_metadata.write().await.remove(bucket); + self.missing_buckets.invalidate(bucket).await; sync_bucket_target_sys(bucket, &bm).await; sync_bucket_durability(bucket, &bm); } MetadataLoadMode::Initial => { let _publish_guard = self - .lock_metadata_publish(bucket, namespace_guard, "initial bucket metadata publish") + .lock_metadata_publish(bucket, namespace_guard, "initial bucket metadata absence publish") .await?; - self.metadata_map - .write() - .await - .entry(bucket.to_string()) - .or_insert_with(|| Arc::new(bm)); + if !self.metadata_map.read().await.contains_key(bucket) { + self.fabricated_metadata.write().await.insert(bucket.to_string()); + self.missing_buckets.invalidate(bucket).await; + } } MetadataLoadMode::Refresh => { self.publish_if_unchanged(bucket, expected, bm, persisted, namespace_guard) @@ -875,6 +1295,26 @@ impl BucketMetadataSys { namespace_guard: &rustfs_lock::NamespaceLockGuard, ) -> Result<()> { if !persisted { + let _publish_guard = self + .lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata absence publish") + .await?; + let mut map = self.metadata_map.write().await; + let unchanged = match (expected, map.get(bucket)) { + (None, None) => true, + (Some(expected), Some(current)) => Arc::ptr_eq(expected, current), + _ => false, + }; + if !unchanged { + return Ok(()); + } + let removed = map.remove(bucket).is_some(); + drop(map); + self.fabricated_metadata.write().await.insert(bucket.to_string()); + self.missing_buckets.invalidate(bucket).await; + if removed { + BucketTargetSys::get().delete(bucket).await; + clear_bucket_durability(bucket); + } return Ok(()); } let _publish_guard = self @@ -892,7 +1332,8 @@ impl BucketMetadataSys { } map.insert(bucket.to_string(), Arc::clone(&metadata)); drop(map); - self.absent_metadata.invalidate(bucket).await; + self.fabricated_metadata.write().await.remove(bucket); + self.missing_buckets.invalidate(bucket).await; sync_bucket_target_sys(bucket, &metadata).await; sync_bucket_durability(bucket, &metadata); Ok(()) @@ -918,8 +1359,8 @@ impl BucketMetadataSys { let mut map = self.metadata_map.write().await; map.insert(bucket.clone(), bm.clone()); drop(map); - // Real metadata supersedes any recorded absence immediately. - self.absent_metadata.invalidate(&bucket).await; + self.fabricated_metadata.write().await.remove(&bucket); + self.missing_buckets.invalidate(&bucket).await; sync_bucket_target_sys(&bucket, &bm).await; sync_bucket_durability(&bucket, &bm); } @@ -937,36 +1378,62 @@ impl BucketMetadataSys { let mut map = self.metadata_map.write().await; let removed = map.remove(bucket).is_some(); drop(map); + let removed_fabricated = self.fabricated_metadata.write().await.remove(bucket); + self.missing_buckets.insert(bucket.to_string(), ()).await; if removed { BucketTargetSys::get().delete(bucket).await; clear_bucket_durability(bucket); } - removed + removed || removed_fabricated } async fn _reset(&mut self) { let mut map = self.metadata_map.write().await; map.clear(); + drop(map); + self.fabricated_metadata.write().await.clear(); + self.missing_buckets.invalidate_all(); } - pub async fn update(&mut self, bucket: &str, config_file: &str, data: Vec) -> Result { - self.update_and_parse(bucket, config_file, data, true).await + /// The `Box::pin`s here and in [`Self::update_checked`] are load-bearing, not + /// style. A bucket-config write nests incarnation resolution (which can drive + /// legacy migration and a peer fan-out), a full metadata load, and `save`, + /// which is itself an object PUT that pulls in the whole erasure write path — + /// and every request that mutates bucket config is already several futures + /// deep. Inlining all of that into one state machine overflows the worker + /// stack in debug builds (rustfs#5648: `SIGABRT`, ~780KiB consumed between + /// `update` and the config read alone). Keep these boxed. + pub async fn update(&self, bucket: &str, config_file: &str, data: Vec) -> Result { + let incarnation_id = Box::pin(self.get_bucket_incarnation_id(bucket)).await?; + Box::pin(self.update_checked(bucket, config_file, data, true, incarnation_id)).await } - pub async fn delete(&mut self, bucket: &str, config_file: &str) -> Result { - self.update_and_parse(bucket, config_file, Vec::new(), false).await + pub async fn delete(&self, bucket: &str, config_file: &str) -> Result { + let incarnation_id = self.get_bucket_incarnation_id(bucket).await?; + self.update_checked(bucket, config_file, Vec::new(), false, incarnation_id) + .await } - async fn update_and_parse(&mut self, bucket: &str, config_file: &str, data: Vec, parse: bool) -> Result { + async fn update_checked( + &self, + bucket: &str, + config_file: &str, + data: Vec, + parse: bool, + expected_incarnation_id: Uuid, + ) -> Result { // Load through this system's own store, the one `save` persists to // (backlog#1052 S7). Reading from the ambient handle instead made the // read and the write of a single read-modify-write able to target // different instances. - let mut bm = Self::load_bucket_metadata_for_update(self.api.clone(), bucket, parse).await?; + let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, parse)).await?; + if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id { + return Err(Error::BucketNotFound(bucket.to_string())); + } let updated = bm.update_config(config_file, data)?; - self.save(bm).await?; + Box::pin(self.save(bm)).await?; Ok(updated) } @@ -975,16 +1442,34 @@ impl BucketMetadataSys { /// [`Self::update`], with the payload computed from the loaded metadata /// instead of supplied up front. Loads through this system's own store so /// the read and the persisted write target the same instance. - async fn update_config_with(&mut self, bucket: &str, config_file: &str, mutate: F) -> Result + async fn update_config_with(&self, bucket: &str, config_file: &str, mutate: F) -> Result where F: FnOnce(&BucketMetadata) -> Result> + Send, { - let mut bm = Self::load_bucket_metadata_for_update(self.api.clone(), bucket, true).await?; + let incarnation_id = self.get_bucket_incarnation_id(bucket).await?; + self.update_config_with_checked(bucket, config_file, mutate, incarnation_id) + .await + } + + async fn update_config_with_checked( + &self, + bucket: &str, + config_file: &str, + mutate: F, + expected_incarnation_id: Uuid, + ) -> Result + where + F: FnOnce(&BucketMetadata) -> Result> + Send, + { + let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, true)).await?; + if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id { + return Err(Error::BucketNotFound(bucket.to_string())); + } let data = mutate(&bm)?; let updated = bm.update_config(config_file, data)?; - self.save(bm).await?; + Box::pin(self.save(bm)).await?; Ok(updated) } @@ -1034,8 +1519,18 @@ impl BucketMetadataSys { Ok(()) } + async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> { + bm.save_with_store(self.api.clone()).await?; + save_bucket_incarnation(self.api.clone(), &bm.name, bm.bucket_incarnation_id).await?; + bm.bucket_incarnation_sidecar = true; + self.set(bm.name.clone(), Arc::new(bm)).await; + Ok(()) + } + async fn bucket_names(&self) -> Vec { - self.metadata_map.read().await.keys().cloned().collect() + let mut names = self.metadata_map.read().await.keys().cloned().collect::>(); + names.extend(self.fabricated_metadata.read().await.iter().cloned()); + names.into_iter().collect() } pub async fn get_config_from_disk(&self, bucket: &str) -> Result { @@ -1098,10 +1593,7 @@ impl BucketMetadataSys { if let Some(bm) = has_bm { Ok((bm, false)) } else { - // A recent lookup already established there is no persisted - // metadata: serve the fabricated default without another - // namespace-lock + erasure-set fanout. - if self.absent_metadata.get(bucket).await.is_some() { + if self.fabricated_metadata.read().await.contains(bucket) || self.missing_buckets.get(bucket).await.is_some() { let mut bm = BucketMetadata::new(bucket); bm.default_timestamps(); return Ok((Arc::new(bm), true)); @@ -1113,13 +1605,12 @@ impl BucketMetadataSys { let load_lock = self.lazy_load_lock(bucket); let _load_guard = load_lock.lock_owned().await; - // Re-check both caches: whoever held the gate before us may have - // already answered this exact question, and repeating the fanout - // is the whole cost this gate exists to avoid. + // Re-check every state: whoever held the gate before us may have + // already answered this exact question. if let Some(bm) = self.metadata_map.read().await.get(bucket).cloned() { return Ok((bm, true)); } - if self.absent_metadata.get(bucket).await.is_some() { + if self.fabricated_metadata.read().await.contains(bucket) || self.missing_buckets.get(bucket).await.is_some() { let mut bm = BucketMetadata::new(bucket); bm.default_timestamps(); return Ok((Arc::new(bm), true)); @@ -1158,16 +1649,6 @@ impl BucketMetadataSys { let bm = Arc::new(bm); - // This lazy path caches only metadata that actually exists on - // this store. A fabricated default must not enter the map: - // `get()` is map-only and fail-closed — the object-lock delete - // gate (`object_lock_delete_check_required`) skips its per-object - // protection stat exactly when the map serves metadata saying the - // bucket has no Object Lock, so caching a fabricated default here - // would turn a metadata miss into an authoritative "no lock" - // answer. (Startup `concurrent_load` still caches fabricated - // defaults for buckets listed on disk — legacy buckets without a - // metadata file — but never lets one replace an existing entry.) if persisted { await_bucket_namespace_operation( Some(&guard), @@ -1197,11 +1678,27 @@ impl BucketMetadataSys { } map.insert(bucket.to_string(), bm.clone()); drop(map); - self.absent_metadata.invalidate(bucket).await; + self.fabricated_metadata.write().await.remove(bucket); + self.missing_buckets.invalidate(bucket).await; sync_bucket_target_sys(bucket, &bm).await; sync_bucket_durability(bucket, &bm); } else { - self.absent_metadata.insert(bucket.to_string(), ()).await; + let exists = self + .bucket_exists(bucket, &guard, "lazy bucket metadata existence check") + .await?; + let _publish_guard = self + .lock_metadata_publish(bucket, &guard, "lazy bucket metadata absence publish") + .await?; + if let Some(current) = self.metadata_map.read().await.get(bucket).cloned() { + return Ok((current, true)); + } + if exists { + self.fabricated_metadata.write().await.insert(bucket.to_string()); + self.missing_buckets.invalidate(bucket).await; + } else { + self.fabricated_metadata.write().await.remove(bucket); + self.missing_buckets.insert(bucket.to_string(), ()).await; + } } Ok((bm, true)) @@ -1230,7 +1727,17 @@ impl BucketMetadataSys { } pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> { - let (bm, _) = self.get_config(bucket).await?; + let bm = match self.get_metadata_authority(bucket).await? { + BucketMetadataAuthority::Authoritative(bm) => bm, + BucketMetadataAuthority::Fabricated => match self.migrate_legacy_metadata(bucket).await? { + BucketMetadataAuthority::Authoritative(bm) => bm, + BucketMetadataAuthority::Fabricated => { + return Err(Error::other(format!("bucket policy metadata is not authoritative: {bucket}"))); + } + BucketMetadataAuthority::MissingBucket => return Err(Error::ConfigNotFound), + }, + BucketMetadataAuthority::MissingBucket => return Err(Error::ConfigNotFound), + }; if let Some(config) = &bm.policy_config { Ok((config.clone(), bm.policy_config_updated_at)) @@ -1244,7 +1751,13 @@ impl BucketMetadataSys { /// Returns the raw JSON string of the bucket policy as originally stored. /// This preserves the exact format of the policy document as it was PUT. pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> { - let (bm, _) = self.get_config(bucket).await?; + let bm = match self.get_metadata_authority(bucket).await? { + BucketMetadataAuthority::Authoritative(bm) => bm, + BucketMetadataAuthority::Fabricated => { + return Err(Error::other(format!("bucket policy metadata is not authoritative: {bucket}"))); + } + BucketMetadataAuthority::MissingBucket => return Err(Error::ConfigNotFound), + }; if bm.policy_config_json.is_empty() { Err(Error::ConfigNotFound) @@ -1286,12 +1799,277 @@ impl BucketMetadataSys { } pub async fn get_object_lock_config(&self, bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> { - let (bm, _) = self.get_config(bucket).await?; + match self.get_object_lock_config_state(bucket).await? { + ObjectLockConfigState::Configured { config, updated_at } => Ok((config, updated_at)), + ObjectLockConfigState::ConfirmedAbsent => Err(Error::ConfigNotFound), + ObjectLockConfigState::Fabricated => { + Err(Error::other(format!("bucket Object Lock metadata is not authoritative: {bucket}"))) + } + } + } - if let Some(config) = &bm.object_lock_config { - Ok((config.clone(), bm.object_lock_config_updated_at)) + pub async fn get_object_lock_config_state(&self, bucket: &str) -> Result { + match self.get_metadata_authority(bucket).await? { + BucketMetadataAuthority::Authoritative(bm) => object_lock_config_state_from_authoritative_metadata(&bm), + BucketMetadataAuthority::Fabricated => match self.migrate_legacy_metadata(bucket).await? { + BucketMetadataAuthority::Authoritative(bm) => object_lock_config_state_from_authoritative_metadata(&bm), + BucketMetadataAuthority::MissingBucket => Err(Error::ConfigNotFound), + BucketMetadataAuthority::Fabricated => Ok(ObjectLockConfigState::Fabricated), + }, + BucketMetadataAuthority::MissingBucket => Err(Error::ConfigNotFound), + } + } + + async fn get_bucket_incarnation_id(&self, bucket: &str) -> Result { + let authority = self.get_metadata_authority(bucket).await?; + if let BucketMetadataAuthority::Authoritative(metadata) = &authority + && metadata.bucket_incarnation_sidecar + && !metadata.bucket_incarnation_id.is_nil() + { + return Ok(metadata.bucket_incarnation_id); + } + + match authority { + BucketMetadataAuthority::Authoritative(_) | BucketMetadataAuthority::Fabricated => { + match self.migrate_legacy_metadata(bucket).await? { + BucketMetadataAuthority::Authoritative(metadata) + if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() => + { + Ok(metadata.bucket_incarnation_id) + } + BucketMetadataAuthority::Authoritative(_) | BucketMetadataAuthority::Fabricated => { + Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}"))) + } + BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())), + } + } + BucketMetadataAuthority::MissingBucket => Err(Error::BucketNotFound(bucket.to_string())), + } + } + + async fn get_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result { + let transaction_lock = self + .api + .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket)) + .await?; + let _transaction_guard = transaction_lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + let incarnation_id = load_bucket_incarnation(self.api.clone(), bucket).await?; + if _transaction_guard.is_lock_lost() { + return Err(Error::other(format!("bucket incarnation metadata transaction lock was lost: {bucket}"))); + } + match incarnation_id { + Some(incarnation_id) if !incarnation_id.is_nil() => Ok(incarnation_id), + _ => Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}"))), + } + } + + async fn get_metadata_authority(&self, bucket: &str) -> Result { + if let Some(bm) = self.metadata_map.read().await.get(bucket).cloned() { + return Ok(BucketMetadataAuthority::Authoritative(bm)); + } + if self.fabricated_metadata.read().await.contains(bucket) { + return Ok(BucketMetadataAuthority::Fabricated); + } + if self.missing_buckets.get(bucket).await.is_some() { + return Ok(BucketMetadataAuthority::MissingBucket); + } + + self.get_config(bucket).await?; + + if let Some(bm) = self.metadata_map.read().await.get(bucket).cloned() { + Ok(BucketMetadataAuthority::Authoritative(bm)) + } else if self.fabricated_metadata.read().await.contains(bucket) { + Ok(BucketMetadataAuthority::Fabricated) + } else if self.missing_buckets.get(bucket).await.is_some() { + Ok(BucketMetadataAuthority::MissingBucket) } else { - Err(Error::ConfigNotFound) + Err(Error::other(format!("bucket metadata authority was not classified: {bucket}"))) + } + } + + async fn migrate_legacy_metadata(&self, bucket: &str) -> Result { + let transaction_lock = self + .api + .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket)) + .await?; + let _transaction_guard = transaction_lock + .get_write_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + + let authority = self + .load_authoritative_metadata_from_disk_under_transaction_lock(bucket) + .await?; + if _transaction_guard.is_lock_lost() { + return Err(Error::other(format!("legacy bucket metadata transaction lock was lost: {bucket}"))); + } + Ok(authority) + } + + async fn load_authoritative_metadata_from_disk_under_transaction_lock( + &self, + bucket: &str, + ) -> Result { + #[cfg(test)] + if self.object_lock_disk_read_errors.write().await.remove(bucket) { + return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}"))); + } + + let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?; + let namespace_guard = namespace_lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + + let bucket_info = match await_bucket_namespace_operation( + Some(&namespace_guard), + bucket, + "legacy bucket metadata existence check", + async { + self.api + .peer_sys + .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) + .await + .map_err(crate::error::StorageError::from) + }, + ) + .await + { + Ok(info) => info, + Err(Error::VolumeNotFound) => { + let _publish_guard = self + .lock_metadata_publish(bucket, &namespace_guard, "missing legacy bucket metadata publish") + .await?; + self.metadata_map.write().await.remove(bucket); + self.fabricated_metadata.write().await.remove(bucket); + self.missing_buckets.insert(bucket.to_string(), ()).await; + return Ok(BucketMetadataAuthority::MissingBucket); + } + Err(err) => return Err(err), + }; + + let (mut metadata, persisted) = await_bucket_namespace_operation( + Some(&namespace_guard), + bucket, + "legacy bucket metadata confirmation", + load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true), + ) + .await?; + if persisted && !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() { + return Err(Error::other(format!( + "bucket incarnation sidecar is missing for new-format metadata: {bucket}" + ))); + } + let needs_migration = !persisted || !metadata.bucket_incarnation_sidecar; + if !persisted { + metadata = BucketMetadata::new(bucket); + metadata.created = bucket_info.created.unwrap_or(OffsetDateTime::UNIX_EPOCH); + } else if metadata.bucket_incarnation_id.is_nil() { + metadata.bucket_incarnation_id = Uuid::new_v4(); + } + if needs_migration { + #[cfg(test)] + if self.legacy_migration_write_error.load(std::sync::atomic::Ordering::Relaxed) { + return Err(Error::other("injected legacy metadata migration write failure")); + } + #[cfg(test)] + if self.legacy_migration_lock_probe.load(std::sync::atomic::Ordering::Relaxed) { + let competing = self.api.new_ns_lock(bucket, bucket).await?; + assert!( + competing.get_write_lock(Duration::from_millis(20)).await.is_err(), + "bucket delete/recreate must not cross the legacy metadata migration fence" + ); + } + save_bucket_incarnation(self.api.clone(), bucket, metadata.bucket_incarnation_id).await?; + metadata.bucket_incarnation_sidecar = true; + if !persisted { + await_bucket_namespace_operation( + Some(&namespace_guard), + bucket, + "legacy bucket metadata migration", + metadata.save_with_store(self.api.clone()), + ) + .await?; + } + } + + if namespace_guard.is_lock_lost() { + return Err(Error::other(format!( + "bucket namespace lock was lost before legacy metadata publish: {bucket}" + ))); + } + let metadata = Arc::new(metadata); + let _publish_guard = self + .lock_metadata_publish(bucket, &namespace_guard, "legacy bucket metadata publish") + .await?; + self.metadata_map + .write() + .await + .insert(bucket.to_string(), Arc::clone(&metadata)); + self.fabricated_metadata.write().await.remove(bucket); + self.missing_buckets.invalidate(bucket).await; + sync_bucket_target_sys(bucket, &metadata).await; + sync_bucket_durability(bucket, &metadata); + Ok(BucketMetadataAuthority::Authoritative(metadata)) + } + + async fn read_authoritative_metadata_from_disk_under_transaction_lock( + &self, + bucket: &str, + ) -> Result { + #[cfg(test)] + if self.object_lock_disk_read_errors.write().await.remove(bucket) { + return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}"))); + } + + let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?; + let namespace_guard = namespace_lock + .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) + .await?; + match await_bucket_namespace_operation( + Some(&namespace_guard), + bucket, + "bucket metadata snapshot existence check", + async { + self.api + .peer_sys + .get_bucket_info(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) + .await + .map_err(crate::error::StorageError::from) + }, + ) + .await + { + Ok(_) => {} + Err(Error::VolumeNotFound) => return Ok(BucketMetadataAuthority::MissingBucket), + Err(err) => return Err(err), + } + + let (metadata, persisted) = await_bucket_namespace_operation( + Some(&namespace_guard), + bucket, + "bucket metadata authoritative snapshot", + load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true), + ) + .await?; + if persisted { + Ok(BucketMetadataAuthority::Authoritative(Arc::new(metadata))) + } else { + Ok(BucketMetadataAuthority::Fabricated) + } + } + + pub(crate) async fn get_authoritative_metadata(&self, bucket: &str) -> Result> { + match self.get_metadata_authority(bucket).await? { + BucketMetadataAuthority::Authoritative(bm) => Ok(bm), + BucketMetadataAuthority::Fabricated => match self.migrate_legacy_metadata(bucket).await? { + BucketMetadataAuthority::Authoritative(bm) => Ok(bm), + BucketMetadataAuthority::Fabricated => { + Err(Error::other(format!("bucket metadata is not authoritative: {bucket}"))) + } + BucketMetadataAuthority::MissingBucket => Err(Error::ConfigNotFound), + }, + BucketMetadataAuthority::MissingBucket => Err(Error::ConfigNotFound), } } @@ -1481,6 +2259,7 @@ mod tests { use super::test_support::isolated_store_over_temp_disks; use super::*; use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials}; + use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; use serial_test::serial; use tokio::time::timeout; @@ -1494,6 +2273,8 @@ mod tests { metadata.versioning_config = None; metadata.replication_config_xml = b"".to_vec(); metadata.replication_config = None; + metadata.object_lock_config_xml = b"".to_vec(); + metadata.object_lock_config = None; sys.set(bucket.to_string(), Arc::new(metadata)).await; assert!( @@ -1504,6 +2285,419 @@ mod tests { sys.get_replication_config(bucket).await.is_err(), "malformed replication metadata must not be reported as ConfigNotFound" ); + assert!( + sys.get_object_lock_config_state(bucket).await.is_err(), + "malformed Object Lock metadata must not be reported as absent" + ); + } + + #[tokio::test] + async fn config_states_distinguish_authoritative_absence_from_fabricated_metadata() { + use std::sync::atomic::Ordering; + + let (dirs, ecstore) = isolated_store_over_temp_disks().await; + let sys = BucketMetadataSys::new(ecstore); + + sys.set("authoritative-empty".to_string(), Arc::new(BucketMetadata::new("authoritative-empty"))) + .await; + assert!(matches!( + sys.get_object_lock_config_state("authoritative-empty").await.unwrap(), + ObjectLockConfigState::ConfirmedAbsent + )); + assert!(matches!(sys.get_bucket_policy("authoritative-empty").await, Err(Error::ConfigNotFound))); + + for dir in &dirs { + std::fs::create_dir_all(dir.path().join("policy-only-legacy")).unwrap(); + } + assert!(matches!(sys.get_bucket_policy("policy-only-legacy").await, Err(Error::ConfigNotFound))); + assert!(matches!( + sys.get_bucket_policy_raw("policy-only-legacy").await, + Err(Error::ConfigNotFound) + )); + + for dir in &dirs { + std::fs::create_dir_all(dir.path().join("raw-policy-only-legacy")).unwrap(); + } + assert!(!matches!( + sys.get_bucket_policy_raw("raw-policy-only-legacy").await, + Err(Error::ConfigNotFound) + )); + + for dir in &dirs { + std::fs::create_dir_all(dir.path().join("fabricated")).unwrap(); + } + let loads_before = sys.lazy_disk_loads.load(Ordering::Relaxed); + sys.legacy_migration_lock_probe.store(true, Ordering::Relaxed); + assert!(matches!( + sys.get_object_lock_config_state("fabricated").await.unwrap(), + ObjectLockConfigState::ConfirmedAbsent + )); + sys.legacy_migration_lock_probe.store(false, Ordering::Relaxed); + assert!(matches!(sys.get_bucket_policy("fabricated").await, Err(Error::ConfigNotFound))); + assert!(matches!(sys.get_bucket_policy_raw("fabricated").await, Err(Error::ConfigNotFound))); + assert!(matches!( + sys.get_object_lock_config_state("fabricated").await.unwrap(), + ObjectLockConfigState::ConfirmedAbsent + )); + assert_eq!( + sys.lazy_disk_loads.load(Ordering::Relaxed), + loads_before + 1, + "legacy metadata migration must not repeat the initial erasure read" + ); + assert!( + !sys.get("fabricated").await.unwrap().bucket_incarnation_id.is_nil(), + "legacy metadata absence migration must persist a bucket incarnation" + ); + + for dir in &dirs { + std::fs::create_dir_all(dir.path().join("migration-write-failure")).unwrap(); + } + sys.legacy_migration_write_error.store(true, Ordering::Relaxed); + assert!(!matches!( + sys.get_bucket_policy_raw("migration-write-failure").await, + Err(Error::ConfigNotFound) + )); + assert!(!matches!( + sys.get_bucket_policy("migration-write-failure").await, + Err(Error::ConfigNotFound) + )); + assert!(sys.get_object_lock_config_state("migration-write-failure").await.is_err()); + assert!(sys.get("migration-write-failure").await.is_err()); + sys.legacy_migration_write_error.store(false, Ordering::Relaxed); + + assert!(matches!( + sys.get_object_lock_config_state("missing-bucket").await, + Err(Error::ConfigNotFound) + )); + assert!(matches!(sys.get_bucket_policy("missing-bucket").await, Err(Error::ConfigNotFound))); + assert!(matches!(sys.get_bucket_policy_raw("missing-bucket").await, Err(Error::ConfigNotFound))); + + let policy_json = br#"{ + "Version":"2012-10-17", + "Statement":[{ + "Effect":"Allow", + "Principal":{"AWS":"*"}, + "Action":["s3:GetObject"], + "Resource":["arn:aws:s3:::configured-policy/*"] + }] + }"#; + let mut configured_policy = BucketMetadata::new("configured-policy"); + configured_policy.policy_config_json = policy_json.to_vec(); + sys.set("configured-policy".to_string(), Arc::new(configured_policy)).await; + sys.get_bucket_policy("configured-policy") + .await + .expect("configured bucket policy JSON should parse"); + + let mut legacy = BucketMetadata::new("legacy-lock-enabled"); + legacy.lock_enabled = true; + sys.set("legacy-lock-enabled".to_string(), Arc::new(legacy)).await; + let ObjectLockConfigState::Configured { config, .. } = + sys.get_object_lock_config_state("legacy-lock-enabled").await.unwrap() + else { + panic!("legacy lock-enabled metadata must remain configured"); + }; + assert_eq!( + config.object_lock_enabled.as_ref().map(|value| value.as_str()), + Some(ObjectLockEnabled::ENABLED) + ); + assert!(config.rule.is_none()); + } + + #[test] + fn authoritative_object_lock_state_rejects_invalid_retention_periods() { + use s3s::dto::{DefaultRetention, ObjectLockRule}; + + for (days, years) in [(Some(0), None), (Some(-1), None), (Some(1), Some(1)), (None, None)] { + let mut metadata = BucketMetadata::new("invalid-object-lock"); + metadata.object_lock_config = Some(ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: Some(ObjectLockRule { + default_retention: Some(DefaultRetention { + days, + mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), + years, + }), + }), + }); + + assert!( + object_lock_config_state_from_authoritative_metadata(&metadata).is_err(), + "invalid retention days={days:?} years={years:?} must fail closed" + ); + } + } + + #[test] + fn authoritative_object_lock_state_rejects_invalid_enabled_mode_and_rule() { + use s3s::dto::{DefaultRetention, ObjectLockRule}; + + for enabled in [None, Some(ObjectLockEnabled::from("Disabled".to_string()))] { + let mut metadata = BucketMetadata::new("invalid-object-lock-enabled"); + metadata.lock_enabled = true; + metadata.object_lock_config = Some(ObjectLockConfiguration { + object_lock_enabled: enabled, + rule: None, + }); + assert!(object_lock_config_state_from_authoritative_metadata(&metadata).is_err()); + } + + let mut invalid_mode = BucketMetadata::new("invalid-object-lock-mode"); + invalid_mode.object_lock_config = Some(ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: Some(ObjectLockRule { + default_retention: Some(DefaultRetention { + days: Some(1), + mode: Some(ObjectLockRetentionMode::from("invalid".to_string())), + years: None, + }), + }), + }); + assert!(object_lock_config_state_from_authoritative_metadata(&invalid_mode).is_err()); + + let mut missing_retention = BucketMetadata::new("missing-object-lock-retention"); + missing_retention.object_lock_config = Some(ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: Some(ObjectLockRule { default_retention: None }), + }); + assert!(object_lock_config_state_from_authoritative_metadata(&missing_retention).is_err()); + } + + #[test] + fn object_lock_ever_enabled_marker_survives_config_removal() { + let mut metadata = BucketMetadata::new("lock-config-removed"); + let config = ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }; + metadata + .update_config( + crate::bucket::metadata::OBJECT_LOCK_CONFIG, + crate::bucket::utils::serialize(&config).unwrap(), + ) + .unwrap(); + metadata + .update_config(crate::bucket::metadata::OBJECT_LOCK_CONFIG, Vec::new()) + .unwrap(); + + let ObjectLockConfigState::Configured { config, .. } = + object_lock_config_state_from_authoritative_metadata(&metadata).unwrap() + else { + panic!("an ever-enabled bucket must remain Object Lock configured"); + }; + assert_eq!( + config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str), + Some(ObjectLockEnabled::ENABLED) + ); + } + + #[tokio::test] + async fn site_replication_empty_object_lock_update_preserves_ever_enabled_state_after_reload() { + use crate::bucket::metadata::OBJECT_LOCK_CONFIG; + use crate::bucket::object_lock::objectlock_sys::ensure_recursive_force_delete_allowed_for_state; + + let (dirs, ecstore) = isolated_store_over_temp_disks().await; + let bucket = "site-repl-lock-removal"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).unwrap(); + } + + let node = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone()))); + let config = ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }; + update_with_sys( + Arc::clone(&node), + bucket, + OBJECT_LOCK_CONFIG, + crate::bucket::utils::serialize(&config).unwrap(), + ) + .await + .expect("initial Object Lock config should persist"); + + // Site replication represents an incoming `object_lock_config: None` + // as deletion of the config payload. That must not clear the durable + // ever-enabled marker. + delete_with_sys(Arc::clone(&node), bucket, OBJECT_LOCK_CONFIG) + .await + .expect("empty site-replication update should persist"); + + let restarted = BucketMetadataSys::new(ecstore); + restarted + .reload_from_store(bucket) + .await + .expect("persisted metadata should reload after restart"); + let state = restarted + .get_object_lock_config_state(bucket) + .await + .expect("ever-enabled state should remain authoritative"); + assert!(matches!(state, ObjectLockConfigState::Configured { .. })); + assert!( + ensure_recursive_force_delete_allowed_for_state(bucket, &state).is_err(), + "force delete must remain forbidden after config removal and reload" + ); + } + + #[tokio::test] + async fn legacy_nil_bucket_incarnation_is_persisted_once_and_fail_closed_on_write_error() { + use std::sync::atomic::Ordering; + + let (dirs, ecstore) = isolated_store_over_temp_disks().await; + let sys = BucketMetadataSys::new(ecstore); + + for bucket in ["legacy-nil-incarnation", "legacy-nil-incarnation-write-failure"] { + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).unwrap(); + } + let mut metadata = BucketMetadata::new(bucket); + metadata.bucket_incarnation_id = Uuid::nil(); + sys.persist_and_set(metadata).await.unwrap(); + } + + sys.legacy_migration_lock_probe.store(true, Ordering::Relaxed); + let migrated = sys + .get_bucket_incarnation_id("legacy-nil-incarnation") + .await + .expect("legacy nil incarnation should migrate under the namespace fence"); + sys.legacy_migration_lock_probe.store(false, Ordering::Relaxed); + assert!(!migrated.is_nil()); + let persisted = sys.get_config_from_disk("legacy-nil-incarnation").await.unwrap(); + assert_eq!(persisted.bucket_incarnation_id, migrated); + assert!(persisted.bucket_incarnation_sidecar); + assert_eq!( + sys.get_bucket_incarnation_id("legacy-nil-incarnation").await.unwrap(), + migrated, + "the durable incarnation must remain stable after migration" + ); + + sys.legacy_migration_write_error.store(true, Ordering::Relaxed); + assert!( + sys.get_bucket_incarnation_id("legacy-nil-incarnation-write-failure") + .await + .is_err(), + "a failed incarnation migration must fail closed" + ); + sys.legacy_migration_write_error.store(false, Ordering::Relaxed); + assert!( + sys.get_config_from_disk("legacy-nil-incarnation-write-failure") + .await + .unwrap() + .bucket_incarnation_id + .is_nil(), + "a failed migration must not fabricate authority in memory or on disk" + ); + } + + #[tokio::test] + async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() { + let (dirs, ecstore) = isolated_store_over_temp_disks().await; + let sys = BucketMetadataSys::new(ecstore.clone()); + let bucket = "old-node-incarnation-rewrite"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).unwrap(); + } + + let metadata = BucketMetadata::new(bucket); + let incarnation = metadata.bucket_incarnation_id; + sys.persist_new_and_set(metadata).await.unwrap(); + + let mut old_node_rewrite = sys.get_config_from_disk(bucket).await.unwrap(); + old_node_rewrite.bucket_incarnation_id = Uuid::nil(); + old_node_rewrite.save_with_store(ecstore.clone()).await.unwrap(); + sys.metadata_map.write().await.clear(); + + assert_eq!( + sys.get_bucket_incarnation_id_from_disk(bucket).await.unwrap(), + incarnation, + "rewriting only .metadata.bin must not replace sidecar authority" + ); + + let mut mismatched_rewrite = sys.get_config_from_disk(bucket).await.unwrap(); + mismatched_rewrite.bucket_incarnation_id = Uuid::new_v4(); + mismatched_rewrite.save_with_store(ecstore).await.unwrap(); + let err = sys + .get_config_from_disk(bucket) + .await + .expect_err("a valid but mismatched embedded incarnation must fail closed"); + assert!(err.to_string().contains("does not match")); + } + + #[tokio::test] + async fn corrupt_bucket_incarnation_sidecar_fails_closed() { + let (dirs, ecstore) = isolated_store_over_temp_disks().await; + let sys = BucketMetadataSys::new(ecstore.clone()); + let bucket = "corrupt-incarnation-sidecar"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).unwrap(); + } + + let metadata = BucketMetadata::new(bucket); + sys.persist_new_and_set(metadata).await.unwrap(); + crate::config::com::save_config( + ecstore.clone(), + &format!( + "{}/{bucket}/{}", + crate::disk::BUCKET_META_PREFIX, + crate::bucket::metadata::BUCKET_INCARNATION_FILE + ), + vec![0_u8; 15], + ) + .await + .unwrap(); + sys.metadata_map.write().await.clear(); + + let err = sys + .get_bucket_incarnation_id_from_disk(bucket) + .await + .expect_err("corrupt sidecar must never fall back to the embedded msgpack UUID"); + assert!(err.to_string().contains("persisted bucket incarnation is invalid")); + + crate::config::com::save_config( + ecstore, + &format!( + "{}/{bucket}/{}", + crate::disk::BUCKET_META_PREFIX, + crate::bucket::metadata::BUCKET_INCARNATION_FILE + ), + Uuid::nil().as_bytes().to_vec(), + ) + .await + .unwrap(); + let err = sys + .get_bucket_incarnation_id_from_disk(bucket) + .await + .expect_err("a nil sidecar must fail closed"); + assert!(err.to_string().contains("persisted bucket incarnation is nil")); + } + + #[tokio::test] + async fn missing_bucket_incarnation_sidecar_for_new_metadata_fails_closed() { + let (dirs, ecstore) = isolated_store_over_temp_disks().await; + let sys = BucketMetadataSys::new(ecstore.clone()); + let bucket = "missing-incarnation-sidecar"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).unwrap(); + } + + let metadata = BucketMetadata::new(bucket); + sys.persist_new_and_set(metadata).await.unwrap(); + crate::config::com::delete_config( + ecstore, + &format!( + "{}/{bucket}/{}", + crate::disk::BUCKET_META_PREFIX, + crate::bucket::metadata::BUCKET_INCARNATION_FILE + ), + ) + .await + .unwrap(); + sys.metadata_map.write().await.clear(); + + let err = sys + .get_object_lock_config_state(bucket) + .await + .expect_err("new-format metadata without its sidecar must fail closed"); + assert!(err.to_string().contains("sidecar is missing")); } /// Concurrent cache misses for one bucket must collapse into a single disk @@ -1544,11 +2738,9 @@ mod tests { } /// Pins the fail-closed caching contract of the lazy `get_config` path - /// and the refresh no-replace rule: fabricated defaults are returned but - /// never served by the map-only `get()`, persisted metadata is cached on - /// lazy load (superseding a recorded absence), a refresh-load miss never - /// replaces an existing entry or heals a deleted bucket, and initial load - /// still heals buckets discovered from storage. + /// and refresh authority invalidation: fabricated defaults are never + /// served by map-only `get()`, persisted metadata supersedes absence, and + /// refresh never heals a deleted bucket or preserves stale authority. #[tokio::test] async fn get_config_never_caches_fabricated_defaults_as_authoritative() { let (dirs, ecstore) = isolated_store_over_temp_disks().await; @@ -1575,12 +2767,12 @@ mod tests { // (b) Persisting real metadata supersedes the recorded absence, and a // lazy reload after a map wipe re-caches it. - let mut persisted = BucketMetadata::new("absent-bucket"); - persisted.policy_config_json = b"persisted-marker".to_vec(); - sys.persist_and_set(persisted).await.expect("metadata should persist"); for dir in &dirs { std::fs::create_dir_all(dir.path().join("absent-bucket")).expect("persisted bucket directory should be created"); } + let mut persisted = BucketMetadata::new("absent-bucket"); + persisted.policy_config_json = b"persisted-marker".to_vec(); + sys.persist_new_and_set(persisted).await.expect("metadata should persist"); sys.metadata_map.write().await.clear(); let _ = sys .get_config("absent-bucket") @@ -1619,15 +2811,16 @@ mod tests { } let mut old_fenced = BucketMetadata::new(fenced_bucket); old_fenced.policy_config_json = b"old-fenced-generation".to_vec(); - sys.persist_and_set(old_fenced).await.unwrap(); + sys.persist_new_and_set(old_fenced).await.unwrap(); sys.metadata_map.write().await.remove(fenced_bucket); sys.lazy_load_lock_probe.store(true, std::sync::atomic::Ordering::Relaxed); let (loaded, _) = sys.get_config(fenced_bucket).await.unwrap(); sys.lazy_load_lock_probe.store(false, std::sync::atomic::Ordering::Relaxed); assert_eq!(loaded.policy_config_json, b"old-fenced-generation".to_vec()); - // (e) A refresh-load miss for a bucket that still exists must not - // replace an existing entry with a fabricated default. + // (e) A refresh-load miss invalidates stale authority without caching + // a fabricated default in the authoritative map. A later real publish + // restores authority. let mut kept = BucketMetadata::new("kept-bucket"); kept.policy_config_json = b"kept-marker".to_vec(); sys.set("kept-bucket".to_string(), Arc::new(kept)).await; @@ -1638,15 +2831,17 @@ mod tests { let refresh_targets = vec!["kept-bucket".to_string()]; sys.concurrent_load(&refresh_targets, &mut failed, MetadataLoadMode::Refresh) .await; - let kept = sys - .get("kept-bucket") - .await - .expect("existing entry must survive a refresh miss"); - assert_eq!( - kept.policy_config_json, - b"kept-marker".to_vec(), - "a fabricated refresh default must not replace real metadata" - ); + assert!(sys.get("kept-bucket").await.is_err()); + assert!(matches!( + sys.get_object_lock_config_state("kept-bucket").await.unwrap(), + ObjectLockConfigState::ConfirmedAbsent + )); + let restored = BucketMetadata::new("kept-bucket"); + sys.set("kept-bucket".to_string(), Arc::new(restored)).await; + assert!(matches!( + sys.get_object_lock_config_state("kept-bucket").await.unwrap(), + ObjectLockConfigState::ConfirmedAbsent + )); // (f) A stale cache entry for a physically deleted bucket must be // removed without recreating the bucket during periodic refresh. @@ -1744,6 +2939,15 @@ mod tests { dirs.iter().all(|dir| dir.path().join("initial-bucket").is_dir()), "initial load must heal buckets discovered from storage" ); + assert!( + sys.get("initial-bucket").await.is_err(), + "initial discovery must not cache fabricated metadata as authoritative" + ); + assert!(matches!( + sys.get_object_lock_config_state("initial-bucket").await.unwrap(), + ObjectLockConfigState::ConfirmedAbsent + )); + assert!(sys.get("initial-bucket").await.is_ok()); } #[tokio::test] @@ -1808,19 +3012,16 @@ mod tests { #[tokio::test] async fn update_config_with_persists_tagging_rewrite_across_disk_reload() { use crate::bucket::metadata::BUCKET_TAGGING_CONFIG; - use crate::storage_api_contracts::bucket::MakeBucketOptions; use s3s::dto::Tag; - let (_dirs, ecstore) = isolated_store_over_temp_disks().await; + let (dirs, ecstore) = isolated_store_over_temp_disks().await; let bucket = "swift-tagging-bucket"; - ecstore - .peer_sys - .make_bucket(bucket, &MakeBucketOptions::default()) - .await - .expect("bucket volume should be created"); - let mut sys = BucketMetadataSys::new(ecstore); - sys.persist_and_set(BucketMetadata::new(bucket)) + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created"); + } + let sys = BucketMetadataSys::new(ecstore); + sys.persist_new_and_set(BucketMetadata::new(bucket)) .await .expect("initial metadata should persist"); @@ -1877,13 +3078,16 @@ mod tests { use crate::bucket::metadata::BUCKET_TAGGING_CONFIG; use s3s::dto::Tag; - let (_dirs, ecstore) = isolated_store_over_temp_disks().await; + let (dirs, ecstore) = isolated_store_over_temp_disks().await; let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore))); let bucket = "swift-tagging-concurrent"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created"); + } sys.read() .await - .persist_and_set(BucketMetadata::new(bucket)) + .persist_new_and_set(BucketMetadata::new(bucket)) .await .expect("initial metadata should persist"); @@ -1971,10 +3175,21 @@ mod tests { // reload converges the cache to disk truth. let mut persisted = BucketMetadata::new("reload-bucket"); persisted.policy_config_json = b"persisted-marker".to_vec(); - sys.persist_and_set(persisted).await.expect("metadata should persist"); + let lock_config = ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }; + persisted + .update_config( + crate::bucket::metadata::OBJECT_LOCK_CONFIG, + crate::bucket::utils::serialize(&lock_config).unwrap(), + ) + .unwrap(); for dir in &dirs { std::fs::create_dir_all(dir.path().join("reload-bucket")).expect("physical bucket should exist before reload"); } + let persisted_incarnation = persisted.bucket_incarnation_id; + sys.persist_new_and_set(persisted).await.expect("metadata should persist"); let mut stale = BucketMetadata::new("reload-bucket"); stale.policy_config_json = b"stale-cache-marker".to_vec(); sys.set("reload-bucket".to_string(), Arc::new(stale)).await; @@ -1990,6 +3205,14 @@ mod tests { b"persisted-marker".to_vec(), "a reload must converge the cache to the persisted disk state" ); + assert_eq!( + cached.bucket_incarnation_id, persisted_incarnation, + "peer reload must publish the persisted bucket incarnation" + ); + assert!(matches!( + sys.get_object_lock_config_state("reload-bucket").await.unwrap(), + ObjectLockConfigState::Configured { .. } + )); } /// Two metadata systems over one backing store: the in-process stand-in @@ -2003,6 +3226,117 @@ mod tests { (dirs, node_a, node_b) } + #[tokio::test] + #[serial] + async fn disk_incarnation_read_detects_stale_cache_until_peer_reload() { + let (dirs, node_a, node_b) = two_nodes_over_one_store().await; + let bucket = "cross-node-incarnation-refresh"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).unwrap(); + } + + let old = BucketMetadata::new(bucket); + let old_incarnation = old.bucket_incarnation_id; + node_a.read().await.persist_new_and_set(old).await.unwrap(); + + let replacement = BucketMetadata::new(bucket); + let replacement_incarnation = replacement.bucket_incarnation_id; + assert_ne!(old_incarnation, replacement_incarnation); + node_b.read().await.persist_new_and_set(replacement).await.unwrap(); + + let node_a_sys = node_a.read().await.clone(); + + assert_eq!( + node_a_sys.get(bucket).await.unwrap().bucket_incarnation_id, + old_incarnation, + "node A should start with the predecessor incarnation cached" + ); + assert_eq!( + node_a_sys.get_bucket_incarnation_id_from_disk(bucket).await.unwrap(), + replacement_incarnation, + "commit-time incarnation validation must use persisted authority" + ); + assert_eq!( + node_a_sys.get(bucket).await.unwrap().bucket_incarnation_id, + old_incarnation, + "an incarnation-only read must not publish a partial metadata snapshot" + ); + node_a_sys.reload_from_store(bucket).await.unwrap(); + assert_eq!( + node_a_sys.get(bucket).await.unwrap().bucket_incarnation_id, + replacement_incarnation, + "peer reload must converge the complete cached metadata snapshot" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn stale_config_request_cannot_mutate_a_recreated_bucket() { + use crate::bucket::metadata::BUCKET_TAGGING_CONFIG; + + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let sys = bucket_metadata_sys_of(&store.ctx).unwrap(); + let bucket = "stale-config-request"; + + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + store.delete_bucket(bucket, &DeleteBucketOptions::default()).await.unwrap(); + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + let new_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + assert_ne!(old_incarnation, new_incarnation); + + let err = + update_with_sys_expected(sys.clone(), bucket, BUCKET_TAGGING_CONFIG, b"".to_vec(), Some(old_incarnation)) + .await + .expect_err("a request authorized for the deleted incarnation must fail closed"); + assert!(matches!(err, Error::BucketNotFound(name) if name == bucket)); + + let persisted = sys.read().await.get_config_from_disk(bucket).await.unwrap(); + assert_eq!(persisted.bucket_incarnation_id, new_incarnation); + assert!(persisted.tagging_config_xml.is_empty()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[serial] + async fn bucket_delete_waits_for_config_mutation_fence() { + use crate::bucket::metadata::BUCKET_TAGGING_CONFIG; + use s3s::dto::Tag; + + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let sys = bucket_metadata_sys_of(&store.ctx).unwrap(); + let bucket = "delete-vs-config-mutation"; + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + + let guard = acquire_config_write_guard(sys.clone(), bucket).await.unwrap(); + let delete = tokio::spawn({ + let store = store.clone(); + async move { store.delete_bucket(bucket, &DeleteBucketOptions::default()).await } + }); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!(!delete.is_finished(), "DeleteBucket must wait for the config mutation lifecycle fence"); + + let tagging = crate::bucket::utils::serialize(&Tagging { + tag_set: vec![Tag { + key: Some("generation".to_string()), + value: Some("old".to_string()), + }], + }) + .unwrap(); + update_under_config_write_guard(sys, &guard, BUCKET_TAGGING_CONFIG, tagging) + .await + .unwrap(); + assert!(!delete.is_finished()); + drop(guard); + + timeout(Duration::from_secs(10), delete) + .await + .expect("DeleteBucket should resume after the config fence is released") + .expect("DeleteBucket task should join") + .expect("DeleteBucket should succeed"); + } + /// Writers on different nodes updating *different* config files of one /// bucket must both survive. Each rewrites the whole metadata blob, so /// without a lock spanning the read-modify-write the later save carries @@ -2013,12 +3347,15 @@ mod tests { async fn concurrent_config_writes_from_separate_nodes_do_not_lose_writes() { use crate::bucket::metadata::{BUCKET_POLICY_CONFIG, BUCKET_TAGGING_CONFIG}; - let (_dirs, node_a, node_b) = two_nodes_over_one_store().await; + let (dirs, node_a, node_b) = two_nodes_over_one_store().await; let bucket = "cross-node-config-writes"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created"); + } node_a .read() .await - .persist_and_set(BucketMetadata::new(bucket)) + .persist_new_and_set(BucketMetadata::new(bucket)) .await .expect("initial metadata should persist"); @@ -2081,12 +3418,15 @@ mod tests { async fn bucket_metadata_transaction_lock_blocks_a_concurrent_config_write() { use crate::bucket::metadata::BUCKET_TAGGING_CONFIG; - let (_dirs, node_a, node_b) = two_nodes_over_one_store().await; + let (dirs, node_a, node_b) = two_nodes_over_one_store().await; let bucket = "cross-node-transaction-lock"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created"); + } node_a .read() .await - .persist_and_set(BucketMetadata::new(bucket)) + .persist_new_and_set(BucketMetadata::new(bucket)) .await .expect("initial metadata should persist"); diff --git a/crates/ecstore/src/bucket/object_lock/objectlock_sys.rs b/crates/ecstore/src/bucket/object_lock/objectlock_sys.rs index f8209ea6e..b85c6976b 100644 --- a/crates/ecstore/src/bucket/object_lock/objectlock_sys.rs +++ b/crates/ecstore/src/bucket/object_lock/objectlock_sys.rs @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::bucket::metadata_sys::get_object_lock_config; +use crate::bucket::metadata_sys::{ObjectLockConfigState, get_object_lock_config, get_object_lock_config_state}; use crate::bucket::object_lock::objectlock; +use crate::error::{Error, Result, StorageError}; use crate::object_api::ObjectInfo; -use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode}; +use s3s::dto::{Date, DefaultRetention, ObjectLockConfiguration, ObjectLockLegalHoldStatus, ObjectLockRetentionMode}; +use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE}; use std::sync::Arc; use time::OffsetDateTime; @@ -37,6 +39,20 @@ impl BucketObjectLockSys { } } +pub(crate) fn ensure_recursive_force_delete_allowed_for_state(bucket: &str, state: &ObjectLockConfigState) -> Result<()> { + match state { + ObjectLockConfigState::ConfirmedAbsent => Ok(()), + ObjectLockConfigState::Configured { .. } => Err(StorageError::InvalidArgument( + bucket.to_string(), + String::new(), + "force-delete is forbidden on Object Locking enabled buckets".to_string(), + )), + ObjectLockConfigState::Fabricated => { + Err(Error::other(format!("bucket Object Lock metadata is not authoritative: {bucket}"))) + } + } +} + /// Check if a retention period is still active based on mode and retain_until_date pub fn is_retention_active(mode: &str, retain_until_date: Option<&s3s::dto::Date>) -> bool { if mode != ObjectLockRetentionMode::COMPLIANCE && mode != ObjectLockRetentionMode::GOVERNANCE { @@ -205,71 +221,122 @@ fn check_retention_blocks_deletion( None } +/// Check an object's lock metadata using an already resolved bucket Object +/// Lock configuration. `None` means the configuration is confirmed absent. +/// /// # S3 Standard Behavior /// - COMPLIANCE mode: Cannot be deleted even with bypass header /// - GOVERNANCE mode: Can be deleted if bypass_governance is true (caller must verify s3:BypassGovernanceRetention permission) /// - Legal Hold: Cannot be bypassed regardless of mode -pub async fn check_object_lock_for_deletion( - bucket: &str, +pub(crate) fn check_object_lock_for_deletion_with_config( + config: Option<&ObjectLockConfiguration>, obj_info: &ObjectInfo, bypass_governance: bool, -) -> Option { +) -> Result> { if obj_info.delete_marker { - return None; + return Ok(None); } - // 1. Check legal hold - cannot be bypassed (reuse has_legal_hold) - if has_legal_hold(&obj_info.user_defined) { - return Some(ObjectLockBlockReason::LegalHold); - } - - // 2. Check explicit retention - let explicit_ret = objectlock::get_object_retention_meta(&obj_info.user_defined); - if let Some(mode) = &explicit_ret.mode { - let mode_str = mode.as_str(); - if is_retention_active(mode_str, explicit_ret.retain_until_date.as_ref()) - && let Some(reason) = check_retention_blocks_deletion( - mode_str, - explicit_ret.retain_until_date.map(OffsetDateTime::from), - bypass_governance, - ) - { - return Some(reason); + if let Some(status) = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()) { + if status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::ON) { + return Ok(Some(ObjectLockBlockReason::LegalHold)); + } + if !status.eq_ignore_ascii_case(ObjectLockLegalHoldStatus::OFF) { + return Err(Error::other("persisted object legal-hold metadata is invalid")); } } - // 3. Check default retention only if no explicit retention is set - if explicit_ret.mode.is_none() - && let Some(default_retention) = BucketObjectLockSys::get(bucket).await + let mode = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_MODE.as_str()); + let retain_until = obj_info.user_defined.get(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()); + let explicit_ret = match (mode, retain_until) { + (None, None) => None, + (Some(mode), Some(retain_until)) => { + let mode = + objectlock::parse_ret_mode(mode).ok_or_else(|| Error::other("persisted object retention mode is invalid"))?; + let retain_until = OffsetDateTime::parse(retain_until, &time::format_description::well_known::Iso8601::DEFAULT) + .map(Date::from) + .map_err(|_| Error::other("persisted object retention date is invalid"))?; + Some((mode, retain_until)) + } + _ => return Err(Error::other("persisted object retention metadata is incomplete")), + }; + + if let Some((mode, retain_until)) = &explicit_ret { + let mode_str = mode.as_str(); + if is_retention_active(mode_str, Some(retain_until)) + && let Some(reason) = + check_retention_blocks_deletion(mode_str, Some(OffsetDateTime::from(retain_until.clone())), bypass_governance) + { + return Ok(Some(reason)); + } + } + + if explicit_ret.is_none() + && let Some(default_retention) = config.and_then(|config| config.rule.as_ref()?.default_retention.as_ref()) && let Some(mode) = &default_retention.mode { let mode_str = mode.as_str(); if mode_str == ObjectLockRetentionMode::COMPLIANCE || mode_str == ObjectLockRetentionMode::GOVERNANCE { // Calculate retention expiration date from object modification time - if let Some(mod_time) = obj_info.mod_time { - let now = objectlock::utc_now_ntp(); - let retain_until = if let Some(days) = default_retention.days { - mod_time.saturating_add(time::Duration::days(days as i64)) - } else { - let years = default_retention.years?; - add_years(mod_time, years) - }; + let mod_time = obj_info + .mod_time + .ok_or_else(|| Error::other("persisted object modification time is missing"))?; + let now = objectlock::utc_now_ntp(); + let retain_until = if let Some(days) = default_retention.days { + mod_time.saturating_add(time::Duration::days(i64::from(days))) + } else { + let years = default_retention + .years + .ok_or_else(|| Error::other("persisted bucket Object Lock retention period is invalid"))?; + add_years(mod_time, years) + }; - if retain_until.unix_timestamp() > now.unix_timestamp() - && let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance) - { - return Some(reason); - } + if retain_until.unix_timestamp() > now.unix_timestamp() + && let Some(reason) = check_retention_blocks_deletion(mode_str, Some(retain_until), bypass_governance) + { + return Ok(Some(reason)); } } } - None + Ok(None) +} + +pub(crate) fn check_object_lock_for_deletion_with_state( + state: &ObjectLockConfigState, + obj_info: &ObjectInfo, + bypass_governance: bool, +) -> Result> { + match state { + ObjectLockConfigState::Configured { config, .. } => { + check_object_lock_for_deletion_with_config(Some(config), obj_info, bypass_governance) + } + ObjectLockConfigState::ConfirmedAbsent => check_object_lock_for_deletion_with_config(None, obj_info, bypass_governance), + ObjectLockConfigState::Fabricated => Err(Error::other("bucket Object Lock metadata is not authoritative")), + } +} + +/// Compatibility wrapper for callers that predate fallible metadata lookup. +/// An authority/read/parse failure is represented as a blocking reason rather +/// than the old fail-open `None` result. +pub async fn check_object_lock_for_deletion( + bucket: &str, + obj_info: &ObjectInfo, + bypass_governance: bool, +) -> Option { + match get_object_lock_config_state(bucket) + .await + .and_then(|state| check_object_lock_for_deletion_with_state(&state, obj_info, bypass_governance)) + { + Ok(reason) => reason, + Err(_) => Some(ObjectLockBlockReason::LegalHold), + } } #[cfg(test)] mod tests { use super::*; + use s3s::dto::{ObjectLockEnabled, ObjectLockRule}; use time::{Date, Month, PrimitiveDateTime, Time}; fn make_datetime(year: i32, month: u8, day: u8) -> OffsetDateTime { @@ -278,6 +345,160 @@ mod tests { PrimitiveDateTime::new(date, time).assume_utc() } + fn default_retention_config(mode: &'static str) -> ObjectLockConfiguration { + ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: Some(ObjectLockRule { + default_retention: Some(DefaultRetention { + mode: Some(ObjectLockRetentionMode::from_static(mode)), + days: Some(30), + years: None, + }), + }), + } + } + + #[test] + fn deletion_with_config_blocks_active_default_compliance_even_with_bypass() { + let config = default_retention_config(ObjectLockRetentionMode::COMPLIANCE); + let obj_info = ObjectInfo { + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }; + + let result = check_object_lock_for_deletion_with_config(Some(&config), &obj_info, true); + + assert!(matches!(result, Ok(Some(ObjectLockBlockReason::Retention { .. })))); + } + + #[test] + fn deletion_with_config_allows_active_default_governance_with_bypass() { + let config = default_retention_config(ObjectLockRetentionMode::GOVERNANCE); + let obj_info = ObjectInfo { + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }; + + assert!(matches!( + check_object_lock_for_deletion_with_config(Some(&config), &obj_info, true), + Ok(None) + )); + } + + #[test] + fn deletion_with_default_retention_rejects_missing_object_mod_time() { + let config = default_retention_config(ObjectLockRetentionMode::COMPLIANCE); + + let err = check_object_lock_for_deletion_with_config(Some(&config), &ObjectInfo::default(), false) + .expect_err("default retention needs an authoritative object modification time"); + + assert!(err.to_string().contains("modification time")); + } + + #[test] + fn deletion_with_confirmed_absence_still_blocks_explicit_compliance() { + let retain_until = OffsetDateTime::now_utc() + time::Duration::days(30); + let mut user_defined = std::collections::HashMap::new(); + user_defined.insert("x-amz-object-lock-mode".to_string(), ObjectLockRetentionMode::COMPLIANCE.to_string()); + user_defined.insert( + "x-amz-object-lock-retain-until-date".to_string(), + retain_until + .format(&time::format_description::well_known::Rfc3339) + .expect("retain-until date should format"), + ); + let obj_info = ObjectInfo { + user_defined: Arc::new(user_defined), + ..Default::default() + }; + + let result = check_object_lock_for_deletion_with_config(None, &obj_info, true); + + assert!(matches!(result, Ok(Some(ObjectLockBlockReason::Retention { .. })))); + } + + #[test] + fn deletion_with_fabricated_bucket_metadata_fails_closed() { + let err = check_object_lock_for_deletion_with_state(&ObjectLockConfigState::Fabricated, &ObjectInfo::default(), false) + .expect_err("non-authoritative Object Lock metadata must block deletion"); + + assert!(err.to_string().contains("not authoritative")); + } + + #[test] + fn recursive_force_delete_with_fabricated_bucket_metadata_fails_closed() { + let err = ensure_recursive_force_delete_allowed_for_state("bucket", &ObjectLockConfigState::Fabricated) + .expect_err("non-authoritative Object Lock metadata must block recursive deletion"); + + assert!(err.to_string().contains("not authoritative")); + } + + #[test] + fn deletion_rejects_incomplete_persisted_retention_metadata() { + let mut user_defined = std::collections::HashMap::new(); + user_defined.insert( + X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), + ObjectLockRetentionMode::COMPLIANCE.to_string(), + ); + let obj_info = ObjectInfo { + user_defined: Arc::new(user_defined), + ..Default::default() + }; + + let err = check_object_lock_for_deletion_with_config(None, &obj_info, false) + .expect_err("mode without retain-until date must fail closed"); + + assert!(err.to_string().contains("incomplete")); + } + + #[test] + fn deletion_rejects_each_malformed_persisted_retention_shape() { + let valid_date = (OffsetDateTime::now_utc() + time::Duration::days(30)) + .format(&time::format_description::well_known::Rfc3339) + .expect("retain-until date should format"); + let cases = [ + ("invalid mode", Some("INVALID"), Some(valid_date.as_str()), "retention mode"), + ( + "invalid date", + Some(ObjectLockRetentionMode::COMPLIANCE), + Some("not-a-date"), + "retention date", + ), + ("date only", None, Some(valid_date.as_str()), "incomplete"), + ]; + + for (case, mode, retain_until, expected) in cases { + let mut user_defined = std::collections::HashMap::new(); + if let Some(mode) = mode { + user_defined.insert(X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), mode.to_string()); + } + if let Some(retain_until) = retain_until { + user_defined.insert(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until.to_string()); + } + let obj_info = ObjectInfo { + user_defined: Arc::new(user_defined), + ..Default::default() + }; + + let err = check_object_lock_for_deletion_with_config(None, &obj_info, false).expect_err(case); + assert!(err.to_string().contains(expected), "unexpected {case} error: {err}"); + } + } + + #[test] + fn deletion_rejects_invalid_persisted_legal_hold_metadata() { + let mut user_defined = std::collections::HashMap::new(); + user_defined.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "INVALID".to_string()); + let obj_info = ObjectInfo { + user_defined: Arc::new(user_defined), + ..Default::default() + }; + + let err = check_object_lock_for_deletion_with_config(None, &obj_info, false) + .expect_err("invalid legal-hold value must fail closed"); + + assert!(err.to_string().contains("legal-hold")); + } + #[test] fn test_add_years_normal() { // Normal case: add 1 year to a regular date diff --git a/crates/ecstore/src/bucket/quota/checker.rs b/crates/ecstore/src/bucket/quota/checker.rs index f8e21a75b..2bfe2173f 100644 --- a/crates/ecstore/src/bucket/quota/checker.rs +++ b/crates/ecstore/src/bucket/quota/checker.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::{BucketQuota, QuotaCheckResult, QuotaError, QuotaOperation}; -use crate::bucket::metadata_sys::{BucketMetadataSys, update}; +use crate::bucket::metadata_sys::{BucketMetadataSys, update, update_if_incarnation}; use crate::data_usage::get_bucket_usage_memory; use rustfs_common::metrics::Metric; use rustfs_config::QUOTA_CONFIG_FILE; @@ -145,14 +145,35 @@ impl QuotaChecker { } pub async fn set_quota_config(&mut self, bucket: &str, quota: BucketQuota) -> Result { + self.set_quota_config_for_incarnation(bucket, quota, None).await + } + + pub async fn set_quota_config_if_incarnation( + &mut self, + bucket: &str, + quota: BucketQuota, + expected_incarnation_id: uuid::Uuid, + ) -> Result { + self.set_quota_config_for_incarnation(bucket, quota, Some(expected_incarnation_id)) + .await + } + + async fn set_quota_config_for_incarnation( + &mut self, + bucket: &str, + quota: BucketQuota, + expected_incarnation_id: Option, + ) -> Result { let json_data = serde_json::to_vec("a).map_err(|e| QuotaError::InvalidConfig { reason: format!("Failed to serialize quota config: {}", e), })?; let start_time = Instant::now(); - let updated_at = update(bucket, QUOTA_CONFIG_FILE, json_data) - .await - .map_err(QuotaError::StorageError)?; + let updated_at = match expected_incarnation_id { + Some(incarnation_id) => update_if_incarnation(bucket, QUOTA_CONFIG_FILE, json_data, incarnation_id).await, + None => update(bucket, QUOTA_CONFIG_FILE, json_data).await, + } + .map_err(QuotaError::StorageError)?; rustfs_common::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed()); Ok(updated_at) diff --git a/crates/ecstore/src/client/object_handlers_common.rs b/crates/ecstore/src/client/object_handlers_common.rs index 44fa67db4..fa4739252 100644 --- a/crates/ecstore/src/client/object_handlers_common.rs +++ b/crates/ecstore/src/client/object_handlers_common.rs @@ -26,8 +26,15 @@ use crate::object_api::ObjectOptions; use crate::storage_api_contracts::object::{ObjectOperations as _, ObjectToDelete}; use crate::store::ECStore; use rustfs_lock::MAX_DELETE_LIST; +use uuid::Uuid; -pub async fn delete_object_versions(api: &Arc, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) { +pub async fn delete_object_versions( + api: &Arc, + bucket: &str, + to_del: &[ObjectToDelete], + _lc_event: lifecycle::Event, + bucket_incarnation_id: Uuid, +) { let delete_config_snapshot = match ReplicationObjectBridge::delete_request_config(api, bucket).await { Ok(snapshot) => Arc::new(snapshot), Err(err) => { @@ -59,6 +66,7 @@ pub async fn delete_object_versions(api: &Arc, bucket: &str, to_del: &[ to_del.to_vec(), ObjectOptions { delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)), + expected_bucket_incarnation_id: Some(bucket_incarnation_id), ..Default::default() }, ) diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 057de9d0b..f74379069 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -435,6 +435,22 @@ where Ok(data) } +pub(crate) async fn read_config_no_lock_preserve_empty_with_metadata(api: Arc, file: &str) -> Result<(Vec, ObjectInfo)> +where + S: EcstoreObjectIO, +{ + read_config_with_metadata_inner( + api, + file, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + true, + ) + .await +} + pub async fn read_config_with_metadata(api: Arc, file: &str, opts: &ObjectOptions) -> Result<(Vec, ObjectInfo)> where S: ObjectIO< @@ -2594,9 +2610,10 @@ mod tests { use super::{ SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config, - lookup_configs, new_and_save_server_config, read_config, read_config_preserve_empty, read_config_with_metadata, - read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot, - save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut, + lookup_configs, new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, + read_config_preserve_empty, read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, + save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation, + server_config_transaction_lock_path, storage_class_kvs_mut, }; use crate::config::{audit, heal, notify, oidc, scanner}; use crate::disk::endpoint::Endpoint; @@ -5027,10 +5044,15 @@ mod tests { .expect_err("the existing config contract treats empty objects as missing"); assert!(matches!(err, Error::ConfigNotFound)); - let data = read_config_preserve_empty(store, "config/empty.json") + let data = read_config_preserve_empty(store.clone(), "config/empty.json") .await .expect("payload-validating callers must observe the empty object"); assert!(data.is_empty()); + + let (data, _) = read_config_no_lock_preserve_empty_with_metadata(store, "config/empty.json") + .await + .expect("no-lock payload-validating callers must observe the empty object"); + assert!(data.is_empty()); } #[async_trait::async_trait] diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 7e09d5eaa..aef124c46 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -18,13 +18,13 @@ use crate::bucket::{ lifecycle::{ bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{ - LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule, eval_action_from_lifecycle, + LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle, lifecycle_delete_all_versions_blocked_by_replication, }, + get_expiry_configs, lifecycle::IlmAction, }, metadata_sys, - object_lock::objectlock_sys::BucketObjectLockSys, }; use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; use crate::config::com::{CONFIG_PREFIX, read_config, read_config_no_lock, save_config, save_config_with_opts}; @@ -60,7 +60,7 @@ use rustfs_common::defer; use rustfs_common::heal_channel::HealOpts; use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; -use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration}; +use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt::Display; @@ -2192,7 +2192,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement( bucket: &str, version: &rustfs_filemeta::FileInfo, lifecycle_config: Option<&BucketLifecycleConfiguration>, - lock_retention: Option, + object_lock_config: Option<&ObjectLockConfiguration>, apply_actions: bool, event_source: &LcEventSrc, ) -> Result { @@ -2202,12 +2202,16 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement( let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await; let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned); - let event = eval_action_from_lifecycle(lifecycle_config, lock_retention, &object_info).await; + let event = eval_action_from_lifecycle(lifecycle_config, object_lock_config, &object_info).await; match event.action { IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => { if apply_actions && object_info.is_remote() { - let _ = apply_expiry_on_transitioned_object(store, &object_info, &event, event_source).await; + let Ok(bucket_incarnation_id) = store.bucket_incarnation_id_from_disk(bucket).await else { + return Ok(false); + }; + let _ = + apply_expiry_on_transitioned_object(store, &object_info, &event, event_source, bucket_incarnation_id).await; } Ok(false) } @@ -2215,7 +2219,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement( if lifecycle_delete_all_versions_blocked_by_replication(store.clone(), bucket, &object_info.name, action).await? { return Ok(false); } - let applied = !apply_actions || apply_expiry_rule(&event, event_source, &object_info).await; + let applied = !apply_actions || apply_expiry_rule_in(store, &event, event_source, &object_info).await; resolve_data_movement_lifecycle_expiry_result(action, apply_actions, applied) } _ => Ok(false), @@ -2647,7 +2651,7 @@ impl ECStore { } #[allow(unused_assignments, clippy::too_many_arguments)] - #[tracing::instrument(skip(self, set, _worker_permit, lifecycle_config, lock_retention, replication_config))] + #[tracing::instrument(skip(self, set, _worker_permit, lifecycle_config, object_lock_config, replication_config))] async fn decommission_entry( self: &Arc, rx: CancellationToken, @@ -2657,7 +2661,7 @@ impl ECStore { set: Arc, _worker_permit: OwnedSemaphorePermit, lifecycle_config: Option, - lock_retention: Option, + object_lock_config: Option, replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, ) -> Result<()> { debug!( @@ -2708,7 +2712,7 @@ impl ECStore { &bucket, version, lifecycle_config.as_ref(), - lock_retention.clone(), + object_lock_config.as_ref(), true, &LcEventSrc::Decom, ) @@ -3113,7 +3117,7 @@ impl ECStore { let mut listing_workers = Vec::with_capacity(pool.disk_set.len()); let mut lifecycle_config = None; - let mut lock_retention = None; + let mut object_lock_config = None; let mut replication_config = None; if bi.name != RUSTFS_META_BUCKET { @@ -3122,8 +3126,9 @@ impl ECStore { "versioning", BucketVersioningSys::get(&bi.name).await, )?; - lifecycle_config = runtime_sources::bucket_lifecycle_config(&bi.name).await; - lock_retention = BucketObjectLockSys::get(&bi.name).await; + let expiry_configs = get_expiry_configs(self, &bi.name).await?; + lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone()); + object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone()); replication_config = resolve_decommission_optional_bucket_config_result( &bi.name, "replication", @@ -3155,7 +3160,7 @@ impl ECStore { let workers = workers.clone(); let set = set.clone(); let lifecycle_config = lifecycle_config.clone(); - let lock_retention = lock_retention.clone(); + let object_lock_config = object_lock_config.clone(); let replication_config = replication_config.clone(); let entry_error = entry_error.clone(); let callback_rx = rx.clone(); @@ -3165,7 +3170,7 @@ impl ECStore { let workers = workers.clone(); let set = set.clone(); let lifecycle_config = lifecycle_config.clone(); - let lock_retention = lock_retention.clone(); + let object_lock_config = object_lock_config.clone(); let replication_config = replication_config.clone(); let entry_error = entry_error.clone(); let callback_rx = callback_rx.clone(); @@ -3227,7 +3232,7 @@ impl ECStore { set, worker_permit, lifecycle_config, - lock_retention, + object_lock_config, replication_config, ) .await @@ -3960,10 +3965,11 @@ impl ECStore { for set in &pool.disk_set { for bucket_info in &buckets { let mut lifecycle_config = None; - let mut lock_retention = None; + let mut object_lock_config = None; if bucket_info.name != RUSTFS_META_BUCKET { - lifecycle_config = runtime_sources::bucket_lifecycle_config(&bucket_info.name).await; - lock_retention = BucketObjectLockSys::get(&bucket_info.name).await; + let expiry_configs = get_expiry_configs(self, &bucket_info.name).await?; + lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone()); + object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone()); } let versions_found = Arc::new(AtomicUsize::new(0)); @@ -3973,7 +3979,7 @@ impl ECStore { let entry_error_cb = entry_error.clone(); let bucket_name = bucket_info.name.clone(); let lifecycle_config_cb = lifecycle_config.clone(); - let lock_retention_cb = lock_retention.clone(); + let object_lock_config_cb = object_lock_config.clone(); let store = Arc::clone(self); let callback_rx_cb = callback_rx.clone(); @@ -3982,7 +3988,7 @@ impl ECStore { let entry_error = entry_error_cb.clone(); let bucket_name = bucket_name.clone(); let lifecycle_config = lifecycle_config_cb.clone(); - let lock_retention = lock_retention_cb.clone(); + let object_lock_config = object_lock_config_cb.clone(); let store = Arc::clone(&store); let callback_rx = callback_rx_cb.clone(); Box::pin(async move { @@ -4024,7 +4030,7 @@ impl ECStore { &bucket_name, version, lifecycle_config.as_ref(), - lock_retention.clone(), + object_lock_config.as_ref(), false, &LcEventSrc::Decom, ) diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index c2f0f7749..efe36f732 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -14,7 +14,7 @@ // limitations under the License. use crate::disk::error_reduce::count_errs; -use crate::error::{Error, Result}; +use crate::error::{Error, Result, is_all_volume_not_found, is_err_object_not_found, is_err_strict_volume_not_found}; use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets}; use crate::multipart_listing::paginate_multipart_listing; use crate::storage_api_contracts::{ @@ -71,6 +71,10 @@ type WalkOptions = StorageWalkOptions bool>; const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4; +fn is_idempotent_delete_prefix_error(err: &Error) -> bool { + is_err_object_not_found(err) || is_err_strict_volume_not_found(err) +} + #[derive(Debug, Clone)] pub struct Sets { pub id: Uuid, @@ -339,8 +343,18 @@ impl Sets { futures.push(set.delete_object(bucket, object, opt.clone())); } - if let Some(err) = join_all(futures).await.into_iter().find_map(Result::err) { - return Err(err); + let errs = join_all(futures) + .await + .into_iter() + .map(|result| result.err()) + .collect::>(); + if is_all_volume_not_found(&errs) { + return Err(StorageError::BucketNotFound(bucket.to_string())); + } + for err in errs.into_iter().flatten() { + if !is_idempotent_delete_prefix_error(&err) { + return Err(err); + } } Ok(()) @@ -816,8 +830,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for Sets { let upload_id_marker = upload_id_marker.clone(); let delimiter = delimiter.clone(); async move { - set.list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, per_set_limit) - .await + // ECStore owns the bucket lifecycle fence and calls the + // incarnation-aware pool helper. This lower-level trait + // surface has no ECStore guard to propagate. + set.list_multipart_uploads_for_incarnation( + bucket, + prefix, + key_marker, + upload_id_marker, + delimiter, + per_set_limit, + None, + ) + .await } }) .buffer_unordered(LIST_MULTIPART_SETS_CONCURRENCY) @@ -1276,6 +1301,19 @@ mod tests { ); } + #[test] + fn delete_prefix_error_classification_only_ignores_absence() { + assert!(is_idempotent_delete_prefix_error(&StorageError::FileNotFound)); + assert!(is_idempotent_delete_prefix_error(&StorageError::ObjectNotFound( + "bucket".to_string(), + "prefix".to_string() + ))); + assert!(is_idempotent_delete_prefix_error(&StorageError::VolumeNotFound)); + assert!(is_idempotent_delete_prefix_error(&StorageError::BucketNotFound("bucket".to_string()))); + assert!(!is_idempotent_delete_prefix_error(&StorageError::DiskNotFound)); + assert!(!is_idempotent_delete_prefix_error(&StorageError::ErasureWriteQuorum)); + } + #[tokio::test] async fn sets_get_pool_and_set_returns_matching_coordinates() { let format = FormatV3::new(2, 2); @@ -1393,6 +1431,161 @@ mod tests { (temp_dirs, sets) } + #[tokio::test] + async fn delete_prefix_surfaces_a_hard_error_from_any_set() { + let (_temp_dirs, sets) = two_set_test_sets().await; + let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple()); + sets.make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created across both sets"); + + let healthy_disks = sets.disk_set[0].disks.read().await.clone(); + for disk in healthy_disks.iter().flatten() { + disk.write_all(&bucket, "blocked/prefix/object", bytes::Bytes::from_static(b"data")) + .await + .expect("healthy set should contain the prefix"); + } + + let failing_disks = sets.disk_set[1].disks.read().await.clone(); + for disk in failing_disks.iter().flatten() { + disk.write_all(&bucket, "blocked", bytes::Bytes::from_static(b"not-a-directory")) + .await + .expect("failing set should contain a parent file"); + } + + let err = sets + .delete_object( + &bucket, + "blocked/prefix", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("a hard failure from one set must not be reported as success"); + + match err { + StorageError::PrefixAccessDenied(error_bucket, error_prefix) => { + assert_eq!(error_bucket, bucket); + assert_eq!(error_prefix, "blocked/prefix"); + } + other => panic!("unexpected recursive delete error: {other:?}"), + } + for disk in healthy_disks.iter().flatten() { + assert!( + matches!(disk.read_all(&bucket, "blocked/prefix/object").await, Err(DiskError::FileNotFound)), + "the healthy set should still complete its prefix deletion" + ); + } + } + + #[tokio::test] + async fn delete_prefix_keeps_a_missing_bucket_idempotent_across_sets() { + let (_temp_dirs, sets) = two_set_test_sets().await; + let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple()); + sets.make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created across both sets"); + + let healthy_disks = sets.disk_set[0].disks.read().await.clone(); + for disk in healthy_disks.iter().flatten() { + disk.write_all(&bucket, "existing/prefix/object", bytes::Bytes::from_static(b"data")) + .await + .expect("healthy set should contain the prefix"); + } + let missing_bucket_disks = sets.disk_set[1].disks.read().await.clone(); + for disk in missing_bucket_disks.iter().flatten() { + disk.delete_volume(&bucket, true) + .await + .expect("the bucket should be removed from one set"); + } + + sets.delete_object( + &bucket, + "existing/prefix", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("a missing bucket on one set should remain an idempotent success"); + for disk in healthy_disks.iter().flatten() { + assert!( + matches!(disk.read_all(&bucket, "existing/prefix/object").await, Err(DiskError::FileNotFound)), + "the healthy set should still complete its prefix deletion" + ); + } + } + + #[tokio::test] + async fn delete_prefix_preserves_a_completely_missing_bucket_error() { + let (_temp_dirs, sets) = two_set_test_sets().await; + let bucket = format!("delete-prefix-missing-{}", Uuid::new_v4().simple()); + + let err = sets + .delete_object( + &bucket, + "missing/prefix", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("a completely missing bucket must not be reported as a successful object deletion"); + + assert_eq!(err, StorageError::BucketNotFound(bucket)); + } + + #[tokio::test] + async fn delete_prefix_fails_when_one_set_is_entirely_offline() { + let (_temp_dirs, sets) = two_set_test_sets().await; + let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple()); + sets.make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created across both sets"); + + let online_disks = sets.disk_set[0].disks.read().await.clone(); + let offline_disks = sets.disk_set[1].disks.read().await.clone(); + for disk in online_disks.iter().chain(offline_disks.iter()).flatten() { + disk.write_all(&bucket, "offline/prefix/object", bytes::Bytes::from_static(b"data")) + .await + .expect("each set should contain the prefix before the outage"); + } + *sets.disk_set[1].disks.write().await = vec![None, None]; + + let err = sets + .delete_object( + &bucket, + "offline/prefix", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("an entirely offline set must make the recursive delete fail"); + + assert!( + matches!(err, StorageError::InsufficientWriteQuorum(ref error_bucket, ref error_prefix) + if error_bucket == &bucket && error_prefix == "offline/prefix"), + "unexpected offline-set error: {err:?}" + ); + for disk in online_disks.iter().flatten() { + assert!(matches!( + disk.read_all(&bucket, "offline/prefix/object").await, + Err(DiskError::FileNotFound) + )); + } + for disk in offline_disks.iter().flatten() { + disk.read_all(&bucket, "offline/prefix/object") + .await + .expect("the offline set's untouched prefix must still be present"); + } + } + #[tokio::test] async fn set_format_heal_accepts_quorum_from_a_nonzero_set() { let (_temp_dirs, sets) = two_set_test_sets().await; diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index 5f045b8e3..ab7b825e5 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -567,6 +567,7 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( ObjectOptions { delete_prefix: true, delete_prefix_object: true, + data_movement: true, no_lock: true, ..Default::default() }, diff --git a/crates/ecstore/src/disk/error.rs b/crates/ecstore/src/disk/error.rs index 8f3eb2ced..7c08115a2 100644 --- a/crates/ecstore/src/disk/error.rs +++ b/crates/ecstore/src/disk/error.rs @@ -321,6 +321,14 @@ fn io_error_chain_contains_kind(io_error: &std::io::Error, kind: std::io::ErrorK impl From for DiskError { fn from(e: std::io::Error) -> Self { + if let Some(error) = e.get_ref().and_then(|source| source.downcast_ref::()) { + if error.is_remote_file_not_found() { + return DiskError::FileNotFound; + } + if error.is_remote_volume_not_found() { + return DiskError::VolumeNotFound; + } + } match e.downcast::() { Ok(disk_error) => disk_error, Err(io_error) => DiskError::Io(io_error), @@ -1044,6 +1052,19 @@ mod tests { assert!(!DiskError::FileNotFound.is_internode_http_status(429)); } + #[test] + fn test_internode_missing_errors_preserve_disk_error_types() { + let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error()); + let volume_missing = DiskError::from(rustfs_rio::new_test_remote_volume_not_found_http_io_error()); + let unmarked_server_error = DiskError::from(rustfs_rio::new_test_internode_http_io_error( + rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR), + )); + + assert_eq!(file_missing, DiskError::FileNotFound); + assert_eq!(volume_missing, DiskError::VolumeNotFound); + assert!(matches!(unmarked_server_error, DiskError::Io(_))); + } + #[test] fn test_metacache_output_stream_closed_classification_survives_clone() { let disk_error = DiskError::metacache_output_stream_closed(); diff --git a/crates/ecstore/src/ecstore_validation_blackbox.rs b/crates/ecstore/src/ecstore_validation_blackbox.rs index b36fdde42..2531b39a8 100644 --- a/crates/ecstore/src/ecstore_validation_blackbox.rs +++ b/crates/ecstore/src/ecstore_validation_blackbox.rs @@ -1,9 +1,10 @@ +use crate::bucket::metadata_sys::ObjectLockConfigState; use crate::disk::endpoint::Endpoint; use crate::disk::format::FormatV3; use crate::disk::{DiskAPI, DiskOption, DiskStore, WalkDirOptions, new_disk}; use crate::error::Error; use crate::io_support::rio::HashReader; -use crate::object_api::{BLOCK_SIZE_V2, ObjectOptions, PutObjReader}; +use crate::object_api::{BLOCK_SIZE_V2, ObjectLockConfigSnapshot, ObjectOptions, PutObjReader}; use crate::set_disk::SetDisks; use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; @@ -357,6 +358,7 @@ async fn blackbox_delete_marker_hides_object_body_without_erasing_prior_version_ let opts = ObjectOptions { no_lock: true, version_suspended: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), ..Default::default() }; @@ -624,6 +626,9 @@ mod old_current_size_backfill { ObjectOptions { no_lock: true, versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), ..Default::default() }, ) diff --git a/crates/ecstore/src/layout/endpoints.rs b/crates/ecstore/src/layout/endpoints.rs index 24c789283..9a9b026d0 100644 --- a/crates/ecstore/src/layout/endpoints.rs +++ b/crates/ecstore/src/layout/endpoints.rs @@ -1744,9 +1744,11 @@ mod test { #[tokio::test] async fn system_resolver_negative_result_reaches_the_dns_allowlist() { - let err = get_host_ip(Host::Domain("rustfs-startup-negative.invalid")) - .await - .expect_err("the reserved .invalid domain must not resolve"); + let Err(err) = get_host_ip(Host::Domain("rustfs-startup-negative.invalid")).await else { + // Some corporate and ISP resolvers synthesize an address for + // unknown names, including the reserved .invalid suffix. + return; + }; assert!( is_retryable_dns_error(&err), "system resolver error kind {:?} and message {err:?} must retain retry provenance", diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 738a09e23..553fc0c29 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -20,43 +20,193 @@ use crate::storage_api_contracts::{ }, }; -#[derive(Clone, Default)] -pub struct DeleteLockFence { +#[derive(Clone)] +pub struct NamespaceLockFence { signals: Arc>>, #[cfg(test)] - forced_lost: bool, + forced_lost: Arc, } -impl Debug for DeleteLockFence { +impl Debug for NamespaceLockFence { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("DeleteLockFence") + f.debug_struct("NamespaceLockFence") .field("signal_count", &self.signals.len()) .finish() } } -impl DeleteLockFence { - pub(crate) fn new(signals: Vec>) -> Self { +impl NamespaceLockFence { + fn new() -> Self { Self { - signals: Arc::new(signals), + signals: Arc::default(), #[cfg(test)] - forced_lost: false, + forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)), } } pub(crate) fn is_lock_lost(&self) -> bool { #[cfg(test)] - if self.forced_lost { + if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) { return true; } self.signals.iter().any(|signal| signal.is_lost()) } + pub(crate) fn add_signal(&mut self, signal: Arc) { + Arc::make_mut(&mut self.signals).push(signal); + } + + fn extend(&mut self, other: &Self) { + if Arc::ptr_eq(&self.signals, &other.signals) { + return; + } + Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned()); + #[cfg(test)] + if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) { + self.forced_lost.store(true, std::sync::atomic::Ordering::Release); + } + } + #[cfg(test)] pub(crate) fn lost_for_test() -> Self { + let fence = Self::new(); + fence.forced_lost.store(true, std::sync::atomic::Ordering::Release); + fence + } + + #[cfg(test)] + pub(crate) fn loss_handle_for_test() -> (Self, Arc) { + let fence = Self::new(); + (fence.clone(), Arc::clone(&fence.forced_lost)) + } +} + +#[derive(Debug)] +pub struct ObjectLockConfigSnapshot { + store_id: Option, + bucket: Option, + bucket_incarnation_id: Option, + config_revision: Option, + state: crate::bucket::metadata_sys::ObjectLockConfigState, + lifecycle_fence: NamespaceLockFence, + _lifecycle_guard: Option, + metadata_transaction_guard: Option, +} + +impl ObjectLockConfigSnapshot { + pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self { Self { - signals: Arc::default(), - forced_lost: true, + store_id: None, + bucket: None, + bucket_incarnation_id: None, + config_revision: None, + state, + lifecycle_fence: NamespaceLockFence::new(), + _lifecycle_guard: None, + metadata_transaction_guard: None, + } + } + + pub(crate) fn for_store_bucket( + store_id: Uuid, + bucket: &str, + bucket_incarnation_id: Uuid, + config_revision: OffsetDateTime, + state: crate::bucket::metadata_sys::ObjectLockConfigState, + ) -> Self { + Self { + store_id: Some(store_id), + bucket: Some(bucket.to_string()), + bucket_incarnation_id: Some(bucket_incarnation_id), + config_revision: Some(config_revision), + state, + lifecycle_fence: NamespaceLockFence::new(), + _lifecycle_guard: None, + metadata_transaction_guard: None, + } + } + + pub(crate) fn for_guarded_store_bucket( + store_id: Uuid, + bucket: &str, + bucket_incarnation_id: Uuid, + config_revision: OffsetDateTime, + state: crate::bucket::metadata_sys::ObjectLockConfigState, + lifecycle_guard: rustfs_lock::NamespaceLockGuard, + metadata_transaction_guard: rustfs_lock::NamespaceLockGuard, + ) -> Self { + let mut lifecycle_fence = NamespaceLockFence::new(); + if let Some(signal) = lifecycle_guard.lock_lost_signal() { + lifecycle_fence.add_signal(signal); + } + Self { + store_id: Some(store_id), + bucket: Some(bucket.to_string()), + bucket_incarnation_id: Some(bucket_incarnation_id), + config_revision: Some(config_revision), + state, + lifecycle_fence, + _lifecycle_guard: Some(lifecycle_guard), + metadata_transaction_guard: Some(metadata_transaction_guard), + } + } + + pub(crate) fn for_store_bucket_under_lifecycle_fence( + store_id: Uuid, + bucket: &str, + bucket_incarnation_id: Uuid, + config_revision: OffsetDateTime, + state: crate::bucket::metadata_sys::ObjectLockConfigState, + lifecycle_fence: NamespaceLockFence, + metadata_transaction_guard: rustfs_lock::NamespaceLockGuard, + ) -> Self { + Self { + store_id: Some(store_id), + bucket: Some(bucket.to_string()), + bucket_incarnation_id: Some(bucket_incarnation_id), + config_revision: Some(config_revision), + state, + lifecycle_fence, + _lifecycle_guard: None, + metadata_transaction_guard: Some(metadata_transaction_guard), + } + } + + pub(crate) fn is_for_store_bucket( + &self, + store_id: Uuid, + bucket: &str, + bucket_incarnation_id: Uuid, + config_revision: OffsetDateTime, + ) -> bool { + self.store_id == Some(store_id) + && self.bucket.as_deref() == Some(bucket) + && self.bucket_incarnation_id == Some(bucket_incarnation_id) + && self.config_revision == Some(config_revision) + } + + pub fn state(&self) -> &crate::bucket::metadata_sys::ObjectLockConfigState { + &self.state + } + + pub(crate) fn is_valid_for_destructive_put(&self, store_id: Uuid, bucket: &str, bucket_incarnation_id: Uuid) -> bool { + self.store_id == Some(store_id) + && self.bucket.as_deref() == Some(bucket) + && self.bucket_incarnation_id == Some(bucket_incarnation_id) + && self.config_revision.is_some() + && !self.lifecycle_fence.is_lock_lost() + && self + .metadata_transaction_guard + .as_ref() + .is_some_and(|guard| !guard.is_lock_lost()) + } + + pub(crate) fn add_lock_fences(&self, opts: &mut ObjectOptions) { + opts.bucket_lifecycle_lock_fence + .get_or_insert_with(NamespaceLockFence::new) + .extend(&self.lifecycle_fence); + if let Some(guard) = self.metadata_transaction_guard.as_ref() { + opts.add_namespace_lock_guard(guard); } } } @@ -73,6 +223,8 @@ pub struct ObjectOptions { pub version_id: Option, /// RustFS-only compare-and-set condition checked under the object write lock. pub expected_current_version_id: Option, + /// Persisted bucket incarnation observed before authorization. + pub expected_bucket_incarnation_id: Option, pub no_lock: bool, /// True when an upper layer already holds the object read lock before /// forwarding a no_lock read to the set layer. @@ -96,7 +248,10 @@ pub struct ObjectOptions { pub delete_replication: Option, pub delete_replication_config_snapshot: Option>, - pub delete_lock_fence: Option, + pub namespace_lock_fence: Option, + /// Proves an upper layer holds the bucket lifecycle sentinel. A separate + /// fence avoids recursively acquiring the read lock behind a queued writer. + pub bucket_lifecycle_lock_fence: Option, pub replication_request: bool, pub delete_marker: bool, pub synthetic_version_id: bool, @@ -108,6 +263,9 @@ pub struct ObjectOptions { pub eval_metadata: Option>, pub object_lock_retention: Option, pub object_lock_delete: Option, + /// Authoritative bucket Object Lock snapshot installed inside `ECStore` + /// before a destructive commit reaches the set layer. + pub object_lock_config_snapshot: Option>, pub want_checksum: Option, pub skip_verify_bitrot: bool, @@ -118,6 +276,33 @@ pub struct ObjectOptions { } impl ObjectOptions { + pub(crate) fn overwrites_existing_version(&self) -> bool { + self.version_id.is_some() || !self.versioned || self.version_suspended + } + + pub(crate) fn add_namespace_lock_lost_signal(&mut self, signal: Arc) { + self.namespace_lock_fence + .get_or_insert_with(NamespaceLockFence::new) + .add_signal(signal); + } + + pub(crate) fn ensure_namespace_lock_fence(&mut self) { + self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new); + } + + pub fn add_namespace_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) { + if let Some(signal) = guard.lock_lost_signal() { + self.add_namespace_lock_lost_signal(signal); + } + } + + pub fn add_bucket_lifecycle_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) { + let fence = self.bucket_lifecycle_lock_fence.get_or_insert_with(NamespaceLockFence::new); + if let Some(signal) = guard.lock_lost_signal() { + fence.add_signal(signal); + } + } + pub fn set_delete_replication_state(&mut self, dsc: ReplicateDecision) { let mut rs = ReplicationState { replicate_decision_str: dsc.to_string(), @@ -565,8 +750,17 @@ impl ObjectInfo { delimiter: Option, after_version_marker: Option, ) -> Vec { - Self::from_meta_cache_entries_sorted_versions_with_purge(entries, bucket, prefix, delimiter, after_version_marker, false) - .await + Self::from_meta_cache_entries_sorted_versions_with_purge( + entries, + bucket, + prefix, + delimiter, + after_version_marker, + false, + false, + ) + .await + .0 } pub(crate) async fn from_meta_cache_entries_sorted_versions_for_lifecycle( @@ -576,8 +770,40 @@ impl ObjectInfo { delimiter: Option, after_version_marker: Option, ) -> Vec { - Self::from_meta_cache_entries_sorted_versions_with_purge(entries, bucket, prefix, delimiter, after_version_marker, true) - .await + Self::from_meta_cache_entries_sorted_versions_with_purge( + entries, + bucket, + prefix, + delimiter, + after_version_marker, + true, + false, + ) + .await + .0 + } + + pub(crate) async fn from_meta_cache_entries_sorted_versions_for_recursive_delete( + entries: &MetaCacheEntriesSorted, + bucket: &str, + prefix: &str, + delimiter: Option, + after_version_marker: Option, + ) -> Result> { + let (objects, error) = Self::from_meta_cache_entries_sorted_versions_with_purge( + entries, + bucket, + prefix, + delimiter, + after_version_marker, + true, + true, + ) + .await; + match error { + Some(error) => Err(error), + None => Ok(objects), + } } async fn from_meta_cache_entries_sorted_versions_with_purge( @@ -587,7 +813,8 @@ impl ObjectInfo { delimiter: Option, after_version_marker: Option, include_version_purge: bool, - ) -> Vec { + fail_on_decode_error: bool, + ) -> (Vec, Option) { let vcfg = get_versioning_config(bucket).await.ok(); let mut objects = Vec::with_capacity(entries.entries().len()); let mut prev_prefix = ""; @@ -623,6 +850,9 @@ impl ObjectInfo { let file_infos = match entry.file_info_versions(bucket) { Ok(res) => res, Err(err) => { + if fail_on_decode_error { + return (objects, Some(err.into())); + } warn!("file_info_versions err {:?}", err); continue; } @@ -674,7 +904,7 @@ impl ObjectInfo { } } - objects + (objects, None) } pub async fn from_meta_cache_entries_sorted_infos( @@ -836,6 +1066,25 @@ fn versions_after_marker(file_infos: &rustfs_filemeta::FileInfoVersions, marker: #[cfg(test)] mod tests { use super::*; + + #[test] + fn object_lock_config_snapshot_is_bound_to_store_bucket_and_incarnation() { + let store_id = Uuid::new_v4(); + let incarnation_id = Uuid::new_v4(); + let snapshot = ObjectLockConfigSnapshot::for_store_bucket( + store_id, + "source-bucket", + incarnation_id, + OffsetDateTime::UNIX_EPOCH, + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + ); + + assert!(snapshot.is_for_store_bucket(store_id, "source-bucket", incarnation_id, OffsetDateTime::UNIX_EPOCH)); + assert!(!snapshot.is_for_store_bucket(Uuid::new_v4(), "source-bucket", incarnation_id, OffsetDateTime::UNIX_EPOCH)); + assert!(!snapshot.is_for_store_bucket(store_id, "other-bucket", incarnation_id, OffsetDateTime::UNIX_EPOCH)); + assert!(!snapshot.is_for_store_bucket(store_id, "source-bucket", Uuid::new_v4(), OffsetDateTime::UNIX_EPOCH)); + assert!(!snapshot.is_for_store_bucket(store_id, "source-bucket", incarnation_id, OffsetDateTime::now_utc())); + } use rustfs_filemeta::{FileInfo, FileMeta, MetaCacheEntry, TRANSITION_COMPLETE}; fn inline_fast_path_object(size: i64, versioned: bool) -> ObjectInfo { @@ -951,10 +1200,11 @@ mod tests { #[test] fn object_info_replication_helpers_parse_target_status_and_reset_headers() { let reset_key = rustfs_utils::http::internal_key_rustfs("replication-reset-arn:target-a"); + let user_defined = HashMap::from([(reset_key, "reset-id".to_string())]); let object = ObjectInfo { replication_status_internal: Some("arn:target-a=COMPLETED;arn:target-b=FAILED;".to_string()), version_purge_status_internal: Some("arn:target-a=PENDING;".to_string()), - user_defined: Arc::new(HashMap::from([(reset_key, "reset-id".to_string())])), + user_defined: Arc::new(user_defined), ..Default::default() }; diff --git a/crates/ecstore/src/services/rebalance/entry.rs b/crates/ecstore/src/services/rebalance/entry.rs index d5c5e6740..4fc043286 100644 --- a/crates/ecstore/src/services/rebalance/entry.rs +++ b/crates/ecstore/src/services/rebalance/entry.rs @@ -118,7 +118,7 @@ impl ECStore { &bucket, version, bucket_configs.lifecycle_config.as_ref(), - bucket_configs.lock_retention.clone(), + bucket_configs.object_lock_config.as_ref(), true, &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Rebal, ) @@ -384,7 +384,7 @@ impl ECStore { ); let pool = clone_arc_by_index(self.pools.as_slice(), pool_index, "invalid rebalance pool index")?; - let bucket_configs = Arc::new(load_rebalance_bucket_configs(&bucket).await?); + let bucket_configs = Arc::new(load_rebalance_bucket_configs(self, &bucket).await?); let mut jobs = Vec::new(); let entry_error = Arc::new(tokio::sync::Mutex::new(None::)); diff --git a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs index bc6006907..401d39c8f 100644 --- a/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs +++ b/crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs @@ -34,20 +34,19 @@ use super::migration::{ rebalance_delete_marker_opts, }; use super::worker::{ - ensure_rebalance_listing_disks_available, is_transient_rebalance_error, load_rebalance_bucket_configs, - parse_rebalance_max_attempts, rebalance_listing_retry_delay, rebalance_migration_retry_delay, - resolve_load_rebalance_stats_update_result, resolve_rebalance_bucket_error, resolve_rebalance_bucket_result, - resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result, - resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, resolve_rebalance_migrate_result_error, - resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result, - resolve_rebalance_terminal_error, resolve_rebalance_worker_result, send_rebalance_done_signal, - should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete, should_defer_rebalance_entry_failure, - should_retry_rebalance_listing, should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks, - wait_rebalance_listing_retry, with_rebalance_entry_context, + ensure_rebalance_listing_disks_available, is_transient_rebalance_error, parse_rebalance_max_attempts, + rebalance_listing_retry_delay, rebalance_migration_retry_delay, resolve_load_rebalance_stats_update_result, + resolve_rebalance_bucket_error, resolve_rebalance_bucket_result, resolve_rebalance_entry_cleanup_delete_result, + resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, + resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result, + resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error, resolve_rebalance_worker_result, + send_rebalance_done_signal, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete, + should_defer_rebalance_entry_failure, should_retry_rebalance_listing, should_skip_rebalance_delete_marker, + wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context, }; use super::{ - DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketOutcome, - RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats, + DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketConfigs, + RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats, }; use crate::bucket::replication::{ReplicationState, ReplicationStatusType, replication_state_to_filemeta}; use crate::data_movement; @@ -2097,13 +2096,11 @@ fn test_resolve_rebalance_optional_bucket_config_result_wraps_other_errors() { ); } -#[tokio::test] -async fn test_load_rebalance_bucket_configs_skips_meta_bucket_lookup() { - let configs = load_rebalance_bucket_configs(RUSTFS_META_BUCKET) - .await - .expect("meta bucket config loading should short-circuit"); +#[test] +fn test_rebalance_meta_bucket_configs_are_empty() { + let configs = RebalanceBucketConfigs::default(); assert!(configs.lifecycle_config.is_none()); - assert!(configs.lock_retention.is_none()); + assert!(configs.object_lock_config.is_none()); assert!(configs.replication_config.is_none()); } diff --git a/crates/ecstore/src/services/rebalance/types.rs b/crates/ecstore/src/services/rebalance/types.rs index 4c42cbf45..bf6fb3bb8 100644 --- a/crates/ecstore/src/services/rebalance/types.rs +++ b/crates/ecstore/src/services/rebalance/types.rs @@ -37,7 +37,7 @@ pub type RStats = Vec>; #[derive(Debug, Default)] pub(super) struct RebalanceBucketConfigs { pub(super) lifecycle_config: Option, - pub(super) lock_retention: Option, + pub(super) object_lock_config: Option, pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>, } diff --git a/crates/ecstore/src/services/rebalance/worker.rs b/crates/ecstore/src/services/rebalance/worker.rs index b33de11e7..f7a352513 100644 --- a/crates/ecstore/src/services/rebalance/worker.rs +++ b/crates/ecstore/src/services/rebalance/worker.rs @@ -10,8 +10,8 @@ use crate::disk::error::DiskError; use crate::error::{ Error, is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down, }; -use crate::runtime::sources as runtime_sources; use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; +use crate::store::ECStore; use rand::RngExt as _; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use std::sync::Arc; @@ -376,7 +376,7 @@ pub(super) fn resolve_rebalance_optional_bucket_config_result( } } -pub(super) async fn load_rebalance_bucket_configs(bucket: &str) -> Result { +pub(super) async fn load_rebalance_bucket_configs(api: &ECStore, bucket: &str) -> Result { if bucket == crate::disk::RUSTFS_META_BUCKET { return Ok(RebalanceBucketConfigs::default()); } @@ -387,9 +387,10 @@ pub(super) async fn load_rebalance_bucket_configs(bucket: &str) -> Result>, write_quorum: usize) -> disk::error::Result<()> { + let has_existing_volume = results + .iter() + .any(|result| matches!(result, Ok(()) | Err(DiskError::FileNotFound))); + let volume_not_found_count = results + .iter() + .filter(|result| matches!(result, Err(DiskError::VolumeNotFound))) + .count(); + let errs = results + .into_iter() + .map(|result| result.err().filter(|err| !DiskError::is_err_object_not_found(err))) + .collect::>(); + + if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { + return Err(err); + } + if !has_existing_volume && volume_not_found_count >= write_quorum { + return Err(DiskError::VolumeNotFound); + } + + Ok(()) + } + pub(in crate::set_disk) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> { let disks = self.get_disks_internal().await; let write_quorum = disks.len() / 2 + 1; @@ -3813,18 +3836,12 @@ impl SetDisks { ) .await } else { - Ok(()) + Err(DiskError::DiskNotFound) } }); } - let errs = join_all(futures).await.into_iter().map(|v| v.err()).collect::>(); - - if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) { - return Err(err); - } - - Ok(()) + Self::reduce_delete_prefix_results(join_all(futures).await, write_quorum) } /// Scan a single disk's copy of `prefix` and decide whether it is an orphan @@ -5744,16 +5761,147 @@ mod tests { } #[tokio::test] - async fn delete_prefix_removes_present_disks_and_ignores_missing_disk_slots() { + async fn delete_prefix_succeeds_when_present_disks_reach_quorum() { let bucket = "delete-prefix-bucket"; - let (_dir, disk) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"payload".as_slice())]).await; - let set = io_primitives_test_set(vec![Some(disk.clone()), None], 1).await; + let (_dir1, disk1) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"one".as_slice())]).await; + let (_dir2, disk2) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"two".as_slice())]).await; + let (_dir3, disk3) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"three".as_slice())]).await; + let set = io_primitives_test_set(vec![Some(disk1.clone()), Some(disk2.clone()), Some(disk3.clone()), None], 2).await; set.delete_prefix(bucket, "prefix") .await - .expect("missing disk slots should not block prefix deletion"); + .expect("three successful disks should meet a four-disk write quorum"); - assert!(matches!(disk.read_all(bucket, "prefix/object.txt").await, Err(DiskError::FileNotFound))); + for disk in [disk1, disk2, disk3] { + assert!(matches!(disk.read_all(bucket, "prefix/object.txt").await, Err(DiskError::FileNotFound))); + } + } + + #[tokio::test] + async fn delete_prefix_counts_confirmed_absence_toward_quorum() { + let bucket = "delete-prefix-confirmed-absence"; + let (_dir1, disk1) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"one".as_slice())]).await; + let (_dir2, disk2) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"two".as_slice())]).await; + let (_dir3, disk3) = read_multiple_test_disk(bucket, &[]).await; + let (_dir4, disk4) = read_multiple_test_disk(bucket, &[]).await; + disk3 + .delete_volume(bucket, true) + .await + .expect("third disk bucket should be absent"); + disk4 + .delete_volume(bucket, true) + .await + .expect("fourth disk bucket should be absent"); + let set = io_primitives_test_set(vec![Some(disk1.clone()), Some(disk2.clone()), Some(disk3), Some(disk4)], 2).await; + + set.delete_prefix(bucket, "prefix") + .await + .expect("successful deletes and confirmed absence should jointly meet quorum"); + + for disk in [disk1, disk2] { + assert!(matches!(disk.read_all(bucket, "prefix/object.txt").await, Err(DiskError::FileNotFound))); + } + } + + #[test] + fn delete_prefix_result_reduction_preserves_existing_volume_evidence() { + assert_eq!( + SetDisks::reduce_delete_prefix_results( + vec![ + Err(DiskError::FileNotFound), + Err(DiskError::FileNotFound), + Err(DiskError::FileNotFound), + Err(DiskError::DiskNotFound), + ], + 3, + ), + Ok(()) + ); + assert_eq!( + SetDisks::reduce_delete_prefix_results( + vec![ + Err(DiskError::FileNotFound), + Err(DiskError::VolumeNotFound), + Err(DiskError::VolumeNotFound), + Err(DiskError::VolumeNotFound), + ], + 3, + ), + Ok(()) + ); + assert_eq!( + SetDisks::reduce_delete_prefix_results( + vec![ + Ok(()), + Err(DiskError::VolumeNotFound), + Err(DiskError::VolumeNotFound), + Err(DiskError::VolumeNotFound), + ], + 3, + ), + Ok(()) + ); + assert_eq!( + SetDisks::reduce_delete_prefix_results( + vec![ + Err(DiskError::VolumeNotFound), + Err(DiskError::VolumeNotFound), + Err(DiskError::VolumeNotFound), + Err(DiskError::VolumeNotFound), + ], + 3, + ), + Err(DiskError::VolumeNotFound) + ); + assert_eq!( + SetDisks::reduce_delete_prefix_results( + vec![Ok(()), Ok(()), Err(DiskError::DiskNotFound), Err(DiskError::DiskNotFound)], + 3, + ), + Err(DiskError::ErasureWriteQuorum) + ); + assert_eq!( + SetDisks::reduce_delete_prefix_results( + vec![ + Ok(()), + Err(DiskError::FileAccessDenied), + Err(DiskError::FileAccessDenied), + Err(DiskError::FileAccessDenied), + ], + 3, + ), + Err(DiskError::FileAccessDenied) + ); + } + + #[tokio::test] + async fn delete_prefix_fails_at_quorum_minus_one() { + let bucket = "delete-prefix-quorum-minus-one"; + let (_dir1, disk1) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"one".as_slice())]).await; + let (_dir2, disk2) = read_multiple_test_disk(bucket, &[("prefix/object.txt", b"two".as_slice())]).await; + let set = io_primitives_test_set(vec![Some(disk1.clone()), Some(disk2.clone()), None, None], 2).await; + + let err = set + .delete_prefix(bucket, "prefix") + .await + .expect_err("two successful disks must not meet a four-disk write quorum"); + + assert_eq!(err, DiskError::ErasureWriteQuorum); + for disk in [disk1, disk2] { + assert!(matches!(disk.read_all(bucket, "prefix/object.txt").await, Err(DiskError::FileNotFound))); + } + } + + #[tokio::test] + async fn delete_prefix_fails_when_all_disk_slots_are_missing() { + let set = io_primitives_test_set(vec![None, None, None, None], 2).await; + + let err = set + .delete_prefix("delete-prefix-offline", "prefix") + .await + .expect_err("an entirely offline set must not report a successful deletion"); + + assert_eq!(err, DiskError::ErasureWriteQuorum); } #[tokio::test] diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 3581a8cba..22a034998 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -45,7 +45,10 @@ use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; use crate::bucket::metadata_sys; -use crate::bucket::object_lock::objectlock_sys::{check_object_lock_for_deletion, check_retention_for_modification}; +use crate::bucket::metadata_sys::ObjectLockConfigState; +use crate::bucket::object_lock::objectlock_sys::{ + check_object_lock_for_deletion_with_config, check_object_lock_for_deletion_with_state, check_retention_for_modification, +}; use crate::bucket::replication::{ ReplicateDecision, ReplicationObjectBridge, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, @@ -107,7 +110,7 @@ use crate::{ SnapshotLeaseToken, UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, }, error::{StorageError, to_object_err}, - object_api::{GetObjectReader, ObjectInfo, PutObjReader}, + object_api::{GetObjectReader, NamespaceLockFence, ObjectInfo, ObjectLockConfigSnapshot, PutObjReader}, // event::name::EventName, services::event_notification::{EventArgs, send_event}, store::init_format::{ @@ -155,9 +158,9 @@ use rustfs_utils::http::headers::{ CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _, }; use rustfs_utils::http::{ - SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, - SUFFIX_RESTORE_OPERATION_ID, contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker, - remove_header_map, + SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, + SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, contains_key_str, get_header_map, get_str, insert_str, + is_object_encryption_marker, remove_header_map, }; use rustfs_utils::{ HashAlgorithm, @@ -669,6 +672,7 @@ fn release_materialized_read_lock(bucket: &str, object: &str, read_lock_guard: O pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap) { metadata.remove(RUSTFS_MULTIPART_BUCKET_KEY); metadata.remove(RUSTFS_MULTIPART_OBJECT_KEY); + rustfs_utils::http::metadata_compat::remove_str(metadata, SUFFIX_BUCKET_INCARNATION_ID); } fn should_persist_encryption_original_size(metadata: &HashMap) -> bool { @@ -956,6 +960,8 @@ pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCl pub(crate) use ops::object::body_cache_plaintext_len; #[cfg(test)] pub(crate) use ops::object::cleanup_rejected_transition_upload_durably; +#[cfg(test)] +pub(crate) use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause}; mod read; mod replication; pub(crate) mod shard_source; @@ -3771,7 +3777,49 @@ pub(crate) fn object_lock_delete_check_required(bucket_meta: Option<&crate::buck bucket_meta.is_none_or(|meta| meta.object_locking()) } -async fn check_object_lock_delete(bucket: &str, object: &str, obj_info: &ObjectInfo, opts: &ObjectOptions) -> Result<()> { +fn restore_expiry_snapshot_matches(obj_info: &ObjectInfo, opts: &ObjectOptions) -> bool { + let expected = &opts.transition; + expected.expire_restored + && expected.status == TRANSITION_COMPLETE + && obj_info.transitioned_object.status == TRANSITION_COMPLETE + && !obj_info.transitioned_object.name.is_empty() + && !obj_info.transitioned_object.tier.is_empty() + && expected.tier == obj_info.transitioned_object.tier + && expected.expected_remote_name == obj_info.transitioned_object.name + && expected.expected_remote_version_id == obj_info.transitioned_object.version_id + && !expected.etag.is_empty() + && obj_info.etag.as_deref() == Some(expected.etag.as_str()) + && expected.expected_data_dir.is_some() + && expected.expected_data_dir == obj_info.data_dir + && obj_info.restore_expires == Some(expected.restore_expiry) + && !obj_info.restore_ongoing + // Deliberately no `restore_expiry <= now` clause. Whether the restored + // copy is due to expire is the ILM evaluator's decision, already made + // when it emitted DeleteRestoredAction; re-deriving it here only adds a + // way for a legitimate action to be rejected. The stale-event risk it + // looks like it covers is already covered above: a re-restore rewrites + // `restore_expires`, so a replayed event fails the equality check. + && match obj_info.version_id { + Some(version_id) => opts.version_id.as_deref().and_then(|value| Uuid::parse_str(value).ok()) == Some(version_id), + None => opts.version_id.is_none(), + } +} + +async fn check_object_lock_delete( + ctx: &InstanceContext, + bucket: &str, + object: &str, + obj_info: &ObjectInfo, + opts: &ObjectOptions, +) -> Result<()> { + if crate::bucket::utils::is_meta_bucketname(bucket) { + return Ok(()); + } + if opts.transition.expire_restored { + return restore_expiry_snapshot_matches(obj_info, opts) + .then_some(()) + .ok_or(StorageError::PreconditionFailed); + } if set_disk_delete_creates_delete_marker(opts) { return Ok(()); } @@ -3780,16 +3828,48 @@ async fn check_object_lock_delete(bucket: &str, object: &str, obj_info: &ObjectI .object_lock_delete .as_ref() .is_some_and(|delete_opts| delete_opts.bypass_governance); - if check_object_lock_for_deletion(bucket, obj_info, bypass_governance) - .await - .is_some() - { + let blocked = match opts.object_lock_config_snapshot.as_deref() { + Some(snapshot) => check_object_lock_for_deletion_with_state(snapshot.state(), obj_info, bypass_governance)?.is_some(), + None => { + let state = metadata_sys::get_object_lock_config_state_in(ctx, bucket).await?; + check_object_lock_for_deletion_with_state(&state, obj_info, bypass_governance)?.is_some() + } + }; + if blocked { return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string())); } Ok(()) } +fn ensure_delete_commit_locks_held( + lock_guard: Option<&ObjectLockDiagGuard>, + bucket: &str, + object: &str, + opts: &ObjectOptions, +) -> Result<()> { + if lock_guard.is_some_and(ObjectLockDiagGuard::is_lock_lost) + || opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "delete_object_commit", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + + Ok(()) +} + fn set_disk_delete_creates_delete_marker(opts: &ObjectOptions) -> bool { opts.version_id.is_none() && opts.versioned && !opts.version_suspended } @@ -5672,7 +5752,7 @@ mod tests { .await .expect("outer write lock should be acquired"); - timeout( + let result = timeout( Duration::from_secs(1), set_disks.delete_object( "bucket", @@ -5684,8 +5764,14 @@ mod tests { ), ) .await - .expect("broad prefix delete must not wait on a literal prefix namespace lock") - .expect("empty test disks should allow broad prefix cleanup"); + .expect("broad prefix delete must not wait on a literal prefix namespace lock"); + + if let Err(err) = result { + assert!( + !err.to_string().to_ascii_lowercase().contains("lock"), + "broad prefix delete returned a lock error: {err}" + ); + } } #[tokio::test(flavor = "multi_thread")] @@ -5705,7 +5791,7 @@ mod tests { .await .expect("outer write lock should be acquired"); - timeout( + let result = timeout( Duration::from_secs(1), set_disks.delete_object( "bucket", @@ -5719,8 +5805,14 @@ mod tests { ), ) .await - .expect("no_lock exact prefix delete path must not wait for the outer lock") - .expect("empty test disks should allow exact prefix cleanup"); + .expect("no_lock exact prefix delete path must not wait for the outer lock"); + + if let Err(err) = result { + assert!( + !err.to_string().to_ascii_lowercase().contains("lock"), + "no_lock exact prefix delete returned a lock error: {err}" + ); + } } #[tokio::test(flavor = "multi_thread")] @@ -8556,16 +8648,105 @@ mod tests { let opts = ObjectOptions { version_id: Some(Uuid::new_v4().to_string()), versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), ..Default::default() }; - let err = check_object_lock_delete("bucket", "object", &obj_info, &opts) + let err = check_object_lock_delete(&bootstrap_ctx(), "bucket", "object", &obj_info, &opts) .await .expect_err("COMPLIANCE retention must block explicit version deletion"); assert!(matches!(err, StorageError::PrefixAccessDenied(_, _))); } + #[tokio::test] + async fn test_check_object_lock_delete_allows_retained_restored_copy_expiry() { + let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60); + let restore_expiry = OffsetDateTime::now_utc() - Duration::from_secs(1); + let version_id = Uuid::new_v4(); + let data_dir = Uuid::new_v4(); + let obj_info = ObjectInfo { + version_id: Some(version_id), + data_dir: Some(data_dir), + etag: Some("etag".to_string()), + transitioned_object: TransitionedObject { + name: "remote-object".to_string(), + tier: "tier".to_string(), + status: TRANSITION_COMPLETE.to_string(), + ..Default::default() + }, + restore_expires: Some(restore_expiry), + user_defined: Arc::new(HashMap::from([ + ( + X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), + s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(), + ), + ( + X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), + retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(), + ), + ])), + ..Default::default() + }; + let opts = ObjectOptions { + version_id: Some(version_id.to_string()), + versioned: true, + transition: crate::bucket::lifecycle::lifecycle::TransitionOptions { + status: TRANSITION_COMPLETE.to_string(), + tier: "tier".to_string(), + etag: "etag".to_string(), + expected_data_dir: Some(data_dir), + expected_remote_name: "remote-object".to_string(), + restore_expiry, + expire_restored: true, + ..Default::default() + }, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), + ..Default::default() + }; + + check_object_lock_delete(&bootstrap_ctx(), "bucket", "object", &obj_info, &opts) + .await + .expect("restore expiry only strips the local copy and must preserve the retained logical version"); + } + + #[tokio::test] + async fn test_check_object_lock_delete_rejects_stale_restored_copy_expiry() { + let restore_expiry = OffsetDateTime::now_utc() - Duration::from_secs(1); + let data_dir = Uuid::new_v4(); + let obj_info = ObjectInfo { + data_dir: Some(data_dir), + etag: Some("etag".to_string()), + transitioned_object: TransitionedObject { + name: "remote-object".to_string(), + tier: "tier".to_string(), + status: TRANSITION_COMPLETE.to_string(), + ..Default::default() + }, + restore_expires: Some(restore_expiry + Duration::from_secs(60)), + ..Default::default() + }; + let opts = ObjectOptions { + versioned: true, + transition: crate::bucket::lifecycle::lifecycle::TransitionOptions { + status: TRANSITION_COMPLETE.to_string(), + tier: "tier".to_string(), + etag: "etag".to_string(), + expected_data_dir: Some(data_dir), + expected_remote_name: "remote-object".to_string(), + restore_expiry, + expire_restored: true, + ..Default::default() + }, + ..Default::default() + }; + + let err = check_object_lock_delete(&bootstrap_ctx(), "bucket", "object", &obj_info, &opts) + .await + .expect_err("a renewed restored copy must reject the stale expiry task"); + assert!(matches!(err, StorageError::PreconditionFailed)); + } + #[tokio::test] async fn test_check_object_lock_delete_allows_versioned_delete_marker_creation() { let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60); @@ -8590,7 +8771,7 @@ mod tests { ..Default::default() }; - check_object_lock_delete("bucket", "object", &obj_info, &opts) + check_object_lock_delete(&bootstrap_ctx(), "bucket", "object", &obj_info, &opts) .await .expect("versioned delete marker creation should not delete the locked version"); } @@ -9881,6 +10062,7 @@ mod tests { let payload = (0..(BLOCK_SIZE_V2 + 17)).map(|idx| (idx % 251) as u8).collect::>(); let opts = ObjectOptions { no_lock: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), ..Default::default() }; @@ -10006,7 +10188,12 @@ mod tests { let bucket = "snapshot-streaming-delete"; let object = "object"; let body = vec![0x41; 2 * 1024 * 1024]; - let opts = ObjectOptions::default(); + let opts = ObjectOptions { + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }; set_disks .make_bucket(bucket, &MakeBucketOptions::default()) @@ -10058,7 +10245,12 @@ mod tests { let bucket = "snapshot-streaming-delete-objects"; let object = "object"; let body = vec![0x41; 2 * 1024 * 1024]; - let opts = ObjectOptions::default(); + let opts = ObjectOptions { + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }; set_disks .make_bucket(bucket, &MakeBucketOptions::default()) @@ -10919,6 +11111,71 @@ mod tests { assert_eq!(restored, payload); } + /// The other half of the suspended-versioning delete contract: a client + /// that drains such a bucket lists the null delete marker and then purges + /// it as `?versionId=null`, which is what `nuke_bucket` does before + /// `DeleteBucket`. That purge must succeed — if it is rejected the marker's + /// `xl.meta` survives, and `DeleteBucket`'s raw disk scan then reports + /// `BucketNotEmpty` for a bucket the client has already emptied. + #[tokio::test] + async fn set_level_explicit_null_version_delete_purges_the_null_delete_marker() { + let set_disks = make_local_bucket_test_set_disks().await; + let bucket = "bucket-null-marker-purge"; + let object = "object.txt"; + // The delete path reads versioned/suspended from the bucket-config + // snapshot, not from `opts`, so inject a real Suspended config — + // otherwise `from_file_info` never synthesizes the null version id and + // the branch under test is not reached. + let suspended = crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test( + s3s::dto::VersioningConfiguration { + status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)), + ..Default::default() + }, + None, + ); + let opts = ObjectOptions { + no_lock: true, + version_suspended: true, + delete_replication_config_snapshot: Some(Arc::new(suspended)), + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut reader = PutObjReader::from_vec(b"suspended version body".to_vec()); + set_disks + .put_object(bucket, object, &mut reader, &opts) + .await + .expect("suspended-version object should be written"); + + let marker = set_disks + .delete_object(bucket, object, opts.clone()) + .await + .expect("version-suspended delete should create a null marker"); + assert!(marker.delete_marker); + assert_eq!(marker.version_id, Some(Uuid::nil())); + + let (_deleted, errs) = set_disks + .delete_objects( + bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + version_id: Some(Uuid::nil()), + ..Default::default() + }], + opts.clone(), + ) + .await; + + assert!( + errs.iter().all(Option::is_none), + "explicit null-version purge of the null delete marker must succeed, got {errs:?}" + ); + } + #[tokio::test] async fn set_level_version_suspended_delete_creates_null_delete_marker() { let set_disks = make_local_bucket_test_set_disks().await; @@ -10927,6 +11184,7 @@ mod tests { let opts = ObjectOptions { no_lock: true, version_suspended: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), ..Default::default() }; @@ -11016,6 +11274,9 @@ mod tests { let missing = "missing.txt"; let opts = ObjectOptions { no_lock: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), ..Default::default() }; diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 349bfc863..e1a748892 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -162,6 +162,86 @@ fn map_upload_id_metadata_error(bucket: &str, object: &str, upload_id: &str, err err.into() } +fn multipart_bucket_incarnation_id(metadata: &HashMap) -> Result> { + let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) else { + if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) { + return Err(Error::other("invalid multipart bucket incarnation metadata")); + } + return Ok(None); + }; + let incarnation = Uuid::parse_str(value).map_err(|_| Error::other("invalid multipart bucket incarnation metadata"))?; + if incarnation.is_nil() { + return Err(Error::other("invalid multipart bucket incarnation metadata")); + } + Ok(Some(incarnation)) +} + +fn multipart_bucket_incarnation_matches(metadata: &HashMap, expected: Uuid) -> bool { + matches!(multipart_bucket_incarnation_id(metadata), Ok(Some(actual)) if actual == expected) +} + +fn validate_multipart_bucket_incarnation( + metadata: &HashMap, + bucket: &str, + object: &str, + upload_id: &str, + expected: Option, + upload_initiated: Option, + bucket_created: Option, +) -> Result<()> { + let Some(expected) = expected else { + return Ok(()); + }; + match multipart_bucket_incarnation_id(metadata) { + Ok(Some(actual)) if actual == expected => return Ok(()), + Ok(None) + if matches!( + (upload_initiated, bucket_created), + (Some(upload_initiated), Some(bucket_created)) if upload_initiated >= bucket_created + ) => + { + return Ok(()); + } + _ => {} + } + Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned())) +} + +async fn ensure_multipart_bucket_incarnation( + ctx: &crate::runtime::instance::InstanceContext, + fi: &FileInfo, + bucket: &str, + object: &str, + upload_id: &str, + expected: Option, +) -> Result<()> { + let bucket_created = if expected.is_some() && matches!(multipart_bucket_incarnation_id(&fi.metadata), Ok(None)) { + Some(metadata_sys::created_at_in(ctx, bucket).await?) + } else { + None + }; + validate_multipart_bucket_incarnation(&fi.metadata, bucket, object, upload_id, expected, fi.mod_time, bucket_created) +} + +fn ensure_multipart_bucket_lifecycle_lock_held(bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let Some(fence) = opts.bucket_lifecycle_lock_fence.as_ref() else { + if opts.expected_bucket_incarnation_id.is_some() && !crate::bucket::utils::is_meta_bucketname(bucket) { + return Err(Error::other("multipart bucket lifecycle lock fence is missing")); + } + return Ok(()); + }; + if fence.is_lock_lost() { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "multipart_bucket_generation", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + Ok(()) +} + fn empty_upload_fallback_possible(successful_responses: usize, errs: &[Option]) -> bool { successful_responses == 0 && errs.iter().any(|err| matches!(err, Some(DiskError::FileNotFound))) @@ -451,6 +531,200 @@ impl SetDisks { Ok((fi, parts_metadata)) } + + #[allow(clippy::too_many_arguments)] + pub(crate) async fn list_multipart_uploads_for_incarnation( + &self, + bucket: &str, + prefix: &str, + key_marker: Option, + upload_id_marker: Option, + delimiter: Option, + max_uploads: usize, + expected_incarnation_id: Option, + ) -> Result { + let disks = self.disks.read().await.clone(); + if disks.is_empty() { + return Err(Error::ErasureReadQuorum); + } + let discovery_quorum = if self.default_parity_count == 0 { + disks.len() + } else { + (disks.len() / 2).max(1) + }; + let mut discovery_errors = (0..disks.len()).map(|_| Some(DiskError::DiskNotFound)).collect::>(); + let mut candidate_counts = HashMap::::new(); + let mut discovery_tasks = JoinSet::new(); + for (index, disk) in disks.iter().enumerate() { + let disk = disk.clone(); + let bucket = bucket.to_string(); + discovery_tasks.spawn(async move { + let result = match disk { + Some(disk) => multipart_upload_paths_on_disk(disk, &bucket).await, + None => Err(DiskError::DiskNotFound), + }; + (index, result) + }); + } + + while let Some(task_result) = discovery_tasks.join_next().await { + let Ok((index, result)) = task_result else { + continue; + }; + match result { + Ok(paths) => { + discovery_errors[index] = None; + for path in paths { + *candidate_counts.entry(path).or_insert(0) += 1; + } + } + Err(err) => discovery_errors[index] = Some(err), + } + } + + if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) { + return Err(to_object_err(err.into(), vec![bucket, prefix])); + } + + let candidate_paths = candidate_counts + .into_iter() + .filter_map(|(path, count)| (count >= discovery_quorum).then_some(path)) + .collect::>(); + let listed_uploads = stream::iter(candidate_paths) + .map(|upload_path| { + let disks = &disks; + async move { + let (sha_dir, raw_upload_id) = upload_path + .rsplit_once('/') + .filter(|(sha_dir, upload_id)| !sha_dir.is_empty() && !upload_id.is_empty()) + .ok_or(DiskError::CorruptedFormat)?; + let (parts_metadata, errs) = Self::read_all_fileinfo( + disks, + bucket, + RUSTFS_META_MULTIPART_BUCKET, + &upload_path, + "", + false, + false, + false, + ) + .await?; + let missing_metadata = errs + .iter() + .filter(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::VolumeNotFound))) + .count(); + if missing_metadata >= discovery_quorum { + if expected_incarnation_id.is_some() { + return Ok(None); + } + // Completion moves the authoritative upload metadata into the + // committed object before it removes the staging directory. A + // crash in that window intentionally leaves a reclaimable + // upload directory whose object name can still be proven for + // an exact-key listing by matching the namespace hash. + if !prefix.is_empty() && sha_dir == Self::get_multipart_sha_dir(bucket, prefix) { + let initiated = raw_upload_id + .rsplit_once('x') + .and_then(|(_, timestamp)| timestamp.parse::().ok()) + .and_then(|timestamp| OffsetDateTime::from_unix_timestamp_nanos(timestamp).ok()); + return Ok(Some(MultipartInfo { + bucket: bucket.to_owned(), + object: prefix.to_owned(), + upload_id: runtime_sources::deployment_upload_id(raw_upload_id), + initiated, + ..Default::default() + })); + } + return Ok(None); + } + let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?; + let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?; + if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { + return Err(err); + } + let (_, mod_time, etag) = Self::list_online_disks(disks, &parts_metadata, &errs, read_quorum); + let file_info = Self::pick_valid_fileinfo(&parts_metadata, mod_time, etag, read_quorum)?; + if expected_incarnation_id + .is_some_and(|expected| !multipart_bucket_incarnation_matches(&file_info.metadata, expected)) + { + return Ok(None); + } + + let object = match ( + file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY), + file_info.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY), + ) { + (Some(stored_bucket), Some(object)) if stored_bucket == bucket && !object.is_empty() => object.clone(), + _ => return Err(DiskError::CorruptedFormat), + }; + if !object.starts_with(prefix) { + return Ok(None); + } + + let initiated = raw_upload_id + .rsplit_once('x') + .and_then(|(_, timestamp)| timestamp.parse::().ok()) + .and_then(|timestamp| OffsetDateTime::from_unix_timestamp_nanos(timestamp).ok()) + .or(file_info.mod_time); + + Ok(Some(MultipartInfo { + bucket: bucket.to_owned(), + object, + upload_id: runtime_sources::deployment_upload_id(raw_upload_id), + initiated, + ..Default::default() + })) + } + }) + .buffer_unordered(MULTIPART_LIST_IO_CONCURRENCY) + .collect::>>>() + .await; + + let mut uploads = Vec::with_capacity(listed_uploads.len()); + for result in listed_uploads { + if let Some(upload) = result.map_err(Error::from)? { + uploads.push(upload); + } + } + + let mut common_prefixes = HashSet::new(); + let mut unfolded_uploads = Vec::with_capacity(uploads.len()); + let delimiter_value = delimiter.as_deref().filter(|delimiter| !delimiter.is_empty()); + for upload in uploads { + let Some(delimiter) = delimiter_value else { + unfolded_uploads.push(upload); + continue; + }; + let suffix = upload.object.strip_prefix(prefix).ok_or(DiskError::CorruptedFormat)?; + if let Some((common_prefix, _)) = suffix.split_once(delimiter) { + common_prefixes.insert(format!("{prefix}{common_prefix}{delimiter}")); + } else { + unfolded_uploads.push(upload); + } + } + + let page = paginate_multipart_listing( + unfolded_uploads, + common_prefixes.into_iter().collect(), + key_marker.as_deref(), + key_marker.as_ref().and(upload_id_marker.as_deref()), + max_uploads, + false, + ); + + Ok(ListMultipartsInfo { + key_marker: key_marker.to_owned(), + upload_id_marker: upload_id_marker.to_owned(), + next_key_marker: page.next_key_marker, + next_upload_id_marker: page.next_upload_id_marker, + max_uploads, + is_truncated: page.is_truncated, + uploads: page.uploads, + common_prefixes: page.common_prefixes, + prefix: prefix.to_owned(), + delimiter: delimiter.to_owned(), + }) + } } #[async_trait::async_trait] @@ -498,6 +772,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?; + ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) + .await?; let write_quorum = fi.write_quorum(self.default_write_quorum()); @@ -747,7 +1023,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .await?, ) }; - self.check_upload_id_exists(bucket, object, upload_id, false).await?; + let (commit_fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?; + ensure_multipart_bucket_incarnation( + &self.ctx, + &commit_fi, + bucket, + object, + upload_id, + opts.expected_bucket_incarnation_id, + ) + .await?; #[cfg(test)] pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await; if _upload_commit_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { @@ -759,6 +1044,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { achieved: 0, }); } + ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; let _ = self .rename_part( @@ -820,6 +1106,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .acquire_multipart_upload_read_lock("list_object_parts", bucket, object, upload_id, opts) .await?; let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?; + ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) + .await?; let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); @@ -927,6 +1215,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ret.next_part_number_marker = ret.parts.last().map(|v| v.part_num).unwrap_or_default(); } + ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; Ok(ret) } @@ -940,179 +1229,44 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { delimiter: Option, max_uploads: usize, ) -> Result { - let disks = self.disks.read().await.clone(); - if disks.is_empty() { - return Err(Error::ErasureReadQuorum); - } - let discovery_quorum = if self.default_parity_count == 0 { - disks.len() + let bucket_lifecycle_guard = if crate::bucket::utils::is_meta_bucketname(bucket) { + None } else { - (disks.len() / 2).max(1) + Some( + metadata_sys::object_store_in(&self.ctx) + .await? + .acquire_bucket_lifecycle_read_lock(bucket) + .await?, + ) }; - let mut discovery_errors = (0..disks.len()).map(|_| Some(DiskError::DiskNotFound)).collect::>(); - let mut candidate_counts = HashMap::::new(); - let mut discovery_tasks = JoinSet::new(); - for (index, disk) in disks.iter().enumerate() { - let disk = disk.clone(); - let bucket = bucket.to_string(); - discovery_tasks.spawn(async move { - let result = match disk { - Some(disk) => multipart_upload_paths_on_disk(disk, &bucket).await, - None => Err(DiskError::DiskNotFound), - }; - (index, result) + let expected_incarnation_id = if bucket_lifecycle_guard.is_some() { + Some(metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?) + } else { + None + }; + let result = self + .list_multipart_uploads_for_incarnation( + bucket, + prefix, + key_marker, + upload_id_marker, + delimiter, + max_uploads, + expected_incarnation_id, + ) + .await; + + if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "multipart_bucket_generation", + bucket: bucket.to_string(), + object: prefix.to_string(), + required: 1, + achieved: 0, }); } - while let Some(task_result) = discovery_tasks.join_next().await { - let Ok((index, result)) = task_result else { - continue; - }; - match result { - Ok(paths) => { - discovery_errors[index] = None; - for path in paths { - *candidate_counts.entry(path).or_insert(0) += 1; - } - } - Err(err) => discovery_errors[index] = Some(err), - } - } - - if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) { - return Err(to_object_err(err.into(), vec![bucket, prefix])); - } - - let candidate_paths = candidate_counts - .into_iter() - .filter_map(|(path, count)| (count >= discovery_quorum).then_some(path)) - .collect::>(); - let listed_uploads = stream::iter(candidate_paths) - .map(|upload_path| { - let disks = &disks; - async move { - let (sha_dir, raw_upload_id) = upload_path - .rsplit_once('/') - .filter(|(sha_dir, upload_id)| !sha_dir.is_empty() && !upload_id.is_empty()) - .ok_or(DiskError::CorruptedFormat)?; - let (parts_metadata, errs) = Self::read_all_fileinfo( - disks, - bucket, - RUSTFS_META_MULTIPART_BUCKET, - &upload_path, - "", - false, - false, - false, - ) - .await?; - let missing_metadata = errs - .iter() - .filter(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::VolumeNotFound))) - .count(); - if missing_metadata >= discovery_quorum { - // Completion moves the authoritative upload metadata into the - // committed object before it removes the staging directory. A - // crash in that window intentionally leaves a reclaimable - // upload directory whose object name can still be proven for - // an exact-key listing by matching the namespace hash. - if !prefix.is_empty() && sha_dir == Self::get_multipart_sha_dir(bucket, prefix) { - let initiated = raw_upload_id - .rsplit_once('x') - .and_then(|(_, timestamp)| timestamp.parse::().ok()) - .and_then(|timestamp| OffsetDateTime::from_unix_timestamp_nanos(timestamp).ok()); - return Ok(Some(MultipartInfo { - bucket: bucket.to_owned(), - object: prefix.to_owned(), - upload_id: runtime_sources::deployment_upload_id(raw_upload_id), - initiated, - ..Default::default() - })); - } - return Ok(None); - } - let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?; - let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?; - if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { - return Err(err); - } - let (_, mod_time, etag) = Self::list_online_disks(disks, &parts_metadata, &errs, read_quorum); - let file_info = Self::pick_valid_fileinfo(&parts_metadata, mod_time, etag, read_quorum)?; - - let object = match ( - file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY), - file_info.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY), - ) { - (Some(stored_bucket), Some(object)) if stored_bucket == bucket && !object.is_empty() => object.clone(), - _ => return Err(DiskError::CorruptedFormat), - }; - if !object.starts_with(prefix) { - return Ok(None); - } - - let initiated = raw_upload_id - .rsplit_once('x') - .and_then(|(_, timestamp)| timestamp.parse::().ok()) - .and_then(|timestamp| OffsetDateTime::from_unix_timestamp_nanos(timestamp).ok()) - .or(file_info.mod_time); - - Ok(Some(MultipartInfo { - bucket: bucket.to_owned(), - object, - upload_id: runtime_sources::deployment_upload_id(raw_upload_id), - initiated, - ..Default::default() - })) - } - }) - .buffer_unordered(MULTIPART_LIST_IO_CONCURRENCY) - .collect::>>>() - .await; - - let mut uploads = Vec::with_capacity(listed_uploads.len()); - for result in listed_uploads { - if let Some(upload) = result.map_err(Error::from)? { - uploads.push(upload); - } - } - - let mut common_prefixes = HashSet::new(); - let mut unfolded_uploads = Vec::with_capacity(uploads.len()); - let delimiter_value = delimiter.as_deref().filter(|delimiter| !delimiter.is_empty()); - for upload in uploads { - let Some(delimiter) = delimiter_value else { - unfolded_uploads.push(upload); - continue; - }; - let suffix = upload.object.strip_prefix(prefix).ok_or(DiskError::CorruptedFormat)?; - if let Some((common_prefix, _)) = suffix.split_once(delimiter) { - common_prefixes.insert(format!("{prefix}{common_prefix}{delimiter}")); - } else { - unfolded_uploads.push(upload); - } - } - - let page = paginate_multipart_listing( - unfolded_uploads, - common_prefixes.into_iter().collect(), - key_marker.as_deref(), - key_marker.as_ref().and(upload_id_marker.as_deref()), - max_uploads, - false, - ); - - Ok(ListMultipartsInfo { - key_marker: key_marker.to_owned(), - upload_id_marker: upload_id_marker.to_owned(), - next_key_marker: page.next_key_marker, - next_upload_id_marker: page.next_upload_id_marker, - max_uploads, - is_truncated: page.is_truncated, - uploads: page.uploads, - common_prefixes: page.common_prefixes, - prefix: prefix.to_owned(), - delimiter: delimiter.to_owned(), - }) + result } #[tracing::instrument(skip(self))] @@ -1225,6 +1379,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { user_defined.insert(RUSTFS_MULTIPART_BUCKET_KEY.to_string(), bucket.to_string()); user_defined.insert(RUSTFS_MULTIPART_OBJECT_KEY.to_string(), object.to_string()); + if let Some(incarnation_id) = opts.expected_bucket_incarnation_id { + insert_str(&mut user_defined, SUFFIX_BUCKET_INCARNATION_ID, incarnation_id.to_string()); + } let (shuffle_disks, mut parts_metadatas) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi); let mod_time = opts.mod_time.unwrap_or_else(OffsetDateTime::now_utc); @@ -1243,6 +1400,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let upload_path = Self::get_upload_id_dir(bucket, object, upload_uuid.as_str()); + ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; Self::write_unique_file_info( &shuffle_disks, bucket, @@ -1278,6 +1436,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .check_upload_id_exists(bucket, object, upload_id, false) .await .map_err(|e| to_object_err(e, vec![bucket, object, upload_id]))?; + ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) + .await?; + ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; Ok(MultipartInfo { bucket: bucket.to_owned(), @@ -1297,6 +1458,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { .acquire_multipart_upload_write_lock("abort_multipart_upload", bucket, object, upload_id, opts) .await?; let (fi, _) = self.check_upload_id_exists(bucket, object, upload_id, true).await?; + ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) + .await?; + ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id); self.delete_all_with_quorum( @@ -1348,6 +1512,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let expected_restore_operation_id = restore_commit_operation_id_from_metadata(&opts.user_defined)?; let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?; + ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id) + .await?; let has_layout_candidate = range_seek_rollout_enabled && fi .data_dir @@ -1786,6 +1952,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { achieved: 0, }); } + ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; self.require_current_restore_operation_id( bucket, @@ -1979,6 +2146,107 @@ mod tests { use tempfile::TempDir; use tokio::sync::{Notify, RwLock}; + #[test] + fn multipart_bucket_incarnation_metadata_is_consistent_and_non_nil() { + let incarnation = Uuid::new_v4(); + let mut metadata = HashMap::new(); + insert_str(&mut metadata, SUFFIX_BUCKET_INCARNATION_ID, incarnation.to_string()); + assert_eq!(multipart_bucket_incarnation_id(&metadata).unwrap(), Some(incarnation)); + + metadata.insert("x-minio-internal-bucket-incarnation-id".to_string(), Uuid::new_v4().to_string()); + assert!(multipart_bucket_incarnation_id(&metadata).is_err()); + + let mut nil_metadata = HashMap::new(); + insert_str(&mut nil_metadata, SUFFIX_BUCKET_INCARNATION_ID, Uuid::nil().to_string()); + assert!(multipart_bucket_incarnation_id(&nil_metadata).is_err()); + } + + #[test] + fn multipart_bucket_incarnation_gate_accepts_only_current_or_same_lifetime_legacy_uploads() { + let expected = Uuid::new_v4(); + let stale = Uuid::new_v4(); + let bucket_created = OffsetDateTime::now_utc(); + let upload_initiated = bucket_created + time::Duration::seconds(1); + let mut current_metadata = HashMap::new(); + insert_str(&mut current_metadata, SUFFIX_BUCKET_INCARNATION_ID, expected.to_string()); + assert!(multipart_bucket_incarnation_matches(¤t_metadata, expected)); + validate_multipart_bucket_incarnation(¤t_metadata, "bucket", "object", "upload", Some(expected), None, None) + .expect("a matching stamped upload should pass"); + + let missing_metadata = HashMap::new(); + assert!(!multipart_bucket_incarnation_matches(&missing_metadata, expected)); + validate_multipart_bucket_incarnation( + &missing_metadata, + "bucket", + "object", + "upload", + Some(expected), + Some(upload_initiated), + Some(bucket_created), + ) + .expect("a legacy upload initiated during the current bucket lifetime should pass"); + assert!(matches!( + validate_multipart_bucket_incarnation( + &missing_metadata, + "bucket", + "object", + "upload", + Some(expected), + Some(bucket_created - time::Duration::seconds(1)), + Some(bucket_created), + ), + Err(StorageError::InvalidUploadID(..)) + )); + assert!(matches!( + validate_multipart_bucket_incarnation( + &missing_metadata, + "bucket", + "object", + "upload", + Some(expected), + None, + Some(bucket_created), + ), + Err(StorageError::InvalidUploadID(..)) + )); + + let mut stale_metadata = HashMap::new(); + insert_str(&mut stale_metadata, SUFFIX_BUCKET_INCARNATION_ID, stale.to_string()); + assert!(!multipart_bucket_incarnation_matches(&stale_metadata, expected)); + assert!(matches!( + validate_multipart_bucket_incarnation( + &stale_metadata, + "bucket", + "object", + "upload", + Some(expected), + Some(upload_initiated), + Some(bucket_created), + ), + Err(StorageError::InvalidUploadID(..)) + )); + } + + #[test] + fn multipart_commit_rejects_missing_or_lost_bucket_lifecycle_fence() { + let expected = Uuid::new_v4(); + let missing = ObjectOptions { + expected_bucket_incarnation_id: Some(expected), + ..Default::default() + }; + assert!(ensure_multipart_bucket_lifecycle_lock_held("bucket", "object", &missing).is_err()); + + let lost = ObjectOptions { + expected_bucket_incarnation_id: Some(expected), + bucket_lifecycle_lock_fence: Some(NamespaceLockFence::lost_for_test()), + ..Default::default() + }; + assert!(matches!( + ensure_multipart_bucket_lifecycle_lock_held("bucket", "object", &lost), + Err(StorageError::NamespaceLockQuorumUnavailable { .. }) + )); + } + struct SetupTypeGuard { previous: SetupType, } @@ -3721,7 +3989,7 @@ mod tests { // A single page must never return more than max_uploads entries. let max_uploads = 2usize; let page = set_disks - .list_multipart_uploads(bucket, object, None, None, None, max_uploads) + .list_multipart_uploads_for_incarnation(bucket, object, None, None, None, max_uploads, None) .await .expect("list should succeed"); assert_eq!( @@ -3738,7 +4006,7 @@ mod tests { // Exact boundary: max_uploads == total must not falsely report truncation. let exact = set_disks - .list_multipart_uploads(bucket, object, None, None, None, total) + .list_multipart_uploads_for_incarnation(bucket, object, None, None, None, total, None) .await .expect("list should succeed"); assert_eq!(exact.uploads.len(), total, "exact boundary must return every upload"); @@ -3756,7 +4024,15 @@ mod tests { let mut pages = 0usize; loop { let page = set_disks - .list_multipart_uploads(bucket, object, key_marker.clone(), upload_id_marker.clone(), None, 1) + .list_multipart_uploads_for_incarnation( + bucket, + object, + key_marker.clone(), + upload_id_marker.clone(), + None, + 1, + None, + ) .await .expect("list should succeed"); assert!(page.uploads.len() <= 1, "max_uploads=1 must never return more than one upload"); @@ -3802,7 +4078,7 @@ mod tests { expected.sort(); let all = set_disks - .list_multipart_uploads(bucket, "", None, None, None, 1000) + .list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None) .await .expect("bucket-wide multipart listing should succeed"); let listed = all @@ -3814,14 +4090,14 @@ mod tests { assert!(!all.is_truncated); let logs = set_disks - .list_multipart_uploads(bucket, "logs/", None, None, None, 1000) + .list_multipart_uploads_for_incarnation(bucket, "logs/", None, None, None, 1000, None) .await .expect("prefix multipart listing should succeed"); assert_eq!(logs.uploads.len(), 3); assert!(logs.uploads.iter().all(|upload| upload.object.starts_with("logs/"))); let exact = set_disks - .list_multipart_uploads(bucket, "logs/a.bin", None, None, None, 1000) + .list_multipart_uploads_for_incarnation(bucket, "logs/a.bin", None, None, None, 1000, None) .await .expect("exact-key multipart listing should remain supported"); assert_eq!(exact.uploads.len(), 2); @@ -3851,7 +4127,15 @@ mod tests { let mut listed = Vec::new(); for _ in 0..expected.len() { let page = set_disks - .list_multipart_uploads(bucket, "logs/", key_marker.clone(), upload_id_marker.clone(), None, 1) + .list_multipart_uploads_for_incarnation( + bucket, + "logs/", + key_marker.clone(), + upload_id_marker.clone(), + None, + 1, + None, + ) .await .expect("multipart page should succeed"); assert_eq!(page.uploads.len(), 1); @@ -3868,7 +4152,7 @@ mod tests { assert_eq!(listed, expected); let key_only = set_disks - .list_multipart_uploads(bucket, "logs/", Some("logs/a.bin".to_string()), None, None, 1000) + .list_multipart_uploads_for_incarnation(bucket, "logs/", Some("logs/a.bin".to_string()), None, None, 1000, None) .await .expect("key-only marker should succeed"); assert_eq!( @@ -3881,7 +4165,7 @@ mod tests { ); let upload_only = set_disks - .list_multipart_uploads(bucket, "logs/", None, Some(expected[0].1.clone()), None, 1000) + .list_multipart_uploads_for_incarnation(bucket, "logs/", None, Some(expected[0].1.clone()), None, 1000, None) .await .expect("an upload marker without a key marker should be ignored"); assert_eq!(upload_only.uploads.len(), expected.len()); @@ -3909,7 +4193,7 @@ mod tests { } let first = set_disks - .list_multipart_uploads(bucket, "logs/", None, None, Some("/".to_string()), 2) + .list_multipart_uploads_for_incarnation(bucket, "logs/", None, None, Some("/".to_string()), 2, None) .await .expect("delimiter multipart listing should succeed"); assert_eq!(first.uploads.len(), 1); @@ -3920,13 +4204,14 @@ mod tests { assert!(first.next_upload_id_marker.is_none()); let second = set_disks - .list_multipart_uploads( + .list_multipart_uploads_for_incarnation( bucket, "logs/", first.next_key_marker, first.next_upload_id_marker, Some("/".to_string()), 2, + None, ) .await .expect("delimiter continuation should succeed"); @@ -3936,7 +4221,7 @@ mod tests { assert!(!second.is_truncated); let exact_boundary = set_disks - .list_multipart_uploads(bucket, "logs/", None, None, Some("/".to_string()), 4) + .list_multipart_uploads_for_incarnation(bucket, "logs/", None, None, Some("/".to_string()), 4, None) .await .expect("delimiter exact boundary should succeed"); assert_eq!(exact_boundary.uploads.len(), 2); @@ -3944,6 +4229,48 @@ mod tests { assert!(!exact_boundary.is_truncated); } + #[tokio::test] + async fn list_multipart_uploads_hides_uploads_from_another_incarnation() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-incarnation-list-bucket"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let incarnation = Uuid::new_v4(); + let (fence, _loss_handle) = NamespaceLockFence::loss_handle_for_test(); + let current = set_disks + .new_multipart_upload( + bucket, + object, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + bucket_lifecycle_lock_fence: Some(fence), + ..Default::default() + }, + ) + .await + .expect("current multipart upload should be created"); + set_disks + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("legacy multipart upload should be created"); + + let unscoped = set_disks + .list_multipart_uploads_for_incarnation(bucket, object, None, None, None, 1000, None) + .await + .expect("unscoped multipart listing should succeed"); + assert_eq!(unscoped.uploads.len(), 2); + + let scoped = set_disks + .list_multipart_uploads_for_incarnation(bucket, object, None, None, None, 1000, Some(incarnation)) + .await + .expect("incarnation-scoped multipart listing should succeed"); + assert_eq!(scoped.uploads.len(), 1); + assert_eq!(scoped.uploads[0].upload_id, current.upload_id); + } + /// Recursively collect every file named `file_name` under the multipart /// staging bucket on a single disk. Used to observe whether a failed commit /// left the per-part metadata intact for a retry. @@ -4425,7 +4752,7 @@ mod tests { async fn upload_is_listed(set_disks: &Arc, bucket: &str, object: &str, upload_id: &str) -> bool { let page = set_disks - .list_multipart_uploads(bucket, object, None, None, None, 1000) + .list_multipart_uploads_for_incarnation(bucket, object, None, None, None, 1000, None) .await .expect("listing multipart uploads should succeed"); page.uploads.iter().any(|u| u.upload_id == upload_id) diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index ad3d68797..146850c69 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -21,6 +21,7 @@ use super::super::*; use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards}; +use crate::bucket::utils::is_meta_bucketname; use crate::set_disk::read::GetObjectDownstreamWriter; use crate::bucket::lifecycle::{ @@ -39,19 +40,86 @@ use crate::bucket::lifecycle::{ save_transition_transaction_record, }, }; -use crate::bucket::replication::{DeleteReplicationConfigSnapshot, VersionPurgeStatusType, version_purge_status_to_filemeta}; +use crate::bucket::replication::{ + DeleteReplicationConfigSnapshot, VersionPurgeStatusType, replication_state_to_filemeta, version_purge_status_to_filemeta, +}; use crate::diagnostics::get::GetObjectFailureReason; use crate::disk::OldCurrentSize; use crate::error::is_err_invalid_upload_id; -use crate::object_api::DeleteLockFence; +use crate::object_api::NamespaceLockFence; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; use crate::services::tier::tier::{TierConfigMgr, TierOperationLease}; use crate::store::ECStore; +use crate::store::utils::clean_metadata; use futures::FutureExt as _; use http::HeaderValue; use rustfs_utils::path::decode_dir_object; use std::future::Future; +fn is_restore_control_metadata(key: &str) -> bool { + key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str()) + || key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_EXPIRY_DAYS) + || key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_REQUEST_DATE) + || rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_RESTORE_OPERATION_ID) + .is_some_and(|remainder| remainder.is_empty()) +} + +fn restore_metadata_update_preserves_protected_metadata( + existing: &HashMap, + replacement: &HashMap, +) -> bool { + let mut existing = existing.clone(); + clean_metadata(&mut existing); + let mut replacement = replacement.clone(); + clean_metadata(&mut replacement); + let existing_count = existing.keys().filter(|key| !is_restore_control_metadata(key)).count(); + let replacement_count = replacement.keys().filter(|key| !is_restore_control_metadata(key)).count(); + existing_count == replacement_count + && existing + .iter() + .filter(|(key, _)| !is_restore_control_metadata(key)) + .all(|(key, value)| replacement.get(key) == Some(value)) +} + +#[cfg(test)] +mod restore_metadata_update_tests { + use super::*; + + #[test] + fn restore_metadata_update_cannot_change_retention_or_user_metadata() { + let mut existing = HashMap::from([ + ("etag".to_string(), "etag-value".to_string()), + ("x-amz-meta-owner".to_string(), "alice".to_string()), + ("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string()), + ]); + let mut replacement = existing.clone(); + replacement.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"true\"".to_string()); + rustfs_utils::http::metadata_compat::insert_str( + &mut replacement, + SUFFIX_RESTORE_OPERATION_ID, + Uuid::new_v4().to_string(), + ); + assert!(restore_metadata_update_preserves_protected_metadata(&existing, &replacement)); + + replacement.insert("x-amz-object-lock-mode".to_string(), "GOVERNANCE".to_string()); + assert!(!restore_metadata_update_preserves_protected_metadata(&existing, &replacement)); + + replacement.clone_from(&existing); + replacement.insert("x-amz-meta-owner".to_string(), "mallory".to_string()); + assert!(!restore_metadata_update_preserves_protected_metadata(&existing, &replacement)); + + existing.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"false\"".to_string()); + replacement.clone_from(&existing); + replacement.remove(X_AMZ_RESTORE.as_str()); + assert!(restore_metadata_update_preserves_protected_metadata(&existing, &replacement)); + } +} + +#[cfg(test)] +mod delete_replication_transport_tests { + use super::*; +} + fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result { coding::Erasure::try_new_with_options(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, uses_legacy) .map_err(Error::from) @@ -894,6 +962,14 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn } impl SetDisks { + async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid) -> Result<()> { + let current = metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?; + if current != expected { + return Err(StorageError::BucketNotFound(bucket.to_string())); + } + Ok(()) + } + /// `put_object` plus the destination key's previous current-version size, /// quorum-reduced from the dst `xl.meta` copies `rename_data` reads while /// committing (rustfs/backlog#1009). `None` means unknown (mixed-version @@ -916,9 +992,21 @@ impl SetDisks { let disks = self.get_disks_internal().await; let mut object_lock_guard = None; + let mut bucket_lifecycle_guard = None; if opts.http_preconditions.is_some() { if !opts.no_lock { + if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id + && opts.bucket_lifecycle_lock_fence.is_none() + { + bucket_lifecycle_guard = Some( + metadata_sys::object_store_in(&self.ctx) + .await? + .acquire_bucket_lifecycle_read_lock(bucket) + .await?, + ); + self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?; + } object_lock_guard = Some( self.acquire_write_lock_diag("put_object_precondition", bucket, object) .await?, @@ -1245,8 +1333,23 @@ impl SetDisks { } if !opts.no_lock && object_lock_guard.is_none() { + #[cfg(test)] + pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await; + if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id + && opts.bucket_lifecycle_lock_fence.is_none() + { + bucket_lifecycle_guard = Some( + metadata_sys::object_store_in(&self.ctx) + .await? + .acquire_bucket_lifecycle_read_lock(bucket) + .await?, + ); + self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?; + } object_lock_guard = Some(self.acquire_write_lock_diag("put_object_commit", bucket, object).await?); } + #[cfg(test)] + pause_put_object_commit(bucket, object, PutObjectCommitPause::AfterNamespace).await; // Generate ordinary PUT timestamps under the commit lock so version // ordering follows durable commit ordering when writers queued on @@ -1287,12 +1390,42 @@ impl SetDisks { } } - // Phase 2 (backlog#899): fence the commit on lock loss. If the refresh - // heartbeat has observed a refresh-quorum loss, another writer may have - // re-acquired this object's lock; committing now would race a double-write. - // Abort with a retryable error *before* rename_data makes the new version - // durable — once rename_data returns Ok the write is committed and must - // never be aborted (that would violate "a committed write is not lost"). + if let Some(version_id) = opts.version_id.as_deref() + && !is_meta_bucketname(bucket) + { + let current = self + .get_object_info( + bucket, + object, + &ObjectOptions { + version_id: Some(version_id.to_string()), + no_lock: true, + metadata_cache_safe: false, + versioned: opts.versioned, + version_suspended: opts.version_suspended, + ..Default::default() + }, + ) + .await; + match current { + Ok(existing) => { + let object_lock_config = opts.object_lock_config_snapshot.as_deref().ok_or_else(|| { + Error::other("explicit-version PUT is missing its Object Lock configuration snapshot") + })?; + if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() { + return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string())); + } + } + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => return Err(err), + } + } + + self.require_current_restore_operation_id(bucket, object, opts, expected_restore_operation_id, "put_object_commit") + .await?; + + // Fence every commit-time read before entering rename_data. Once + // rename_data returns Ok the write is durable and must not be aborted. if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { return Err(StorageError::NamespaceLockQuorumUnavailable { mode: "put_object_commit", @@ -1303,8 +1436,34 @@ impl SetDisks { }); } - self.require_current_restore_operation_id(bucket, object, opts, expected_restore_operation_id, "put_object_commit") - .await?; + if opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "put_object_outer_lock", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + + if opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "put_object_bucket_generation", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } let rename_stage_start = Instant::now(); let (online_disks, _, op_old_dir, cleanup_disks, old_current_size) = Self::rename_data( @@ -2417,6 +2576,170 @@ fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: boo requested && fleet_confirmed && fleet_proof_valid } +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum PutObjectCommitPause { + BeforeNamespace, + AfterNamespace, + BeforeMetadata, +} + +#[cfg(test)] +struct PutObjectCommitBarrierState { + bucket: String, + object: String, + pause: PutObjectCommitPause, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +pub(crate) struct PutObjectCommitBarrier { + state: Arc, +} + +#[cfg(test)] +static PUT_OBJECT_COMMIT_BARRIER: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +impl PutObjectCommitBarrier { + pub(crate) fn install(bucket: &str, object: &str, pause: PutObjectCommitPause) -> Self { + let state = Arc::new(PutObjectCommitBarrierState { + bucket: bucket.to_string(), + object: object.to_string(), + pause, + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = PUT_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(Vec::new())) + .lock() + .expect("put object commit barrier mutex should not poison"); + assert!( + !slot.iter().any(|current| { + current.bucket == state.bucket && current.object == state.object && current.pause == state.pause + }), + "put object commit barrier must be unique for a bucket, object, and pause" + ); + slot.push(Arc::clone(&state)); + drop(slot); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("put object should reach the deterministic commit barrier"); + } + + pub(crate) fn release(&self) { + self.state.release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for PutObjectCommitBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = PUT_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(Vec::new())) + .lock() + .expect("put object commit barrier mutex should not poison"); + slot.retain(|state| !Arc::ptr_eq(state, &self.state)); + } +} + +#[cfg(test)] +async fn pause_put_object_commit(bucket: &str, object: &str, pause: PutObjectCommitPause) { + let barrier = PUT_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(Vec::new())) + .lock() + .expect("put object commit barrier mutex should not poison") + .iter() + .find(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + +#[cfg(test)] +struct DeleteObjectCommitBarrierState { + bucket: String, + object: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +struct DeleteObjectCommitBarrier { + state: Arc, +} + +#[cfg(test)] +static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +impl DeleteObjectCommitBarrier { + fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(DeleteObjectCommitBarrierState { + bucket: bucket.to_string(), + object: object.to_string(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = DELETE_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete object commit barrier mutex should not poison"); + assert!(slot.is_none(), "delete object commit barrier must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + async fn wait_until_paused(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("delete object should reach the deterministic commit barrier"); + } + + fn release(&self) { + self.state.release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for DeleteObjectCommitBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = DELETE_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete object commit barrier mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +async fn pause_delete_object_commit(bucket: &str, object: &str) { + let barrier = DELETE_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete object commit barrier mutex should not poison") + .as_ref() + .filter(|barrier| barrier.bucket == bucket && barrier.object == object) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + fn persisted_transition_version_with_gate( remote_version: &str, remote_version_state_writer_enabled: bool, @@ -2951,6 +3274,49 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { return Err(to_object_err(Error::MethodNotAllowed, vec![src_bucket, src_object])); } + let restore_metadata_update = src_info.metadata_only + && src_bucket == dst_bucket + && src_object == dst_object + && src_opts.version_id == dst_opts.version_id + && src_info + .user_defined + .keys() + .any(|key| key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str())) + && restore_metadata_update_preserves_protected_metadata(&fi.metadata, src_info.user_defined.as_ref()); + if let Some(dst_version_id) = dst_opts.version_id.as_deref() + && !is_meta_bucketname(dst_bucket) + && !restore_metadata_update + { + let object_lock_config = dst_opts + .object_lock_config_snapshot + .as_deref() + .ok_or_else(|| Error::other("explicit-version copy is missing its Object Lock configuration snapshot"))?; + let current = self + .get_object_info( + dst_bucket, + dst_object, + &ObjectOptions { + version_id: Some(dst_version_id.to_string()), + no_lock: true, + metadata_cache_safe: false, + versioned: dst_opts.versioned, + version_suspended: dst_opts.version_suspended, + ..Default::default() + }, + ) + .await; + match current { + Ok(existing) + if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() => + { + return Err(StorageError::PrefixAccessDenied(dst_bucket.to_string(), dst_object.to_string())); + } + Ok(_) => {} + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => return Err(err), + } + } + let version_id = { if src_info.version_only { if let Some(vid) = &dst_opts.version_id { @@ -2974,6 +3340,25 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { fi.version_id = version_id; fi.versioned = src_opts.versioned || src_opts.version_suspended; + if _lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || dst_opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || dst_opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "copy_object_commit", + bucket: dst_bucket.to_string(), + object: dst_object.to_string(), + required: 1, + achieved: 0, + }); + } + if src_info.version_only { let inline_data = fi.inline_data(); @@ -3235,11 +3620,25 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } - // Resolve Object Lock once; replication still reads each matching - // object's tags below while the batch write lock is held. - let object_lock_checks_required = - object_lock_delete_check_required(metadata_sys::get_in(&self.ctx, bucket).await.ok().as_deref()); - + let object_lock_config = if is_meta_bucketname(bucket) { + Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent)) + } else { + match opts.object_lock_config_snapshot.clone() { + Some(snapshot) => snapshot, + None => match metadata_sys::get_object_lock_config_state_in(&self.ctx, bucket).await { + Ok(state) => Arc::new(ObjectLockConfigSnapshot::new(state)), + Err(err) => { + let message = err.to_string(); + for (index, item) in del_errs.iter_mut().enumerate() { + if locked_objects.contains(&objects[index].object_name) { + *item = Some(Error::other(message.clone())); + } + } + return (del_objects, del_errs); + } + }, + } + }; let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new(); let mut journal_entries: Vec<(usize, Jentry)> = Vec::new(); @@ -3255,18 +3654,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { .versioning_config() .delete_state(replication_object_name.as_str()); let check_opts = ObjectOptions { - version_id: version_id.map(|version_id| version_id.to_string()), + version_id: dobj.version_id.map(|version_id| version_id.to_string()), versioned, version_suspended, object_lock_delete: opts.object_lock_delete.clone(), + object_lock_config_snapshot: Some(Arc::clone(&object_lock_config)), no_lock: true, ..Default::default() }; + // A missing bucket Object Lock configuration does not prove that + // persisted object metadata is clean: migrated or corrupt data may + // still carry an explicit retention or legal hold. Only a real + // delete-marker creation can skip the object-level WORM check. + let object_lock_check_required = !is_meta_bucketname(bucket) && !set_disk_delete_creates_delete_marker(&check_opts); let replicate_delete = delete_config_snapshot.has_active_rule(&replication_object_name); let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id; let replication_needs_source = replicate_delete && (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name)); - let (goi, gerr) = if object_lock_checks_required || replication_needs_source || opts.tier_delete_journal_api.is_some() + let (goi, gerr) = if object_lock_check_required || replication_needs_source || opts.tier_delete_journal_api.is_some() { let (goi, _write_quorum, gerr) = self.get_object_info_and_quorum(bucket, &dobj.object_name, &check_opts).await; (goi, gerr) @@ -3276,10 +3681,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let source_missing = gerr .as_ref() .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)); - let explicit_delete_marker = opts.tier_delete_journal_api.is_some() - && dobj.version_id.is_some() + // Normalize both sides before comparing. `goi.version_id` is the + // client-facing identity, where `from_file_info` synthesizes + // `Some(Uuid::nil())` for a null version on a versioned or + // versioning-suspended bucket; `version_id` is the storage identity, + // where an explicit `?versionId=null` maps to `None`. Comparing them + // raw makes an explicit null-version delete of a null delete marker + // look like a version mismatch, so the `MethodNotAllowed` from the + // lookup below is recorded as a delete failure and the marker is + // never purged — the bucket then stays non-empty on disk forever. + let explicit_delete_marker = dobj.version_id.is_some() && goi.delete_marker - && goi.version_id == version_id + && delete_file_info_version_id(goi.version_id) == version_id && matches!(gerr.as_ref(), Some(StorageError::MethodNotAllowed)); if let Some(err) = gerr.as_ref() && !source_missing @@ -3288,9 +3701,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { del_errs[i] = Some(err.clone()); continue; } - if object_lock_checks_required + if object_lock_check_required && !source_missing - && let Err(err) = check_object_lock_delete(bucket, &dobj.object_name, &goi, &check_opts).await + && let Err(err) = check_object_lock_delete(&self.ctx, bucket, &dobj.object_name, &goi, &check_opts).await { del_errs[i] = Some(err); continue; @@ -3417,7 +3830,17 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { vers.push(fi_vers); } - if opts.delete_lock_fence.as_ref().is_some_and(DeleteLockFence::is_lock_lost) { + let rollback_dir = Uuid::new_v4(); + + let disks = self.disks.read().await; + + let disks = disks.clone(); + + if opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { if dist_erasure { self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await; } @@ -3451,20 +3874,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { fi_vers.versions.retain(|fi| del_errs[fi.idx].is_none()); } vers.retain(|fi_vers| !fi_vers.versions.is_empty()); - - let rollback_dir = Uuid::new_v4(); - - let disks = self.disks.read().await; - - let disks = disks.clone(); - let mut futures = Vec::with_capacity(disks.len()); + let lock_lost_during_commit = Arc::new(std::sync::atomic::AtomicBool::new(false)); // let mut errors = Vec::with_capacity(disks.len()); for disk in disks.iter() { let vers = vers.clone(); + let namespace_lock_fence = opts.namespace_lock_fence.clone(); + let lock_lost_during_commit = Arc::clone(&lock_lost_during_commit); futures.push(async move { + if namespace_lock_fence.as_ref().is_some_and(NamespaceLockFence::is_lock_lost) { + lock_lost_during_commit.store(true, std::sync::atomic::Ordering::Release); + return (0..vers.len()).map(|_| Some(DiskError::DiskOngoingReq)).collect(); + } if let Some(disk) = disk { disk.delete_versions( bucket, @@ -3536,6 +3959,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } } + if lock_lost_during_commit.load(std::sync::atomic::Ordering::Acquire) { + for (index, object) in objects.iter().enumerate() { + if del_errs[index].is_none() { + del_errs[index] = Some(Error::NamespaceLockQuorumUnavailable { + mode: "delete_objects_commit", + bucket: bucket.to_string(), + object: decode_dir_object(&object.object_name), + required: 1, + achieved: 0, + }); + } + } + } + self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks); let mut rollback_futures = Vec::new(); @@ -3680,6 +4117,59 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { None }; if opts.delete_prefix { + if opts.delete_prefix_object && !is_meta_bucketname(bucket) { + let object_lock_config = if opts.data_movement { + None + } else { + Some(match opts.object_lock_config_snapshot.as_deref() { + Some(snapshot) => snapshot.state().clone(), + None => metadata_sys::get_object_lock_config_state_in(&self.ctx, bucket).await?, + }) + }; + if let Some(versions) = self.load_file_info_versions_exact(bucket, object).await? { + let bypass_governance = opts + .object_lock_delete + .as_ref() + .is_some_and(|delete_opts| delete_opts.bypass_governance); + if let Some(object_lock_config) = object_lock_config.as_ref() { + for version in versions.versions.iter().chain(versions.free_versions.iter()) { + let object_info = ObjectInfo::from_file_info(version, bucket, object, true); + if check_object_lock_for_deletion_with_state(object_lock_config, &object_info, bypass_governance)? + .is_some() + { + return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string())); + } + } + } + for version in &versions.versions { + ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + let mut delete_request = FileInfo { + name: object.to_string(), + version_id: version.version_id, + ..Default::default() + }; + delete_request.set_tier_free_version_id(&Uuid::new_v4().to_string()); + self.delete_object_version(bucket, object, &delete_request, false).await?; + } + for version in &versions.free_versions { + ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; + let mut delete_request = FileInfo { + name: object.to_string(), + version_id: version.version_id, + deleted: true, + ..Default::default() + }; + delete_request.set_tier_free_version(); + self.delete_object_version(bucket, object, &delete_request, false).await?; + } + } + self.invalidate_get_object_metadata_cache(bucket, object).await; + return Ok(ObjectInfo::default()); + } + if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id { + self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?; + } + ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; self.delete_prefix(bucket, object) .await .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; @@ -3732,7 +4222,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { if version_found { opts.precondition_check(&goi)?; - check_object_lock_delete(bucket, object, &goi, &opts).await?; + check_object_lock_delete(&self.ctx, bucket, object, &goi, &opts).await?; } if opts.transition.expire_restored { @@ -3753,6 +4243,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { expire_restored: true, ..Default::default() }; + ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; self.delete_object_version(bucket, object, &dfi, false) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -3824,6 +4315,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { None }; + ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; self.delete_object_version(bucket, object, &fi, should_force_delete_marker_for_missing_version(&opts)) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -3855,6 +4347,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { dfi.set_skip_tier_free_version(); } + #[cfg(test)] + pause_delete_object_commit(bucket, object).await; + ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?; self.delete_object_version(bucket, object, &dfi, opts.delete_marker) .await .map_err(|e| to_object_err(e, vec![bucket, object]))?; @@ -3921,6 +4416,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { self.invalidate_get_object_metadata_cache(bucket, object).await; // Guard lock for metadata update + #[cfg(test)] + pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeMetadata).await; let _lock_guard = if !opts.no_lock { Some(self.acquire_write_lock_diag("put_object_metadata", bucket, object).await?) } else { @@ -3991,6 +4488,28 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { fi.version_id = Uuid::parse_str(version_id).ok(); } + if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id { + self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?; + } + if _lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + || opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "put_object_metadata_commit", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + self.update_object_meta(bucket, object, fi.clone(), &online_disks) .await .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; @@ -4463,15 +4982,42 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // _lock_guard = guard_opt; // } let self_ = self.clone(); + let restore_header_self = self_.clone(); let set_restore_header_fn = async move |oi: &mut ObjectInfo, rerr: Option| -> Result<()> { if rerr.is_none() { return Ok(()); } - self.update_restore_metadata(bucket, object, oi, opts).await?; + restore_header_self.update_restore_metadata(bucket, object, oi, opts).await?; Err(rerr.unwrap()) }; let mut oi = ObjectInfo::default(); - let fi = self_.clone().get_object_fileinfo(bucket, object, opts, true, false).await; + let bucket_lifecycle_guard = if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id + && opts.bucket_lifecycle_lock_fence.is_none() + { + let guard = metadata_sys::object_store_in(&self.ctx) + .await? + .acquire_bucket_lifecycle_read_lock(bucket) + .await?; + self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?; + Some(guard) + } else { + None + }; + if opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "restore_object_bucket_generation", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + let fi = self.clone().get_object_fileinfo(bucket, object, opts, true, false).await; + drop(bucket_lifecycle_guard); if let Err(err) = fi { return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await; } @@ -4511,6 +5057,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // hold and time out. put_restore_opts builds fresh options that default // no_lock=false, so propagate it explicitly here. ropts.no_lock = opts.no_lock; + ropts.expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id; + ropts.bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone(); + ropts.namespace_lock_fence = opts.namespace_lock_fence.clone(); + ropts.object_lock_config_snapshot = opts.object_lock_config_snapshot.clone(); if oi.parts.len() == 1 { let mut opts = opts.clone(); opts.part_number = Some(1); @@ -4605,7 +5155,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { fail_restore_multipart_at(RestoreMultipartFailurePoint::PutPart)?; let p_info = self_ .clone() - .put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ObjectOptions::default()) + .put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ropts) .await?; #[cfg(test)] let p_info = if restore_multipart_failure_is(RestoreMultipartFailurePoint::SizeMismatch) { @@ -4649,6 +5199,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { &ObjectOptions { mod_time: oi.mod_time, version_id: oi.version_id.map(|version| version.to_string()), + expected_bucket_incarnation_id: opts.expected_bucket_incarnation_id, + bucket_lifecycle_lock_fence: opts.bucket_lifecycle_lock_fence.clone(), user_defined: restore_commit_metadata, // Inherit the restore write lock (see ropts.no_lock above): // the commit phase re-acquires this object's write lock. @@ -5097,7 +5649,16 @@ mod metadata_mutation_generation_tests { let (_, delete_key) = put_and_prime(&set_disks, bucket, delete_object, b"DELETE body").await; let delete_probe = MetadataCacheInvalidationProbe::install(bucket, delete_object); set_disks - .delete_object(bucket, delete_object, ObjectOptions::default()) + .delete_object( + bucket, + delete_object, + ObjectOptions { + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) .await .expect("DELETE should succeed"); assert_eq!(delete_probe.count(), 2, "DELETE must invalidate before mutation and after commit"); @@ -5354,6 +5915,132 @@ mod transition_commit_failure_tests { ); } + #[tokio::test] + #[serial_test::serial] + async fn single_part_restore_keeps_object_lock_snapshot_for_versioned_and_suspended_objects() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "restore-single-part-versioned-bucket"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + for (case, versioned, version_suspended) in [("versioned", true, false), ("suspended", false, true)] { + let object = format!("{case}.bin"); + let payload = format!("single-part restore must preserve the {case} object identity") + .repeat(1024) + .into_bytes(); + let mut reader = PutObjReader::from_vec(payload.clone()); + let original = set_disks + .put_object( + bucket, + &object, + &mut reader, + &ObjectOptions { + versioned, + version_suspended, + ..Default::default() + }, + ) + .await + .expect("source object should be written"); + let version_id = original + .version_id + .expect("versioned and suspended objects should resolve an explicit version"); + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + set_disks + .transition_object( + bucket, + &object, + &ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: Some(version_id.to_string()), + versioned, + version_suspended, + mod_time: original.mod_time, + ..Default::default() + }, + ) + .await + .expect("source object should transition before restore"); + + set_disks + .clone() + .restore_transitioned_object( + bucket, + &object, + &ObjectOptions { + transition: TransitionOptions { + restore_request: RestoreRequest { + days: Some(1), + ..Default::default() + }, + ..Default::default() + }, + version_id: Some(version_id.to_string()), + versioned, + version_suspended, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await + .expect("single-part restore should commit the selected version"); + + let restored = set_disks + .get_object_info( + bucket, + &object, + &ObjectOptions { + version_id: Some(version_id.to_string()), + versioned, + version_suspended, + ..Default::default() + }, + ) + .await + .expect("restored version should remain readable"); + assert_eq!(restored.version_id, Some(version_id), "restore must preserve the selected {case} version"); + let restore_header = restored + .user_defined + .get(s3s::header::X_AMZ_RESTORE.as_str()) + .expect("restored version must carry its completed restore status"); + let restore_status = parse_restore_obj_status(restore_header).expect("restore status must be valid"); + assert!(!restore_status.on_going(), "restored {case} version must not remain in progress"); + + backend.external_remove(&restored.transitioned_object.name).await; + let mut restored_body = Vec::new(); + set_disks + .get_object_reader( + bucket, + &object, + None, + HeaderMap::new(), + &ObjectOptions { + version_id: Some(version_id.to_string()), + versioned, + version_suspended, + ..Default::default() + }, + ) + .await + .expect("restored local version should remain readable after the remote copy is removed") + .stream + .read_to_end(&mut restored_body) + .await + .expect("restored local body should drain"); + assert_eq!(restored_body, payload, "restored {case} body must come from the committed local copy"); + } + } + #[tokio::test] #[serial_test::serial] async fn local_commit_failure_returns_error_and_preserves_remote_candidate() { @@ -7850,6 +8537,540 @@ mod put_object_tmp_cleanup_tests { drop(temp_dirs); } + + #[tokio::test] + async fn put_object_no_lock_aborts_after_outer_namespace_lock_loss() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-lost-outer-lock"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut reader = PutObjReader::from_vec(vec![3u8; 4096]); + let err = set_disks + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + no_lock: true, + namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()), + ..Default::default() + }, + ) + .await + .expect_err("a lost outer namespace lock must abort before rename"); + + assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. })); + assert!( + set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .is_err(), + "the object must not become visible after the outer lock is lost" + ); + } + + #[tokio::test] + async fn put_object_aborts_when_bucket_lifecycle_fence_is_lost_after_commit_lock() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-lost-bucket-lifecycle-lock"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace); + let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test(); + let put_store = Arc::clone(&set_disks); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(b"must-not-commit".to_vec()); + put_store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + versioned: true, + bucket_lifecycle_lock_fence: Some(fence), + ..Default::default() + }, + ) + .await + }); + + barrier.wait_until_paused().await; + loss_handle.store(true, std::sync::atomic::Ordering::Release); + barrier.release(); + let err = put + .await + .expect("PUT task should join") + .expect_err("bucket lifecycle lock loss at commit must abort the PUT"); + assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. })); + assert!( + set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .is_err(), + "object must remain absent after bucket lifecycle lock loss" + ); + } + + #[tokio::test] + async fn metadata_copy_no_lock_aborts_after_outer_namespace_lock_loss() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "copy-lost-outer-lock"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut initial_reader = PutObjReader::from_vec(b"original".to_vec()); + set_disks + .put_object(bucket, object, &mut initial_reader, &ObjectOptions::default()) + .await + .expect("initial object should be written"); + let mut copy_info = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("initial metadata should be readable"); + copy_info.metadata_only = true; + Arc::make_mut(&mut copy_info.user_defined).insert("x-amz-meta-proof".to_string(), "mutated".to_string()); + + let err = set_disks + .copy_object( + bucket, + object, + bucket, + object, + &mut copy_info, + &ObjectOptions::default(), + &ObjectOptions { + no_lock: true, + namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()), + ..Default::default() + }, + ) + .await + .expect_err("metadata copy must not commit after its outer namespace lock is lost"); + assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. })); + + let current = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("original metadata should remain readable"); + assert!(!current.user_defined.contains_key("x-amz-meta-proof")); + } + + #[tokio::test] + async fn explicit_version_overwrite_rechecks_object_lock_under_commit_lock() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-explicit-version-object-lock"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let retain_until = (OffsetDateTime::now_utc() + time::Duration::days(1)) + .format(&time::format_description::well_known::Rfc3339) + .unwrap(); + let cases = [ + ( + "compliance", + HashMap::from([ + ( + X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), + s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(), + ), + (X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until.clone()), + ]), + ), + ( + "governance", + HashMap::from([ + ( + X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), + s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(), + ), + (X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), retain_until), + ]), + ), + ( + "legal-hold", + HashMap::from([( + X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), + s3s::dto::ObjectLockLegalHoldStatus::ON.to_string(), + )]), + ), + ]; + + for (case, lock_metadata) in cases { + let object = format!("{case}-object"); + let original_body = format!("original-{case}").into_bytes(); + let mut initial_reader = PutObjReader::from_vec(original_body.clone()); + let initial = set_disks + .put_object( + bucket, + &object, + &mut initial_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("initial version should be written"); + let version_id = initial + .version_id + .expect("versioned PUT should return a version ID") + .to_string(); + + let barrier = PutObjectCommitBarrier::install(bucket, &object, PutObjectCommitPause::BeforeNamespace); + let overwrite_store = Arc::clone(&set_disks); + let overwrite_object = object.clone(); + let overwrite_version = version_id.clone(); + let overwrite = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(format!("replacement-{case}").into_bytes()); + overwrite_store + .put_object( + bucket, + &overwrite_object, + &mut reader, + &ObjectOptions { + version_id: Some(overwrite_version), + versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await + }); + + barrier.wait_until_paused().await; + set_disks + .put_object_metadata( + bucket, + &object, + &ObjectOptions { + version_id: Some(version_id.clone()), + versioned: true, + eval_metadata: Some(lock_metadata), + ..Default::default() + }, + ) + .await + .expect("Object Lock metadata should win the race before the overwrite commit lock"); + barrier.release(); + + let err = overwrite + .await + .expect("overwrite task should not panic") + .expect_err("the commit-time Object Lock check must reject the overwrite"); + assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected {case} error: {err}"); + + let mut reader = set_disks + .get_object_reader( + bucket, + &object, + None, + HeaderMap::new(), + &ObjectOptions { + version_id: Some(version_id), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("the protected version should remain readable"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("protected body should drain"); + assert_eq!(body, original_body, "{case} overwrite must not change the protected body"); + drop(barrier); + } + + let object = "commit-lock-proof"; + let mut initial_reader = PutObjReader::from_vec(b"original".to_vec()); + let initial = set_disks + .put_object( + bucket, + object, + &mut initial_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("initial lock-proof version should be written"); + let version_id = initial + .version_id + .expect("versioned PUT should return a version ID") + .to_string(); + let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace); + let overwrite_store = Arc::clone(&set_disks); + let overwrite_version = version_id.clone(); + let overwrite = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(b"replacement".to_vec()); + overwrite_store + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + version_id: Some(overwrite_version), + versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await + }); + + barrier.wait_until_paused().await; + let metadata_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeMetadata); + let metadata_store = Arc::clone(&set_disks); + let metadata_version = version_id.clone(); + let mut metadata_update = tokio::spawn(async move { + metadata_store + .put_object_metadata( + bucket, + object, + &ObjectOptions { + version_id: Some(metadata_version), + versioned: true, + eval_metadata: Some(HashMap::from([("proof".to_string(), "blocked".to_string())])), + ..Default::default() + }, + ) + .await + }); + metadata_barrier.wait_until_paused().await; + metadata_barrier.release(); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut metadata_update) + .await + .is_err(), + "metadata update must block while PUT holds the commit lock" + ); + drop(metadata_barrier); + barrier.release(); + overwrite + .await + .expect("overwrite task should not panic") + .expect("unlocked exact-version overwrite should commit"); + metadata_update + .await + .expect("metadata task should not panic") + .expect("metadata update should proceed after PUT releases the commit lock"); + drop(barrier); + } + + #[tokio::test] + async fn explicit_version_overwrite_honors_bucket_default_compliance() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "put-default-compliance-object-lock"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let original_body = b"default-compliance-original".to_vec(); + let mut initial_reader = PutObjReader::from_vec(original_body.clone()); + let initial = set_disks + .put_object( + bucket, + object, + &mut initial_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("initial version should be written"); + let version_id = initial + .version_id + .expect("versioned PUT should return a version ID") + .to_string(); + let default_compliance = ObjectLockConfigState::Configured { + config: s3s::dto::ObjectLockConfiguration { + object_lock_enabled: Some(s3s::dto::ObjectLockEnabled::from_static(s3s::dto::ObjectLockEnabled::ENABLED)), + rule: Some(s3s::dto::ObjectLockRule { + default_retention: Some(s3s::dto::DefaultRetention { + mode: Some(s3s::dto::ObjectLockRetentionMode::from_static( + s3s::dto::ObjectLockRetentionMode::COMPLIANCE, + )), + days: Some(1), + years: None, + }), + }), + }, + updated_at: OffsetDateTime::now_utc(), + }; + let mut replacement = PutObjReader::from_vec(b"replacement".to_vec()); + let err = set_disks + .put_object( + bucket, + object, + &mut replacement, + &ObjectOptions { + version_id: Some(version_id.clone()), + versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(default_compliance))), + ..Default::default() + }, + ) + .await + .expect_err("bucket default COMPLIANCE must block an exact-version overwrite"); + assert!(matches!(err, StorageError::PrefixAccessDenied(_, _))); + + let mut reader = set_disks + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + version_id: Some(version_id), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("the default-protected version should remain readable"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("protected body should drain"); + assert_eq!(body, original_body); + } + + #[tokio::test] + async fn version_only_copy_checks_the_destination_version_object_lock() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "copy-destination-object-lock"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let mut source_reader = PutObjReader::from_vec(b"source-version".to_vec()); + let source = set_disks + .put_object( + bucket, + object, + &mut source_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("source version should be written"); + let source_version = source.version_id.expect("source version ID").to_string(); + + let destination_body = b"protected-destination".to_vec(); + let mut destination_reader = PutObjReader::from_vec(destination_body.clone()); + let destination = set_disks + .put_object( + bucket, + object, + &mut destination_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("destination version should be written"); + let destination_version = destination.version_id.expect("destination version ID").to_string(); + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + version_id: Some(destination_version.clone()), + versioned: true, + eval_metadata: Some(HashMap::from([( + X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), + s3s::dto::ObjectLockLegalHoldStatus::ON.to_string(), + )])), + ..Default::default() + }, + ) + .await + .expect("destination legal hold should be written"); + + let mut source_info = set_disks + .get_object_info( + bucket, + object, + &ObjectOptions { + version_id: Some(source_version.clone()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("source version should be readable"); + source_info.version_only = true; + let err = set_disks + .copy_object( + bucket, + object, + bucket, + object, + &mut source_info, + &ObjectOptions { + version_id: Some(source_version), + versioned: true, + ..Default::default() + }, + &ObjectOptions { + version_id: Some(destination_version.clone()), + versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await + .expect_err("source metadata must not overwrite a locked destination version"); + assert!(matches!(err, StorageError::PrefixAccessDenied(_, _))); + + let mut reader = set_disks + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + version_id: Some(destination_version), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("protected destination should remain readable"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("destination body should drain"); + assert_eq!(body, destination_body); + } } #[cfg(test)] @@ -8012,6 +9233,7 @@ mod object_tagging_namespace_lock_tests { versioned: true, version_id: Some(version_id), delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::default())), + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), ..Default::default() }; set_disks @@ -8051,6 +9273,9 @@ mod object_tagging_namespace_lock_tests { let object_opts = ObjectOptions { versioned, delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::default())), + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), ..Default::default() }; let original_body = b"original body".to_vec(); @@ -8227,7 +9452,18 @@ mod delete_objects_lock_gating_tests { }, ]; - let (deleted, errs) = set_disks.delete_objects(bucket, objects, ObjectOptions::default()).await; + let (deleted, errs) = set_disks + .delete_objects( + bucket, + objects, + ObjectOptions { + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await; assert_eq!(deleted.len(), 3); assert_eq!(errs.len(), 3); @@ -8290,6 +9526,9 @@ mod delete_objects_lock_gating_tests { ObjectOptions { versioned: true, delete_replication_config_snapshot: Some(snapshot), + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), ..Default::default() }, ) @@ -8391,6 +9630,9 @@ mod delete_objects_lock_gating_tests { ObjectOptions { versioned: true, delete_replication_config_snapshot: Some(snapshot), + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), ..Default::default() }, ) @@ -8469,6 +9711,9 @@ mod delete_objects_lock_gating_tests { ObjectOptions { versioned: true, delete_replication_config_snapshot: Some(snapshot), + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), ..Default::default() }, ) @@ -8502,8 +9747,11 @@ mod delete_objects_lock_gating_tests { }], ObjectOptions { no_lock: true, - delete_lock_fence: Some(DeleteLockFence::lost_for_test()), + namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()), delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::default())), + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), ..Default::default() }, ) @@ -8520,6 +9768,50 @@ mod delete_objects_lock_gating_tests { .expect("object must survive a lost-lock batch"); } + #[tokio::test] + async fn delete_object_aborts_when_outer_lock_is_lost_at_commit() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "single-lost-outer-lock"; + let object = "object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + put_plain_object(&set_disks, bucket, object).await; + + let barrier = DeleteObjectCommitBarrier::install(bucket, object); + let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test(); + let delete_store = Arc::clone(&set_disks); + let delete = tokio::spawn(async move { + delete_store + .delete_object( + bucket, + object, + ObjectOptions { + no_lock: true, + namespace_lock_fence: Some(fence), + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + loss_handle.store(true, std::sync::atomic::Ordering::Release); + barrier.release(); + + let err = delete + .await + .expect("delete task should join") + .expect_err("lock loss at commit must abort the delete"); + assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. })); + set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("object must survive commit-time lock loss"); + } + #[tokio::test] async fn delete_objects_blocks_locked_object_and_deletes_the_rest() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; @@ -8570,7 +9862,18 @@ mod delete_objects_lock_gating_tests { }, ]; - let (_deleted, errs) = set_disks.delete_objects(bucket, objects, ObjectOptions::default()).await; + let (_deleted, errs) = set_disks + .delete_objects( + bucket, + objects, + ObjectOptions { + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await; let lock_err = errs[0] .as_ref() @@ -8629,7 +9932,18 @@ mod delete_objects_lock_gating_tests { object_name: "obj".to_string(), ..Default::default() }]; - let (_deleted, errs) = set_disks.delete_objects(bucket, objects, ObjectOptions::default()).await; + let (_deleted, errs) = set_disks + .delete_objects( + bucket, + objects, + ObjectOptions { + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await; assert!( errs[0].is_some(), @@ -8677,6 +9991,9 @@ mod delete_objects_lock_gating_tests { objects, ObjectOptions { no_lock: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + ObjectLockConfigState::ConfirmedAbsent, + ))), ..Default::default() }, ), diff --git a/crates/ecstore/src/set_disk/replication.rs b/crates/ecstore/src/set_disk/replication.rs index ad84d4842..732666296 100644 --- a/crates/ecstore/src/set_disk/replication.rs +++ b/crates/ecstore/src/set_disk/replication.rs @@ -37,7 +37,10 @@ impl RestoreCleanupIdentity { } fn matches_file_info(&self, fi: &FileInfo, expected_etag: &str) -> bool { - self.version_id == fi.version_id + // Normalize the nil version on both sides: a versioning-suspended object + // is `Some(Uuid::nil())` on one and `None` on the other, so a raw compare + // reports every suspended restore as "changed before finalization". + self.version_id.filter(|version_id| !version_id.is_nil()) == fi.version_id.filter(|version_id| !version_id.is_nil()) && self.data_dir == fi.data_dir && self.mod_time == fi.mod_time && self.size == fi.size diff --git a/crates/ecstore/src/storage_api_contracts/mod.rs b/crates/ecstore/src/storage_api_contracts/mod.rs index d061c257b..8a288c6b1 100644 --- a/crates/ecstore/src/storage_api_contracts/mod.rs +++ b/crates/ecstore/src/storage_api_contracts/mod.rs @@ -9,7 +9,8 @@ pub(crate) mod admin { pub(crate) mod bucket { pub(crate) use rustfs_storage_api::{ - BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp, + BUCKET_LIFECYCLE_LOCK_OBJECT, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, + SRBucketDeleteOp, }; } diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 46c9e7aa0..6055f8bf8 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -20,7 +20,7 @@ use crate::bucket::{ use crate::error::is_err_bucket_not_found; use crate::runtime::sources as runtime_sources; use crate::set_disk::get_lock_acquire_timeout; -use crate::storage_api_contracts::bucket::SRBucketDeleteOp; +use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp}; use crate::storage_api_contracts::namespace::NamespaceLocking as _; use futures::stream::{self, StreamExt}; use std::collections::BTreeMap; @@ -113,6 +113,25 @@ where } } +async fn await_bucket_lifecycle_operation( + lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>, + namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>, + bucket: &str, + operation: &'static str, + future: F, +) -> Result +where + F: Future>, +{ + await_bucket_namespace_operation( + lifecycle_guard, + bucket, + operation, + await_bucket_namespace_operation(namespace_guard, bucket, operation, future), + ) + .await +} + async fn run_bucket_usage_cleanup(guard: Option<&rustfs_lock::NamespaceLockGuard>, bucket: &str, future: F) -> Result<()> where F: Future>, @@ -131,6 +150,50 @@ where } impl ECStore { + pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result { + metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await + } + + pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result { + metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await + } + + /// The object commit path acquires this sentinel before bucket metadata and + /// exact-object namespace locks. + pub(crate) async fn acquire_bucket_lifecycle_read_lock(&self, bucket: &str) -> Result { + let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?; + lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| match err { + rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => { + StorageError::NamespaceLockQuorumUnavailable { + mode: "bucket_lifecycle_read", + bucket: bucket.to_string(), + object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(), + required, + achieved, + } + } + other => StorageError::Lock(other), + }) + } + + pub(crate) async fn acquire_bucket_lifecycle_write_lock(&self, bucket: &str) -> Result { + let lock = self.new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT).await?; + lock.get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(|err| match err { + rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => { + StorageError::NamespaceLockQuorumUnavailable { + mode: "bucket_lifecycle_write", + bucket: bucket.to_string(), + object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(), + required, + achieved, + } + } + other => StorageError::Lock(other), + }) + } + async fn mark_bucket_deleted(&self, bucket: &str) -> Result<()> { let marker_volume = bucket_deleted_marker_volume(bucket); @@ -204,25 +267,39 @@ impl ECStore { } } - async fn rollback_failed_bucket_creation(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) { + async fn rollback_failed_bucket_creation( + &self, + bucket: &str, + lifecycle_guard: Option<&rustfs_lock::NamespaceLockGuard>, + namespace_guard: Option<&rustfs_lock::NamespaceLockGuard>, + ) { let rollback_opts = DeleteBucketOptions { no_lock: true, no_recreate: true, ..Default::default() }; - if let Err(err) = await_bucket_namespace_operation(guard, bucket, "failed bucket creation rollback", async { - self.peer_sys - .delete_bucket(bucket, &rollback_opts) - .await - .map_err(|rollback_err| to_object_err(rollback_err.into(), vec![bucket])) - }) - .await + if let Err(err) = + await_bucket_lifecycle_operation(lifecycle_guard, namespace_guard, bucket, "failed bucket creation rollback", async { + self.peer_sys + .delete_bucket(bucket, &rollback_opts) + .await + .map_err(|rollback_err| to_object_err(rollback_err.into(), vec![bucket])) + }) + .await { warn!( bucket = %bucket, error = ?err, "failed bucket creation rollback did not remove every physical bucket volume" ); + return; + } + if let Err(err) = self.cleanup_deleted_bucket_metadata(bucket, false, namespace_guard).await { + warn!( + bucket = %bucket, + error = ?err, + "failed bucket creation rollback left internal bucket metadata" + ); } } @@ -234,40 +311,112 @@ impl ECStore { return Err(StorageError::BucketNameInvalid(err.to_string())); } + // Lock order: bucket lifecycle -> metadata transaction -> exact bucket namespace. + let bucket_lifecycle_guard = if !opts.no_lock { + Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?) + } else { + None + }; + let metadata_transaction_guard = if !opts.no_lock && !is_meta_bucketname(bucket) { + Some(metadata_sys::acquire_bucket_metadata_transaction_lock_in(&self.ctx, bucket).await?) + } else { + None + }; let ns_guard = if !opts.no_lock { let ns_lock = self.new_ns_lock(bucket, bucket).await?; Some( - ns_lock - .get_write_lock(get_lock_acquire_timeout()) - .await - .map_err(|e| match e { - rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => { - StorageError::NamespaceLockQuorumUnavailable { - mode: "write", - bucket: bucket.to_string(), - object: bucket.to_string(), - required, - achieved, + await_bucket_namespace_operation( + bucket_lifecycle_guard.as_ref(), + bucket, + "bucket namespace lock acquisition", + async { + ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e { + rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => { + StorageError::NamespaceLockQuorumUnavailable { + mode: "write", + bucket: bucket.to_string(), + object: bucket.to_string(), + required, + achieved, + } } - } - other => StorageError::Lock(other), - })?, + other => StorageError::Lock(other), + }) + }, + ) + .await?, ) } else { None }; - let confirmed_missing = match self.peer_sys.get_bucket_info(bucket, &BucketOptions::default()).await { - Ok(_) => false, + let existing_bucket_info = match self.peer_sys.get_bucket_info(bucket, &BucketOptions::default()).await { + Ok(info) => Some(info), Err(err) => { let err: StorageError = err.into(); if is_err_bucket_not_found(&err) { - true + None } else { return Err(to_object_err(err, vec![bucket])); } } }; + let confirmed_missing = existing_bucket_info.is_none(); + let existing_metadata = if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) { + let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?; + if !persisted { + metadata = BucketMetadata::new(bucket); + metadata.created = existing_bucket_info + .as_ref() + .and_then(|info| info.created) + .unwrap_or(OffsetDateTime::UNIX_EPOCH); + } else if !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() { + return Err(Error::other(format!( + "bucket incarnation sidecar is missing for new-format metadata: {bucket}" + ))); + } else if !metadata.bucket_incarnation_sidecar { + metadata.bucket_incarnation_id = Uuid::new_v4(); + } + Some(metadata) + } else { + None + }; + + let mut meta = existing_metadata.unwrap_or_else(|| BucketMetadata::new(bucket)); + let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar; + if confirmed_missing || is_meta_bucketname(bucket) { + meta.set_created(opts.created_at); + + if opts.versioning_enabled { + meta.versioning_config_xml = + crate::bucket::utils::serialize::(&ENABLED_VERSIONING_CONFIG)?; + } + } + + // Object Lock enable is one-way, and it must apply to an existing bucket + // too. Site replication replays make-with-versioning carrying the + // source's lockEnabled against a destination bucket that already exists; + // gating this on `confirmed_missing` returned success while leaving the + // replica unlocked, so replicated versions could be deleted without the + // retention the source enforces. + let lock_newly_enabled = opts.lock_enabled && !meta.lock_enabled; + if opts.lock_enabled { + meta.lock_enabled = true; + meta.object_lock_config_xml = + crate::bucket::utils::serialize::(&ENABLED_OBJECT_LOCK_CONFIG)?; + meta.versioning_config_xml = crate::bucket::utils::serialize::(&ENABLED_VERSIONING_CONFIG)?; + } + + let metadata_persisted_before_physical = confirmed_missing && !is_meta_bucketname(bucket) && opts.lock_enabled; + if metadata_persisted_before_physical { + metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta.clone()).await?; + if bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + return Err(Error::other(format!("bucket metadata creation intent lock was lost: {bucket}"))); + } + } if confirmed_missing && !is_meta_bucketname(bucket) { // Fence every scanner cycle that could have observed the namespace @@ -277,12 +426,23 @@ impl ECStore { self.cleanup_bucket_usage(bucket, ns_guard.as_ref()).await?; } - if let Err(err) = await_bucket_namespace_operation(ns_guard.as_ref(), bucket, "physical bucket creation", async { - self.peer_sys - .make_bucket(bucket, opts) - .await - .map_err(|err| to_object_err(err.into(), vec![bucket])) - }) + if let Err(err) = await_bucket_lifecycle_operation( + bucket_lifecycle_guard.as_ref(), + ns_guard.as_ref(), + bucket, + "physical bucket creation", + await_bucket_namespace_operation( + metadata_transaction_guard.as_ref(), + bucket, + "bucket creation metadata transaction", + async { + self.peer_sys + .make_bucket(bucket, opts) + .await + .map_err(|err| to_object_err(err.into(), vec![bucket])) + }, + ), + ) .await { if is_err_bucket_exists(&err) @@ -300,32 +460,38 @@ impl ECStore { } if !is_err_bucket_exists(&err) && ns_guard.as_ref().is_none_or(|guard| !guard.is_lock_lost()) { error!("make bucket failed: {err}"); - self.rollback_failed_bucket_creation(bucket, ns_guard.as_ref()).await; + self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref()) + .await; } return Err(err); }; - let mut meta = BucketMetadata::new(bucket); - - meta.set_created(opts.created_at); - - if opts.lock_enabled { - meta.object_lock_config_xml = - crate::bucket::utils::serialize::(&ENABLED_OBJECT_LOCK_CONFIG)?; - meta.versioning_config_xml = crate::bucket::utils::serialize::(&ENABLED_VERSIONING_CONFIG)?; + let metadata_result = async { + if metadata_persisted_before_physical { + return Ok(()); + } + if is_meta_bucketname(bucket) { + metadata_sys::set_bucket_metadata_in(&self.ctx, meta).await + } else if existing_incarnation_is_authoritative && !lock_newly_enabled { + metadata_sys::cache_bucket_metadata_in(&self.ctx, meta).await + } else { + metadata_sys::set_new_bucket_metadata_in(&self.ctx, meta).await + } } - - if opts.versioning_enabled { - meta.versioning_config_xml = crate::bucket::utils::serialize::(&ENABLED_VERSIONING_CONFIG)?; + .await; + let metadata_lock_lost = bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || metadata_transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + || ns_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()); + if let Err(err) = metadata_result { + if !metadata_lock_lost { + self.rollback_failed_bucket_creation(bucket, bucket_lifecycle_guard.as_ref(), ns_guard.as_ref()) + .await; + } + return Err(err); + } + if metadata_lock_lost { + return Err(Error::other(format!("bucket metadata initialization lock was lost: {bucket}"))); } - - await_bucket_namespace_operation( - ns_guard.as_ref(), - bucket, - "bucket metadata initialization", - metadata_sys::set_bucket_metadata_in(&self.ctx, meta), - ) - .await?; if confirmed_missing && !is_meta_bucketname(bucket) { // A scanner may have sampled the first fence before the bucket @@ -454,24 +620,34 @@ impl ECStore { return Err(StorageError::BucketNameInvalid(err.to_string())); } + let bucket_lifecycle_guard = if !opts.no_lock { + Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?) + } else { + None + }; let ns_guard = if !opts.no_lock { let ns_lock = self.new_ns_lock(bucket, bucket).await?; Some( - ns_lock - .get_write_lock(get_lock_acquire_timeout()) - .await - .map_err(|e| match e { - rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => { - StorageError::NamespaceLockQuorumUnavailable { - mode: "write", - bucket: bucket.to_string(), - object: bucket.to_string(), - required, - achieved, + await_bucket_namespace_operation( + bucket_lifecycle_guard.as_ref(), + bucket, + "bucket namespace lock acquisition", + async { + ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| match e { + rustfs_lock::error::LockError::QuorumNotReached { required, achieved } => { + StorageError::NamespaceLockQuorumUnavailable { + mode: "write", + bucket: bucket.to_string(), + object: bucket.to_string(), + required, + achieved, + } } - } - other => StorageError::Lock(other), - })?, + other => StorageError::Lock(other), + }) + }, + ) + .await?, ) } else { None @@ -519,7 +695,8 @@ impl ECStore { } if sr_mark_delete { - await_bucket_namespace_operation( + await_bucket_lifecycle_operation( + bucket_lifecycle_guard.as_ref(), ns_guard.as_ref(), bucket, "bucket delete marker creation", @@ -528,12 +705,17 @@ impl ECStore { .await?; } - let delete_result = run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async { - self.peer_sys - .delete_bucket(bucket, &delete_opts) - .await - .map_err(|err| to_object_err(err.into(), vec![bucket])) - }) + let delete_result = await_bucket_namespace_operation( + bucket_lifecycle_guard.as_ref(), + bucket, + "physical bucket deletion", + run_physical_bucket_deletion(ns_guard.as_ref(), bucket, async { + self.peer_sys + .delete_bucket(bucket, &delete_opts) + .await + .map_err(|err| to_object_err(err.into(), vec![bucket])) + }), + ) .await; if let Err(err) = delete_result && (!sr_delete || !is_err_strict_volume_not_found(&err)) @@ -591,7 +773,6 @@ mod tests { use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, SystemTime}; use time::OffsetDateTime; - use tokio::io::AsyncReadExt; use tokio::sync::{Notify, OnceCell}; use tokio_util::sync::CancellationToken; use uuid::Uuid; @@ -1229,6 +1410,86 @@ mod tests { assert!(!any_disk_path_exists(&disk_paths, bucket_deleted_marker_volume(&bucket)).await); } + /// Site replication replays make-with-versioning carrying the source's + /// `lockEnabled` against a destination bucket that already exists. Gating the + /// lock enable on `confirmed_missing` returned success while leaving the + /// replica unlocked, so replicated versions could be deleted without the + /// retention the source enforces. + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn force_create_enables_object_lock_on_an_existing_bucket() { + let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await; + let bucket = format!("bucket-force-lock-{}", Uuid::new_v4().simple()); + + ecstore + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("plain bucket should be created"); + assert!( + !metadata_sys::get_in(&ecstore.ctx, &bucket) + .await + .expect("metadata should load") + .lock_enabled, + "test setup: the bucket must start unlocked" + ); + + ecstore + .make_bucket( + &bucket, + &MakeBucketOptions { + force_create: true, + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("force create with lock_enabled should succeed on an existing bucket"); + + let meta = metadata_sys::get_in(&ecstore.ctx, &bucket) + .await + .expect("metadata should load after the lock enable"); + assert!(meta.lock_enabled, "Object Lock must be enabled on the existing bucket"); + assert!( + !meta.object_lock_config_xml.is_empty(), + "the Object Lock configuration must be persisted, not just the flag" + ); + assert!( + !meta.versioning_config_xml.is_empty(), + "Object Lock requires versioning, so that must be persisted too" + ); + } + + /// `DeleteBucket`'s emptiness check is a raw disk scan (`has_xlmeta_files`), + /// not an S3-level listing, so "the client drained the bucket" and "the + /// bucket is deletable" are two different contracts. Nothing pinned the + /// second one, which is how the s3-tests lane ended up failing 219 cases on + /// `nuke_prefixed_buckets` while every test body passed. + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn bucket_delete_succeeds_after_the_last_object_version_is_deleted() { + let (disk_paths, ecstore) = setup_bucket_delete_test_env().await; + let bucket = format!("bucket-delete-after-drain-{}", Uuid::new_v4().simple()); + let object = "object.txt"; + + create_bucket_with_object(&ecstore, &bucket, object).await; + + ecstore + .delete_object(&bucket, object, ObjectOptions::default()) + .await + .expect("client delete of the only object should succeed"); + + assert!( + !any_disk_has_object_metadata(&disk_paths, &bucket).await, + "deleting the last version must not leave xl.meta on disk: DeleteBucket scans the raw \ + bucket directory, so residue here is reported to clients as BucketNotEmpty" + ); + + ecstore + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("DeleteBucket must succeed once the client has drained the bucket"); + } + #[tokio::test] #[serial] async fn bucket_delete_default_s3_delete_still_rejects_non_empty_bucket() { @@ -1262,11 +1523,11 @@ mod tests { #[tokio::test] #[serial] - async fn bucket_delete_preserves_put_committed_after_empty_scan() { + async fn bucket_delete_fences_put_started_after_empty_scan() { let (_, ecstore) = setup_bucket_delete_test_env().await; let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple()); let object = "committed-after-empty-scan"; - let payload = b"object committed after DeleteBucket empty scan".to_vec(); + let payload = b"object started after DeleteBucket empty scan".to_vec(); ecstore .make_bucket(&bucket, &MakeBucketOptions::default()) @@ -1283,30 +1544,29 @@ mod tests { }); barrier.wait_until_paused().await; - let mut put_reader = PutObjReader::from_vec(payload.clone()); - ecstore - .put_object(&bucket, object, &mut put_reader, &ObjectOptions::default()) - .await - .expect("PUT should commit while DeleteBucket is paused after its empty scan"); + let put_store = ecstore.clone(); + let put_bucket = bucket.clone(); + let mut put = tokio::spawn(async move { + let mut put_reader = PutObjReader::from_vec(payload); + put_store + .put_object(&put_bucket, object, &mut put_reader, &ObjectOptions::default()) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut put).await.is_err(), + "PUT must wait behind the DeleteBucket lifecycle fence" + ); barrier.release(); - let err = delete + delete .await .expect("DeleteBucket task should join") - .expect_err("DeleteBucket must reject a PUT committed after its empty scan"); - assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket)); - - let mut reader = ecstore - .get_object_reader(&bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default()) + .expect("DeleteBucket should commit while holding the lifecycle fence"); + let err = put .await - .expect("committed object should remain readable after DeleteBucket fails"); - let mut restored = Vec::new(); - reader - .stream - .read_to_end(&mut restored) - .await - .expect("object body should remain readable"); - assert_eq!(restored, payload); + .expect("PUT task should join") + .expect_err("PUT must not recreate an object in the deleted bucket"); + assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket)); } #[tokio::test] @@ -1441,8 +1701,11 @@ mod tests { crate::data_usage::store_data_usage_in_backend(snapshot, ecstore.clone()) .await .expect("the usage fixture should be stored"); + crate::bucket::metadata::save_bucket_incarnation(ecstore.clone(), &bucket, Uuid::new_v4()) + .await + .expect("partial create should have an incarnation sidecar"); - ecstore.rollback_failed_bucket_creation(&bucket, None).await; + ecstore.rollback_failed_bucket_creation(&bucket, None, None).await; assert!( ecstore @@ -1452,6 +1715,13 @@ mod tests { .is_err(), "failed-create rollback should remove the partial physical bucket" ); + assert!( + crate::bucket::metadata::load_bucket_incarnation(ecstore.clone(), &bucket) + .await + .expect("rollback sidecar lookup should succeed") + .is_none(), + "failed-create rollback must remove the orphan incarnation sidecar" + ); let persisted = crate::data_usage::load_data_usage_from_backend(ecstore.clone()) .await .expect("the usage snapshot should remain readable"); diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index f8b1e2c26..a5ce7baf4 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -593,11 +593,7 @@ mod tests { tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request}, warm_backend::{TransitionCandidateProbe, WarmBackend}, }, - storage_api_contracts::{ - bucket::{BucketOperations as _, MakeBucketOptions}, - list::ListOperations as _, - object::ObjectOperations as _, - }, + storage_api_contracts::list::ListOperations as _, }; use crate::{ core::pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus}, @@ -606,13 +602,19 @@ mod tests { layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}, services::rebalance::RebalanceMeta, - storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec}, + storage_api_contracts::{ + bucket::{BucketOperations as _, MakeBucketOptions}, + multipart::MultipartOperations as _, + object::{ObjectIO, ObjectOperations as _}, + range::HTTPRangeSpec, + }, }; use http::HeaderMap; use rustfs_config::server_config::KVS; #[cfg(feature = "test-util")] use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase}; use std::{ + collections::HashMap, future::Future, io::Cursor, sync::{ @@ -622,6 +624,7 @@ mod tests { time::Duration, }; use time::OffsetDateTime; + use tokio::io::AsyncReadExt; use tokio_util::sync::CancellationToken; #[derive(Debug)] @@ -1336,6 +1339,1226 @@ mod tests { assert!(!Arc::ptr_eq(&sys_a, &sys_b), "each store must own a distinct bucket metadata system"); } + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn same_name_bucket_recursive_delete_uses_owning_instance_object_lock_state() { + let temp_a = tempfile::tempdir().expect("create temp store dir a"); + let temp_b = tempfile::tempdir().expect("create temp store dir b"); + let (_ctx_a, store_a, _shutdown_a) = + without_storage_class_env(build_isolated_test_store(temp_a.path(), "delete-scope-a", &[4])).await; + let (_ctx_b, store_b, _shutdown_b) = + without_storage_class_env(build_isolated_test_store(temp_b.path(), "delete-scope-b", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store_a.clone(), Vec::new()).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store_b.clone(), Vec::new()).await; + + let bucket = format!("same-name-delete-scope-{}", uuid::Uuid::new_v4()); + store_a + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create plain bucket in store A"); + store_b + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create Object Lock bucket in store B"); + + for store in [&store_a, &store_b] { + let mut reader = PutObjReader::from_vec(b"payload".to_vec()); + store + .put_object( + &bucket, + "prefix/object.bin", + &mut reader, + &ObjectOptions { + versioned: Arc::ptr_eq(store, &store_b), + ..Default::default() + }, + ) + .await + .expect("put same-name object"); + } + + store_a + .delete_object( + &bucket, + "prefix", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("plain store prefix delete should remain allowed"); + + let err = store_b + .delete_object( + &bucket, + "prefix", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("store B must not consult store A's plain bucket metadata"); + assert!(matches!(err, StorageError::InvalidArgument(_, _, _))); + store_b + .get_object_info(&bucket, "prefix/object.bin", &ObjectOptions::default()) + .await + .expect("locked store object must survive the rejected prefix delete"); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn recursive_delete_checks_object_lock_on_every_version_page() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "recursive-delete-pagination", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("recursive-page-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create plain bucket"); + for object in ["prefix/a.bin", "prefix/b.bin"] { + let mut reader = PutObjReader::from_vec(b"plain".to_vec()); + store + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("write first-page object"); + } + let locked_object = "prefix/z-locked.bin"; + let mut reader = PutObjReader::from_vec(b"locked".to_vec()); + store + .put_object( + &bucket, + locked_object, + &mut reader, + &ObjectOptions { + user_defined: HashMap::from([ + ( + s3s::header::X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(), + s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(), + ), + ( + s3s::header::X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(), + "2099-01-01T00:00:00Z".to_string(), + ), + ]), + ..Default::default() + }, + ) + .await + .expect("write second-page locked object"); + + let first_page = store.pools[0].disk_set[0] + .clone() + .inner_list_object_versions_for_recursive_delete(&bucket, "prefix/", None, None, 2) + .await + .expect("list the first recursive-delete validation page"); + assert!(first_page.is_truncated, "the fixture must require a continuation page"); + assert!( + first_page.objects.iter().all(|object| object.name != locked_object), + "the locked object must not be visible on the first page" + ); + let second_page = store.pools[0].disk_set[0] + .clone() + .inner_list_object_versions_for_recursive_delete( + &bucket, + "prefix/", + first_page.next_marker, + first_page.next_version_idmarker, + 2, + ) + .await + .expect("list the recursive-delete continuation page"); + assert!( + second_page.objects.iter().any(|object| object.name == locked_object), + "the continuation page must contain the locked object" + ); + + let err = store + .delete_object( + &bucket, + "prefix/", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("recursive delete must scan the continuation page and find the locked object"); + assert!( + matches!(err, StorageError::PrefixAccessDenied(ref name, ref object) if name == &bucket && object == locked_object), + "unexpected recursive delete error: {err:?}" + ); + for object in ["prefix/a.bin", "prefix/b.bin", locked_object] { + store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("a rejected recursive delete must leave every page intact"); + } + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn delete_holds_bucket_incarnation_sentinel_through_object_commit() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "delete-bucket-incarnation", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("delete-bucket-incarnation-{}", uuid::Uuid::new_v4()); + let object = "same-key.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create initial plain bucket"); + let old_incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("read initial bucket incarnation"); + let mut reader = PutObjReader::from_vec(b"old generation".to_vec()); + store + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("put old-generation object"); + + let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object(&delete_bucket, object, ObjectOptions::default()) + .await + }); + barrier.wait_until_paused().await; + + let recreate_store = Arc::clone(&store); + let recreate_bucket = bucket.clone(); + let mut recreate = tokio::spawn(async move { + recreate_store + .delete_bucket( + &recreate_bucket, + &crate::storage_api_contracts::bucket::DeleteBucketOptions { + force: true, + ..Default::default() + }, + ) + .await?; + recreate_store + .make_bucket( + &recreate_bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await?; + let mut reader = PutObjReader::from_vec(b"new protected generation".to_vec()); + recreate_store + .put_object( + &recreate_bucket, + object, + &mut reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await?; + Ok::<_, Error>(()) + }); + + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(), + "bucket delete and same-name recreation must wait for the in-flight object delete" + ); + barrier.release(); + delete + .await + .expect("delete task should join") + .expect("old-generation delete should complete"); + tokio::time::timeout(Duration::from_secs(10), recreate) + .await + .expect("bucket recreation should finish after delete releases the sentinel") + .expect("bucket recreation task should join") + .expect("bucket recreation should succeed"); + + let stale_delete_err = store + .delete_object( + &bucket, + object, + ObjectOptions { + expected_bucket_incarnation_id: Some(old_incarnation), + versioned: true, + ..Default::default() + }, + ) + .await + .expect_err("an authorization snapshot from the old bucket incarnation must be rejected"); + assert!(matches!(stale_delete_err, StorageError::BucketNotFound(name) if name == bucket)); + + let (_deleted, stale_batch_errors) = store + .delete_objects( + &bucket, + vec![crate::storage_api_contracts::object::ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + ObjectOptions { + expected_bucket_incarnation_id: Some(old_incarnation), + versioned: true, + ..Default::default() + }, + ) + .await; + assert!( + matches!(stale_batch_errors.as_slice(), [Some(StorageError::BucketNotFound(name))] if name.as_str() == bucket), + "batch delete must reject an authorization snapshot from the old bucket incarnation" + ); + + store + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("the new locked-bucket object must survive the old-generation delete"); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn delete_objects_holds_bucket_incarnation_sentinel_through_commit() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "batch-delete-bucket-incarnation", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("batch-incarnation-{}", uuid::Uuid::new_v4()); + let object = "same-key.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create initial bucket"); + let mut reader = PutObjReader::from_vec(b"old generation".to_vec()); + store + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("put old-generation object"); + + let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_objects( + &delete_bucket, + vec![crate::storage_api_contracts::object::ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await + }); + barrier.wait_until_paused().await; + + let recreate_store = Arc::clone(&store); + let recreate_bucket = bucket.clone(); + let mut recreate = tokio::spawn(async move { + recreate_store + .delete_bucket( + &recreate_bucket, + &crate::storage_api_contracts::bucket::DeleteBucketOptions { + force: true, + ..Default::default() + }, + ) + .await?; + recreate_store + .make_bucket(&recreate_bucket, &MakeBucketOptions::default()) + .await?; + let mut reader = PutObjReader::from_vec(b"new generation".to_vec()); + recreate_store + .put_object(&recreate_bucket, object, &mut reader, &ObjectOptions::default()) + .await?; + Ok::<_, Error>(()) + }); + + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut recreate).await.is_err(), + "bucket recreation must wait for the in-flight batch delete" + ); + barrier.release(); + let (_deleted, errors) = delete.await.expect("batch delete task should join"); + assert!( + errors.iter().all(Option::is_none), + "old-generation batch delete should complete: {errors:?}" + ); + tokio::time::timeout(Duration::from_secs(10), recreate) + .await + .expect("bucket recreation should finish after batch delete releases the sentinel") + .expect("bucket recreation task should join") + .expect("bucket recreation should succeed"); + store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("new-generation object must survive the old batch delete"); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn prefix_delete_serializes_with_concurrent_object_lock_metadata_updates() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "prefix-delete-lock-metadata", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let cases = [ + ( + "compliance", + HashMap::from([ + ("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string()), + ("x-amz-object-lock-retain-until-date".to_string(), "2099-01-01T00:00:00Z".to_string()), + ]), + ), + ( + "legal-hold", + HashMap::from([("x-amz-object-lock-legal-hold".to_string(), "ON".to_string())]), + ), + ]; + + for (case, eval_metadata) in cases { + let bucket = format!("prefix-lock-{case}-{}", uuid::Uuid::new_v4()); + let object = "protected-after-scan.bin"; + store + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create Object Lock bucket"); + let mut reader = PutObjReader::from_vec(b"body".to_vec()); + let object_info = store + .put_object( + &bucket, + object, + &mut reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("write unprotected version"); + let version_id = object_info.version_id.expect("versioned PUT should return an ID").to_string(); + + let barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object( + &delete_bucket, + object, + ObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + versioned: true, + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + + let update_store = Arc::clone(&store); + let update_bucket = bucket.clone(); + let mut update = tokio::spawn(async move { + update_store + .put_object_metadata( + &update_bucket, + object, + &ObjectOptions { + version_id: Some(version_id), + versioned: true, + eval_metadata: Some(eval_metadata), + ..Default::default() + }, + ) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(), + "{case} metadata update must wait behind the prefix-delete lifecycle write fence" + ); + + barrier.release(); + delete + .await + .expect("prefix delete task should join") + .expect("prefix delete should remove the unprotected version"); + update + .await + .expect("metadata update task should join") + .expect_err("metadata update must not succeed after the version was deleted"); + assert!( + store + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .is_err(), + "{case} case must not report a successful protected write followed by deletion" + ); + } + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn put_rejects_same_name_bucket_recreated_after_request_snapshot() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "put-bucket-incarnation", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("put-bucket-incarnation-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create initial bucket"); + let old_incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("read initial bucket incarnation"); + + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + object, + crate::set_disk::PutObjectCommitPause::BeforeNamespace, + ); + let put_store = Arc::clone(&store); + let put_bucket = bucket.clone(); + let put = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(b"old generation".to_vec()); + put_store + .put_object( + &put_bucket, + object, + &mut reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + }); + barrier.wait_until_paused().await; + + store + .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) + .await + .expect("delete initial empty bucket while PUT is staged"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("recreate same-name bucket"); + let new_incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("read recreated bucket incarnation"); + assert_ne!(old_incarnation, new_incarnation); + + barrier.release(); + let err = put + .await + .expect("PUT task should join") + .expect_err("old-generation PUT must not commit into the recreated bucket"); + assert!( + matches!(err, StorageError::BucketNotFound(ref name) if name == &bucket), + "unexpected stale PUT error: {err:?}" + ); + assert!( + store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .is_err(), + "the recreated bucket must not contain the staged old-generation object" + ); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn exact_version_put_uses_only_its_own_bucket_object_lock_snapshot() { + let target_temp = tempfile::tempdir().expect("create target store dir"); + let foreign_temp = tempfile::tempdir().expect("create foreign store dir"); + let (target_ctx, target_store, _target_shutdown) = + without_storage_class_env(build_isolated_test_store(target_temp.path(), "put-object-lock-target", &[4])).await; + let (_foreign_ctx, foreign_store, _foreign_shutdown) = + without_storage_class_env(build_isolated_test_store(foreign_temp.path(), "put-object-lock-foreign", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(target_store.clone(), Vec::new()).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(foreign_store.clone(), Vec::new()).await; + + let bucket = format!("put-object-lock-snapshot-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + target_store + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create target Object Lock bucket"); + foreign_store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create same-name plain bucket in foreign store"); + let stale_snapshot = target_store + .object_lock_config_snapshot(&bucket) + .await + .expect("load target snapshot before changing its default retention"); + + let mut metadata = crate::bucket::metadata_sys::get_in(&target_ctx, &bucket) + .await + .expect("load target bucket metadata") + .as_ref() + .clone(); + metadata + .update_config( + crate::bucket::metadata::OBJECT_LOCK_CONFIG, + b"EnabledCOMPLIANCE1".to_vec(), + ) + .expect("set default COMPLIANCE metadata"); + let update_started = Arc::new(tokio::sync::Barrier::new(2)); + let update_ctx = Arc::clone(&target_ctx); + let update_started_task = Arc::clone(&update_started); + let mut update = tokio::spawn(async move { + update_started_task.wait().await; + let _transaction_guard = + crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_lock_in(&update_ctx, &metadata.name).await?; + crate::bucket::metadata_sys::set_bucket_metadata_in(&update_ctx, metadata).await + }); + update_started.wait().await; + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut update).await.is_err(), + "an active Object Lock snapshot must hold the metadata revision read fence" + ); + drop(stale_snapshot); + update + .await + .expect("Object Lock metadata update task should join") + .expect("persist target Object Lock metadata after snapshot release"); + + let original_body = b"protected body".to_vec(); + let mut original_reader = PutObjReader::from_vec(original_body.clone()); + let original = target_store + .put_object( + &bucket, + object, + &mut original_reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("write protected target version"); + let version_id = original + .version_id + .expect("versioned PUT should return a version ID") + .to_string(); + + let mut replacement = PutObjReader::from_vec(b"replacement".to_vec()); + let err = target_store + .put_object( + &bucket, + object, + &mut replacement, + &ObjectOptions { + version_id: Some(version_id.clone()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect_err("ECStore must load default COMPLIANCE before an exact-version overwrite"); + assert!(matches!(err, StorageError::PrefixAccessDenied(_, _))); + + let foreign_snapshot = foreign_store + .object_lock_config_snapshot(&bucket) + .await + .expect("load same-name foreign bucket snapshot"); + let mut foreign_replacement = PutObjReader::from_vec(b"foreign replacement".to_vec()); + target_store + .put_object( + &bucket, + object, + &mut foreign_replacement, + &ObjectOptions { + version_id: Some(version_id.clone()), + versioned: true, + object_lock_config_snapshot: Some(foreign_snapshot), + ..Default::default() + }, + ) + .await + .expect_err("a snapshot from another store must fail closed"); + + let mut reader = target_store + .get_object_reader( + &bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + version_id: Some(version_id), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("protected target version should remain readable"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("protected body should drain"); + assert_eq!(body, original_body); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn force_create_existing_bucket_preserves_incarnation_and_inflight_request() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "force-create-incarnation", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("force-create-incarnation-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create initial bucket"); + let incarnation = store.bucket_incarnation_id(&bucket).await.expect("read initial incarnation"); + + store + .make_bucket( + &bucket, + &MakeBucketOptions { + force_create: true, + ..Default::default() + }, + ) + .await + .expect("idempotent force-create should succeed"); + assert_eq!( + store.bucket_incarnation_id(&bucket).await.unwrap(), + incarnation, + "idempotent force-create must not rotate the bucket generation" + ); + + let mut reader = PutObjReader::from_vec(b"authorized before force-create".to_vec()); + store + .put_object( + &bucket, + "inflight.bin", + &mut reader, + &ObjectOptions { + expected_bucket_incarnation_id: Some(incarnation), + ..Default::default() + }, + ) + .await + .expect("a request authorized before idempotent force-create must still commit"); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn lock_enabled_create_retries_metadata_intent_without_unlocking() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-intent-retry", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("lock-create-intent-retry-{}", uuid::Uuid::new_v4()); + let mut intent = crate::bucket::metadata::BucketMetadata::new(&bucket); + intent.lock_enabled = true; + crate::bucket::metadata_sys::set_new_bucket_metadata_in(&ctx, intent) + .await + .expect("persist pre-visibility lock intent"); + assert!( + store + .peer_sys + .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) + .await + .is_err(), + "the intent must not make the physical bucket visible" + ); + + store + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("retry lock-enabled creation"); + assert!(matches!( + crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket) + .await + .expect("read retried lock state"), + crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. } + )); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn physical_lock_bucket_with_incomplete_metadata_fails_closed() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-incomplete-metadata", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("lock-create-incomplete-metadata-{}", uuid::Uuid::new_v4()); + let mut metadata = crate::bucket::metadata::BucketMetadata::new(&bucket); + metadata.lock_enabled = true; + metadata + .save_with_store(store.clone()) + .await + .expect("persist lock metadata before sidecar"); + store + .peer_sys + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("expose physical bucket after metadata phase"); + + let err = crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket) + .await + .expect_err("missing sidecar must not fabricate an unlocked bucket"); + assert!(err.to_string().contains("sidecar is missing"), "unexpected error: {err}"); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn successful_lock_enabled_create_has_complete_metadata_intent() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "lock-create-complete-metadata", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("lock-create-{}", uuid::Uuid::new_v4()); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create lock-enabled bucket"); + let (metadata, persisted) = crate::bucket::metadata_sys::get_config_from_disk_with_presence_in(&ctx, &bucket) + .await + .expect("read completed metadata intent"); + assert!(persisted && metadata.bucket_incarnation_sidecar && metadata.lock_enabled); + assert!(matches!( + crate::bucket::metadata_sys::get_object_lock_config_state_in(&ctx, &bucket) + .await + .expect("read completed lock state"), + crate::bucket::metadata_sys::ObjectLockConfigState::Configured { .. } + )); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn lifecycle_expiry_rejects_task_from_recreated_bucket_incarnation() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "lifecycle-expiry-incarnation", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("lifecycle-incarnation-{}", uuid::Uuid::new_v4()); + let object = "same-key.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create initial bucket"); + let old_incarnation = store.bucket_incarnation_id(&bucket).await.expect("read old incarnation"); + let mut old_reader = PutObjReader::from_vec(b"old-generation".to_vec()); + store + .put_object(&bucket, object, &mut old_reader, &ObjectOptions::default()) + .await + .expect("put old-generation object"); + let old_object = store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("snapshot old-generation object"); + + store + .delete_object(&bucket, object, ObjectOptions::default()) + .await + .expect("delete old-generation object"); + store + .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) + .await + .expect("delete old bucket"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("recreate bucket"); + let mut new_reader = PutObjReader::from_vec(b"new-generation".to_vec()); + store + .put_object(&bucket, object, &mut new_reader, &ObjectOptions::default()) + .await + .expect("put replacement object"); + + let applied = crate::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_on_non_transitioned_objects( + store.clone(), + &old_object, + &crate::bucket::lifecycle::lifecycle::Event { + action: crate::bucket::lifecycle::lifecycle::IlmAction::DeleteAction, + ..Default::default() + }, + &crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc::Scanner, + old_incarnation, + ) + .await; + assert!(!applied, "an expiry task from the old bucket incarnation must fail closed"); + + let mut reader = store + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("new-generation object must survive stale expiry"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("read replacement object"); + assert_eq!(body, b"new-generation"); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn pre_upgrade_multipart_upload_can_complete_and_abort_in_the_same_bucket_lifetime() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-upgrade-compat", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("multipart-upgrade-compat-{}", uuid::Uuid::new_v4()); + let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid"); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + created_at: Some(created), + ..Default::default() + }, + ) + .await + .expect("create bucket"); + + let complete_object = "complete.bin"; + let complete_upload = store.pools[0] + .new_multipart_upload( + &bucket, + complete_object, + &ObjectOptions { + mod_time: Some(created + time::Duration::seconds(1)), + ..Default::default() + }, + ) + .await + .expect("create a pre-upgrade multipart upload without an incarnation stamp"); + let mut part_reader = PutObjReader::from_vec(b"pre-upgrade multipart body".to_vec()); + let part = store.pools[0] + .put_object_part( + &bucket, + complete_object, + &complete_upload.upload_id, + 1, + &mut part_reader, + &ObjectOptions::default(), + ) + .await + .expect("stage a pre-upgrade multipart part"); + store + .clone() + .complete_multipart_upload( + &bucket, + complete_object, + &complete_upload.upload_id, + vec![crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }], + &ObjectOptions::default(), + ) + .await + .expect("a pre-upgrade upload from the current bucket lifetime should complete"); + store + .get_object_info(&bucket, complete_object, &ObjectOptions::default()) + .await + .expect("completed pre-upgrade upload should be visible"); + + let abort_object = "abort.bin"; + let abort_upload = store.pools[0] + .new_multipart_upload( + &bucket, + abort_object, + &ObjectOptions { + mod_time: Some(created + time::Duration::seconds(2)), + ..Default::default() + }, + ) + .await + .expect("create another pre-upgrade multipart upload"); + store + .abort_multipart_upload(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default()) + .await + .expect("a pre-upgrade upload from the current bucket lifetime should abort"); + let err = store + .get_multipart_info(&bucket, abort_object, &abort_upload.upload_id, &ObjectOptions::default()) + .await + .expect_err("aborted pre-upgrade upload should be removed"); + assert!(matches!(err, StorageError::InvalidUploadID(..))); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn complete_multipart_rejects_legacy_upload_from_recreated_bucket_incarnation() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "multipart-bucket-incarnation", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("multipart-incarnation-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + let created = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed timestamp should be valid"); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + created_at: Some(created), + ..Default::default() + }, + ) + .await + .expect("create initial bucket"); + let old_incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("read initial bucket incarnation"); + let upload = store.pools[0] + .new_multipart_upload( + &bucket, + object, + &ObjectOptions { + mod_time: Some(created + time::Duration::seconds(1)), + ..Default::default() + }, + ) + .await + .expect("create old-generation legacy multipart upload"); + + store + .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) + .await + .expect("delete bucket with an incomplete multipart upload"); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + created_at: Some(created + time::Duration::seconds(2)), + ..Default::default() + }, + ) + .await + .expect("recreate same-name bucket"); + let new_incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("read recreated bucket incarnation"); + assert_ne!(old_incarnation, new_incarnation); + + let err = store + .clone() + .complete_multipart_upload(&bucket, object, &upload.upload_id, Vec::new(), &ObjectOptions::default()) + .await + .expect_err("an upload initiated in the old bucket incarnation must not commit into the replacement"); + assert!( + match &err { + StorageError::BucketNotFound(name) => name == &bucket, + StorageError::InvalidUploadID(name, key, _) => name == &bucket && key == object, + _ => false, + }, + "unexpected stale multipart completion error: {err:?}" + ); + assert!( + store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .is_err(), + "the recreated bucket must not contain the old-generation multipart upload" + ); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn delete_after_bucket_validation_preserves_bucket_not_found() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "delete-missing-bucket", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("delete-missing-bucket-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket before validation"); + store + .get_bucket_info(&bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) + .await + .expect("request-level bucket validation should succeed"); + store + .delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default()) + .await + .expect("delete bucket after validation"); + + let err = store + .delete_object(&bucket, "missing.bin", ObjectOptions::default()) + .await + .expect_err("post-validation bucket deletion must not become an object-not-found error"); + + assert!(matches!(err, StorageError::BucketNotFound(name) if name == bucket)); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn object_lock_metadata_read_error_blocks_every_destructive_delete_shape() { + let temp = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp.path(), "delete-config-read-error", &[4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("delete-config-read-error-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create bucket"); + for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] { + let mut reader = PutObjReader::from_vec(b"must survive".to_vec()); + store + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("put object before metadata corruption"); + } + + crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket) + .await + .expect("inject single-delete metadata read failure"); + + let single_err = store + .delete_object(&bucket, "single.bin", ObjectOptions::default()) + .await + .expect_err("single delete must propagate the metadata read failure"); + assert!( + single_err + .to_string() + .contains("injected Object Lock metadata disk read failure") + ); + + crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket) + .await + .expect("inject batch-delete metadata read failure"); + let (_deleted, batch_errs) = store + .delete_objects( + &bucket, + vec![crate::storage_api_contracts::object::ObjectToDelete { + object_name: "batch.bin".to_string(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await; + assert!( + batch_errs[0] + .as_ref() + .is_some_and(|err| err.to_string().contains("injected Object Lock metadata disk read failure")) + ); + + crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket) + .await + .expect("inject prefix-delete metadata read failure"); + let prefix_err = store + .delete_object( + &bucket, + "prefix", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("prefix delete must propagate the metadata read failure"); + assert!( + prefix_err + .to_string() + .contains("injected Object Lock metadata disk read failure") + ); + + crate::bucket::metadata_sys::inject_object_lock_disk_read_error_in(&ctx, &bucket) + .await + .expect("inject lifecycle-style delete-all metadata read failure"); + let lifecycle_err = store + .delete_object( + &bucket, + "lifecycle.bin", + ObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + ..Default::default() + }, + ) + .await + .expect_err("lifecycle-style delete-all must propagate the metadata read failure"); + assert!( + lifecycle_err + .to_string() + .contains("injected Object Lock metadata disk read failure") + ); + + for object in ["single.bin", "batch.bin", "prefix/child.bin", "lifecycle.bin"] { + store + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("metadata failure must preserve every object"); + } + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 2c41676ac..c8b9ecbbb 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -5819,6 +5819,83 @@ impl Sets { } impl SetDisks { + pub(crate) async fn inner_list_object_versions_for_recursive_delete( + self: Arc, + bucket: &str, + prefix: &str, + marker: Option, + version_marker: Option, + max_keys: i32, + ) -> Result { + let max_keys = normalize_max_keys(max_keys); + if marker.is_none() && version_marker.is_some() { + return Err(StorageError::NotImplemented); + } + + let has_version_marker = version_marker.is_some(); + let version_marker = version_marker.map(parse_version_marker).transpose()?; + let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) }; + let mut opts = ListPathOptions { + bucket: bucket.to_owned(), + prefix: prefix.to_owned(), + limit: effective_max_keys, + marker, + incl_deleted: true, + ask_disks: list_objects_quorum_from_env(), + versioned: true, + include_marker: has_version_marker, + ..Default::default() + }; + opts.parse_marker(); + + let mut list_result = self + .list_path_result(&opts) + .await + .unwrap_or_else(|err| MetaCacheEntriesSortedResult { + err: Some(err.into()), + ..Default::default() + }); + let next_cache_id = list_result.entries.as_ref().and_then(|entries| entries.list_id.clone()); + let disk_has_more = list_result.err.is_none(); + if let Some(err) = list_result.err.take() + && err != rustfs_filemeta::Error::Unexpected + { + return Err(to_object_err(err.into(), vec![bucket, prefix])); + } + if let Some(result) = list_result.entries.as_mut() + && !has_version_marker + { + result.forward_past(opts.marker.clone()); + } + let version_marker = version_marker_for_entries(list_result.entries.as_ref(), opts.marker.as_deref(), version_marker); + let last_scanned_key = last_scanned_entry_name(list_result.entries.as_ref()); + let entries = list_result.entries.unwrap_or_default(); + let get_objects = ObjectInfo::from_meta_cache_entries_sorted_versions_for_recursive_delete( + &entries, + bucket, + prefix, + None, + version_marker, + ) + .await?; + let (objects, prefixes, is_truncated, next_marker, next_version_idmarker) = list_objects_paginate( + get_objects, + &None, + max_keys, + disk_has_more, + next_cache_id.as_deref(), + true, + last_scanned_key.as_deref(), + ); + Ok(ListObjectVersionsInfo { + is_truncated, + next_marker, + next_version_idmarker, + objects, + prefixes, + }) + } + #[allow(clippy::too_many_arguments)] pub async fn inner_list_objects_v2( self: Arc, diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 4e6bf3d15..0c4be2476 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -656,8 +656,18 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore { delimiter: Option, max_uploads: usize, ) -> Result { - self.handle_list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads) - .await + self.handle_list_multipart_uploads( + bucket, + multipart::MultipartUploadListRequest { + prefix: prefix.to_string(), + key_marker, + upload_id_marker, + delimiter, + max_uploads, + expected_incarnation_id: None, + }, + ) + .await } #[instrument(skip(self))] diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index d595d1885..ccf4b5dc1 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -16,8 +16,21 @@ use super::*; use crate::multipart_listing::paginate_multipart_listing; use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::multipart::MultipartOperations as _; +use futures::{StreamExt, stream}; use std::collections::HashSet; +const MULTIPART_LIST_SET_CONCURRENCY: usize = 4; + +#[derive(Clone, Debug)] +pub(super) struct MultipartUploadListRequest { + pub(super) prefix: String, + pub(super) key_marker: Option, + pub(super) upload_id_marker: Option, + pub(super) delimiter: Option, + pub(super) max_uploads: usize, + pub(super) expected_incarnation_id: Option, +} + fn map_multipart_namespace_lock_error( bucket: &str, object: &str, @@ -36,7 +49,137 @@ fn map_multipart_namespace_lock_error( } } +fn ensure_multipart_bucket_lifecycle_guard_held( + guard: Option<&rustfs_lock::NamespaceLockGuard>, + bucket: &str, + object: &str, +) -> Result<()> { + if guard.is_some_and(rustfs_lock::NamespaceLockGuard::is_lock_lost) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "multipart_bucket_generation", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + Ok(()) +} + +async fn list_pool_multipart_uploads_for_incarnation( + pool: &crate::core::sets::Sets, + bucket: &str, + request: &MultipartUploadListRequest, +) -> Result { + let per_set_limit = request.max_uploads.saturating_add(1); + let results = stream::iter(pool.disk_set.iter().cloned()) + .map(|set| { + let request = request.clone(); + async move { + set.list_multipart_uploads_for_incarnation( + bucket, + &request.prefix, + request.key_marker, + request.upload_id_marker, + request.delimiter, + per_set_limit, + request.expected_incarnation_id, + ) + .await + } + }) + .buffer_unordered(MULTIPART_LIST_SET_CONCURRENCY) + .collect::>() + .await; + + let mut uploads = Vec::new(); + let mut common_prefixes = HashSet::new(); + let mut source_truncated = false; + for result in results { + let page = result?; + uploads.extend(page.uploads); + common_prefixes.extend(page.common_prefixes); + source_truncated |= page.is_truncated; + } + + let page = paginate_multipart_listing( + uploads, + common_prefixes.into_iter().collect(), + request.key_marker.as_deref(), + request.key_marker.as_ref().and(request.upload_id_marker.as_deref()), + request.max_uploads, + source_truncated, + ); + + Ok(ListMultipartsInfo { + key_marker: request.key_marker.clone(), + upload_id_marker: request.upload_id_marker.clone(), + next_key_marker: page.next_key_marker, + next_upload_id_marker: page.next_upload_id_marker, + max_uploads: request.max_uploads, + is_truncated: page.is_truncated, + uploads: page.uploads, + common_prefixes: page.common_prefixes, + prefix: request.prefix.clone(), + delimiter: request.delimiter.clone(), + }) +} + impl ECStore { + #[allow(clippy::too_many_arguments)] + pub async fn list_multipart_uploads_for_bucket_incarnation( + &self, + bucket: &str, + prefix: &str, + key_marker: Option, + upload_id_marker: Option, + delimiter: Option, + max_uploads: usize, + expected_incarnation_id: Uuid, + ) -> Result { + self.handle_list_multipart_uploads( + bucket, + MultipartUploadListRequest { + prefix: prefix.to_string(), + key_marker, + upload_id_marker, + delimiter, + max_uploads, + expected_incarnation_id: Some(expected_incarnation_id), + }, + ) + .await + } + + /// Multipart lock order is bucket lifecycle, generation validation, then + /// object/upload locks in the selected set. + async fn guard_multipart_bucket_incarnation( + &self, + bucket: &str, + opts: &ObjectOptions, + ) -> Result<(ObjectOptions, Option)> { + let mut opts = opts.clone(); + if is_meta_bucketname(bucket) { + return Ok((opts, None)); + } + if opts.expected_bucket_incarnation_id.is_none() { + opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?); + } + let guard = if opts.bucket_lifecycle_lock_fence.is_some() { + None + } else { + Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?) + }; + if let Some(guard) = guard.as_ref() { + opts.add_bucket_lifecycle_lock_guard(guard); + } + let current = crate::bucket::metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?; + if opts.expected_bucket_incarnation_id != Some(current) { + return Err(StorageError::BucketNotFound(bucket.to_string())); + } + Ok((opts, guard)) + } + async fn acquire_list_parts_read_lock( &self, bucket: &str, @@ -66,6 +209,8 @@ impl ECStore { opts: &ObjectOptions, ) -> Result { check_list_parts_args(bucket, object, upload_id)?; + let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let opts = &opts; let _object_lock_guard = self.acquire_list_parts_read_lock(bucket, object, opts).await?; @@ -100,22 +245,38 @@ impl ECStore { pub(super) async fn handle_list_multipart_uploads( &self, bucket: &str, - prefix: &str, - key_marker: Option, - upload_id_marker: Option, - delimiter: Option, - max_uploads: usize, + request: MultipartUploadListRequest, ) -> Result { - check_list_multipart_args(bucket, prefix, &key_marker, &upload_id_marker, &delimiter)?; + check_list_multipart_args( + bucket, + &request.prefix, + &request.key_marker, + &request.upload_id_marker, + &request.delimiter, + )?; + let guard_opts = ObjectOptions { + expected_bucket_incarnation_id: request.expected_incarnation_id, + ..Default::default() + }; + let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, &guard_opts).await?; + let expected_incarnation_id = opts.expected_bucket_incarnation_id; - if prefix.is_empty() { + if request.prefix.is_empty() { // TODO: return from cache } if self.single_pool() { - return self.pools[0] - .list_multipart_uploads(bucket, prefix, key_marker, upload_id_marker, delimiter, max_uploads) - .await; + let result = list_pool_multipart_uploads_for_incarnation( + &self.pools[0], + bucket, + &MultipartUploadListRequest { + expected_incarnation_id, + ..request.clone() + }, + ) + .await; + ensure_multipart_bucket_lifecycle_guard_held(bucket_lifecycle_guard.as_ref(), bucket, &request.prefix)?; + return result; } let mut uploads = Vec::new(); @@ -126,16 +287,15 @@ impl ECStore { if self.is_suspended(pool.pool_idx).await { continue; } - let res = pool - .list_multipart_uploads( - bucket, - prefix, - key_marker.clone(), - upload_id_marker.clone(), - delimiter.clone(), - max_uploads, - ) - .await?; + let res = list_pool_multipart_uploads_for_incarnation( + pool, + bucket, + &MultipartUploadListRequest { + expected_incarnation_id, + ..request.clone() + }, + ) + .await?; uploads.extend(res.uploads); common_prefixes.extend(res.common_prefixes); source_truncated |= res.is_truncated; @@ -145,19 +305,21 @@ impl ECStore { // unordered across pools and may exceed the global cap. Re-sort, re-cap, // and derive the truncation markers so a bucket whose uploads span pools // pages correctly instead of being silently reported complete. - let page = merge_multipart_upload_pages(uploads, common_prefixes.into_iter().collect(), max_uploads, source_truncated); + let page = + merge_multipart_upload_pages(uploads, common_prefixes.into_iter().collect(), request.max_uploads, source_truncated); + ensure_multipart_bucket_lifecycle_guard_held(bucket_lifecycle_guard.as_ref(), bucket, &request.prefix)?; Ok(ListMultipartsInfo { - key_marker, - upload_id_marker, + key_marker: request.key_marker, + upload_id_marker: request.upload_id_marker, next_key_marker: page.next_key_marker, next_upload_id_marker: page.next_upload_id_marker, - max_uploads, + max_uploads: request.max_uploads, is_truncated: page.is_truncated, uploads: page.uploads, common_prefixes: page.common_prefixes, - prefix: prefix.to_owned(), - delimiter: delimiter.to_owned(), + prefix: request.prefix, + delimiter: request.delimiter, }) } @@ -180,6 +342,8 @@ impl ECStore { opts: &ObjectOptions, ) -> Result<(MultipartUploadResult, usize)> { check_new_multipart_args(bucket, object)?; + let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let opts = &opts; if self.single_pool() { return self.pools[0] @@ -192,9 +356,19 @@ impl ECStore { if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await { continue; } - let res = pool - .list_multipart_uploads(bucket, object, None, None, None, MAX_UPLOADS_LIST) - .await?; + let res = list_pool_multipart_uploads_for_incarnation( + pool, + bucket, + &MultipartUploadListRequest { + prefix: object.to_string(), + key_marker: None, + upload_id_marker: None, + delimiter: None, + max_uploads: MAX_UPLOADS_LIST, + expected_incarnation_id: opts.expected_bucket_incarnation_id, + }, + ) + .await?; if !res.uploads.is_empty() { let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; @@ -249,6 +423,8 @@ impl ECStore { opts: &ObjectOptions, ) -> Result { check_put_object_part_args(bucket, object, upload_id)?; + let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let opts = &opts; if self.single_pool() { return self.pools[0] @@ -289,6 +465,8 @@ impl ECStore { opts: &ObjectOptions, ) -> Result { check_list_parts_args(bucket, object, upload_id)?; + let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let opts = &opts; if self.single_pool() { return self.pools[0].get_multipart_info(bucket, object, upload_id, opts).await; } @@ -322,6 +500,8 @@ impl ECStore { opts: &ObjectOptions, ) -> Result<()> { check_abort_multipart_args(bucket, object, upload_id)?; + let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let opts = &opts; // TODO: defer DeleteUploadID @@ -360,6 +540,8 @@ impl ECStore { opts: &ObjectOptions, ) -> Result { check_complete_multipart_args(bucket, object, upload_id)?; + let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; + let opts = &opts; if self.single_pool() { return self.pools[0] diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index fb74c86db..78f6ead15 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -23,9 +23,16 @@ use crate::bucket::lifecycle::{ Jentry, attach_tier_delete_source, transitioned_delete_journal_entry_for_source, transitioned_force_delete_journal_entry, }, }; +use crate::bucket::metadata_sys::{ + acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_in, get_cached_bucket_incarnation_id_in, + get_object_lock_config_and_incarnation_from_disk_in, +}; +use crate::bucket::object_lock::objectlock_sys::{ + check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state, +}; use crate::bucket::replication::ReplicationObjectBridge; use crate::disk::OldCurrentSize; -use crate::object_api::DeleteLockFence; +use crate::object_api::{NamespaceLockFence, ObjectLockConfigSnapshot}; use crate::set_disk::{ get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, is_lock_optimization_enabled, is_object_lock_diag_enabled, @@ -43,6 +50,10 @@ use std::{ }; use tokio::io::{AsyncRead, ReadBuf}; +#[cfg(not(test))] +const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000; +#[cfg(test)] +const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2; const FORCE_DELETE_LIST_PAGE_SIZE: i32 = 1_000; fn build_tier_delete_journal_entry( @@ -365,6 +376,13 @@ impl RestoreAcceptGuard { pub fn is_lock_lost(&self) -> bool { self.0.guard.is_lock_lost() } + + pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { + opts.ensure_namespace_lock_fence(); + if let Some(signal) = self.0.lock_lost_signal() { + opts.add_namespace_lock_lost_signal(signal); + } + } } impl Drop for ObjectLockDiagGuard { @@ -485,6 +503,77 @@ fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool opts.versioned && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement } +#[cfg(test)] +struct DeleteAfterObjectLockSnapshotBarrierState { + bucket: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +pub(crate) struct DeleteAfterObjectLockSnapshotBarrier { + state: Arc, +} + +#[cfg(test)] +static DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl DeleteAfterObjectLockSnapshotBarrier { + pub(crate) fn install(bucket: &str) -> Self { + let state = Arc::new(DeleteAfterObjectLockSnapshotBarrierState { + bucket: bucket.to_string(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete snapshot barrier mutex should not poison"); + assert!(slot.is_none(), "delete snapshot barrier must not already be installed"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + self.state.arrived.notified().await; + } + + pub(crate) fn release(&self) { + self.state.release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for DeleteAfterObjectLockSnapshotBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + if let Some(slot) = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER.get() { + let mut slot = slot.lock().expect("delete snapshot barrier mutex should not poison"); + if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) { + *slot = None; + } + } + } +} + +#[cfg(test)] +async fn pause_delete_after_object_lock_snapshot(bucket: &str) { + let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete snapshot barrier mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket) + .cloned(); + if let Some(state) = state { + state.arrived.notify_one(); + state.release.notified().await; + } +} + /// Whether a delete-time lookup miss on a directory key should trigger an orphan /// empty-directory tree purge (issue #4189). /// @@ -643,6 +732,54 @@ fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> { } impl ECStore { + /// Captures Object Lock state once for a batch of PUTs to the same bucket. + /// `handle_put_object` only reuses the token for the same store, bucket, + /// bucket incarnation, and Object Lock configuration revision. + pub async fn object_lock_config_snapshot(&self, bucket: &str) -> Result> { + check_valid_bucket_name(bucket)?; + let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?; + let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?; + let (state, bucket_incarnation_id, config_revision) = + get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?; + if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() { + return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot")); + } + Ok(Arc::new(ObjectLockConfigSnapshot::for_guarded_store_bucket( + self.id, + bucket, + bucket_incarnation_id, + config_revision, + state, + lifecycle_guard, + metadata_guard, + ))) + } + + async fn object_lock_config_snapshot_under_lifecycle_fence( + &self, + bucket: &str, + lifecycle_fence: &NamespaceLockFence, + ) -> Result> { + if lifecycle_fence.is_lock_lost() { + return Err(Error::other("bucket lifecycle lock was lost before loading the Object Lock snapshot")); + } + let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?; + let (state, bucket_incarnation_id, config_revision) = + get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?; + if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() { + return Err(Error::other("bucket lock was lost while loading the Object Lock snapshot")); + } + Ok(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence( + self.id, + bucket, + bucket_incarnation_id, + config_revision, + state, + lifecycle_fence.clone(), + metadata_guard, + ))) + } + /// Resolves a GET's object identity without constructing its body reader. /// /// This is an additive two-stage counterpart to `get_object_reader`. The @@ -748,6 +885,10 @@ impl ECStore { } let guard = self.acquire_object_write_lock(op, bucket, object).await?; + if let Some(signal) = guard.lock_lost_signal() { + opts.add_namespace_lock_lost_signal(signal); + } + opts.ensure_namespace_lock_fence(); opts.no_lock = true; Ok(Some(guard)) @@ -788,9 +929,10 @@ impl ECStore { guards.push(self.acquire_object_write_lock("delete_objects", bucket, object).await?); } opts.no_lock = true; - opts.delete_lock_fence = Some(DeleteLockFence::new( - guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal).collect(), - )); + for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) { + opts.add_namespace_lock_lost_signal(signal); + } + opts.ensure_namespace_lock_fence(); Ok(guards) } @@ -1057,17 +1199,40 @@ impl ECStore { check_put_object_args(bucket, object)?; let object = encode_dir_object(object); + let mut opts = opts.clone(); + if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() { + opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?); + } + if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) { + let expected_incarnation_id = opts + .expected_bucket_incarnation_id + .ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?; + if opts.object_lock_config_snapshot.is_none() { + opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot(bucket).await?); + } + let snapshot = match opts.object_lock_config_snapshot.as_ref() { + Some(snapshot) if snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) => { + Arc::clone(snapshot) + } + _ => { + return Err(Error::other( + "Object Lock snapshot does not hold valid target bucket generation and configuration fences", + )); + } + }; + snapshot.add_lock_fences(&mut opts); + } // Keep PUT atomic-read friendly: SetDisks takes the object write lock only // around precondition checks and the final rename/commit. if self.single_pool() { return self.pools[0] - .put_object_with_old_current_size(bucket, object.as_str(), data, opts) + .put_object_with_old_current_size(bucket, object.as_str(), data, &opts) .await; } let idx = if opts.data_movement && opts.version_id.is_some() { - self.select_data_movement_pool_idx(bucket, &object, data.size(), opts, false) + self.select_data_movement_pool_idx(bucket, &object, data.size(), &opts, false) .await? } else if opts.no_lock { self.get_pool_idx_no_lock(bucket, &object, data.size()).await? @@ -1084,7 +1249,7 @@ impl ECStore { } self.pools[idx] - .put_object_with_old_current_size(bucket, &object, data, opts) + .put_object_with_old_current_size(bucket, &object, data, &opts) .await } @@ -1130,6 +1295,60 @@ impl ECStore { let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]); let mut dst_opts = dst_opts.clone(); + if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() { + dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?); + } + let _bucket_lifecycle_guard = if is_meta_bucketname(dst_bucket) || dst_opts.bucket_lifecycle_lock_fence.is_some() { + None + } else { + Some(self.acquire_bucket_lifecycle_read_lock(dst_bucket).await?) + }; + let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() { + dst_opts.add_bucket_lifecycle_lock_guard(guard); + let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, dst_bucket).await?; + if dst_opts + .expected_bucket_incarnation_id + .is_some_and(|expected| expected != current_incarnation_id) + { + return Err(StorageError::BucketNotFound(dst_bucket.to_string())); + } + Some(current_incarnation_id) + } else { + dst_opts.expected_bucket_incarnation_id + }; + if dst_opts + .bucket_lifecycle_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "copy_object_bucket_generation", + bucket: dst_bucket.to_string(), + object: dst_object.clone(), + required: 1, + achieved: 0, + }); + } + if dst_opts.overwrites_existing_version() && !is_meta_bucketname(dst_bucket) { + let incarnation_id = + current_bucket_incarnation_id.ok_or_else(|| Error::other("copy is missing its bucket incarnation snapshot"))?; + let lifecycle_fence = dst_opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("copy is missing its bucket lifecycle fence"))?; + let snapshot = match dst_opts.object_lock_config_snapshot.as_ref() { + Some(snapshot) => Arc::clone(snapshot), + None => { + self.object_lock_config_snapshot_under_lifecycle_fence(dst_bucket, lifecycle_fence) + .await? + } + }; + if !snapshot.is_valid_for_destructive_put(self.id, dst_bucket, incarnation_id) { + return Err(Error::other("copy Object Lock snapshot does not match the target bucket generation")); + } + snapshot.add_lock_fences(&mut dst_opts); + dst_opts.object_lock_config_snapshot = Some(snapshot); + } let _dst_lock_guard = if cp_src_dst_same && dst_opts.expected_current_version_id.is_none() { self.acquire_object_write_lock_if_needed("copy_object", dst_bucket, &dst_object, &mut dst_opts) .await? @@ -1176,6 +1395,10 @@ impl ECStore { mod_time: dst_opts.mod_time, http_preconditions: dst_opts.http_preconditions.clone(), expected_current_version_id: dst_opts.expected_current_version_id.clone(), + expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id, + namespace_lock_fence: dst_opts.namespace_lock_fence.clone(), + bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(), + object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(), ..Default::default() }; return if let Some(reader) = src_info.put_object_reader.as_mut() { @@ -1206,6 +1429,10 @@ impl ECStore { mod_time: dst_opts.mod_time, http_preconditions: dst_opts.http_preconditions.clone(), expected_current_version_id: dst_opts.expected_current_version_id.clone(), + expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id, + namespace_lock_fence: dst_opts.namespace_lock_fence.clone(), + bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(), + object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(), ..Default::default() }; return self.pools[pool_idx] @@ -1233,6 +1460,10 @@ impl ECStore { mod_time: dst_opts.mod_time, http_preconditions: dst_opts.http_preconditions.clone(), expected_current_version_id: dst_opts.expected_current_version_id.clone(), + expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id, + namespace_lock_fence: dst_opts.namespace_lock_fence.clone(), + bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(), + object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(), ..Default::default() }; @@ -1322,6 +1553,13 @@ impl ECStore { ) -> Result { check_del_obj_args(bucket, object)?; + let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) { + None + } else if opts.delete_prefix { + Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?) + } else { + Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?) + }; let object = if opts.delete_prefix && !opts.delete_prefix_object { object.to_owned() } else { @@ -1330,10 +1568,94 @@ impl ECStore { let object = object.as_str(); let mut opts = opts; opts.tier_delete_journal_api = tier_journal_api.clone(); + if let Some(guard) = _bucket_lifecycle_guard.as_ref() { + opts.add_bucket_lifecycle_lock_guard(guard); + } + + if !is_meta_bucketname(bucket) { + get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?; + } + let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) { + Some(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?) + } else { + None + }; + if let Some(guard) = _object_lock_metadata_guard.as_ref() { + opts.add_namespace_lock_guard(guard); + } + let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() { + let (state, incarnation_id, config_revision) = + get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?; + opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket( + self.id, + bucket, + incarnation_id, + config_revision, + state, + ))); + Some(incarnation_id) + } else { + None + }; + if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id) + && expected != current + { + return Err(StorageError::BucketNotFound(bucket.to_string())); + } + #[cfg(test)] + if current_bucket_incarnation_id.is_some() { + pause_delete_after_object_lock_snapshot(bucket).await; + } if opts.delete_prefix && !opts.delete_prefix_object { // Prefix deletes cover multiple object keys; an exact lock on the prefix string // would not protect child objects. + if !is_meta_bucketname(bucket) { + let state = opts + .object_lock_config_snapshot + .as_deref() + .ok_or_else(|| Error::other("recursive delete is missing its Object Lock configuration snapshot"))? + .state(); + ensure_recursive_force_delete_allowed_for_state(bucket, state)?; + let bypass_governance = opts + .object_lock_delete + .as_ref() + .is_some_and(|delete_opts| delete_opts.bypass_governance); + for pool in &self.pools { + for set in &pool.disk_set { + let mut marker = None; + let mut version_marker = None; + loop { + let page = set + .clone() + .inner_list_object_versions_for_recursive_delete( + bucket, + object, + marker.clone(), + version_marker.clone(), + RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE, + ) + .await?; + for object_info in &page.objects { + if check_object_lock_for_deletion_with_state(state, object_info, bypass_governance)?.is_some() { + return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object_info.name.clone())); + } + } + if !page.is_truncated { + break; + } + let next_marker = page.next_marker.ok_or_else(|| { + Error::other("recursive delete version scan did not return a continuation marker") + })?; + if marker.as_ref() == Some(&next_marker) && version_marker == page.next_version_idmarker { + return Err(Error::other("recursive delete version scan did not advance")); + } + marker = Some(next_marker); + version_marker = page.next_version_idmarker; + } + } + } + } delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?; return Ok(ObjectInfo::default()); } @@ -1344,7 +1666,6 @@ impl ECStore { } else { None }; - if opts.delete_prefix { delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?; return Ok(ObjectInfo::default()); @@ -1531,6 +1852,17 @@ impl ECStore { let mut opts = opts; opts.tier_delete_journal_api = tier_journal_api; + let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) { + None + } else { + match self.acquire_bucket_lifecycle_read_lock(bucket).await { + Ok(guard) => Some(guard), + Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err), + } + }; + if let Some(guard) = _bucket_lifecycle_guard.as_ref() { + opts.add_bucket_lifecycle_lock_guard(guard); + } if opts.delete_replication_config_snapshot.is_none() { match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await { Ok(snapshot) => opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)), @@ -1541,7 +1873,48 @@ impl ECStore { } } } - + if !is_meta_bucketname(bucket) + && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await + { + return return_batch_delete_lock_error(objects.as_slice(), err); + } + let _object_lock_metadata_guard = if is_meta_bucketname(bucket) { + None + } else { + Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await { + Ok(guard) => guard, + Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err), + }) + }; + if let Some(guard) = _object_lock_metadata_guard.as_ref() { + opts.add_namespace_lock_guard(guard); + } + let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() { + let (state, incarnation_id, config_revision) = + match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await { + Ok(snapshot) => snapshot, + Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err), + }; + opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket( + self.id, + bucket, + incarnation_id, + config_revision, + state, + ))); + Some(incarnation_id) + } else { + None + }; + if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id) + && expected != current + { + return return_batch_delete_lock_error(objects.as_slice(), StorageError::BucketNotFound(bucket.to_string())); + } + #[cfg(test)] + if current_bucket_incarnation_id.is_some() { + pause_delete_after_object_lock_snapshot(bucket).await; + } let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await { Ok(guards) => guards, Err(err) => return return_batch_delete_lock_error(objects.as_slice(), err), @@ -1747,6 +2120,45 @@ impl ECStore { opts: &ObjectOptions, ) -> Result<()> { let object = encode_dir_object(object); + let mut opts = transition_restore_pool_opts(opts); + if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() { + opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?); + } + let bucket_lifecycle_guard = if is_meta_bucketname(bucket) { + None + } else { + Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?) + }; + if let Some(guard) = bucket_lifecycle_guard.as_ref() { + opts.add_bucket_lifecycle_lock_guard(guard); + } + if !is_meta_bucketname(bucket) { + let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?; + if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) { + return Err(StorageError::BucketNotFound(bucket.to_string())); + } + } + if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) { + let expected_incarnation_id = opts + .expected_bucket_incarnation_id + .ok_or_else(|| Error::other("restore is missing its bucket incarnation snapshot"))?; + let lifecycle_fence = opts + .bucket_lifecycle_lock_fence + .as_ref() + .ok_or_else(|| Error::other("restore is missing its bucket lifecycle fence"))?; + let snapshot = match opts.object_lock_config_snapshot.as_ref() { + Some(snapshot) => Arc::clone(snapshot), + None => { + self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence) + .await? + } + }; + if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) { + return Err(Error::other("restore Object Lock snapshot does not match the target bucket generation")); + } + snapshot.add_lock_fences(&mut opts); + opts.object_lock_config_snapshot = Some(snapshot); + } // Deliberately NOT holding the object write lock across the tier // copy-back (backlog#1304): non-SELECT restore-vs-restore is // serialized by the accept path's compare-and-set of the ongoing flag @@ -1762,10 +2174,12 @@ impl ECStore { // (#4877) blocked HEAD/get_object_info for the whole copy-back and // self-deadlocked on the inner commits. if self.single_pool() { - return self.pools[0].clone().restore_transitioned_object(bucket, &object, opts).await; + return self.pools[0] + .clone() + .restore_transitioned_object(bucket, &object, &opts) + .await; } - let opts = transition_restore_pool_opts(opts); let (_, idx) = self .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts) .await?; @@ -1784,18 +2198,38 @@ impl ECStore { opts: &ObjectOptions, ) -> Result { let object = encode_dir_object(object); - if self.single_pool() { - return self.pools[0].put_object_metadata(bucket, object.as_str(), opts).await; - } - let mut opts = opts.clone(); opts.metadata_chg = true; + let bucket_lifecycle_guard = if is_meta_bucketname(bucket) { + None + } else { + let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?; + let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?; + if opts + .expected_bucket_incarnation_id + .is_some_and(|expected| expected != current_incarnation_id) + { + return Err(StorageError::BucketNotFound(bucket.to_string())); + } + opts.expected_bucket_incarnation_id = Some(current_incarnation_id); + opts.add_bucket_lifecycle_lock_guard(&guard); + if guard.is_lock_lost() { + return Err(Error::other("bucket lifecycle lock was lost before the metadata update")); + } + Some(guard) + }; + + if self.single_pool() { + return self.pools[0].put_object_metadata(bucket, object.as_str(), &opts).await; + } let (_, idx) = self .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts) .await?; - self.pools[idx].put_object_metadata(bucket, object.as_str(), &opts).await + let result = self.pools[idx].put_object_metadata(bucket, object.as_str(), &opts).await; + drop(bucket_lifecycle_guard); + result } #[instrument(skip(self))] @@ -2885,7 +3319,10 @@ mod tests { assert_eq!(guards.len(), 2, "duplicate object names should share one namespace lock"); assert!(opts.no_lock, "set layer should not reacquire locks already held by ECStore"); - assert!(opts.delete_lock_fence.is_some(), "set layer must receive the outer write-lock loss fence"); + assert!( + opts.namespace_lock_fence.is_some(), + "set layer must receive the outer write-lock loss fence" + ); let alpha_lock = store .handle_new_ns_lock("bucket", "alpha") diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index 68f2e1e67..334b87d85 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -202,13 +202,35 @@ impl ECStore { } pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let mut first_error = None; + let mut first_volume_error = None; + let mut has_success = false; for pool in self.pools.iter() { let mut opts = opts.clone(); opts.delete_prefix = true; - pool.delete_object(bucket, object, opts).await?; + match pool.delete_object(bucket, object, opts).await { + Ok(_) => has_success = true, + Err(err) if is_err_strict_volume_not_found(&err) => { + if first_volume_error.is_none() { + first_volume_error = Some(err); + } + } + Err(err) => { + if first_error.is_none() { + first_error = Some(err); + } + } + } } - Ok(()) + match first_error { + Some(err) => Err(err), + None if has_success => Ok(()), + None => match first_volume_error { + Some(err) => Err(err), + None => Ok(()), + }, + } } pub(super) async fn get_available_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Option { @@ -748,9 +770,148 @@ impl ECStore { mod tests { use super::*; use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env}; + use crate::disk::error::DiskError; + use crate::layout::endpoint::Endpoint; + use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; + use crate::storage_api_contracts::bucket::MakeBucketOptions; use arc_swap::ArcSwap; use rustfs_config::server_config::KVS; use std::sync::Arc; + use tokio_util::sync::CancellationToken; + + #[tokio::test] + async fn delete_prefix_attempts_later_pools_after_an_earlier_pool_error() { + let temp_dir = tempfile::tempdir().expect("multi-pool delete test directory should be created"); + let mut pools = Vec::with_capacity(2); + for (pool_index, drives_per_set) in [2, 4].into_iter().enumerate() { + let mut endpoints = Vec::with_capacity(drives_per_set); + for disk_index in 0..drives_per_set { + let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}")); + tokio::fs::create_dir_all(&disk_path) + .await + .expect("multi-pool delete test disk should be created"); + let mut endpoint = + Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); + endpoint.set_pool_index(pool_index); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } + pools.push(PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set, + endpoints: Endpoints::from(endpoints), + cmd_line: format!("delete-prefix-pool-{pool_index}"), + platform: "test".to_string(), + }); + } + + let endpoint_pools = EndpointServerPools(pools); + let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) + .await + .expect("multi-pool local disks should initialize"); + let shutdown = CancellationToken::new(); + let store = ECStore::new_with_instance_ctx( + "127.0.0.1:0".parse().expect("test address should parse"), + endpoint_pools, + shutdown.clone(), + instance_ctx, + ) + .await + .expect("multi-pool store should initialize"); + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created in both pools"); + + let first_pool_disks = store.pools[0].disk_set[0].disks.read().await.clone(); + for disk in first_pool_disks.iter().flatten() { + disk.write_all(&bucket, "blocked", bytes::Bytes::from_static(b"not-a-directory")) + .await + .expect("first pool should contain a blocking parent file"); + } + let later_pool_disks = store.pools[1].disk_set[0].disks.read().await.clone(); + let later_data_disk = later_pool_disks[0].clone().expect("later pool should have its first disk"); + later_data_disk + .write_all(&bucket, "blocked/prefix/object", bytes::Bytes::from_static(b"data")) + .await + .expect("later pool should contain the prefix on its available disk"); + *store.pools[1].disk_set[0].disks.write().await = vec![Some(later_data_disk.clone()), None, None, None]; + + let err = store + .delete_prefix(&bucket, "blocked/prefix", &ObjectOptions::default()) + .await + .expect_err("the first pool's hard error must be returned"); + + assert!( + matches!(err, StorageError::PrefixAccessDenied(ref error_bucket, ref error_prefix) + if error_bucket == &bucket && error_prefix == "blocked/prefix"), + "unexpected multi-pool delete error: {err:?}" + ); + assert!(matches!( + later_data_disk.read_all(&bucket, "blocked/prefix/object").await, + Err(DiskError::FileNotFound) + )); + + *store.pools[1].disk_set[0].disks.write().await = later_pool_disks.clone(); + for disk in first_pool_disks.iter().flatten() { + disk.write_all(&bucket, "second-blocked", bytes::Bytes::from_static(b"not-a-directory")) + .await + .expect("first pool should contain a second blocking parent file"); + } + for disk in later_pool_disks.iter().flatten() { + disk.write_all(&bucket, "second-blocked/prefix/object", bytes::Bytes::from_static(b"data")) + .await + .expect("later pool should contain the second prefix"); + } + let err = store + .delete_prefix(&bucket, "second-blocked/prefix", &ObjectOptions::default()) + .await + .expect_err("a successful later pool must not override the first pool's hard error"); + assert!( + matches!(err, StorageError::PrefixAccessDenied(ref error_bucket, ref error_prefix) + if error_bucket == &bucket && error_prefix == "second-blocked/prefix"), + "unexpected hard-error plus success result: {err:?}" + ); + for disk in later_pool_disks.iter().flatten() { + assert!(matches!( + disk.read_all(&bucket, "second-blocked/prefix/object").await, + Err(DiskError::FileNotFound) + )); + } + + for disk in later_pool_disks.iter().flatten() { + disk.delete_volume(&bucket, true) + .await + .expect("the bucket should be absent from the later pool"); + } + let healthy_object = "healthy/prefix/object"; + for disk in first_pool_disks.iter().flatten() { + disk.write_all(&bucket, healthy_object, bytes::Bytes::from_static(b"data")) + .await + .expect("the first pool should contain the healthy prefix"); + } + store + .delete_prefix(&bucket, "healthy/prefix", &ObjectOptions::default()) + .await + .expect("one successful pool should make a partially missing bucket idempotent"); + for disk in first_pool_disks.iter().flatten() { + assert!(matches!(disk.read_all(&bucket, healthy_object).await, Err(DiskError::FileNotFound))); + } + + let missing_bucket = format!("delete-prefix-missing-{}", Uuid::new_v4().simple()); + let err = store + .delete_prefix(&missing_bucket, "missing/prefix", &ObjectOptions::default()) + .await + .expect_err("a bucket missing from every pool must remain an error"); + assert_eq!(err, StorageError::BucketNotFound(missing_bucket)); + + shutdown.cancel(); + } fn assert_backend_layout_empty(info: &rustfs_madmin::BackendInfo) { assert!(info.standard_sc_parities.is_empty()); diff --git a/crates/iam/src/sys.rs b/crates/iam/src/sys.rs index ab6f40a2d..07a6aa81f 100644 --- a/crates/iam/src/sys.rs +++ b/crates/iam/src/sys.rs @@ -161,7 +161,8 @@ impl PreparedIamAuth { /// conditions for the provided request args. pub async fn needs_existing_object_tag_for_args(&self, args: &Args<'_>) -> bool { match &self.mode { - PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny => false, + PreparedIamMode::Opa => true, + PreparedIamMode::Owner | PreparedIamMode::Deny => false, PreparedIamMode::Regular { combined_policy } => { policy_needs_existing_object_tag_for_args(combined_policy, args).await } @@ -1099,7 +1100,7 @@ impl IamSys { match Self::policy_plugin_state().await { PolicyPluginState::Ready(_) => { return PreparedIamAuth { - needs_existing_object_tag: false, + needs_existing_object_tag: true, mode: PreparedIamMode::Opa, }; } @@ -1766,6 +1767,7 @@ mod tests { use rustfs_policy::policy::action::{Action, AdminAction, S3Action, StsAction}; use rustfs_policy::policy::policy_uses_existing_object_tag_conditions; use serde_json::Value; + use serial_test::serial; use std::{ collections::{HashMap, HashSet}, sync::{Arc, Mutex}, @@ -1797,6 +1799,42 @@ mod tests { assert!(prepared.combined_policy_for_view().is_none()); } + #[tokio::test] + #[serial] + async fn test_prepare_auth_requests_existing_object_tags_in_opa_mode() { + let store = StsTestMockStore::new(false); + let iam_sys = IamSys::new(IamCache::new(store).await.expect("initialize IAM cache")); + let previous_state = IamSys::::policy_plugin_state().await; + IamSys::::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args { + url: "http://127.0.0.1:8181/v1/data/rustfs/authz/allow".to_string(), + auth_token: String::new(), + })) + .await; + + let claims = HashMap::new(); + let groups = None; + let conditions = HashMap::new(); + let args = Args { + account: "opa-tag-test-user", + groups: &groups, + action: Action::S3Action(S3Action::GetObjectAction), + bucket: "bucket", + conditions: &conditions, + is_owner: false, + object: "tagged-object", + claims: &claims, + deny_only: false, + }; + + let prepared = iam_sys.prepare_auth(&args).await; + let needs_initial_tags = prepared.needs_existing_object_tag; + let needs_secondary_tags = prepared.needs_existing_object_tag_for_args(&args).await; + *get_policy_plugin_state().write().await = previous_state; + + assert!(needs_initial_tags, "OPA mode must request existing object tags before evaluation"); + assert!(needs_secondary_tags, "OPA mode must request existing object tags for secondary actions"); + } + const CUSTOM_STS_CLAIM_POLICY: &str = "custom-sts-claim-getobject"; const CUSTOM_STS_CLAIM_BUCKET: &str = "claim-bucket"; const CUSTOM_STS_CLAIM_POLICY_JSON: &str = r#"{ diff --git a/crates/lifecycle/src/core.rs b/crates/lifecycle/src/core.rs index 9de9bdfd6..ff7d15505 100644 --- a/crates/lifecycle/src/core.rs +++ b/crates/lifecycle/src/core.rs @@ -1061,6 +1061,9 @@ pub struct TransitionOptions { pub status: String, pub tier: String, pub etag: String, + pub expected_data_dir: Option, + pub expected_remote_name: String, + pub expected_remote_version_id: String, pub restore_request: RestoreRequest, pub restore_expiry: OffsetDateTime, pub expire_restored: bool, @@ -1072,6 +1075,9 @@ impl Default for TransitionOptions { status: Default::default(), tier: Default::default(), etag: Default::default(), + expected_data_dir: Default::default(), + expected_remote_name: Default::default(), + expected_remote_version_id: Default::default(), restore_request: Default::default(), restore_expiry: OffsetDateTime::now_utc(), expire_restored: Default::default(), diff --git a/crates/lock/src/namespace/mod.rs b/crates/lock/src/namespace/mod.rs index f0195bd46..96fd73383 100644 --- a/crates/lock/src/namespace/mod.rs +++ b/crates/lock/src/namespace/mod.rs @@ -129,6 +129,15 @@ impl NamespaceLockGuard { } } + /// Share the loss signal when a caller deliberately forwards a mutation + /// under an already-held distributed lock. + pub fn lock_lost_signal(&self) -> Option> { + match self { + Self::Standard(guard) => Some(guard.lock_lost()), + Self::Fast(_) => None, + } + } + /// Resolves when a distributed guard loses refresh quorum. /// /// Local fast locks cannot lose distributed quorum, so their future remains pending. diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 967cf8f02..14289bda6 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -50,6 +50,9 @@ const HTTP_VERSION_10_LABEL: &str = "http/1.0"; const HTTP_VERSION_11_LABEL: &str = "http/1.1"; const HTTP_VERSION_2_LABEL: &str = "h2"; const HTTP_VERSION_UNKNOWN_LABEL: &str = "unknown"; +pub const INTERNODE_DISK_ERROR_HEADER: &str = "x-rustfs-disk-error"; +pub const INTERNODE_FILE_NOT_FOUND: &str = "file-not-found"; +pub const INTERNODE_VOLUME_NOT_FOUND: &str = "volume-not-found"; #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum InternodeHttpErrorKind { @@ -163,15 +166,23 @@ impl std::fmt::Display for InternodeHttpRequestContext { pub struct InternodeHttpError { kind: InternodeHttpErrorKind, context: InternodeHttpRequestContext, + remote_disk_error: Option, #[source] source: Option>, } +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RemoteDiskErrorKind { + FileNotFound, + VolumeNotFound, +} + impl std::fmt::Debug for InternodeHttpError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("InternodeHttpError") .field("kind", &self.kind) .field("context", &self.context) + .field("remote_disk_error_present", &self.remote_disk_error.is_some()) .field("source_present", &self.source.is_some()) .finish() } @@ -186,10 +197,19 @@ impl InternodeHttpError { &self.context } + pub fn is_remote_file_not_found(&self) -> bool { + self.remote_disk_error == Some(RemoteDiskErrorKind::FileNotFound) + } + + pub fn is_remote_volume_not_found(&self) -> bool { + self.remote_disk_error == Some(RemoteDiskErrorKind::VolumeNotFound) + } + fn new(kind: InternodeHttpErrorKind, context: InternodeHttpRequestContext) -> Self { Self { kind, context, + remote_disk_error: None, source: None, } } @@ -213,6 +233,7 @@ impl InternodeHttpError { Self { kind, context, + remote_disk_error: None, source: Some(Box::new(source)), } } @@ -220,6 +241,19 @@ impl InternodeHttpError { fn into_io_error(self) -> io::Error { io::Error::new(self.kind.io_error_kind(), self) } + + fn with_remote_disk_error( + kind: InternodeHttpErrorKind, + context: InternodeHttpRequestContext, + remote_disk_error: RemoteDiskErrorKind, + ) -> Self { + Self { + kind, + context, + remote_disk_error: Some(remote_disk_error), + source: None, + } + } } #[doc(hidden)] @@ -227,6 +261,34 @@ pub fn new_test_internode_http_io_error(kind: InternodeHttpErrorKind) -> io::Err InternodeHttpError::new_for_test(kind).into_io_error() } +#[doc(hidden)] +pub fn new_test_remote_file_not_found_http_io_error() -> io::Error { + InternodeHttpError::with_remote_disk_error( + InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR), + InternodeHttpRequestContext { + method: "GET".to_string(), + target: READ_FILE_STREAM_PATH.to_string(), + operation: Some(INTERNODE_OPERATION_READ_FILE_STREAM), + }, + RemoteDiskErrorKind::FileNotFound, + ) + .into_io_error() +} + +#[doc(hidden)] +pub fn new_test_remote_volume_not_found_http_io_error() -> io::Error { + InternodeHttpError::with_remote_disk_error( + InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR), + InternodeHttpRequestContext { + method: "GET".to_string(), + target: READ_FILE_STREAM_PATH.to_string(), + operation: Some(INTERNODE_OPERATION_READ_FILE_STREAM), + }, + RemoteDiskErrorKind::VolumeNotFound, + ) + .into_io_error() +} + fn add_root_certificates_from_der(builder: reqwest::ClientBuilder, certs_der: &[Vec]) -> reqwest::ClientBuilder { let mut b = builder; for der in certs_der { @@ -726,6 +788,32 @@ fn classify_http_status(status: reqwest::StatusCode) -> InternodeHttpErrorKind { InternodeHttpErrorKind::HttpStatus(status) } +#[derive(Debug)] +struct ClassifiedHttpResponse { + kind: InternodeHttpErrorKind, + remote_disk_error: Option, +} + +fn classify_http_response( + status: reqwest::StatusCode, + headers: &HeaderMap, + operation: Option<&'static str>, +) -> ClassifiedHttpResponse { + let kind = classify_http_status(status); + if status != reqwest::StatusCode::INTERNAL_SERVER_ERROR || operation != Some(INTERNODE_OPERATION_READ_FILE_STREAM) { + return ClassifiedHttpResponse { + kind, + remote_disk_error: None, + }; + } + let remote_disk_error = match headers.get(INTERNODE_DISK_ERROR_HEADER).and_then(|value| value.to_str().ok()) { + Some(INTERNODE_FILE_NOT_FOUND) => Some(RemoteDiskErrorKind::FileNotFound), + Some(INTERNODE_VOLUME_NOT_FOUND) => Some(RemoteDiskErrorKind::VolumeNotFound), + _ => None, + }; + ClassifiedHttpResponse { kind, remote_disk_error } +} + fn internode_reqwest_error(method: &Method, url: &str, operation: Option<&'static str>, err: reqwest::Error) -> io::Error { let context = internode_request_context(method, url, operation); let classified = classify_reqwest_error(&err); @@ -742,10 +830,24 @@ fn internode_classified_error( method: &Method, url: &str, operation: Option<&'static str>, - kind: InternodeHttpErrorKind, + classified: ClassifiedHttpResponse, ) -> io::Error { let context = internode_request_context(method, url, operation); - InternodeHttpError::new(kind, context).into_io_error() + match classified.remote_disk_error { + Some(remote_disk_error) => InternodeHttpError::with_remote_disk_error(classified.kind, context, remote_disk_error), + None => InternodeHttpError::new(classified.kind, context), + } + .into_io_error() +} + +fn internode_kind_error( + method: &Method, + url: &str, + operation: Option<&'static str>, + classified: InternodeHttpErrorKind, +) -> io::Error { + let context = internode_request_context(method, url, operation); + InternodeHttpError::new(classified, context).into_io_error() } fn internode_status_error(method: &Method, url: &str, operation: Option<&'static str>, status: reqwest::StatusCode) -> io::Error { @@ -826,10 +928,11 @@ impl HttpReader { record_internode_http_version(track_internode_metrics, internode_operation, http_version_metric_label(resp.version())); maybe_warn_h2_inert(resp.version()); if resp.status().is_success().not() { + let classified = classify_http_response(resp.status(), resp.headers(), internode_operation); record_internode_operation_duration(track_internode_metrics, internode_operation, request_started.elapsed()); record_internode_error(track_internode_metrics, internode_operation); - record_internode_classified_error(track_internode_metrics, internode_operation, classify_http_status(resp.status())); - return Err(internode_status_error(&method, &url, internode_operation, resp.status())); + record_internode_classified_error(track_internode_metrics, internode_operation, classified.kind); + return Err(internode_classified_error(&method, &url, internode_operation, classified)); } record_internode_outgoing_request(track_internode_metrics, internode_operation); @@ -1073,7 +1176,7 @@ impl HttpWriter { record_internode_error(track_internode_metrics, internode_operation); let classified = classify_reqwest_error(&e); record_internode_classified_error(track_internode_metrics, internode_operation, classified); - let _ = err_tx.send(internode_classified_error(&method_clone, &url_clone, internode_operation, classified)); + let _ = err_tx.send(internode_kind_error(&method_clone, &url_clone, internode_operation, classified)); let io_err = internode_reqwest_error(&method_clone, &url_clone, internode_operation, e); return Err(io_err); } @@ -1677,6 +1780,54 @@ mod tests { } } + #[test] + fn disk_error_wire_tokens_are_stable() { + assert_eq!(INTERNODE_DISK_ERROR_HEADER, "x-rustfs-disk-error"); + assert_eq!(INTERNODE_FILE_NOT_FOUND, "file-not-found"); + assert_eq!(INTERNODE_VOLUME_NOT_FOUND, "volume-not-found"); + } + + #[test] + fn classify_http_response_requires_an_explicit_missing_disk_error() { + let mut headers = HeaderMap::new(); + headers.insert(INTERNODE_DISK_ERROR_HEADER, INTERNODE_FILE_NOT_FOUND.parse().unwrap()); + let classified = classify_http_response( + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + &headers, + Some(INTERNODE_OPERATION_READ_FILE_STREAM), + ); + assert_eq!( + classified.kind, + InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR) + ); + assert_eq!(classified.kind.io_error_kind(), io::ErrorKind::Other); + assert_eq!(classified.kind.metric_label(), "http_status_other"); + assert!(matches!(classified.remote_disk_error, Some(RemoteDiskErrorKind::FileNotFound))); + + headers.insert(INTERNODE_DISK_ERROR_HEADER, INTERNODE_VOLUME_NOT_FOUND.parse().unwrap()); + let classified = classify_http_response( + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + &headers, + Some(INTERNODE_OPERATION_READ_FILE_STREAM), + ); + assert!(matches!(classified.remote_disk_error, Some(RemoteDiskErrorKind::VolumeNotFound))); + + headers.insert(INTERNODE_DISK_ERROR_HEADER, "permission-denied".parse().unwrap()); + let classified = classify_http_response( + reqwest::StatusCode::INTERNAL_SERVER_ERROR, + &headers, + Some(INTERNODE_OPERATION_READ_FILE_STREAM), + ); + assert!(classified.remote_disk_error.is_none()); + headers.insert(INTERNODE_DISK_ERROR_HEADER, INTERNODE_FILE_NOT_FOUND.parse().unwrap()); + let wrong_status = + classify_http_response(reqwest::StatusCode::NOT_FOUND, &headers, Some(INTERNODE_OPERATION_READ_FILE_STREAM)); + assert!(wrong_status.remote_disk_error.is_none()); + let wrong_operation = + classify_http_response(reqwest::StatusCode::INTERNAL_SERVER_ERROR, &headers, Some(INTERNODE_OPERATION_WALK_DIR)); + assert!(wrong_operation.remote_disk_error.is_none()); + } + #[derive(Clone, Default)] struct TestState { head_count: Arc, @@ -2158,7 +2309,7 @@ mod tests { #[test] fn dns_resolution_error_uses_network_io_kind() { - let err = internode_classified_error( + let err = internode_kind_error( &Method::GET, "http://missing.invalid/rustfs/rpc/read_file_stream", Some(INTERNODE_OPERATION_READ_FILE_STREAM), diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index dcb352bdd..3341bba2d 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -348,10 +348,19 @@ pub(crate) async fn enqueue_runtime_newer_noncurrent( event: Event, src: &LcEventSrc, ) -> bool { - ecstore_expiry_state_handle() - .write() - .await - .enqueue_by_newer_noncurrent(bucket, to_delete_objs, event, src) + let Some(store) = ecstore_resolve_object_store_handle() else { + return false; + }; + let Ok(bucket_incarnation_id) = store.bucket_incarnation_id(bucket).await else { + return false; + }; + ecstore_expiry_state_handle().write().await.enqueue_by_newer_noncurrent( + bucket, + to_delete_objs, + event, + src, + bucket_incarnation_id, + ) } pub(crate) async fn queue_replication_heal( diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index d260972a9..9fef7a372 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -703,7 +703,11 @@ mod serial_tests { action: IlmAction::DeleteAction, ..Default::default() }; - expire_transitioned_object(ecstore.clone(), &oi, &lc_event, &LcEventSrc::Scanner) + let bucket_incarnation_id = ecstore + .bucket_incarnation_id(bucket_name.as_str()) + .await + .expect("read bucket incarnation"); + expire_transitioned_object(ecstore.clone(), &oi, &lc_event, &LcEventSrc::Scanner, bucket_incarnation_id) .await .expect("expire_transitioned_object should succeed"); @@ -2186,7 +2190,11 @@ mod serial_tests { action: IlmAction::DeleteRestoredAction, ..Default::default() }; - expire_transitioned_object(ecstore.clone(), &restored, &lc_event, &LcEventSrc::Scanner) + let bucket_incarnation_id = ecstore + .bucket_incarnation_id(bucket_name.as_str()) + .await + .expect("read bucket incarnation"); + expire_transitioned_object(ecstore.clone(), &restored, &lc_event, &LcEventSrc::Scanner, bucket_incarnation_id) .await .expect("restore-expiry cleanup should succeed"); diff --git a/crates/storage-api/src/bucket.rs b/crates/storage-api/src/bucket.rs index c6025e58c..d37a96467 100644 --- a/crates/storage-api/src/bucket.rs +++ b/crates/storage-api/src/bucket.rs @@ -17,6 +17,10 @@ use std::fmt::Debug; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; +/// Namespace-lock object reserved for bucket lifecycle generation fencing. +/// Empty S3 object keys are invalid, so this cannot collide with an object lock. +pub const BUCKET_LIFECYCLE_LOCK_OBJECT: &str = ""; + /// Options for creating a new bucket. #[derive(Debug, Default, Serialize, Deserialize)] pub struct MakeBucketOptions { diff --git a/crates/storage-api/src/lib.rs b/crates/storage-api/src/lib.rs index d4a73c10a..15002fa3b 100644 --- a/crates/storage-api/src/lib.rs +++ b/crates/storage-api/src/lib.rs @@ -51,6 +51,7 @@ pub mod topology; mod replication; pub use admin::{DiskSetSelector, StorageAdminApi}; +pub use bucket::BUCKET_LIFECYCLE_LOCK_OBJECT; pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp}; pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus}; pub use error::{StorageErrorCode, StorageResult}; diff --git a/crates/utils/src/http/metadata_compat.rs b/crates/utils/src/http/metadata_compat.rs index c99d8ed00..12956cf23 100644 --- a/crates/utils/src/http/metadata_compat.rs +++ b/crates/utils/src/http/metadata_compat.rs @@ -42,6 +42,7 @@ pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier"; pub const SUFFIX_TRANSITION_TIER_DESTINATION_ID: &str = "transition-tier-destination-id"; pub const SUFFIX_TRANSITION_TRANSACTION_ID: &str = "transition-transaction-id"; pub const SUFFIX_RESTORE_OPERATION_ID: &str = "restore-operation-id"; +pub const SUFFIX_BUCKET_INCARNATION_ID: &str = "bucket-incarnation-id"; pub const SUFFIX_FREE_VERSION: &str = "free-version"; pub const SUFFIX_PURGESTATUS: &str = "purgestatus"; pub const SUFFIX_REPLICA_STATUS: &str = "replica-status"; diff --git a/rustfs/src/admin/handlers/durability.rs b/rustfs/src/admin/handlers/durability.rs index 98350f7cb..3fc92572e 100644 --- a/rustfs/src/admin/handlers/durability.rs +++ b/rustfs/src/admin/handlers/durability.rs @@ -160,6 +160,9 @@ impl Operation for SetBucketDurabilityHandler { authenticate_admin(&req).await?; let bucket = bucket_from_params(¶ms)?; + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&bucket) + .await + .map_err(|e| s3_error!(InternalError, "failed to capture bucket incarnation: {}", e))?; let body = req .input @@ -175,7 +178,7 @@ impl Operation for SetBucketDurabilityHandler { // System buckets are rejected by the metadata layer (and pinned to // strict by the disk layer regardless). - metadata_sys::update(&bucket, BUCKET_DURABILITY_CONFIG, json) + metadata_sys::update_if_incarnation(&bucket, BUCKET_DURABILITY_CONFIG, json, expected_incarnation_id) .await .map_err(|e| s3_error!(InternalError, "failed to set bucket durability: {}", e))?; @@ -218,8 +221,11 @@ impl Operation for DeleteBucketDurabilityHandler { authenticate_admin(&req).await?; let bucket = bucket_from_params(¶ms)?; + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&bucket) + .await + .map_err(|e| s3_error!(InternalError, "failed to capture bucket incarnation: {}", e))?; - metadata_sys::delete(&bucket, BUCKET_DURABILITY_CONFIG) + metadata_sys::delete_if_incarnation(&bucket, BUCKET_DURABILITY_CONFIG, expected_incarnation_id) .await .map_err(|e| s3_error!(InternalError, "failed to clear bucket durability: {}", e))?; diff --git a/rustfs/src/admin/handlers/quota.rs b/rustfs/src/admin/handlers/quota.rs index 7486e0c9a..35124b038 100644 --- a/rustfs/src/admin/handlers/quota.rs +++ b/rustfs/src/admin/handlers/quota.rs @@ -18,7 +18,7 @@ use crate::admin::auth::{validate_admin_request, validate_admin_request_with_buc use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{current_bucket_metadata_handle, current_object_store_handle}; -use crate::admin::storage_api::bucket::metadata_sys::BucketMetadataSys; +use crate::admin::storage_api::bucket::metadata_sys::{self, BucketMetadataSys}; use crate::admin::storage_api::bucket::quota::checker::QuotaChecker; use crate::admin::storage_api::bucket::quota::{BucketQuota, QuotaError, QuotaOperation}; use crate::auth::{check_key_valid, get_session_token}; @@ -277,6 +277,9 @@ impl Operation for SetBucketQuotaHandler { if bucket.is_empty() { return Err(s3_error!(InvalidRequest, "bucket name is required")); } + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&bucket) + .await + .map_err(|e| s3_error!(InternalError, "failed to capture bucket incarnation: {}", e))?; let body = req .input @@ -297,7 +300,7 @@ impl Operation for SetBucketQuotaHandler { let mut quota_checker = QuotaChecker::new(metadata_sys_lock.clone()); let updated_at = quota_checker - .set_quota_config(&bucket, quota.clone()) + .set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id) .await .map_err(|e| s3_error!(InternalError, "failed to set quota: {}", e))?; @@ -369,7 +372,6 @@ impl Operation for GetBucketQuotaHandler { if bucket.is_empty() { return Err(s3_error!(InvalidRequest, "bucket name is required")); } - validate_admin_request_with_bucket( &req.headers, &cred, @@ -452,6 +454,9 @@ impl Operation for ClearBucketQuotaHandler { if bucket.is_empty() { return Err(s3_error!(InvalidRequest, "bucket name is required")); } + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&bucket) + .await + .map_err(|e| s3_error!(InternalError, "failed to capture bucket incarnation: {}", e))?; info!( event = EVENT_ADMIN_QUOTA_STATE, @@ -471,7 +476,7 @@ impl Operation for ClearBucketQuotaHandler { // Clear quota (set to None) let quota = BucketQuota::new(None); let updated_at = quota_checker - .set_quota_config(&bucket, quota.clone()) + .set_quota_config_if_incarnation(&bucket, quota.clone(), expected_incarnation_id) .await .map_err(|e| s3_error!(InternalError, "failed to clear quota: {}", e))?; diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 223cb484d..26aea24e8 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -6667,6 +6667,7 @@ async fn ensure_site_replication_bucket_targets_with_runtime( local_peer: &PeerInfo, config: Option<&s3s::dto::ReplicationConfiguration>, service_account_secret_key: &str, + expected_incarnation_id: Uuid, ) -> S3Result<()> { let existing = match metadata_sys::list_bucket_targets(bucket).await { Ok(targets) => targets, @@ -6690,11 +6691,9 @@ async fn ensure_site_replication_bucket_targets_with_runtime( if json_targets == existing_json { return Ok(()); } - metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets) + metadata_sys::update_if_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id) .await .map_err(ApiError::from)?; - BucketTargetSys::get().update_all_targets(bucket, Some(&updated)).await; - Ok(()) } @@ -6707,6 +6706,9 @@ async fn bucket_replication_config_for_target_refresh(bucket: &str) -> S3Result< } async fn ensure_site_replication_bucket_targets(bucket: &str) -> S3Result<()> { + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) + .await + .map_err(ApiError::from)?; let _targets_guard = lock_bucket_targets_metadata(bucket).await; let Some(runtime) = runtime_site_replication_targets().await? else { return Ok(()); @@ -6718,6 +6720,7 @@ async fn ensure_site_replication_bucket_targets(bucket: &str) -> S3Result<()> { &runtime.local_peer, config.as_ref(), &runtime.service_account_secret_key, + expected_incarnation_id, ) .await } @@ -6727,6 +6730,7 @@ async fn ensure_site_replication_bucket_replication_config_with_runtime( state: &SiteReplicationState, local_peer: &PeerInfo, service_account_secret_key: &str, + expected_incarnation_id: Uuid, ) -> S3Result<()> { let existing = match metadata_sys::get_replication_config(bucket).await { Ok((existing, _)) => Some(existing), @@ -6771,7 +6775,7 @@ async fn ensure_site_replication_bucket_replication_config_with_runtime( let data = serialize(&ReplicationConfiguration { role, rules }) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize replication failed: {e}")))?; - metadata_sys::update(bucket, BUCKET_REPLICATION_CONFIG, data) + metadata_sys::update_if_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -6782,11 +6786,33 @@ async fn ensure_site_replication_bucket_setup(bucket: &str) -> S3Result { let Some(runtime) = runtime_site_replication_targets().await? else { return Ok(false); }; - ensure_site_replication_bucket_setup_with_runtime(bucket, &runtime).await?; + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) + .await + .map_err(ApiError::from)?; + ensure_site_replication_bucket_setup_with_runtime_for_incarnation(bucket, &runtime, expected_incarnation_id).await?; + Ok(true) +} + +async fn ensure_site_replication_bucket_setup_for_incarnation(bucket: &str, incarnation_id: Uuid) -> S3Result { + let Some(runtime) = runtime_site_replication_targets().await? else { + return Ok(false); + }; + ensure_site_replication_bucket_setup_with_runtime_for_incarnation(bucket, &runtime, incarnation_id).await?; Ok(true) } async fn ensure_site_replication_bucket_setup_with_runtime(bucket: &str, runtime: &SiteReplicationRuntime) -> S3Result<()> { + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) + .await + .map_err(ApiError::from)?; + ensure_site_replication_bucket_setup_with_runtime_for_incarnation(bucket, runtime, expected_incarnation_id).await +} + +async fn ensure_site_replication_bucket_setup_with_runtime_for_incarnation( + bucket: &str, + runtime: &SiteReplicationRuntime, + expected_incarnation_id: Uuid, +) -> S3Result<()> { let _targets_guard = lock_bucket_targets_metadata(bucket).await; let config = bucket_replication_config_for_target_refresh(bucket).await?; ensure_site_replication_bucket_targets_with_runtime( @@ -6795,6 +6821,7 @@ async fn ensure_site_replication_bucket_setup_with_runtime(bucket: &str, runtime &runtime.local_peer, config.as_ref(), &runtime.service_account_secret_key, + expected_incarnation_id, ) .await?; ensure_site_replication_bucket_replication_config_with_runtime( @@ -6802,12 +6829,16 @@ async fn ensure_site_replication_bucket_setup_with_runtime(bucket: &str, runtime &runtime.state, &runtime.local_peer, &runtime.service_account_secret_key, + expected_incarnation_id, ) .await?; Ok(()) } async fn cleanup_removed_site_replication_bucket(bucket: &str, removed_deployment_ids: &HashSet) -> S3Result { + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) + .await + .map_err(ApiError::from)?; let _targets_guard = lock_bucket_targets_metadata(bucket).await; let mut removed = 0usize; @@ -6819,12 +6850,9 @@ async fn cleanup_removed_site_replication_bucket(bucket: &str, removed_deploymen let json_targets = serde_json::to_vec(&updated_targets).map_err(|e| { S3Error::with_message(S3ErrorCode::InternalError, format!("serialize bucket targets failed: {e}")) })?; - metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets) + metadata_sys::update_if_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id) .await .map_err(ApiError::from)?; - BucketTargetSys::get() - .update_all_targets(bucket, Some(&updated_targets)) - .await; removed = removed.saturating_add(removed_targets); } } @@ -6840,11 +6868,11 @@ async fn cleanup_removed_site_replication_bucket(bucket: &str, removed_deploymen let data = serialize(&updated_config).map_err(|e| { S3Error::with_message(S3ErrorCode::InternalError, format!("serialize replication failed: {e}")) })?; - metadata_sys::update(bucket, BUCKET_REPLICATION_CONFIG, data) + metadata_sys::update_if_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; } else { - metadata_sys::delete(bucket, BUCKET_REPLICATION_CONFIG) + metadata_sys::delete_if_incarnation(bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id) .await .map_err(ApiError::from)?; } @@ -7121,6 +7149,9 @@ async fn refresh_bucket_targets_after_endpoint_edit(pending_id: &str, service_ac let buckets = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?; for bucket in buckets { + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&bucket.name) + .await + .map_err(ApiError::from)?; let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await; let state = load_site_replication_state().await?; let Some(pending) = pending_endpoint_refresh(&state).filter(|pending| pending.id == pending_id) else { @@ -7136,6 +7167,7 @@ async fn refresh_bucket_targets_after_endpoint_edit(pending_id: &str, service_ac &local_peer, replication_config.as_ref(), service_account_secret_key, + expected_incarnation_id, ) .await?; } @@ -7205,7 +7237,7 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st return bucket_status; }; let _targets_guard = lock_bucket_targets_metadata(bucket).await; - let _transaction_guard = match metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await { + let transaction_guard = match metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await { Ok(guard) => guard, Err(_) => { bucket_status.status = "failed".to_string(); @@ -7273,12 +7305,12 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st let opts = replication::resync_opts(bucket, target_arn.clone(), resync_id, reset_before); let admission_pool = pool.clone(); let activation_pool = pool.clone(); - let committed_targets = match replication::commit_resync_target( + let _committed_targets = match replication::commit_resync_target( targets, opts, move |opts| async move { admission_pool.admit_bucket_resync(opts).await }, move |encoded| async move { - metadata_sys::update_bucket_targets_under_transaction_lock(bucket, encoded) + metadata_sys::update_bucket_targets_under_transaction_lock(&transaction_guard, bucket, encoded) .await .map(|_| ()) .map_err(|_| { @@ -7304,10 +7336,6 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st return bucket_status; } }; - BucketTargetSys::get() - .update_all_targets(bucket, Some(&committed_targets)) - .await; - bucket_status } @@ -7318,6 +7346,14 @@ async fn cancel_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &s status: "canceled".to_string(), ..Default::default() }; + let expected_incarnation_id = match metadata_sys::capture_bucket_metadata_incarnation(bucket).await { + Ok(incarnation_id) => incarnation_id, + Err(err) => { + bucket_status.status = "failed".to_string(); + bucket_status.err_detail = err.to_string(); + return bucket_status; + } + }; let targets_guard = lock_bucket_targets_metadata(bucket).await; let mut targets = match metadata_sys::list_bucket_targets(bucket).await { @@ -7366,12 +7402,13 @@ async fn cancel_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &s } }; - if let Err(err) = metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets).await { + if let Err(err) = + metadata_sys::update_if_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id).await + { bucket_status.status = "failed".to_string(); bucket_status.err_detail = err.to_string(); return bucket_status; } - BucketTargetSys::get().update_all_targets(bucket, Some(&targets)).await; drop(targets_guard); bucket_status @@ -7491,13 +7528,16 @@ fn bucket_versioning_xml() -> S3Result> { } async fn ensure_site_replication_bucket_versioning(bucket: &str) -> S3Result<()> { + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(bucket) + .await + .map_err(ApiError::from)?; match metadata_sys::get_versioning_config(bucket).await { Ok((config, _)) if config.enabled() => return Ok(()), Ok(_) | Err(StorageError::ConfigNotFound) => {} Err(err) => return Err(ApiError::from(err).into()), } - metadata_sys::update(bucket, BUCKET_VERSIONING_CONFIG, bucket_versioning_xml()?) + metadata_sys::update_if_incarnation(bucket, BUCKET_VERSIONING_CONFIG, bucket_versioning_xml()?, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -7530,6 +7570,9 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { let Some(store) = current_object_store_handle() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&item.bucket) + .await + .map_err(ApiError::from)?; store .get_bucket_info(&item.bucket, &BucketOptions::default()) @@ -7630,11 +7673,11 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { if !skip_config_write { if let Some(data) = data { - metadata_sys::update(&item.bucket, config_file, data) + metadata_sys::update_if_incarnation(&item.bucket, config_file, data, expected_incarnation_id) .await .map_err(ApiError::from)?; } else { - metadata_sys::delete(&item.bucket, config_file) + metadata_sys::delete_if_incarnation(&item.bucket, config_file, expected_incarnation_id) .await .map_err(ApiError::from)?; } @@ -7644,7 +7687,7 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { if item.r#type == "replication-config" { // Rebuild the local outbound rules too: a site that joined an already-replicated // bucket receives this item before it has any `site-repl-*` rule of its own. - ensure_site_replication_bucket_setup(&item.bucket).await?; + ensure_site_replication_bucket_setup_for_incarnation(&item.bucket, expected_incarnation_id).await?; } if item.r#type == "version-config" @@ -7653,7 +7696,7 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { .ok() .is_some_and(|(config, _)| config.enabled()) { - ensure_site_replication_bucket_setup(&item.bucket).await?; + ensure_site_replication_bucket_setup_for_incarnation(&item.bucket, expected_incarnation_id).await?; } Ok(()) @@ -8569,9 +8612,17 @@ impl Operation for SRPeerBucketOpsHandler { ) .await .map_err(ApiError::from)?; - metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, bucket_versioning_xml()?) + let expected_incarnation_id = metadata_sys::capture_bucket_metadata_incarnation(&bucket) .await .map_err(ApiError::from)?; + metadata_sys::update_if_incarnation( + &bucket, + BUCKET_VERSIONING_CONFIG, + bucket_versioning_xml()?, + expected_incarnation_id, + ) + .await + .map_err(ApiError::from)?; } "configure-replication" => { store diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 7bec6c88a..5c81e0e5b 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -2478,7 +2478,7 @@ async fn start_replication_resync(bucket: &str, reset: &ReplicationResetStartReq }; let _targets_guard = lock_bucket_targets_metadata(bucket).await; - let _transaction_guard = metadata_sys::acquire_bucket_metadata_transaction_lock(bucket) + let transaction_guard = metadata_sys::acquire_bucket_metadata_transaction_lock(bucket) .await .map_err(ApiError::from)?; let (config, _) = metadata_sys::get_replication_config(bucket).await.map_err(ApiError::from)?; @@ -2494,7 +2494,7 @@ async fn start_replication_resync(bucket: &str, reset: &ReplicationResetStartReq opts, move |opts| async move { admission_pool.admit_bucket_resync(opts).await }, move |encoded| async move { - metadata_sys::update_bucket_targets_under_transaction_lock(bucket, encoded) + metadata_sys::update_bucket_targets_under_transaction_lock(&transaction_guard, bucket, encoded) .await .map(|_| ()) .map_err(|_| { diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 3feba88a6..451256730 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -286,16 +286,39 @@ pub(crate) mod metadata_sys { crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await } - pub(crate) async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result { + pub(crate) async fn update_if_incarnation( + bucket: &str, + config_file: &str, + data: Vec, + expected_incarnation_id: uuid::Uuid, + ) -> Result { + super::ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, expected_incarnation_id).await + } + + pub(crate) async fn capture_bucket_metadata_incarnation(bucket: &str) -> Result { + super::ecstore_bucket::metadata_sys::capture_bucket_metadata_incarnation(bucket).await + } + + pub(crate) async fn acquire_bucket_metadata_transaction_lock( + bucket: &str, + ) -> Result { crate::storage::storage_api::acquire_bucket_metadata_transaction_lock(bucket).await } - pub(crate) async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Vec) -> Result { - crate::storage::storage_api::update_bucket_targets_under_transaction_lock(bucket, data).await + pub(crate) async fn update_bucket_targets_under_transaction_lock( + guard: &super::ecstore_bucket::metadata_sys::BucketMetadataMutationGuard, + bucket: &str, + data: Vec, + ) -> Result { + crate::storage::storage_api::update_bucket_targets_under_transaction_lock(guard, bucket, data).await } - pub(crate) async fn delete(bucket: &str, config_file: &str) -> Result { - crate::storage::storage_api::delete_bucket_metadata_config(bucket, config_file).await + pub(crate) async fn delete_if_incarnation( + bucket: &str, + config_file: &str, + expected_incarnation_id: uuid::Uuid, + ) -> Result { + super::ecstore_bucket::metadata_sys::delete_if_incarnation(bucket, config_file, expected_incarnation_id).await } pub(crate) async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> { diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 075514dcc..544a6ac1f 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -16,12 +16,11 @@ use super::storage_api::bucket_usecase::ECStore; use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo; -use super::storage_api::bucket_usecase::access::{ReqInfo, authorize_request, req_info_ref}; +use super::storage_api::bucket_usecase::access::{ReqInfo, authorize_request, bucket_config_mutation_incarnation, req_info_ref}; #[cfg(test)] use super::storage_api::bucket_usecase::bucket::target::BucketTarget; use super::storage_api::bucket_usecase::bucket::{ ObjectLockConfigExt as _, VersioningConfigExt as _, - bucket_target_sys::BucketTargetSys, lifecycle::bucket_lifecycle_ops::{ enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, run_stale_multipart_upload_cleanup_once, validate_lifecycle_config, validate_transition_tier, @@ -383,6 +382,29 @@ fn serialize_config(value: &T) -> S3Result> { serialize(value).map_err(to_internal_error) } +async fn update_bucket_config_for_incarnation( + bucket: &str, + config_file: &str, + data: Vec, + expected_incarnation_id: Option, +) -> Result { + match expected_incarnation_id { + Some(incarnation_id) => metadata_sys::update_if_incarnation(bucket, config_file, data, incarnation_id).await, + None => metadata_sys::update(bucket, config_file, data).await, + } +} + +async fn delete_bucket_config_for_incarnation( + bucket: &str, + config_file: &str, + expected_incarnation_id: Option, +) -> Result { + match expected_incarnation_id { + Some(incarnation_id) => metadata_sys::delete_if_incarnation(bucket, config_file, incarnation_id).await, + None => metadata_sys::delete(bucket, config_file).await, + } +} + fn to_internal_error(err: impl Display) -> S3Error { S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")) } @@ -604,10 +626,7 @@ async fn replication_targets_without_config_targets( let mut targets = match metadata_sys::get_bucket_targets_config(bucket).await { Ok(targets) => targets, - Err(StorageError::ConfigNotFound) => { - BucketTargetSys::get().update_all_targets(bucket, None).await; - return Ok(None); - } + Err(StorageError::ConfigNotFound) => return Ok(None), Err(err) => return Err(ApiError::from(err).into()), }; @@ -632,12 +651,16 @@ fn remove_replication_targets_from_config_targets(targets: &mut BucketTargets, t original_len - targets.targets.len() } -async fn write_replication_targets_after_config_delete(bucket: &str, targets: &BucketTargets, removed: usize) -> S3Result<()> { +async fn write_replication_targets_after_config_delete( + bucket: &str, + targets: &BucketTargets, + removed: usize, + expected_incarnation_id: Option, +) -> S3Result<()> { let json_targets = serde_json::to_vec(&targets).map_err(to_internal_error)?; - metadata_sys::update(bucket, BUCKET_TARGETS_FILE, json_targets) + update_bucket_config_for_incarnation(bucket, BUCKET_TARGETS_FILE, json_targets, expected_incarnation_id) .await .map_err(ApiError::from)?; - BucketTargetSys::get().update_all_targets(bucket, Some(targets)).await; info!(bucket = %bucket, removed, "removed replication remote targets referenced by deleted bucket replication config"); Ok(()) @@ -647,10 +670,13 @@ async fn restore_replication_config_after_target_cleanup_failure( bucket: &str, config: &ReplicationConfiguration, cleanup_err: S3Error, + expected_incarnation_id: Option, ) -> S3Error { match serialize(config) { Ok(data) => { - if let Err(restore_err) = metadata_sys::update(bucket, BUCKET_REPLICATION_CONFIG, data).await { + if let Err(restore_err) = + update_bucket_config_for_incarnation(bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id).await + { error!( bucket = %bucket, error = ?restore_err, @@ -1412,6 +1438,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let DeleteBucketEncryptionInput { bucket, .. } = req.input; @@ -1424,7 +1451,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - metadata_sys::delete(&bucket, BUCKET_SSECONFIG) + delete_bucket_config_for_incarnation(&bucket, BUCKET_SSECONFIG, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -1443,6 +1470,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let DeleteBucketCorsInput { bucket, .. } = req.input; @@ -1455,7 +1483,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - metadata_sys::delete(&bucket, BUCKET_CORS_CONFIG) + delete_bucket_config_for_incarnation(&bucket, BUCKET_CORS_CONFIG, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -1474,6 +1502,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let DeleteBucketLifecycleInput { bucket, .. } = req.input; @@ -1486,7 +1515,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - metadata_sys::delete(&bucket, BUCKET_LIFECYCLE_CONFIG) + delete_bucket_config_for_incarnation(&bucket, BUCKET_LIFECYCLE_CONFIG, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -1505,6 +1534,7 @@ impl DefaultBucketUsecase { req: S3Request, ) -> S3Result> { record_s3_op(S3Operation::DeleteBucketPolicy); + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let DeleteBucketPolicyInput { bucket, .. } = req.input; @@ -1517,7 +1547,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - metadata_sys::delete(&bucket, BUCKET_POLICY_CONFIG) + delete_bucket_config_for_incarnation(&bucket, BUCKET_POLICY_CONFIG, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -1535,6 +1565,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let DeleteBucketReplicationInput { bucket, .. } = req.input; @@ -1558,14 +1589,21 @@ impl DefaultBucketUsecase { None }; - metadata_sys::delete(&bucket, BUCKET_REPLICATION_CONFIG) + delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id) .await .map_err(ApiError::from)?; if let Some((targets, removed)) = updated_targets - && let Err(err) = write_replication_targets_after_config_delete(&bucket, &targets, removed).await + && let Err(err) = + write_replication_targets_after_config_delete(&bucket, &targets, removed, expected_incarnation_id).await { if let Some(config) = replication_config.as_ref() { - return Err(restore_replication_config_after_target_cleanup_failure(&bucket, config, err).await); + return Err(restore_replication_config_after_target_cleanup_failure( + &bucket, + config, + err, + expected_incarnation_id, + ) + .await); } return Err(err); } @@ -1588,10 +1626,11 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let DeleteBucketTaggingInput { bucket, .. } = req.input; - metadata_sys::delete(&bucket, BUCKET_TAGGING_CONFIG) + delete_bucket_config_for_incarnation(&bucket, BUCKET_TAGGING_CONFIG, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -1611,6 +1650,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let DeletePublicAccessBlockInput { bucket, .. } = req.input; @@ -1623,7 +1663,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - metadata_sys::delete(&bucket, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG) + delete_bucket_config_for_incarnation(&bucket, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -2039,6 +2079,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let PutBucketEncryptionInput { bucket, @@ -2077,7 +2118,7 @@ impl DefaultBucketUsecase { .map_err(ApiError::from)?; let data = serialize_config(&server_side_encryption_configuration)?; - metadata_sys::update(&bucket, BUCKET_SSECONFIG, data) + update_bucket_config_for_incarnation(&bucket, BUCKET_SSECONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -2099,6 +2140,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let PutBucketLifecycleConfigurationInput { bucket, @@ -2138,7 +2180,7 @@ impl DefaultBucketUsecase { input_cfg.expiry_updated_at = Some(Timestamp::from(time::OffsetDateTime::now_utc())); let data = serialize_config(&input_cfg)?; - metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data) + update_bucket_config_for_incarnation(&bucket, BUCKET_LIFECYCLE_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -2194,6 +2236,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_region = req.region.clone(); let request_context = req.extensions.get::().cloned(); @@ -2215,7 +2258,7 @@ impl DefaultBucketUsecase { .map_err(ApiError::from)?; let data = serialize_config(¬ification_configuration)?; - metadata_sys::update(&bucket, BUCKET_NOTIFICATION_CONFIG, data) + update_bucket_config_for_incarnation(&bucket, BUCKET_NOTIFICATION_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -2269,6 +2312,7 @@ impl DefaultBucketUsecase { req: S3Request, ) -> S3Result> { record_s3_op(S3Operation::PutBucketPolicy); + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let PutBucketPolicyInput { bucket, policy, .. } = req.input; @@ -2319,7 +2363,7 @@ impl DefaultBucketUsecase { let data = policy.as_bytes().to_vec(); - metadata_sys::update(&bucket, BUCKET_POLICY_CONFIG, data) + update_bucket_config_for_incarnation(&bucket, BUCKET_POLICY_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -2336,6 +2380,7 @@ impl DefaultBucketUsecase { #[instrument(level = "debug", skip(self))] pub async fn execute_put_bucket_cors(&self, req: S3Request) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let PutBucketCorsInput { bucket, @@ -2353,7 +2398,7 @@ impl DefaultBucketUsecase { .map_err(ApiError::from)?; let data = serialize_config(&cors_configuration)?; - metadata_sys::update(&bucket, BUCKET_CORS_CONFIG, data) + update_bucket_config_for_incarnation(&bucket, BUCKET_CORS_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -2373,6 +2418,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let PutBucketReplicationInput { bucket, @@ -2395,7 +2441,7 @@ impl DefaultBucketUsecase { let targets_guard = lock_bucket_targets_metadata(&bucket).await; validate_bucket_replication_update(&bucket, &replication_configuration).await?; let data = serialize_config(&replication_configuration)?; - metadata_sys::update(&bucket, BUCKET_REPLICATION_CONFIG, data) + update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; drop(targets_guard); @@ -2418,6 +2464,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let PutPublicAccessBlockInput { bucket, @@ -2435,7 +2482,7 @@ impl DefaultBucketUsecase { .map_err(ApiError::from)?; let data = serialize_config(&public_access_block_configuration)?; - metadata_sys::update(&bucket, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, data) + update_bucket_config_for_incarnation(&bucket, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -2449,6 +2496,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let PutBucketTaggingInput { bucket, tagging, .. } = req.input; @@ -2463,7 +2511,7 @@ impl DefaultBucketUsecase { let data = serialize_config(&tagging)?; - metadata_sys::update(&bucket, BUCKET_TAGGING_CONFIG, data) + update_bucket_config_for_incarnation(&bucket, BUCKET_TAGGING_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -2484,6 +2532,7 @@ impl DefaultBucketUsecase { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); let PutBucketVersioningInput { bucket, @@ -2495,7 +2544,7 @@ impl DefaultBucketUsecase { let data = serialize_config(&versioning_configuration)?; - metadata_sys::update(&bucket, BUCKET_VERSIONING_CONFIG, data) + update_bucket_config_for_incarnation(&bucket, BUCKET_VERSIONING_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -3297,7 +3346,7 @@ mod tests { let req = build_request(input, Method::PUT); let usecase = DefaultBucketUsecase::without_context(); - let err = usecase.execute_create_bucket(req).await.unwrap_err(); + let err = Box::pin(usecase.execute_create_bucket(req)).await.unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InternalError); } diff --git a/rustfs/src/app/delete_objects_stat_gating_test.rs b/rustfs/src/app/delete_objects_stat_gating_test.rs index f68776969..16096df50 100644 --- a/rustfs/src/app/delete_objects_stat_gating_test.rs +++ b/rustfs/src/app/delete_objects_stat_gating_test.rs @@ -21,16 +21,15 @@ //! - buckets created with Object Lock keep the held-lock stat and the #4297 //! delete protection (explicit-version deletes of retained objects are //! rejected); -//! - buckets without Object Lock take the gated (stat-skipping) path and must -//! behave exactly as before: unversioned batch deletes remove objects and -//! report per-key results, versioned batch deletes still create delete -//! markers and preserve the underlying version. +//! - delete-marker creation can skip the held-lock stat, while destructive +//! deletes still inspect legacy or corrupt explicit Object Lock metadata even +//! when the bucket configuration is confirmed absent. use super::gating_test_env::shared_gating_ecstore; use super::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions}; use super::storage_api::test::contract::object::{ObjectIO as _, ObjectOperations as _}; use super::storage_api::test::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; -use crate::storage::storage_api::{StorageObjectLockDeleteOptions, StorageObjectToDelete as ObjectToDelete}; +use crate::storage::storage_api::{StorageError, StorageObjectLockDeleteOptions, StorageObjectToDelete as ObjectToDelete}; use serial_test::serial; use uuid::Uuid; @@ -98,10 +97,7 @@ async fn object_lock_bucket_batch_delete_keeps_held_lock_protection() { ) .await; - assert!( - errs[0].is_some(), - "explicit-version delete of a COMPLIANCE-retained object must be rejected on lock buckets" - ); + assert!(matches!(errs[0], Some(StorageError::PrefixAccessDenied(_, _)))); ecstore .get_object_info( @@ -117,6 +113,454 @@ async fn object_lock_bucket_batch_delete_keeps_held_lock_protection() { .expect("retained version must survive the batch delete"); } +#[tokio::test] +#[serial] +async fn object_lock_batch_delete_preserves_explicit_null_version_protection() { + let ecstore = shared_gating_ecstore().await; + let bucket = format!("explicit-null-lock-{}", Uuid::new_v4()); + + ecstore + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create object-lock bucket"); + let mut reader = PutObjReader::from_vec(b"retained null version".to_vec()); + ecstore + .put_object( + &bucket, + "null.bin", + &mut reader, + &ObjectOptions { + version_suspended: true, + user_defined: compliance_retention_metadata(), + ..Default::default() + }, + ) + .await + .expect("put retained null version"); + + let (_deleted, errs) = ecstore + .delete_objects( + &bucket, + vec![ObjectToDelete { + object_name: "null.bin".to_string(), + version_id: Some(Uuid::nil()), + ..Default::default() + }], + ObjectOptions { + versioned: true, + object_lock_delete: Some(StorageObjectLockDeleteOptions { + bypass_governance: false, + }), + ..Default::default() + }, + ) + .await; + + assert!(matches!(errs[0], Some(StorageError::PrefixAccessDenied(_, _)))); + ecstore + .get_object_info( + &bucket, + "null.bin", + &ObjectOptions { + version_id: Some(Uuid::nil().to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("retained null version must survive the batch delete"); +} + +#[tokio::test] +#[serial] +async fn recursive_force_delete_is_blocked_for_object_lock_bucket() { + let ecstore = shared_gating_ecstore().await; + let bucket = format!("force-delete-lock-{}", Uuid::new_v4()); + + ecstore + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create object-lock bucket"); + + let mut reader = PutObjReader::from_vec(b"protected payload".to_vec()); + ecstore + .put_object( + &bucket, + "protected/object.bin", + &mut reader, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("put object under protected prefix"); + + let err = ecstore + .delete_object( + &bucket, + "protected", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("recursive force-delete must be rejected for Object Lock buckets"); + + assert!(matches!(err, StorageError::InvalidArgument(_, _, _))); + ecstore + .get_object_info(&bucket, "protected/object.bin", &ObjectOptions::default()) + .await + .expect("rejected recursive delete must preserve the protected object"); +} + +#[tokio::test] +#[serial] +async fn lifecycle_style_delete_all_versions_rechecks_each_retained_version() { + let ecstore = shared_gating_ecstore().await; + let bucket = format!("delete-all-lock-{}", Uuid::new_v4()); + + ecstore + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create object-lock bucket"); + + let mut reader = PutObjReader::from_vec(b"retained payload".to_vec()); + let put_info = ecstore + .put_object( + &bucket, + "retained.bin", + &mut reader, + &ObjectOptions { + versioned: true, + user_defined: compliance_retention_metadata(), + ..Default::default() + }, + ) + .await + .expect("put retained object"); + let version_id = put_info.version_id.expect("lock bucket writes must be versioned"); + + let err = ecstore + .delete_object( + &bucket, + "retained.bin", + ObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + versioned: true, + ..Default::default() + }, + ) + .await + .expect_err("delete-all must recheck every retained version under the object lock"); + + assert!(matches!(err, StorageError::PrefixAccessDenied(_, _))); + ecstore + .get_object_info( + &bucket, + "retained.bin", + &ObjectOptions { + version_id: Some(version_id.to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("retained version must survive rejected delete-all"); +} + +#[tokio::test] +#[serial] +async fn malformed_persisted_retention_metadata_blocks_version_delete() { + let ecstore = shared_gating_ecstore().await; + let bucket = format!("malformed-retention-{}", Uuid::new_v4()); + + ecstore + .make_bucket( + &bucket, + &MakeBucketOptions { + lock_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create object-lock bucket"); + + let mut malformed = std::collections::HashMap::new(); + malformed.insert("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string()); + let mut reader = PutObjReader::from_vec(b"must survive".to_vec()); + let put_info = ecstore + .put_object( + &bucket, + "malformed.bin", + &mut reader, + &ObjectOptions { + versioned: true, + user_defined: malformed, + ..Default::default() + }, + ) + .await + .expect("storage fixture should persist the malformed boundary value"); + let version_id = put_info.version_id.expect("lock bucket writes must be versioned"); + + ecstore + .delete_object( + &bucket, + "malformed.bin", + ObjectOptions { + version_id: Some(version_id.to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect_err("incomplete persisted retention metadata must fail closed"); + + ecstore + .get_object_info( + &bucket, + "malformed.bin", + &ObjectOptions { + version_id: Some(version_id.to_string()), + versioned: true, + ..Default::default() + }, + ) + .await + .expect("malformed retained object must survive the rejected delete"); +} + +#[tokio::test] +#[serial] +async fn recursive_force_delete_remains_allowed_for_plain_bucket() { + let ecstore = shared_gating_ecstore().await; + let bucket = format!("force-delete-plain-{}", Uuid::new_v4()); + + ecstore + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create plain bucket"); + + for object in ["prefix/a.bin", "prefix/b.bin"] { + let mut reader = PutObjReader::from_vec(b"plain payload".to_vec()); + ecstore + .put_object(&bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("put object under plain prefix"); + } + + ecstore + .delete_object( + &bucket, + "prefix", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect("recursive force-delete should remain allowed for a plain bucket"); + + for object in ["prefix/a.bin", "prefix/b.bin"] { + ecstore + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect_err("recursive force-delete should remove every matching object"); + } +} + +#[tokio::test] +#[serial] +async fn plain_bucket_explicit_retention_blocks_every_destructive_delete_shape() { + let ecstore = shared_gating_ecstore().await; + let bucket = format!("plain-explicit-lock-{}", Uuid::new_v4()); + ecstore + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create plain bucket fixture"); + + for object in ["batch.bin", "delete-all.bin", "prefix/retained.bin"] { + let mut reader = PutObjReader::from_vec(b"legacy retained payload".to_vec()); + ecstore + .put_object( + &bucket, + object, + &mut reader, + &ObjectOptions { + user_defined: compliance_retention_metadata(), + ..Default::default() + }, + ) + .await + .expect("storage fixture should persist explicit retention metadata"); + } + + let (_deleted, errors) = ecstore + .delete_objects( + &bucket, + vec![ObjectToDelete { + object_name: "batch.bin".to_string(), + ..Default::default() + }], + ObjectOptions::default(), + ) + .await; + assert!(matches!(errors[0], Some(StorageError::PrefixAccessDenied(_, _)))); + + let delete_all_error = ecstore + .delete_object( + &bucket, + "delete-all.bin", + ObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + ..Default::default() + }, + ) + .await + .expect_err("delete-all must inspect explicit retention in a plain bucket"); + assert!(matches!(delete_all_error, StorageError::PrefixAccessDenied(_, _))); + + let prefix_error = ecstore + .delete_object( + &bucket, + "prefix", + ObjectOptions { + delete_prefix: true, + ..Default::default() + }, + ) + .await + .expect_err("recursive force-delete must inspect every matching object"); + assert!(matches!(prefix_error, StorageError::PrefixAccessDenied(_, _))); + + for object in ["batch.bin", "delete-all.bin", "prefix/retained.bin"] { + ecstore + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect("retained object must survive every rejected delete shape"); + } +} + +#[tokio::test] +#[serial] +async fn batch_delete_can_purge_an_explicit_delete_marker_version() { + let ecstore = shared_gating_ecstore().await; + let bucket = format!("batch-delete-marker-{}", Uuid::new_v4()); + ecstore + .make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create versioned bucket"); + + let deleted = ecstore + .delete_object( + &bucket, + "marker.bin", + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("create a delete marker"); + let marker_version = deleted.version_id.expect("delete marker should have a version id"); + + let (_deleted, errors) = ecstore + .delete_objects( + &bucket, + vec![ObjectToDelete { + object_name: "marker.bin".to_string(), + version_id: Some(marker_version), + ..Default::default() + }], + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await; + assert!(errors[0].is_none(), "explicit delete-marker purge must succeed: {:?}", errors[0]); +} + +#[tokio::test] +#[serial] +async fn delete_all_exact_object_preserves_a_retained_child_key() { + let ecstore = shared_gating_ecstore().await; + let bucket = format!("delete-all-child-{}", Uuid::new_v4()); + ecstore + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create plain bucket fixture"); + + let mut parent_reader = PutObjReader::from_vec(b"parent".to_vec()); + ecstore + .put_object(&bucket, "foo", &mut parent_reader, &ObjectOptions::default()) + .await + .expect("put exact parent object"); + let mut child_reader = PutObjReader::from_vec(b"retained child".to_vec()); + ecstore + .put_object( + &bucket, + "foo/bar", + &mut child_reader, + &ObjectOptions { + user_defined: compliance_retention_metadata(), + ..Default::default() + }, + ) + .await + .expect("put retained child object"); + + ecstore + .delete_object( + &bucket, + "foo", + ObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + ..Default::default() + }, + ) + .await + .expect("delete-all should remove only the exact object"); + + ecstore + .get_object_info(&bucket, "foo", &ObjectOptions::default()) + .await + .expect_err("exact parent should be deleted"); + ecstore + .get_object_info(&bucket, "foo/bar", &ObjectOptions::default()) + .await + .expect("retained child key must survive exact delete-all"); +} + #[tokio::test] #[serial] async fn non_lock_versioned_bucket_batch_delete_still_creates_delete_marker() { diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 1b773f1d1..9a73f2189 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -15,7 +15,10 @@ //! Multipart application use-case contracts. use super::storage_api::multipart_usecase::ECStore; -use super::storage_api::multipart_usecase::access::{has_bypass_governance_header, replication_request_authorized}; +use super::storage_api::multipart_usecase::access::{ + apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, has_bypass_governance_header, + replication_request_authorized, +}; use super::storage_api::multipart_usecase::bucket::quota::checker::QuotaChecker; use super::storage_api::multipart_usecase::bucket::{ lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate}, @@ -52,8 +55,8 @@ use super::storage_api::multipart_usecase::sse::{ DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, authorize_sse_kms_object_read, build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, - get_buffer_size_opt_in, map_get_object_reader_error, mark_encrypted_multipart_metadata, sse_decryption, - sse_prepare_encryption, + get_buffer_size_opt_in, load_bucket_object_lock_config_state, map_get_object_reader_error, mark_encrypted_multipart_metadata, + sse_decryption, sse_prepare_encryption, }; use super::storage_api::multipart_usecase::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; use crate::app::object_data_cache::{ @@ -61,7 +64,8 @@ use crate::app::object_data_cache::{ invalidate_object_data_cache_after_delete_success, invalidate_object_data_cache_before_mutation, }; use crate::app::object_usecase::{ - build_put_like_object_lock_metadata, map_quota_check_outcome, validate_existing_object_lock_for_write, + acquire_copy_bucket_lifecycle_locks, build_put_like_object_lock_metadata, map_quota_check_outcome, + validate_existing_object_lock_for_write, }; use crate::app::runtime_sources::{ AppContext, current_app_context, current_object_data_cache_for_context, current_object_store_handle_for_context, @@ -343,6 +347,8 @@ impl DefaultMultipartUsecase { req: S3Request, ) -> S3Result> { record_s3_op(S3Operation::AbortMultipartUpload); + let mut opts = ObjectOptions::default(); + apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?; let AbortMultipartUploadInput { bucket, key, upload_id, .. } = req.input; @@ -351,8 +357,6 @@ impl DefaultMultipartUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let opts = &ObjectOptions::default(); - // Special handling for abort_multipart_upload: Per AWS S3 API specification, this operation // should return NoSuchUpload (404) when the upload_id doesn't exist, even if the format // appears invalid. This differs from other multipart operations (upload_part, list_parts, @@ -360,7 +364,7 @@ impl DefaultMultipartUsecase { // The lenient validation matches AWS S3 behavior where format validation is relaxed for // abort operations to avoid leaking information about upload_id format requirements. match store - .abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), opts) + .abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), &opts) .await { Ok(_) => { @@ -388,7 +392,7 @@ impl DefaultMultipartUsecase { S3Operation::CompleteMultipartUpload, ); let replication_authorized = replication_request_authorized(&req); - let input = req.input; + let input = req.input.clone(); let CompleteMultipartUploadInput { multipart_upload, bucket, @@ -454,6 +458,7 @@ impl DefaultMultipartUsecase { let mut opts = get_complete_multipart_upload_opts_with_replication_authorization(&req.headers, replication_authorized) .map_err(ApiError::from)?; + apply_bucket_generation_guard(&req, &bucket, &mut opts)?; let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await; let version_suspended = BucketVersioningSys::prefix_suspended(&bucket, &key).await; opts.versioned = versioned; @@ -495,7 +500,7 @@ impl DefaultMultipartUsecase { }; let multipart_info = store - .get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default()) + .get_multipart_info(&bucket, &key, &upload_id, &opts) .await .map_err(ApiError::from)?; EncryptionRequest { @@ -715,17 +720,22 @@ impl DefaultMultipartUsecase { let mut metadata = create_multipart_upload_metadata(input_metadata, &req.headers, tagging, storage_class.as_ref()); let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some(); + let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?; if let Some(object_lock_metadata) = build_put_like_object_lock_metadata( &bucket, + &object_lock_config_state, object_lock_legal_hold_status, object_lock_mode, object_lock_retain_until_date, - ) - .await? - { + )? { metadata.extend(object_lock_metadata); } - apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?; + apply_bucket_default_lock_retention( + &bucket, + &object_lock_config_state, + &mut metadata, + has_explicit_object_lock_retention, + )?; let (header_sse_customer_algorithm, header_sse_customer_key, header_sse_customer_key_md5) = extract_ssec_params_from_headers(&req.headers)?; let sse_customer_algorithm = sse_customer_algorithm.or(header_sse_customer_algorithm); @@ -771,6 +781,7 @@ impl DefaultMultipartUsecase { put_opts_with_replication_authorization(&bucket, &key, version_id, &req.headers, metadata, replication_authorized) .await .map_err(ApiError::from)?; + apply_bucket_generation_guard(&req, &bucket, &mut opts)?; let dsc = must_replicate_object(&bucket, &key, &mt2, "".to_string(), opts.delete_marker_replication_status(), opts.clone()) @@ -835,6 +846,8 @@ impl DefaultMultipartUsecase { #[instrument(level = "debug", skip(self, req))] pub async fn execute_upload_part(&self, req: S3Request) -> S3Result> { + let mut opts = ObjectOptions::default(); + apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?; let input = req.input; let UploadPartInput { body, @@ -881,7 +894,6 @@ impl DefaultMultipartUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; - let mut opts = ObjectOptions::default(); let fi = store .get_multipart_info(&bucket, &key, &upload_id, &opts) .await @@ -1095,6 +1107,8 @@ impl DefaultMultipartUsecase { &self, req: S3Request, ) -> S3Result> { + let mut opts = ObjectOptions::default(); + apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?; let ListMultipartUploadsInput { bucket, prefix, @@ -1110,13 +1124,29 @@ impl DefaultMultipartUsecase { key_marker, max_uploads, } = parse_list_multipart_uploads_params(prefix, key_marker, max_uploads)?; - let Some(store) = self.object_store() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; + // `apply_bucket_generation_guard` tolerates a missing guard (only the S3 + // access layer installs one), so resolve the current generation rather + // than failing the request. Listing is filtered by this value, so a + // stale one simply hides foreign-incarnation uploads, as intended. + let expected_incarnation_id = match opts.expected_bucket_incarnation_id { + Some(incarnation_id) => incarnation_id, + None => store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)?, + }; + let result = store - .list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads) + .list_multipart_uploads_for_bucket_incarnation( + &bucket, + &prefix, + key_marker, + upload_id_marker, + delimiter, + max_uploads, + expected_incarnation_id, + ) .await .map_err(ApiError::from)?; @@ -1124,6 +1154,8 @@ impl DefaultMultipartUsecase { } pub async fn execute_list_parts(&self, req: S3Request) -> S3Result> { + let mut opts = ObjectOptions::default(); + apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?; let ListPartsInput { bucket, key, @@ -1140,14 +1172,7 @@ impl DefaultMultipartUsecase { }; let res = store - .list_object_parts( - &bucket, - &key, - &upload_id, - params.part_number_marker, - params.max_parts, - &ObjectOptions::default(), - ) + .list_object_parts(&bucket, &key, &upload_id, params.part_number_marker, params.max_parts, &opts) .await .map_err(ApiError::from)?; @@ -1161,6 +1186,17 @@ impl DefaultMultipartUsecase { ) -> S3Result> { // Captured before `req.input` is destructured below. let copy_principal = SseKmsPrincipal::from_request(&req); + let source_bucket = match &req.input.copy_source { + CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)), + CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)), + CopySource::Bucket { bucket, .. } => bucket.to_string(), + }; + let mut source_generation_opts = ObjectOptions::default(); + apply_copy_source_bucket_generation_guard(&req, &source_bucket, &mut source_generation_opts)?; + let expected_source_incarnation_id = source_generation_opts.expected_bucket_incarnation_id; + let mut destination_generation_opts = ObjectOptions::default(); + apply_bucket_generation_guard(&req, &req.input.bucket, &mut destination_generation_opts)?; + let expected_destination_incarnation_id = destination_generation_opts.expected_bucket_incarnation_id; let UploadPartCopyInput { bucket, key, @@ -1203,8 +1239,47 @@ impl DefaultMultipartUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; + let (source_bucket_lifecycle_guard, destination_bucket_lifecycle_guard_storage) = + acquire_copy_bucket_lifecycle_locks(store.as_ref(), &src_bucket, &bucket).await?; + let current_source_incarnation_id = store + .bucket_incarnation_id_from_disk(&src_bucket) + .await + .map_err(ApiError::from)?; + if expected_source_incarnation_id.is_some_and(|expected| expected != current_source_incarnation_id) { + return Err(ApiError::from(StorageError::BucketNotFound(src_bucket.clone())).into()); + } + let current_destination_incarnation_id = if src_bucket == bucket { + current_source_incarnation_id + } else { + store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)? + }; + if expected_destination_incarnation_id.is_some_and(|expected| expected != current_destination_incarnation_id) { + return Err(ApiError::from(StorageError::BucketNotFound(bucket.clone())).into()); + } + let destination_bucket_lifecycle_guard = destination_bucket_lifecycle_guard_storage + .as_ref() + .unwrap_or(&source_bucket_lifecycle_guard); + if source_bucket_lifecycle_guard.is_lock_lost() || destination_bucket_lifecycle_guard.is_lock_lost() { + return Err(ApiError::from(StorageError::NamespaceLockQuorumUnavailable { + mode: "copy_bucket_generation", + bucket: bucket.clone(), + object: key.clone(), + required: 1, + achieved: 0, + }) + .into()); + } + let mut dst_opts = ObjectOptions { + expected_bucket_incarnation_id: Some(current_destination_incarnation_id), + ..Default::default() + }; + if src_bucket != bucket { + dst_opts.add_bucket_lifecycle_lock_guard(&source_bucket_lifecycle_guard); + } + dst_opts.add_bucket_lifecycle_lock_guard(destination_bucket_lifecycle_guard); + let mp_info = store - .get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default()) + .get_multipart_info(&bucket, &key, &upload_id, &dst_opts) .await .map_err(ApiError::from)?; EncryptionRequest { @@ -1233,8 +1308,19 @@ impl DefaultMultipartUsecase { version_id: src_opts.version_id.clone(), versioned: src_opts.versioned, version_suspended: src_opts.version_suspended, + expected_bucket_incarnation_id: Some(current_source_incarnation_id), ..Default::default() }; + if source_bucket_lifecycle_guard.is_lock_lost() { + return Err(ApiError::from(StorageError::NamespaceLockQuorumUnavailable { + mode: "copy_source_bucket_generation", + bucket: src_bucket.clone(), + object: src_key.clone(), + required: 1, + achieved: 0, + }) + .into()); + } let src_reader = store .get_object_reader(&src_bucket, &src_key, rs.clone(), h, &get_opts) @@ -1400,10 +1486,7 @@ impl DefaultMultipartUsecase { let mut reader = PutObjReader::new(reader); - let dst_opts = ObjectOptions { - user_defined: dst_user_defined, - ..Default::default() - }; + dst_opts.user_defined = dst_user_defined; let part_info = store .put_object_part(&bucket, &key, &upload_id, part_id, &mut reader, &dst_opts) diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 62a804996..8226bc33b 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -20,7 +20,8 @@ use rustfs_io_metrics::buffered_write; use crate::storage_api::table::get_bucket_metadata; use super::storage_api::object_usecase::access::{ - PostObjectRequestMarker, authorize_request, has_bypass_governance_header, recursive_force_delete_is_authorized, + PostObjectRequestMarker, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_request, + has_bypass_governance_header, load_bucket_generation_from_store, recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref, }; use super::storage_api::object_usecase::bucket::quota::checker::QuotaChecker; @@ -68,8 +69,7 @@ use super::storage_api::object_usecase::data_usage::{ use super::storage_api::object_usecase::deadlock_detector; use super::storage_api::object_usecase::ecfs::FS; use super::storage_api::object_usecase::error::{ - DiskError, Error as EcstoreError, StorageError, is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, - is_err_version_not_found, + Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, }; use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children}; use super::storage_api::object_usecase::helper::{OperationHelper, build_event_resp_elements, spawn_background_with_context}; @@ -83,8 +83,8 @@ use super::storage_api::object_usecase::object_utils::to_s3s_etag; use super::storage_api::object_usecase::options::{ copy_dst_opts_with_replication_authorization, copy_src_opts, del_opts_with_versioning, extract_metadata, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_content_sha256_with_query, get_opts, - namespace_reserved_user_metadata, normalize_content_encoding_for_storage, put_opts_with_replication_authorization, - validate_archive_content_encoding, + namespace_reserved_user_metadata, normalize_content_encoding_for_storage, preserve_unclassified_user_metadata, + put_opts_with_replication_authorization, validate_archive_content_encoding, }; use super::storage_api::object_usecase::request_context::{self, spawn_traced}; use super::storage_api::object_usecase::s3_api::multipart::parse_list_parts_params; @@ -95,16 +95,16 @@ use super::storage_api::object_usecase::sse::{ DecryptionRequest, EncryptionRequest, SSEType, SseKmsPrincipal, apply_bucket_default_lock_retention, authorize_sse_kms_object_read, build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, - get_buffer_size_opt_in, map_get_object_reader_error, sse_decryption, sse_encryption, + get_buffer_size_opt_in, load_bucket_object_lock_config_state, map_get_object_reader_error, sse_decryption, sse_encryption, + validate_bucket_object_lock_enabled_state, }; use super::storage_api::object_usecase::storage_class as storageclass; use super::storage_api::object_usecase::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; use super::storage_api::object_usecase::{ECStore, OldCurrentSize}; use super::storage_api::object_usecase::{ RFC1123, check_preconditions, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, - remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, - validate_bucket_object_lock_enabled, validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, - validate_ssec_for_read, wrap_response_with_cors, + remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_object_key, + validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors, }; use crate::app::runtime_sources::{ AppContext, current_app_context, current_notify_interface_for_context, current_object_data_cache_for_context, @@ -198,8 +198,9 @@ use tracing::{debug, error, instrument, warn}; use uuid::Uuid; use super::storage_api::object_usecase::{ - GetObjectReader, StorageDeletedObject, StorageObjectInfo as ObjectInfo, StorageObjectLockDeleteOptions, - StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader, + BUCKET_LIFECYCLE_LOCK_OBJECT, GetObjectReader, StorageDeletedObject, StorageObjectInfo as ObjectInfo, + StorageObjectLockDeleteOptions, StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, + StoragePutObjReader as PutObjReader, }; use crate::app::object_data_cache::{ ColdFillCoordinateOutcome, ColdFillDiskPermitOwner, ColdFillError, ColdFillProducer, GetObjectBodyCacheLookup, @@ -2480,6 +2481,8 @@ type DeleteSnapshotTestHook = (String, Arc, Arc>> = OnceLock::new(); #[cfg(test)] static DELETE_SOURCE_TEST_HOOK: OnceLock>> = OnceLock::new(); +#[cfg(test)] +static DELETE_OBJECTS_AUTH_TEST_HOOK: OnceLock>> = OnceLock::new(); #[cfg(test)] pub(crate) fn install_delete_snapshot_test_hook( @@ -2543,6 +2546,37 @@ async fn wait_for_delete_source_test_hook(bucket: &str) { } } +#[cfg(test)] +pub(crate) fn install_delete_objects_auth_test_hook( + bucket: String, + loaded: Arc, + resume: Arc, +) { + *DELETE_OBJECTS_AUTH_TEST_HOOK + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("delete objects auth test hook lock should not be poisoned") = Some((bucket, loaded, resume)); +} + +#[cfg(test)] +async fn wait_for_delete_objects_auth_test_hook(bucket: &str) { + let hook = { + let mut slot = DELETE_OBJECTS_AUTH_TEST_HOOK + .get_or_init(|| Mutex::new(None)) + .lock() + .expect("delete objects auth test hook lock should not be poisoned"); + if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) { + slot.take() + } else { + None + } + }; + if let Some((_bucket, loaded, resume)) = hook { + loaded.wait().await; + resume.wait().await; + } +} + fn build_put_object_expiration_header(event: &lifecycle::Event) -> Option { if !event.action.delete() { return None; @@ -2659,6 +2693,48 @@ where .map_err(|err| ApiError::from(copy_namespace_lock_error(bucket, &object, "write", err)).into()) } +pub(crate) async fn acquire_copy_bucket_lifecycle_lock(store: &S, bucket: &str) -> S3Result +where + S: NamespaceLocking + ?Sized, +{ + let lock = store + .new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT) + .await + .map_err(ApiError::from)?; + lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| { + ApiError::from(copy_namespace_lock_error( + bucket, + BUCKET_LIFECYCLE_LOCK_OBJECT, + "bucket_lifecycle_read", + err, + )) + .into() + }) +} + +pub(crate) async fn acquire_copy_bucket_lifecycle_locks( + store: &S, + source_bucket: &str, + destination_bucket: &str, +) -> S3Result<(NamespaceLockGuard, Option)> +where + S: NamespaceLocking + ?Sized, +{ + if source_bucket == destination_bucket { + return Ok((acquire_copy_bucket_lifecycle_lock(store, source_bucket).await?, None)); + } + + if source_bucket < destination_bucket { + let source_guard = acquire_copy_bucket_lifecycle_lock(store, source_bucket).await?; + let destination_guard = acquire_copy_bucket_lifecycle_lock(store, destination_bucket).await?; + Ok((source_guard, Some(destination_guard))) + } else { + let destination_guard = acquire_copy_bucket_lifecycle_lock(store, destination_bucket).await?; + let source_guard = acquire_copy_bucket_lifecycle_lock(store, source_bucket).await?; + Ok((source_guard, Some(destination_guard))) + } +} + const AMZ_SNOWBALL_EXTRACT_COMPAT: &str = "X-Amz-Snowball-Auto-Extract"; #[cfg(test)] const AMZ_SNOWBALL_PREFIX_INTERNAL: &str = "X-Amz-Meta-Rustfs-Snowball-Prefix"; @@ -2772,37 +2848,124 @@ fn map_extract_archive_error(err: impl std::fmt::Display) -> S3Error { s3_error!(InvalidArgument, "Failed to process archive entry: {}", err) } +#[derive(Debug, Default)] +struct ExtractEntryPaxAuthorization { + headers: HeaderMap, + object_lock_legal_hold_status: Option, + object_lock_mode: Option, + object_lock_retain_until_date: Option, +} + async fn apply_extract_entry_pax_extensions( entry: &mut tokio_tar::Entry>, + bucket: &str, + object_name: &str, + object_lock_config_state: &metadata_sys::ObjectLockConfigState, metadata: &mut HashMap, opts: &mut ObjectOptions, -) -> S3Result<()> +) -> S3Result where R: AsyncRead + Send + Unpin + 'static, { let Some(extensions) = entry.pax_extensions().await.map_err(map_extract_archive_error)? else { - return Ok(()); + return Ok(ExtractEntryPaxAuthorization::default()); }; + let mut pax_headers = HeaderMap::new(); + let mut pax_version_id = None; for ext in extensions { let ext = ext.map_err(map_extract_archive_error)?; let key = ext.key().map_err(map_extract_archive_error)?; let value = ext.value().map_err(map_extract_archive_error)?; if let Some(meta_key) = key.strip_prefix("minio.metadata.") { - let meta_key = meta_key.strip_prefix("x-amz-meta-").unwrap_or(meta_key); if !meta_key.is_empty() { - metadata.insert(meta_key.to_string(), value.to_string()); + let name = http::HeaderName::from_bytes(meta_key.as_bytes()) + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball PAX metadata header"))?; + let header_value = HeaderValue::from_str(value) + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball PAX metadata value"))?; + preserve_unclassified_user_metadata(metadata, name.as_str(), value); + pax_headers.insert(name, header_value); } continue; } if key == "minio.versionId" && !value.is_empty() { - opts.version_id = Some(value.to_string()); + if Uuid::parse_str(value).is_err() { + return Err(s3_error!(InvalidArgument, "Invalid Snowball PAX version ID")); + } + pax_version_id = Some(value.to_string()); } } - Ok(()) + let has_replica_status = pax_headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS); + if let Some(value) = pax_headers.get(AMZ_BUCKET_REPLICATION_STATUS) { + let status = value + .to_str() + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball replication status"))?; + if !status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str()) { + return Err(s3_error!(InvalidArgument, "Invalid Snowball replication status")); + } + pax_headers.insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA")); + } + + let authorization_headers = pax_headers.clone(); + + let object_lock_mode = pax_headers + .remove(AMZ_OBJECT_LOCK_MODE_LOWER) + .map(|value| { + value + .to_str() + .map(|value| ObjectLockMode::from(value.to_string())) + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball Object Lock mode")) + }) + .transpose()?; + let object_lock_retain_until_date = pax_headers + .remove(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER) + .map(|value| { + let value = value + .to_str() + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball Object Lock retain-until date"))?; + OffsetDateTime::parse(value, &Rfc3339) + .map(Timestamp::from) + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball Object Lock retain-until date")) + }) + .transpose()?; + let object_lock_legal_hold_status = pax_headers + .remove(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER) + .map(|value| { + value + .to_str() + .map(|value| ObjectLockLegalHoldStatus::from(value.to_string())) + .map_err(|_| s3_error!(InvalidArgument, "Invalid Snowball Object Lock legal-hold status")) + }) + .transpose()?; + opts.version_id = pax_version_id; + + extract_metadata_from_mime_with_object_name(&pax_headers, metadata, false, Some(object_name)); + if has_replica_status { + metadata.retain(|key, _| !key.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)); + metadata.insert( + AMZ_BUCKET_REPLICATION_STATUS.to_string(), + ReplicationStatusType::Replica.as_str().to_string(), + ); + } + if let Some(object_lock_metadata) = build_put_like_object_lock_metadata( + bucket, + object_lock_config_state, + object_lock_legal_hold_status.clone(), + object_lock_mode.clone(), + object_lock_retain_until_date.clone(), + )? { + metadata.extend(object_lock_metadata); + } + + Ok(ExtractEntryPaxAuthorization { + headers: authorization_headers, + object_lock_legal_hold_status, + object_lock_mode, + object_lock_retain_until_date, + }) } fn insert_expires_metadata(metadata: &mut HashMap, expires: Option<&Timestamp>) -> S3Result<()> { @@ -2866,6 +3029,7 @@ fn apply_put_request_metadata( tagging: Option, storage_class: Option, ) -> S3Result<()> { + namespace_reserved_user_metadata(metadata); apply_standard_object_metadata( metadata, cache_control.as_deref(), @@ -2922,8 +3086,9 @@ fn response_storage_class_for_object_attributes( )) } -async fn apply_put_request_object_lock_opts( +fn apply_put_request_object_lock_opts( bucket: &str, + object_lock_config_state: &metadata_sys::ObjectLockConfigState, object_lock_legal_hold_status: Option, object_lock_mode: Option, object_lock_retain_until_date: Option, @@ -2931,12 +3096,11 @@ async fn apply_put_request_object_lock_opts( ) -> S3Result<()> { if let Some(eval_metadata) = build_put_like_object_lock_metadata( bucket, + object_lock_config_state, object_lock_legal_hold_status, object_lock_mode, object_lock_retain_until_date, - ) - .await? - { + )? { opts.eval_metadata = Some(eval_metadata); } @@ -2948,8 +3112,9 @@ async fn apply_put_request_object_lock_opts( pub(crate) const ERR_OBJECT_LOCK_RETENTION_HEADERS_MUST_BE_PAIRED: &str = "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied"; -pub(crate) async fn build_put_like_object_lock_metadata( +pub(crate) fn build_put_like_object_lock_metadata( bucket: &str, + object_lock_config_state: &metadata_sys::ObjectLockConfigState, object_lock_legal_hold_status: Option, object_lock_mode: Option, object_lock_retain_until_date: Option, @@ -2972,7 +3137,7 @@ pub(crate) async fn build_put_like_object_lock_metadata( (None, None) => None, }; - validate_bucket_object_lock_enabled(bucket).await?; + validate_bucket_object_lock_enabled_state(bucket, object_lock_config_state)?; let mut eval_metadata = parse_object_lock_retention(retention)?; eval_metadata.extend(parse_object_lock_legal_hold( @@ -4861,6 +5026,7 @@ impl DefaultObjectUsecase { let mut metadata = metadata.unwrap_or_default(); let has_explicit_object_lock_retention = object_lock_mode.is_some() || object_lock_retain_until_date.is_some(); + let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?; apply_put_request_metadata( &mut metadata, &req.headers, @@ -4875,7 +5041,12 @@ impl DefaultObjectUsecase { tagging, storage_class.clone(), )?; - apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?; + apply_bucket_default_lock_retention( + &bucket, + &object_lock_config_state, + &mut metadata, + has_explicit_object_lock_retention, + )?; let mut opts: ObjectOptions = put_opts_with_replication_authorization( &bucket, @@ -4887,14 +5058,15 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; + apply_bucket_generation_guard(&req, &bucket, &mut opts)?; apply_put_request_object_lock_opts( &bucket, + &object_lock_config_state, object_lock_legal_hold_status, object_lock_mode, object_lock_retain_until_date, &mut opts, - ) - .await?; + )?; // rustfs/backlog#1009: the pre-PUT lookup has exactly two consumers — // the existing-object WORM validation and usage accounting's @@ -4904,7 +5076,7 @@ impl DefaultObjectUsecase { // replication), the lookup is skipped and accounting is backfilled from // the dst xl.meta that rename_data already reads, saving a full-disk // metadata fanout per PUT. - let prelookup_required = version_id.is_some() || put_prelookup_worm_gate(&bucket).await; + let prelookup_required = version_id.is_some() || object_lock_checks_required(&bucket).await; // Outer None = prelookup skipped (accounting comes from the commit // backfill); Some(inner) = the previous current size as observed by the // lookup, with the pre-#1009 semantics kept bit-for-bit. @@ -6154,6 +6326,7 @@ impl DefaultObjectUsecase { version_suspended: src_opts.version_suspended, ..Default::default() }; + apply_copy_source_bucket_generation_guard(&req, &src_bucket, &mut src_get_opts)?; let mut dst_opts = copy_dst_opts_with_replication_authorization( &bucket, @@ -6165,6 +6338,7 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; + apply_bucket_generation_guard(&req, &bucket, &mut dst_opts)?; let cp_src_dst_same = path_join_buf(&[&src_bucket, &src_key]) == path_join_buf(&[&bucket, &key]); let expected_current_version_id = expected_current_version_id(&req.headers)?; @@ -6180,6 +6354,63 @@ impl DefaultObjectUsecase { let Some(store) = self.object_store() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; + let (source_bucket_lifecycle_guard, destination_bucket_lifecycle_guard_storage) = + acquire_copy_bucket_lifecycle_locks(store.as_ref(), &src_bucket, &bucket).await?; + let current_source_incarnation_id = store + .bucket_incarnation_id_from_disk(&src_bucket) + .await + .map_err(ApiError::from)?; + if src_get_opts + .expected_bucket_incarnation_id + .is_some_and(|expected| expected != current_source_incarnation_id) + { + return Err(ApiError::from(StorageError::BucketNotFound(src_bucket.clone())).into()); + } + let current_destination_incarnation_id = if src_bucket == bucket { + current_source_incarnation_id + } else { + store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)? + }; + if dst_opts + .expected_bucket_incarnation_id + .is_some_and(|expected| expected != current_destination_incarnation_id) + { + return Err(ApiError::from(StorageError::BucketNotFound(bucket.clone())).into()); + } + let destination_bucket_lifecycle_guard = destination_bucket_lifecycle_guard_storage + .as_ref() + .unwrap_or(&source_bucket_lifecycle_guard); + if source_bucket_lifecycle_guard.is_lock_lost() || destination_bucket_lifecycle_guard.is_lock_lost() { + return Err(ApiError::from(StorageError::NamespaceLockQuorumUnavailable { + mode: "copy_bucket_generation", + bucket: bucket.clone(), + object: key.clone(), + required: 1, + achieved: 0, + }) + .into()); + } + src_get_opts.expected_bucket_incarnation_id = Some(current_source_incarnation_id); + dst_opts.expected_bucket_incarnation_id = Some(current_destination_incarnation_id); + if src_bucket != bucket { + dst_opts.add_bucket_lifecycle_lock_guard(&source_bucket_lifecycle_guard); + } + dst_opts.add_bucket_lifecycle_lock_guard(destination_bucket_lifecycle_guard); + + // Bucket metadata uses the bucket name as its namespace-lock key. Load + // every copy-time bucket snapshot before a same-object key can collide + // with that key (for example, copying `bucket/bucket` onto itself). + let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); + let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?; + if cp_src_dst_same && key == bucket && expected_current_version_id.is_none() { + dst_opts.object_lock_config_snapshot = + Some(store.object_lock_config_snapshot(&bucket).await.map_err(ApiError::from)?); + } + let mut current_opts: ObjectOptions = internal_object_info_lookup_opts( + get_opts(&bucket, &key, dest_version_id.clone(), None, &req.headers) + .await + .map_err(ApiError::from)?, + ); let _self_copy_lock_guard = if cp_src_dst_same && expected_current_version_id.is_none() { let guard = acquire_self_copy_namespace_lock(store.as_ref(), &bucket, &key).await?; @@ -6190,13 +6421,11 @@ impl DefaultObjectUsecase { } else { None }; + if let Some(guard) = _self_copy_lock_guard.as_ref() { + dst_opts.add_namespace_lock_guard(guard); + } dst_opts.expected_current_version_id = expected_current_version_id.clone(); - let mut current_opts: ObjectOptions = internal_object_info_lookup_opts( - get_opts(&bucket, &key, dest_version_id.clone(), None, &req.headers) - .await - .map_err(ApiError::from)?, - ); if _self_copy_lock_guard.is_some() { current_opts.no_lock = true; } @@ -6221,7 +6450,6 @@ impl DefaultObjectUsecase { } }; - let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); let mut effective_sse = requested_sse.or_else(|| { if has_explicit_ssec { return None; @@ -6259,6 +6487,17 @@ impl DefaultObjectUsecase { let copy_principal = SseKmsPrincipal::from_request(&req); + if source_bucket_lifecycle_guard.is_lock_lost() { + return Err(ApiError::from(StorageError::NamespaceLockQuorumUnavailable { + mode: "copy_source_bucket_generation", + bucket: src_bucket.clone(), + object: src_key.clone(), + required: 1, + achieved: 0, + }) + .into()); + } + let gr = store .get_object_reader(&src_bucket, &src_key, None, h, &src_get_opts) .await @@ -6411,15 +6650,19 @@ impl DefaultObjectUsecase { remove_object_lock_metadata_for_copy(&mut user_defined); if let Some(object_lock_metadata) = build_put_like_object_lock_metadata( &bucket, + &object_lock_config_state, object_lock_legal_hold_status, object_lock_mode, object_lock_retain_until_date, - ) - .await? - { + )? { user_defined.extend(object_lock_metadata); } - apply_bucket_default_lock_retention(&bucket, &mut user_defined, has_explicit_object_lock_retention).await?; + apply_bucket_default_lock_retention( + &bucket, + &object_lock_config_state, + &mut user_defined, + has_explicit_object_lock_retention, + )?; let mut write_plan = WritePlan::new(); let mut reader = if should_compress { @@ -6492,6 +6735,7 @@ impl DefaultObjectUsecase { .copy_object(&src_bucket, &src_key, &bucket, &key, &mut src_info, &src_opts, &dst_opts) .await .map_err(ApiError::from)?; + drop(_self_copy_lock_guard); maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await; let _ = invalidate_object_data_cache_after_copy_success(&cache_adapter, &bucket, &key).await; @@ -6622,9 +6866,11 @@ impl DefaultObjectUsecase { let Some(store) = self.object_store() else { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; + // Capture the bucket generation before per-object authorization, but + // do not expose a bucket-state error unless at least one object is authorized. + let bucket_generation = load_bucket_generation_from_store(store.as_ref(), &req, &bucket).await; let bypass_governance = has_bypass_governance_header(&req.headers); - let bucket_lock_enabled = bucket_object_locking_enabled(&bucket).await; #[derive(Default, Clone)] struct DeleteResult { @@ -6638,7 +6884,6 @@ impl DefaultObjectUsecase { struct AuthorizedDelete { idx: usize, object: ObjectToDelete, - version_id: Option, } let mut authorized_deletes = Vec::with_capacity(delete.objects.len()); @@ -6665,11 +6910,11 @@ impl DefaultObjectUsecase { } let auth_res = authorize_request(&mut req, Action::S3Action(S3Action::DeleteObjectAction)).await; - if let Err(e) = auth_res { + if auth_res.is_err() { delete_results[idx].error = Some(s3s::dto::Error { code: Some("AccessDenied".to_string()), key: Some(obj_id.key.clone()), - message: Some(e.to_string()), + message: Some("Access Denied".to_string()), version_id: version_id.clone(), }); continue; @@ -6677,11 +6922,11 @@ impl DefaultObjectUsecase { if bypass_governance { let auth_res = authorize_request(&mut req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await; - if let Err(e) = auth_res { + if auth_res.is_err() { delete_results[idx].error = Some(s3s::dto::Error { code: Some("AccessDenied".to_string()), key: Some(obj_id.key.clone()), - message: Some(e.to_string()), + message: Some("Access Denied".to_string()), version_id: version_id.clone(), }); continue; @@ -6707,7 +6952,7 @@ impl DefaultObjectUsecase { }; delete_results[idx].synthetic_version_id = synthetic_version_id; - authorized_deletes.push(AuthorizedDelete { idx, object, version_id }); + authorized_deletes.push(AuthorizedDelete { idx, object }); } if authorized_deletes.is_empty() { @@ -6720,6 +6965,10 @@ impl DefaultObjectUsecase { let _ = helper.complete(&result); return result; } + #[cfg(test)] + wait_for_delete_objects_auth_test_hook(&bucket).await; + req.extensions.insert(bucket_generation?); + let bucket_lock_enabled = object_lock_checks_required(&bucket).await; let delete_config_snapshot = Arc::new( load_delete_config_snapshot(store.as_ref(), &bucket) @@ -6735,7 +6984,6 @@ impl DefaultObjectUsecase { idx: usize, object: ObjectToDelete, opts: ObjectOptions, - version_id: Option, skip_stat: bool, } @@ -6743,7 +6991,7 @@ impl DefaultObjectUsecase { // configuration after every candidate has passed authorization. let mut prepared_deletes: Vec = Vec::with_capacity(authorized_deletes.len()); for authorized in authorized_deletes { - let AuthorizedDelete { idx, object, version_id } = authorized; + let AuthorizedDelete { idx, object } = authorized; let metadata = extract_metadata(&req.headers); let opts: ObjectOptions = del_opts_with_versioning( @@ -6768,7 +7016,6 @@ impl DefaultObjectUsecase { idx, object, opts, - version_id, skip_stat, }); } @@ -6780,16 +7027,12 @@ impl DefaultObjectUsecase { version_suspended: bool, size: i64, existing: Option, - blocked: Option, } - // Phase 2 (bounded concurrency, backlog#929 / HP-8): the per-object - // pre-delete stat plus the admission checks that consume it. Entries - // are independent per key, and `buffered` preserves input order so the - // per-key result mapping below is identical to the previous serial - // loop. The authoritative object-lock enforcement stays in the - // set_disk layer under the held write lock (#4297); the check here is - // the same early, advisory rejection as before. + // Phase 2 (bounded concurrency, backlog#929 / HP-8): collect the + // metadata needed for accounting and tier cleanup. Entries are + // independent per key, and `buffered` preserves input order. Object + // Lock admission is enforced later in set_disk under the write lock. let store_ref = &store; let bucket_ref = bucket.as_str(); let admitted_deletes: Vec = @@ -6798,7 +7041,6 @@ impl DefaultObjectUsecase { idx, mut object, opts, - version_id, skip_stat, } = prepared; let synthetic_version_id = object.version_id.is_none() && is_dir_object(&object.object_name); @@ -6814,28 +7056,6 @@ impl DefaultObjectUsecase { } }; - if !skip_stat - && !source_missing - && !delete_creates_delete_marker(&opts) - && let Some(block_reason) = check_object_lock_for_deletion(bucket_ref, &goi, bypass_governance).await - { - let blocked_key = object.object_name.clone(); - return Ok(AdmittedDelete { - idx, - object, - versioned: opts.versioned, - version_suspended: opts.version_suspended, - size: 0, - existing: None, - blocked: Some(s3s::dto::Error { - code: Some("AccessDenied".to_string()), - key: Some(blocked_key), - message: Some(block_reason.error_message()), - version_id, - }), - }); - } - let size = goi.size; if synthetic_version_id { @@ -6850,7 +7070,6 @@ impl DefaultObjectUsecase { version_suspended: opts.version_suspended, size, existing, - blocked: None, }) })) .buffered(DELETE_OBJECTS_PRE_STAT_CONCURRENCY) @@ -6865,11 +7084,6 @@ impl DefaultObjectUsecase { let mut existing_object_infos = Vec::new(); let mut object_versioning = Vec::new(); for admitted in admitted_deletes { - if let Some(err) = admitted.blocked { - delete_results[admitted.idx].error = Some(err); - continue; - } - object_sizes.push(admitted.size); object_to_delete_idx.push(admitted.idx); object_versioning.push((admitted.versioned, admitted.version_suspended)); @@ -6883,29 +7097,22 @@ impl DefaultObjectUsecase { .collect::>(); invalidate_object_data_cache_objects_before_mutation(&cache_adapter, &bucket, cache_keys_before_delete.iter()).await; + let mut storage_delete_opts = ObjectOptions { + versioned: version_cfg.enabled(), + version_suspended: version_cfg.suspended(), + delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)), + object_lock_delete: Some(StorageObjectLockDeleteOptions { bypass_governance }), + ..Default::default() + }; + apply_bucket_generation_guard(&req, &bucket, &mut storage_delete_opts)?; let (dobjs, errs) = store - .delete_objects_with_tier_delete_journal( - &bucket, - object_to_delete.clone(), - ObjectOptions { - versioned: version_cfg.enabled(), - version_suspended: version_cfg.suspended(), - delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)), - object_lock_delete: Some(StorageObjectLockDeleteOptions { bypass_governance }), - ..Default::default() - }, - ) + .delete_objects_with_tier_delete_journal(&bucket, object_to_delete.clone(), storage_delete_opts) .await; let _manager = get_concurrency_manager(); let _bucket_clone = bucket.clone(); let _deleted_objects = dobjs.clone(); - if is_all_buckets_not_found( - &errs - .iter() - .map(|v| v.as_ref().map(|v| v.clone().into())) - .collect::>>() as &[Option], - ) { + if !errs.is_empty() && errs.iter().all(|err| err.as_ref().is_some_and(is_err_bucket_not_found)) { let result = Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "Bucket not found".to_string())); let _ = helper.complete(&result); return result; @@ -6937,10 +7144,11 @@ impl DefaultObjectUsecase { } if let Some(err) = err.clone() { + let api_error = ApiError::from(err); delete_results[didx].error = Some(s3s::dto::Error { - code: Some(err.to_string()), + code: Some(api_error.code.as_str().to_string()), key: Some(object_to_delete[i].object_name.clone()), - message: Some(err.to_string()), + message: Some(api_error.message), version_id: delete_response_version_id( object_to_delete[i].version_id, delete_results[didx].synthetic_version_id, @@ -7057,7 +7265,6 @@ impl DefaultObjectUsecase { // Validate object key validate_object_key(&key, "DELETE")?; - validate_table_catalog_object_mutation(&bucket, &key).await?; let replica = req .headers @@ -7076,6 +7283,7 @@ impl DefaultObjectUsecase { "Recursive force-delete is restricted to internal or administrative requests", )); } + validate_table_catalog_object_mutation(&bucket, &key).await?; // Establish bucket existence before any bucket-metadata work (matches // PUT/GET): nonexistent buckets fail here instead of paying the @@ -7107,15 +7315,9 @@ impl DefaultObjectUsecase { opts.object_lock_delete = Some(StorageObjectLockDeleteOptions { bypass_governance: has_bypass_governance_header(&req.headers), }); + apply_bucket_generation_guard(&req, &bucket, &mut opts)?; let force_delete = opts.delete_prefix; - if opts.delete_prefix && bucket_object_locking_enabled(&bucket).await { - return Err(S3Error::with_message( - S3ErrorCode::Custom("force-delete is forbidden on Object Locking enabled buckets".into()), - "force-delete is forbidden on Object Locking enabled buckets", - )); - } - // let mut vid = opts.version_id.clone(); if replica { @@ -7139,23 +7341,9 @@ impl DefaultObjectUsecase { let replicate_force_delete = force_delete && !replica && has_active_delete_rule(&delete_config_snapshot, &key); let mut force_delete_intent = None; - // Check Object Lock retention before deletion - // TODO: Future optimization (separate PR) - If performance becomes critical under high delete load: - // 1. Add a lightweight get_object_lock_info() that only fetches retention metadata - // 2. Or use combined get-and-delete in storage layer with retention check callback let get_opts = opts.clone(); let existing_object_info = match store.get_object_info(&bucket, &key, &get_opts).await { - Ok(obj_info) => { - // Check for bypass governance retention header (permission already verified in access.rs) - let bypass_governance = has_bypass_governance_header(&req.headers); - - if !delete_creates_delete_marker(&opts) - && let Some(block_reason) = check_object_lock_for_deletion(&bucket, &obj_info, bypass_governance).await - { - return Err(S3Error::with_message(S3ErrorCode::AccessDenied, block_reason.error_message())); - } - Some(obj_info) - } + Ok(obj_info) => Some(obj_info), Err(err) => { // If object not found, allow deletion to proceed (will return 204 No Content) if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { @@ -7216,6 +7404,18 @@ impl DefaultObjectUsecase { return result; } + if matches!(&err, StorageError::PrefixAccessDenied(_, _)) + && let Some(existing_object_info) = existing_object_info.as_ref() + && let Some(reason) = check_object_lock_for_deletion( + &bucket, + existing_object_info, + has_bypass_governance_header(&req.headers), + ) + .await + { + return Err(S3Error::with_message(S3ErrorCode::AccessDenied, reason.error_message())); + } + return Err(ApiError::from(err).into()); } } @@ -7728,6 +7928,20 @@ impl DefaultObjectUsecase { let mut opts = post_restore_opts(&version_id_str, &bucket, &object) .await .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrPostRestoreOpts".into()), "restore object failed."))?; + apply_bucket_generation_guard(&req, &bucket, &mut opts)?; + // `apply_bucket_generation_guard` deliberately tolerates a missing guard + // (only the S3 access layer installs one), so this must not hard-require + // it. Resolve the current generation instead, exactly as the copy path + // does. The fence is unaffected: the value is re-read from disk and + // compared below, before the restore is admitted. + let restore_bucket_incarnation_id = match opts.expected_bucket_incarnation_id { + Some(incarnation_id) => incarnation_id, + None => { + let incarnation_id = store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)?; + opts.expected_bucket_incarnation_id = Some(incarnation_id); + incarnation_id + } + }; // SELECT-type restores skip both the ongoing check and the metadata // write below, so the accept guard would protect nothing for them — @@ -7744,6 +7958,10 @@ impl DefaultObjectUsecase { // in-flight commit on the same object) is transient — answer 503 // SlowDown so SDK clients back off and retry instead of treating it // as a hard failure. + let restore_bucket_lifecycle_guard = Some(acquire_copy_bucket_lifecycle_lock(store.as_ref(), &bucket).await?); + if store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)? != restore_bucket_incarnation_id { + return Err(ApiError::from(StorageError::BucketNotFound(bucket.clone())).into()); + } let accept_guard = if is_select { None } else { @@ -7841,6 +8059,19 @@ impl DefaultObjectUsecase { return Err(S3Error::with_message(S3ErrorCode::SlowDown, "restore object failed.")); } + let mut restore_dst_opts = ObjectOptions { + version_id: obj_info_.version_id.map(|v| v.to_string()), + mod_time: obj_info_.mod_time, + no_lock: true, + expected_bucket_incarnation_id: Some(restore_bucket_incarnation_id), + ..Default::default() + }; + if let Some(guard) = restore_bucket_lifecycle_guard.as_ref() { + restore_dst_opts.add_bucket_lifecycle_lock_guard(guard); + } + if let Some(guard) = accept_guard.as_ref() { + guard.add_namespace_lock_fence(&mut restore_dst_opts); + } store .clone() .copy_object( @@ -7855,12 +8086,7 @@ impl DefaultObjectUsecase { no_lock: true, ..Default::default() }, - &ObjectOptions { - version_id: obj_info_.version_id.map(|v| v.to_string()), - mod_time: obj_info_.mod_time, - no_lock: true, - ..Default::default() - }, + &restore_dst_opts, ) .await .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrCopyObject".into()), "restore object failed."))?; @@ -7884,6 +8110,7 @@ impl DefaultObjectUsecase { // The accept decision is committed; release the object write lock so // the background copy-back and concurrent reads are never blocked on it. drop(accept_guard); + drop(restore_bucket_lifecycle_guard); // Handle output location for SELECT requests if let Some(output_location) = &rreq.output_location @@ -7927,6 +8154,7 @@ impl DefaultObjectUsecase { version_id: version_id_clone, versioned, version_suspended, + expected_bucket_incarnation_id: Some(restore_bucket_incarnation_id), user_defined: restore_operation_metadata, ..Default::default() }; @@ -7989,6 +8217,9 @@ impl DefaultObjectUsecase { return Err(s3_error!(NotImplemented, "SSE-KMS is not supported for extract uploads")); } let replication_authorized = replication_request_authorized(&req); + let mut bucket_generation_opts = ObjectOptions::default(); + apply_bucket_generation_guard(&req, &req.input.bucket, &mut bucket_generation_opts)?; + let expected_bucket_incarnation_id = bucket_generation_opts.expected_bucket_incarnation_id; let input = req.input; let PutObjectInput { @@ -8161,6 +8392,8 @@ impl DefaultObjectUsecase { let mut wrote_any_entry = false; let mut extracted_entry_count = 0usize; let mut total_unpacked_size = 0u64; + let object_lock_config_snapshot = store.object_lock_config_snapshot(&bucket).await.map_err(ApiError::from)?; + let object_lock_config_state = object_lock_config_snapshot.state(); while let Some(entry) = entries.next().await { let mut f = match entry { @@ -8219,8 +8452,6 @@ impl DefaultObjectUsecase { req_info.object = Some(fpath.clone()); req_info.version_id = None; } - authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await?; - let entry_size = f.header().size().unwrap_or_default(); validate_put_object_extract_entry_size(&fpath, entry_size, extract_limits)?; total_unpacked_size = total_unpacked_size @@ -8259,7 +8490,12 @@ impl DefaultObjectUsecase { tagging.clone(), storage_class.clone(), )?; - apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?; + apply_bucket_default_lock_retention( + &bucket, + object_lock_config_state, + &mut metadata, + has_explicit_object_lock_retention, + )?; let mut opts = put_opts_with_replication_authorization( &bucket, &fpath, @@ -8270,7 +8506,40 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; - apply_extract_entry_pax_extensions(&mut f, &mut metadata, &mut opts).await?; + opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id; + opts.object_lock_config_snapshot = Some(Arc::clone(&object_lock_config_snapshot)); + let pax_authorization = + apply_extract_entry_pax_extensions(&mut f, &bucket, &fpath, object_lock_config_state, &mut metadata, &mut opts) + .await?; + for (name, value) in &pax_authorization.headers { + auth_req.headers.insert(name.clone(), value.clone()); + } + if let Some(version_id) = opts.version_id.as_ref() { + req_info_mut(&mut auth_req)?.version_id = Some(version_id.clone()); + } + authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectAction)).await?; + if pax_authorization.object_lock_mode.is_some() || pax_authorization.object_lock_retain_until_date.is_some() { + authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?; + } + if pax_authorization.object_lock_legal_hold_status.is_some() { + authorize_request(&mut auth_req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await?; + } + if opts.version_id.is_some() || pax_authorization.headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS) { + authorize_request(&mut auth_req, Action::S3Action(S3Action::ReplicateObjectAction)).await?; + } + let effective_object_lock_legal_hold_status = pax_authorization + .object_lock_legal_hold_status + .clone() + .or_else(|| object_lock_legal_hold_status.clone()); + let (effective_object_lock_mode, effective_object_lock_retain_until_date) = + if pax_authorization.object_lock_mode.is_some() || pax_authorization.object_lock_retain_until_date.is_some() { + ( + pax_authorization.object_lock_mode.clone(), + pax_authorization.object_lock_retain_until_date.clone(), + ) + } else { + (object_lock_mode.clone(), object_lock_retain_until_date.clone()) + }; if archive_entry_mod_time.is_some() { opts.mod_time = archive_entry_mod_time; } @@ -8307,12 +8576,12 @@ impl DefaultObjectUsecase { }; apply_put_request_object_lock_opts( &bucket, - object_lock_legal_hold_status.clone(), - object_lock_mode.clone(), - object_lock_retain_until_date.clone(), + object_lock_config_state, + effective_object_lock_legal_hold_status, + effective_object_lock_mode, + effective_object_lock_retain_until_date, &mut opts, - ) - .await?; + )?; if let Some(material) = sse_encryption(EncryptionRequest { bucket: &bucket, key: &fpath, @@ -8460,21 +8729,12 @@ fn object_attributes_requested(object_attributes: &[ObjectAttributes], name: &'s }) } -async fn bucket_object_locking_enabled(bucket: &str) -> bool { +/// Fail closed when deciding whether an object-lock-sensitive operation may +/// skip its existing-object lookup. +pub(super) async fn object_lock_checks_required(bucket: &str) -> bool { get_bucket_metadata(bucket) .await - .is_ok_and(|metadata| metadata.object_locking()) -} - -/// Fail-closed WORM gate for skipping the pre-PUT lookup -/// (rustfs/backlog#1009): a bucket-metadata read failure counts as "locking -/// enabled" so the existing-object lock validation can never silently -/// disappear on a degraded metadata subsystem. -pub(super) async fn put_prelookup_worm_gate(bucket: &str) -> bool { - match get_bucket_metadata(bucket).await { - Ok(metadata) => metadata.object_locking(), - Err(_) => true, - } + .map_or(true, |metadata| metadata.object_locking()) } /// rustfs/backlog#1009: map the rename_data old-size backfill onto the @@ -8493,16 +8753,17 @@ mod tests { use super::*; use http::{Extensions, HeaderMap, HeaderName, HeaderValue, Method, Uri}; use s3s::dto::{ - Delete, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination, - ExistingObjectReplication, ExistingObjectReplicationStatus, ObjectIdentifier, ReplicaModifications, - ReplicaModificationsStatus, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, RestoreRequest, - SourceSelectionCriteria, + DefaultRetention, Delete, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, + DeleteReplicationStatus, Destination, ExistingObjectReplication, ExistingObjectReplicationStatus, ObjectIdentifier, + ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRule, ReplicaModifications, ReplicaModificationsStatus, + ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, RestoreRequest, SourceSelectionCriteria, }; use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use std::task::{Context, Poll}; use tokio::io::{AsyncRead, ReadBuf}; + use tokio_tar::{Builder, EntryType, Header}; #[test] fn delete_response_version_id_preserves_null_and_synthetic_semantics() { @@ -8690,27 +8951,388 @@ mod tests { assert!(lookup_opts.no_lock); } + #[test] + fn put_request_user_metadata_cannot_suppress_bucket_default_retention() { + let mut metadata = + HashMap::from([(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), ObjectLockRetentionMode::GOVERNANCE.to_string())]); + apply_put_request_metadata( + &mut metadata, + &HeaderMap::new(), + "object", + None, + None, + None, + None, + None, + None, + None, + None, + None, + ) + .unwrap(); + + let state = metadata_sys::ObjectLockConfigState::Configured { + config: ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: Some(ObjectLockRule { + default_retention: Some(DefaultRetention { + mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), + days: Some(1), + years: None, + }), + }), + }, + updated_at: OffsetDateTime::now_utc(), + }; + apply_bucket_default_lock_retention("bucket", &state, &mut metadata, false).unwrap(); + + assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("COMPLIANCE")); + assert!(metadata.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)); + assert_eq!(metadata.get("x-amz-meta-x-amz-object-lock-mode").map(String::as_str), Some("GOVERNANCE")); + } + + fn pax_record(key: &str, value: &[u8]) -> Vec { + let body_len = 1 + key.len() + 1 + value.len() + 1; + let mut len = body_len + 1; + loop { + let actual_len = len.to_string().len() + body_len; + if actual_len == len { + break; + } + len = actual_len; + } + + let mut record = format!("{len} {key}=").into_bytes(); + record.extend_from_slice(value); + record.push(b'\n'); + assert_eq!(record.len(), len); + record + } + #[tokio::test] - async fn build_put_like_object_lock_metadata_rejects_mode_without_retain_until_date() { + async fn snowball_pax_rejects_unpaired_object_lock_retention() { + let record = pax_record("minio.metadata.x-amz-object-lock-mode", b"GOVERNANCE"); + let mut builder = Builder::new(Vec::new()); + let mut extension = Header::new_ustar(); + extension.set_size(record.len() as u64); + extension.set_entry_type(EntryType::XHeader); + builder.append_data(&mut extension, "pax", &record[..]).await.unwrap(); + + let mut file = Header::new_ustar(); + file.set_size(0); + builder.append_data(&mut file, "object", &b""[..]).await.unwrap(); + let mut archive = Archive::new(std::io::Cursor::new(builder.into_inner().await.unwrap())); + let mut entries = archive.entries().unwrap(); + let mut entry = entries.next().await.unwrap().unwrap(); + let mut metadata = HashMap::from([ + (AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), ObjectLockRetentionMode::COMPLIANCE.to_string()), + (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2030-01-01T00:00:00Z".to_string()), + ]); + let mut opts = ObjectOptions::default(); + let state = metadata_sys::ObjectLockConfigState::Configured { + config: ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }, + updated_at: OffsetDateTime::now_utc(), + }; + + let err = apply_extract_entry_pax_extensions(&mut entry, "bucket", "object", &state, &mut metadata, &mut opts) + .await + .unwrap_err(); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("COMPLIANCE")); + } + + #[tokio::test] + async fn snowball_pax_privileged_fields_require_independent_authorization() { + let mut retention = pax_record("minio.metadata.X-Amz-Object-Lock-Mode", b"GOVERNANCE"); + retention.extend(pax_record("minio.metadata.X-Amz-Object-Lock-Retain-Until-Date", b"2099-01-01T00:00:00Z")); + let cases = [ + ("retention", retention, (true, false, false)), + ( + "legal-hold", + pax_record("minio.metadata.X-Amz-Object-Lock-Legal-Hold", b"ON"), + (false, true, false), + ), + ( + "version-id", + pax_record("minio.versionId", Uuid::nil().to_string().as_bytes()), + (false, false, true), + ), + ( + "replication-status", + pax_record("minio.metadata.x-amz-replication-status", b"REPLICA"), + (false, false, true), + ), + ]; + let state = metadata_sys::ObjectLockConfigState::Configured { + config: ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }, + updated_at: OffsetDateTime::now_utc(), + }; + + for (case, record, expected) in cases { + let mut builder = Builder::new(Vec::new()); + let mut extension = Header::new_ustar(); + extension.set_size(record.len() as u64); + extension.set_entry_type(EntryType::XHeader); + builder.append_data(&mut extension, "pax", &record[..]).await.unwrap(); + let mut file = Header::new_ustar(); + file.set_size(0); + builder.append_data(&mut file, "object", &b""[..]).await.unwrap(); + let mut archive = Archive::new(std::io::Cursor::new(builder.into_inner().await.unwrap())); + let mut entries = archive.entries().unwrap(); + let mut entry = entries.next().await.unwrap().unwrap(); + + let mut opts = ObjectOptions::default(); + let authorization = + apply_extract_entry_pax_extensions(&mut entry, "bucket", "object", &state, &mut HashMap::new(), &mut opts) + .await + .unwrap(); + + assert_eq!( + ( + authorization.object_lock_mode.is_some() || authorization.object_lock_retain_until_date.is_some(), + authorization.object_lock_legal_hold_status.is_some(), + opts.version_id.is_some() || authorization.headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS), + ), + expected, + "{case} must request only its own additional authorization" + ); + match case { + "retention" => { + assert!(authorization.headers.contains_key(AMZ_OBJECT_LOCK_MODE_LOWER)); + assert!(authorization.headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)); + } + "legal-hold" => assert!(authorization.headers.contains_key(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER)), + "version-id" => assert_eq!(opts.version_id.as_deref(), Some(Uuid::nil().to_string().as_str())), + "replication-status" => assert_eq!( + authorization + .headers + .get(AMZ_BUCKET_REPLICATION_STATUS) + .and_then(|value| value.to_str().ok()), + Some("REPLICA") + ), + _ => unreachable!(), + } + } + } + + #[tokio::test] + async fn snowball_pax_rejects_invalid_retention_and_replication_values() { + let mut invalid_mode = pax_record("minio.metadata.X-Amz-Object-Lock-Mode", b"INVALID"); + invalid_mode.extend(pax_record("minio.metadata.X-Amz-Object-Lock-Retain-Until-Date", b"2099-01-01T00:00:00Z")); + let mut invalid_date = pax_record("minio.metadata.X-Amz-Object-Lock-Mode", b"COMPLIANCE"); + invalid_date.extend(pax_record("minio.metadata.X-Amz-Object-Lock-Retain-Until-Date", b"not-a-date")); + let cases = [ + ("invalid-mode", invalid_mode), + ("invalid-date", invalid_date), + ( + "invalid-replication-status", + pax_record("minio.metadata.x-amz-replication-status", b"INVALID"), + ), + ("invalid-version-id", pax_record("minio.versionId", b"not-a-uuid")), + ]; + let state = metadata_sys::ObjectLockConfigState::Configured { + config: ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }, + updated_at: OffsetDateTime::now_utc(), + }; + + for (case, record) in cases { + let mut builder = Builder::new(Vec::new()); + let mut extension = Header::new_ustar(); + extension.set_size(record.len() as u64); + extension.set_entry_type(EntryType::XHeader); + builder.append_data(&mut extension, "pax", &record[..]).await.unwrap(); + let mut file = Header::new_ustar(); + file.set_size(0); + builder.append_data(&mut file, "object", &b""[..]).await.unwrap(); + let mut archive = Archive::new(std::io::Cursor::new(builder.into_inner().await.unwrap())); + let mut entries = archive.entries().unwrap(); + let mut entry = entries.next().await.unwrap().unwrap(); + + let err = apply_extract_entry_pax_extensions( + &mut entry, + "bucket", + "object", + &state, + &mut HashMap::new(), + &mut ObjectOptions::default(), + ) + .await + .unwrap_err(); + + assert!( + err.code() == &S3ErrorCode::InvalidArgument || err.code() == &S3ErrorCode::MalformedXML, + "{case}" + ); + } + } + + #[tokio::test] + async fn snowball_pax_preserves_canonical_minio_metadata_and_valid_retention() { + let mut record = pax_record("minio.metadata.Content-Type", b"text/plain"); + record.extend(pax_record("minio.metadata.X-Amz-Meta-Owner", b"alice")); + record.extend(pax_record("minio.metadata.project", b"alpha-demo")); + record.extend(pax_record("minio.versionId", Uuid::nil().to_string().as_bytes())); + record.extend(pax_record("minio.metadata.x-amz-replication-status", b"REPLICA")); + record.extend(pax_record("minio.metadata.X-Amz-Object-Lock-Mode", b"GOVERNANCE")); + record.extend(pax_record("minio.metadata.X-Amz-Object-Lock-Retain-Until-Date", b"2099-01-01T00:00:00Z")); + record.extend(pax_record("minio.metadata.X-Amz-Object-Lock-Legal-Hold", b"ON")); + let mut builder = Builder::new(Vec::new()); + let mut extension = Header::new_ustar(); + extension.set_size(record.len() as u64); + extension.set_entry_type(EntryType::XHeader); + builder.append_data(&mut extension, "pax", &record[..]).await.unwrap(); + + let mut file = Header::new_ustar(); + file.set_size(0); + builder.append_data(&mut file, "object.txt", &b""[..]).await.unwrap(); + let mut archive = Archive::new(std::io::Cursor::new(builder.into_inner().await.unwrap())); + let mut entries = archive.entries().unwrap(); + let mut entry = entries.next().await.unwrap().unwrap(); + let mut metadata = HashMap::from([ + (AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), ObjectLockRetentionMode::COMPLIANCE.to_string()), + (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2030-01-01T00:00:00Z".to_string()), + ]); + let mut opts = ObjectOptions::default(); + let state = metadata_sys::ObjectLockConfigState::Configured { + config: ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }, + updated_at: OffsetDateTime::now_utc(), + }; + + let authorization = + apply_extract_entry_pax_extensions(&mut entry, "bucket", "object.txt", &state, &mut metadata, &mut opts) + .await + .unwrap(); + + assert_eq!(metadata.get("content-type").map(String::as_str), Some("text/plain")); + assert_eq!(metadata.get("owner").map(String::as_str), Some("alice")); + assert_eq!(metadata.get("project").map(String::as_str), Some("alpha-demo")); + assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE")); + assert_eq!( + metadata.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER).map(String::as_str), + Some("2099-01-01T00:00:00Z") + ); + assert_eq!(metadata.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("ON")); + assert!(metadata.contains_key("x-rustfs-internal-objectlock-legalhold-timestamp")); + assert!(metadata.contains_key("x-minio-internal-objectlock-legalhold-timestamp")); + assert_eq!(metadata.get(AMZ_BUCKET_REPLICATION_STATUS).map(String::as_str), Some("REPLICA")); + assert_eq!(opts.version_id.as_deref(), Some("00000000-0000-0000-0000-000000000000")); + assert!(authorization.object_lock_mode.is_some()); + assert!(authorization.object_lock_retain_until_date.is_some()); + assert!(authorization.object_lock_legal_hold_status.is_some()); + assert!(opts.version_id.is_some()); + assert!(authorization.headers.contains_key(AMZ_BUCKET_REPLICATION_STATUS)); + } + + #[tokio::test] + async fn snowball_pax_rejects_legal_hold_without_bucket_object_lock() { + let record = pax_record("minio.metadata.X-Amz-Object-Lock-Legal-Hold", b"ON"); + let mut builder = Builder::new(Vec::new()); + let mut extension = Header::new_ustar(); + extension.set_size(record.len() as u64); + extension.set_entry_type(EntryType::XHeader); + builder.append_data(&mut extension, "pax", &record[..]).await.unwrap(); + + let mut file = Header::new_ustar(); + file.set_size(0); + builder.append_data(&mut file, "object", &b""[..]).await.unwrap(); + let mut archive = Archive::new(std::io::Cursor::new(builder.into_inner().await.unwrap())); + let mut entries = archive.entries().unwrap(); + let mut entry = entries.next().await.unwrap().unwrap(); + let mut metadata = HashMap::new(); + + let err = apply_extract_entry_pax_extensions( + &mut entry, + "bucket", + "object", + &metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + &mut metadata, + &mut ObjectOptions::default(), + ) + .await + .unwrap_err(); + + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(!metadata.contains_key(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER)); + } + + #[tokio::test] + async fn snowball_pax_rejects_invalid_legal_hold_status() { + let record = pax_record("minio.metadata.X-Amz-Object-Lock-Legal-Hold", b"INVALID"); + let mut builder = Builder::new(Vec::new()); + let mut extension = Header::new_ustar(); + extension.set_size(record.len() as u64); + extension.set_entry_type(EntryType::XHeader); + builder.append_data(&mut extension, "pax", &record[..]).await.unwrap(); + + let mut file = Header::new_ustar(); + file.set_size(0); + builder.append_data(&mut file, "object", &b""[..]).await.unwrap(); + let mut archive = Archive::new(std::io::Cursor::new(builder.into_inner().await.unwrap())); + let mut entries = archive.entries().unwrap(); + let mut entry = entries.next().await.unwrap().unwrap(); + let mut metadata = HashMap::new(); + let state = metadata_sys::ObjectLockConfigState::Configured { + config: ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: None, + }, + updated_at: OffsetDateTime::now_utc(), + }; + + let err = apply_extract_entry_pax_extensions( + &mut entry, + "bucket", + "object", + &state, + &mut metadata, + &mut ObjectOptions::default(), + ) + .await + .unwrap_err(); + + assert_eq!(err.code(), &S3ErrorCode::MalformedXML); + assert!(!metadata.contains_key(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER)); + } + + #[test] + fn build_put_like_object_lock_metadata_rejects_mode_without_retain_until_date() { let err = build_put_like_object_lock_metadata( "test-bucket", + &metadata_sys::ObjectLockConfigState::ConfirmedAbsent, None, Some(ObjectLockMode::from_static(ObjectLockMode::GOVERNANCE)), None, ) - .await .unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); assert_eq!(err.message(), Some(ERR_OBJECT_LOCK_RETENTION_HEADERS_MUST_BE_PAIRED)); } - #[tokio::test] - async fn build_put_like_object_lock_metadata_rejects_retain_until_date_without_mode() { + #[test] + fn build_put_like_object_lock_metadata_rejects_retain_until_date_without_mode() { let retain_until = Timestamp::from(OffsetDateTime::now_utc().add(time::Duration::days(1))); - let err = build_put_like_object_lock_metadata("test-bucket", None, None, Some(retain_until)) - .await - .unwrap_err(); + let err = build_put_like_object_lock_metadata( + "test-bucket", + &metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + None, + None, + Some(retain_until), + ) + .unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); assert_eq!(err.message(), Some(ERR_OBJECT_LOCK_RETENTION_HEADERS_MUST_BE_PAIRED)); @@ -13061,6 +13683,91 @@ mod tests { assert_ne!(err.code(), &S3ErrorCode::InvalidRequest); } + #[tokio::test] + #[serial_test::serial] + async fn execute_self_copy_when_object_name_equals_bucket_observes_lock_order() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; + use s3s::access::S3Access as _; + + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + if current_app_context().is_none() { + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + } + let ambient = current_app_context().expect("self-copy lock-order test requires an AppContext"); + let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms())); + let server_ctx = crate::app::runtime_sources::ServerContextSlot::new(); + assert!(server_ctx.install(Arc::clone(&context))); + let fs = FS::with_server_ctx(server_ctx); + + let bucket = format!("self-copy-lock-order-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create self-copy test bucket"); + let payload = b"object whose key equals its bucket".to_vec(); + let mut reader = PutObjReader::from_vec(payload.clone()); + store + .put_object(&bucket, &bucket, &mut reader, &ObjectOptions::default()) + .await + .expect("put object whose key equals its bucket"); + + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let mut bucket_metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("self-copy bucket metadata should be cached")) + .clone(); + bucket_metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("self-copy policy should parse")); + bucket_metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), bucket_metadata) + .await + .expect("publish self-copy test policy"); + + let input = CopyObjectInput::builder() + .copy_source(CopySource::Bucket { + bucket: bucket.clone().into(), + key: bucket.clone().into(), + version_id: None, + }) + .bucket(bucket.clone()) + .key(bucket.clone()) + .metadata_directive(Some(MetadataDirective::from_static(MetadataDirective::REPLACE))) + .metadata(Some(HashMap::from([("lock-order".to_string(), "verified".to_string())]))) + .build() + .expect("self-copy input should build"); + let mut req = build_request(input, Method::PUT); + req.extensions.insert(crate::storage::access::ReqInfo::default()); + fs.copy_object(&mut req) + .await + .expect("authorize self-copy whose object key equals its bucket"); + + let response = tokio::time::timeout( + std::time::Duration::from_secs(30), + DefaultObjectUsecase::with_context(Some(context)).execute_copy_object(req), + ) + .await + .expect("lifecycle, authority, and exact object locks must not deadlock") + .expect("self-copy whose object key equals its bucket should succeed"); + assert!(response.output.copy_object_result.is_some()); + let info = store + .get_object_info(&bucket, &bucket, &ObjectOptions::default()) + .await + .expect("self-copied object should remain readable"); + assert_eq!(info.size, payload.len() as i64); + + store + .delete_bucket( + &bucket, + &DeleteBucketOptions { + force: true, + ..Default::default() + }, + ) + .await + .expect("clean up self-copy test bucket"); + } + #[tokio::test] async fn execute_copy_object_allows_tiered_self_copy_with_storage_class_change() { let input = CopyObjectInput::builder() @@ -13327,6 +14034,94 @@ mod tests { assert_eq!(err.message(), Some("Not init")); } + #[tokio::test] + #[serial_test::serial] + async fn execute_delete_objects_rejects_bucket_recreated_after_authorization() { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; + + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + if current_app_context().is_none() { + crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await; + } + let context = current_app_context().expect("delete objects generation test requires an AppContext"); + let bucket = format!("delete-objects-generation-{}", Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create authorized bucket generation"); + let mut reader = PutObjReader::from_vec(b"old generation".to_vec()); + store + .put_object(&bucket, "object", &mut reader, &ObjectOptions::default()) + .await + .expect("put old-generation object"); + + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:DeleteObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("authorized bucket metadata should be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("publish test bucket policy"); + + let input = DeleteObjectsInput::builder() + .bucket(bucket.clone()) + .delete(Delete { + objects: vec![ObjectIdentifier { + key: "object".to_string(), + version_id: None, + ..Default::default() + }], + quiet: None, + }) + .build() + .expect("delete objects input should build"); + let mut req = build_request(input, Method::POST); + req.extensions.insert(crate::storage::access::ReqInfo::default()); + let loaded = Arc::new(tokio::sync::Barrier::new(2)); + let resume = Arc::new(tokio::sync::Barrier::new(2)); + install_delete_objects_auth_test_hook(bucket.clone(), Arc::clone(&loaded), Arc::clone(&resume)); + + let usecase = DefaultObjectUsecase::with_context(Some(context)); + let delete = tokio::spawn(async move { usecase.execute_delete_objects(req).await }); + loaded.wait().await; + + store + .delete_bucket( + &bucket, + &DeleteBucketOptions { + force: true, + ..Default::default() + }, + ) + .await + .expect("delete authorized bucket generation"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("recreate same bucket name"); + let mut reader = PutObjReader::from_vec(b"new generation".to_vec()); + store + .put_object(&bucket, "object", &mut reader, &ObjectOptions::default()) + .await + .expect("put new-generation object"); + resume.wait().await; + + let err = delete + .await + .expect("delete objects task should join") + .expect_err("old authorization must not delete from the recreated bucket"); + assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket); + store + .get_object_info(&bucket, "object", &ObjectOptions::default()) + .await + .expect("new-generation object must survive the stale batch request"); + } + #[tokio::test] async fn execute_delete_object_allows_non_force_request_without_req_info_until_store_lookup() { let input = DeleteObjectInput::builder() diff --git a/rustfs/src/app/put_prelookup_gating_test.rs b/rustfs/src/app/put_prelookup_gating_test.rs index 743fa1a27..3f6261336 100644 --- a/rustfs/src/app/put_prelookup_gating_test.rs +++ b/rustfs/src/app/put_prelookup_gating_test.rs @@ -13,7 +13,7 @@ // limitations under the License. //! Regression coverage for rustfs/backlog#1009: the PUT path skips its -//! pre-PUT `get_object_info` only when `put_prelookup_worm_gate` proves the +//! pre-PUT `get_object_info` only when `object_lock_checks_required` proves the //! existing-object WORM validation is a no-op. These tests pin the gate's //! truth table against a real 4-disk `ECStore` with the bucket metadata sys //! initialized, mirroring the HP-8 delete-gating fixture: @@ -24,7 +24,7 @@ //! so a degraded metadata subsystem can never silently drop the WORM check. use super::gating_test_env::shared_gating_ecstore; -use super::object_usecase::put_prelookup_worm_gate; +use super::object_usecase::object_lock_checks_required; use super::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions}; use serial_test::serial; use uuid::Uuid; @@ -47,7 +47,7 @@ async fn worm_gate_keeps_prelookup_for_object_lock_bucket() { .expect("create object-lock bucket"); assert!( - put_prelookup_worm_gate(&bucket).await, + object_lock_checks_required(&bucket).await, "an object-lock bucket must keep the pre-PUT lookup" ); } @@ -64,7 +64,7 @@ async fn worm_gate_allows_skip_for_plain_bucket() { .expect("create plain bucket"); assert!( - !put_prelookup_worm_gate(&bucket).await, + !object_lock_checks_required(&bucket).await, "a bucket without object locking must take the prelookup-skip path" ); } @@ -76,7 +76,7 @@ async fn worm_gate_fails_closed_when_bucket_metadata_is_unavailable() { let missing_bucket = format!("put-gate-missing-{}", Uuid::new_v4()); assert!( - put_prelookup_worm_gate(&missing_bucket).await, + object_lock_checks_required(&missing_bucket).await, "a bucket-metadata lookup failure must fail closed and keep the pre-PUT lookup" ); } diff --git a/rustfs/src/app/runtime_sources.rs b/rustfs/src/app/runtime_sources.rs index 3b2475986..2321969e7 100644 --- a/rustfs/src/app/runtime_sources.rs +++ b/rustfs/src/app/runtime_sources.rs @@ -16,6 +16,8 @@ use crate::app::object_data_cache::ObjectDataCacheAdapter; #[cfg(test)] use crate::app::storage_api::runtime_sources::TierConfigMgr; use crate::runtime_sources as root_runtime_sources; +#[cfg(test)] +pub(crate) use crate::runtime_sources::ServerContextSlot; pub(crate) use crate::runtime_sources::{ AppContext, current_encryption_service, current_endpoints_handle, current_notification_system, current_object_data_cache_handle_for_context, current_object_store_handle_for_context, diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 494ebed16..486d1fdcb 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -216,8 +216,9 @@ pub(crate) mod runtime_sources { pub(crate) mod access { pub(crate) use crate::storage::storage_api::access_consumer::{ - PostObjectRequestMarker, ReqInfo, authorize_request, has_bypass_governance_header, recursive_force_delete_is_authorized, - replication_request_authorized, req_info_mut, req_info_ref, + PostObjectRequestMarker, ReqInfo, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, + authorize_request, bucket_config_mutation_incarnation, has_bypass_governance_header, load_bucket_generation_from_store, + recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref, }; } @@ -267,10 +268,6 @@ pub(crate) mod bucket { ) } - pub(crate) mod bucket_target_sys { - pub(crate) type BucketTargetSys = crate::storage::storage_api::ecstore_bucket::bucket_target_sys::BucketTargetSys; - } - pub(crate) mod lifecycle { pub(crate) mod bucket_lifecycle_audit { pub(crate) type LcEventSrc = @@ -409,6 +406,7 @@ pub(crate) mod bucket { use super::target::BucketTargets; pub(crate) type BucketMetadataSys = crate::storage::storage_api::ecstore_bucket::metadata_sys::BucketMetadataSys; + pub(crate) type ObjectLockConfigState = crate::storage::storage_api::ecstore_bucket::metadata_sys::ObjectLockConfigState; #[cfg(test)] pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { @@ -422,6 +420,19 @@ pub(crate) mod bucket { crate::storage::storage_api::delete_bucket_metadata_config(bucket, config_file).await } + pub(crate) async fn delete_if_incarnation( + bucket: &str, + config_file: &str, + expected_incarnation_id: uuid::Uuid, + ) -> Result { + crate::storage::storage_api::delete_bucket_metadata_config_if_incarnation( + bucket, + config_file, + Some(expected_incarnation_id), + ) + .await + } + pub(crate) async fn get_bucket_policy( bucket: &str, ) -> Result<(BucketPolicy, OffsetDateTime), crate::storage::storage_api::StorageError> { @@ -499,6 +510,21 @@ pub(crate) mod bucket { ) -> Result { crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await } + + pub(crate) async fn update_if_incarnation( + bucket: &str, + config_file: &str, + data: Vec, + expected_incarnation_id: uuid::Uuid, + ) -> Result { + crate::storage::storage_api::update_bucket_metadata_config_if_incarnation( + bucket, + config_file, + data, + Some(expected_incarnation_id), + ) + .await + } } pub(crate) mod object_lock { @@ -517,14 +543,11 @@ pub(crate) mod bucket { } pub(crate) mod objectlock_sys { - pub(crate) type ObjectLockBlockReason = - crate::storage::storage_api::ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason; - pub(crate) async fn check_object_lock_for_deletion( bucket: &str, - obj_info: &crate::storage::storage_api::StorageObjectInfo, + obj_info: &crate::storage::storage_api::ObjectInfo, bypass_governance: bool, - ) -> Option { + ) -> Option { crate::storage::storage_api::ecstore_bucket::object_lock::objectlock_sys::check_object_lock_for_deletion( bucket, obj_info, @@ -899,8 +922,7 @@ pub(crate) mod ecfs { pub(crate) mod error { pub(crate) use crate::storage::storage_api::{ - DiskError, StorageError, is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, - is_err_version_not_found, + StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, }; pub(crate) type Error = StorageError; @@ -938,7 +960,7 @@ pub(crate) mod options { extract_metadata_from_mime, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts_with_replication_authorization, get_content_sha256_with_query, get_opts, namespace_reserved_user_metadata, normalize_content_encoding_for_storage, parse_copy_source_range, - put_opts_with_replication_authorization, validate_archive_content_encoding, + preserve_unclassified_user_metadata, put_opts_with_replication_authorization, validate_archive_content_encoding, }; } @@ -949,8 +971,9 @@ pub(crate) mod request_context { pub(crate) mod sse { pub(crate) use crate::storage::storage_api::sse_consumer::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, - authorize_sse_kms_object_read, extract_server_side_encryption_from_headers, get_buffer_size_opt_in, sse_decryption, - sse_encryption, sse_prepare_encryption, + authorize_sse_kms_object_read, extract_server_side_encryption_from_headers, get_buffer_size_opt_in, + load_bucket_object_lock_config_state, sse_decryption, sse_encryption, sse_prepare_encryption, + validate_bucket_object_lock_enabled_state, }; pub(crate) use crate::storage::storage_api::sse_consumer::{ EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, @@ -1030,6 +1053,8 @@ pub(crate) mod bucket_usecase { } pub(crate) mod object_usecase { + pub(crate) use super::storage_contracts::BUCKET_LIFECYCLE_LOCK_OBJECT; + pub(crate) mod object_cache { #[cfg(test)] pub(crate) use crate::storage::storage_api::ecstore_object::GetObjectBodySource; @@ -1067,9 +1092,8 @@ pub(crate) mod object_usecase { ECStore, GetObjectReader, OldCurrentSize, RFC1123, StorageDeletedObject, StorageObjectInfo, StorageObjectLockDeleteOptions, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, - remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, - validate_bucket_object_lock_enabled, validate_object_key, validate_sse_headers_for_read, validate_sse_headers_for_write, - validate_ssec_for_read, wrap_response_with_cors, + remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_exists, validate_object_key, + validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors, }; } @@ -1123,7 +1147,9 @@ pub(crate) mod test { pub(crate) use super::EndpointServerPools; pub(crate) mod contract { pub(crate) mod bucket { - pub(crate) use super::super::super::storage_contracts::{BucketOperations, BucketOptions, MakeBucketOptions}; + pub(crate) use super::super::super::storage_contracts::{ + BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, + }; } pub(crate) mod heal { diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 7b30012ab..7ca56d505 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::ECStore; +use super::ObjectOptions; use super::ecfs::FS; use super::{ - PolicySys, ReplicationStatusType, StorageError, get_bucket_metadata, get_bucket_policy_raw, get_public_access_block_config, - is_err_bucket_not_found, + ECStore, PolicySys, ReplicationStatusType, StorageError, get_bucket_metadata, get_bucket_policy_raw, + get_lock_acquire_timeout, get_public_access_block_config, is_err_bucket_not_found, }; use crate::auth::{ check_key_valid_with_context, get_condition_values_with_client_info, get_condition_values_with_query_and_client_info, @@ -26,7 +26,9 @@ use crate::error::ApiError; use crate::license::license_check; use crate::server::RemoteAddr; use crate::storage::request_context::RequestContext; -use crate::storage::storage_api::access_consumer::contract::bucket::BucketOperations; +use crate::storage::storage_api::contract::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT; +use crate::storage::storage_api::contract::namespace::NamespaceLocking as _; +use crate::storage::storage_api::runtime_sources_consumer::ServerContextSlot; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use http::HeaderMap; use metrics::counter; @@ -97,6 +99,220 @@ pub(crate) fn recursive_force_delete_is_authorized(headers: &HeaderMap, is_owner #[derive(Clone, Debug)] pub(crate) struct PostObjectRequestMarker; +#[derive(Clone, Debug)] +pub(crate) struct BucketGenerationGuard { + bucket: String, + incarnation_id: uuid::Uuid, +} + +#[derive(Clone, Debug)] +pub(crate) struct BucketConfigMutationSnapshot { + bucket: String, + incarnation_id: uuid::Uuid, +} + +pub(crate) fn bucket_config_mutation_incarnation(req: &S3Request, bucket: &str) -> S3Result> { + let Some(snapshot) = req.extensions.get::() else { + if req.extensions.get::>().is_some() { + return Err(s3_error!(InternalError, "bucket config mutation snapshot is missing")); + } + return Ok(None); + }; + if snapshot.bucket != bucket { + return Err(s3_error!(InternalError, "bucket config mutation snapshot does not match request bucket")); + } + Ok(Some(snapshot.incarnation_id)) +} + +async fn load_bucket_config_mutation_snapshot( + fs: &FS, + req: &S3Request, + bucket: &str, +) -> S3Result { + let store = fs + .server_ctx() + .object_store() + .ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?; + let lock = store + .new_ns_lock(bucket, BUCKET_LIFECYCLE_LOCK_OBJECT) + .await + .map_err(ApiError::from)?; + let _lifecycle_guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(|err| { + ApiError::from(match err { + rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable { + mode: "bucket_config_mutation", + bucket: bucket.to_string(), + object: BUCKET_LIFECYCLE_LOCK_OBJECT.to_string(), + required, + achieved, + }, + other => StorageError::Lock(other), + }) + })?; + let incarnation_id = load_bucket_generation_from_store(store.as_ref(), req, bucket) + .await? + .incarnation_id; + Ok(BucketConfigMutationSnapshot { + bucket: bucket.to_string(), + incarnation_id, + }) +} + +async fn authorize_bucket_config_mutation(fs: &FS, req: &mut S3Request, action: Action) -> S3Result<()> { + let bucket = req_info_ref(req)? + .bucket + .clone() + .ok_or_else(|| s3_error!(InternalError, "bucket config mutation request has no bucket"))?; + let snapshot = load_bucket_config_mutation_snapshot(fs, req, &bucket).await; + authorize_request(req, action).await?; + req.extensions.insert(snapshot?); + Ok(()) +} + +#[derive(Clone, Debug)] +struct CopySourceBucketGenerationGuard { + bucket: String, + incarnation_id: uuid::Uuid, +} + +#[cfg(test)] +type RestoreAuthorizationTestHook = (String, std::sync::Arc, std::sync::Arc); + +#[cfg(test)] +static RESTORE_AUTHORIZATION_TEST_HOOK: OnceLock>> = OnceLock::new(); + +#[cfg(test)] +fn install_restore_authorization_test_hook( + bucket: String, + authorized: std::sync::Arc, + resume: std::sync::Arc, +) { + *RESTORE_AUTHORIZATION_TEST_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("restore authorization test hook lock should not be poisoned") = Some((bucket, authorized, resume)); +} + +#[cfg(test)] +async fn wait_for_restore_authorization_test_hook(bucket: &str) { + let hook = { + let mut slot = RESTORE_AUTHORIZATION_TEST_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("restore authorization test hook lock should not be poisoned"); + if slot.as_ref().is_some_and(|(expected_bucket, _, _)| expected_bucket == bucket) { + slot.take() + } else { + None + } + }; + if let Some((_bucket, authorized, resume)) = hook { + authorized.wait().await; + resume.wait().await; + } +} + +#[derive(Clone, Debug)] +enum PendingDeleteBucketGenerationGuard { + Ready(BucketGenerationGuard), + Failed { code: S3ErrorCode, message: String }, +} + +impl PendingDeleteBucketGenerationGuard { + fn from_result(result: S3Result) -> Self { + match result { + Ok(guard) => Self::Ready(guard), + Err(err) => Self::Failed { + code: err.code().clone(), + message: err.message().unwrap_or_else(|| err.code().as_str()).to_string(), + }, + } + } +} + +pub(crate) async fn load_bucket_generation_from_store( + store: &crate::storage::storage_api::ECStore, + req: &S3Request, + bucket: &str, +) -> S3Result { + if let Some(existing) = req.extensions.get::() { + return if existing.bucket == bucket { + Ok(existing.clone()) + } else { + Err(s3_error!(InternalError, "bucket generation guard does not match request bucket")) + }; + } + + let incarnation_id = store.bucket_incarnation_id(bucket).await.map_err(ApiError::from)?; + Ok(BucketGenerationGuard { + bucket: bucket.to_string(), + incarnation_id, + }) +} + +async fn load_bucket_generation(fs: &FS, req: &S3Request, bucket: &str) -> S3Result { + let store = fs + .server_ctx() + .object_store() + .ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?; + load_bucket_generation_from_store(store.as_ref(), req, bucket).await +} + +async fn load_copy_source_bucket_generation(fs: &FS, bucket: &str) -> S3Result { + let store = fs + .server_ctx() + .object_store() + .ok_or_else(|| s3_error!(InternalError, "object store is not initialized"))?; + let incarnation_id = store.bucket_incarnation_id(bucket).await.map_err(ApiError::from)?; + Ok(CopySourceBucketGenerationGuard { + bucket: bucket.to_string(), + incarnation_id, + }) +} + +pub(crate) fn apply_bucket_generation_guard(req: &S3Request, bucket: &str, opts: &mut ObjectOptions) -> S3Result<()> { + let pending_guard = req.extensions.get::(); + let guard = match pending_guard { + Some(PendingDeleteBucketGenerationGuard::Ready(guard)) => Some(guard), + Some(PendingDeleteBucketGenerationGuard::Failed { code, message }) => { + return Err(S3Error::with_message(code.clone(), message.clone())); + } + None => req.extensions.get::(), + }; + let Some(guard) = guard else { + if req.extensions.get::>().is_some() { + return Err(s3_error!(InternalError, "bucket generation guard is missing")); + } + return Ok(()); + }; + if guard.bucket != bucket { + return Err(s3_error!(InternalError, "bucket generation guard does not match request bucket")); + } + opts.expected_bucket_incarnation_id = Some(guard.incarnation_id); + Ok(()) +} + +pub(crate) fn apply_copy_source_bucket_generation_guard( + req: &S3Request, + bucket: &str, + opts: &mut ObjectOptions, +) -> S3Result<()> { + let Some(guard) = req.extensions.get::() else { + if req.extensions.get::>().is_some() { + return Err(s3_error!(InternalError, "copy source bucket generation guard is missing")); + } + return Ok(()); + }; + if guard.bucket != bucket { + return Err(s3_error!( + InternalError, + "copy source bucket generation guard does not match request bucket" + )); + } + opts.expected_bucket_incarnation_id = Some(guard.incarnation_id); + Ok(()) +} + pub(crate) fn req_info_ref(req: &S3Request) -> S3Result<&ReqInfo> { req.extensions .get::() @@ -1132,38 +1348,49 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn abort_multipart_upload(&self, req: &mut S3Request) -> S3Result<()> { + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); - authorize_request(req, Action::S3Action(S3Action::AbortMultipartUploadAction)).await + authorize_request(req, Action::S3Action(S3Action::AbortMultipartUploadAction)).await?; + req.extensions.insert(bucket_generation?); + Ok(()) } /// Checks whether the CompleteMultipartUpload request has accesses to the resources. /// /// This method returns `Ok(())` by default. async fn complete_multipart_upload(&self, req: &mut S3Request) -> S3Result<()> { + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); authorize_request(req, complete_multipart_upload_authorize_action()).await?; - authorize_replication_only_put_headers(req).await + authorize_replication_only_put_headers(req).await?; + req.extensions.insert(bucket_generation?); + Ok(()) } /// Checks whether the CopyObject request has accesses to the resources. /// /// This method returns `Ok(())` by default. async fn copy_object(&self, req: &mut S3Request) -> S3Result<()> { - { - let (src_bucket, src_key, version_id) = match &req.input.copy_source { - CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)), - CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)), - CopySource::Bucket { bucket, key, version_id } => { - (bucket.to_string(), key.to_string(), version_id.as_ref().map(|v| v.to_string())) - } - }; + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; + let (src_bucket, src_key, version_id) = match &req.input.copy_source { + CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)), + CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)), + CopySource::Bucket { bucket, key, version_id } => { + (bucket.to_string(), key.to_string(), version_id.as_ref().map(|v| v.to_string())) + } + }; + let source_bucket_generation = load_copy_source_bucket_generation(self, &src_bucket).await; + { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(src_bucket.clone()); req_info.object = Some(src_key.clone()); @@ -1173,6 +1400,7 @@ impl S3Access for FS { // s3:GetObjectVersion, not s3:GetObject. authorize_request(req, versioned_read_action(version_id.as_deref())).await?; } + req.extensions.insert(source_bucket_generation?); let req_info = ext_req_info_mut(&mut req.extensions)?; @@ -1192,11 +1420,14 @@ impl S3Access for FS { authorize_request(req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?; } + req.extensions.insert(bucket_generation?); Ok(()) } /// Checks whether the CreateMultipartUpload request has accesses to the resources. async fn create_multipart_upload(&self, req: &mut S3Request) -> S3Result<()> { + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); @@ -1213,6 +1444,7 @@ impl S3Access for FS { authorize_request(req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?; } + req.extensions.insert(bucket_generation?); Ok(()) } @@ -1248,7 +1480,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::DeleteBucketCorsAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::DeleteBucketCorsAction)).await } /// Checks whether the DeleteBucketEncryption request has accesses to the resources. @@ -1258,7 +1490,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketEncryptionAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketEncryptionAction)).await } /// Checks whether the DeleteBucketIntelligentTieringConfiguration request has accesses to the resources. @@ -1288,7 +1520,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketLifecycleAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketLifecycleAction)).await } /// Checks whether the DeleteBucketMetricsConfiguration request has accesses to the resources. @@ -1315,7 +1547,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::DeleteBucketPolicyAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::DeleteBucketPolicyAction)).await } /// Checks whether the DeleteBucketReplication request has accesses to the resources. @@ -1325,7 +1557,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutReplicationConfigurationAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutReplicationConfigurationAction)).await } /// Checks whether the DeleteBucketTagging request has accesses to the resources. @@ -1335,7 +1567,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketTaggingAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketTaggingAction)).await } /// Checks whether the DeleteBucketWebsite request has accesses to the resources. @@ -1345,18 +1577,24 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, bucket_website_config_authorize_action()).await + authorize_bucket_config_mutation(self, req, bucket_website_config_authorize_action()).await } /// Checks whether the DeleteObject request has accesses to the resources. /// /// This method returns `Ok(())` by default. async fn delete_object(&self, req: &mut S3Request) -> S3Result<()> { - if let Some(store) = runtime_sources::current_object_store_handle() - && let Err(err) = store.get_bucket_info(&req.input.bucket, &Default::default()).await - && is_err_bucket_not_found(&err) + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; + // Preserve DeleteObject's established NoSuchBucket response instead of + // letting policy lookup turn a missing bucket into AccessDenied. + if let Err(err) = &bucket_generation + && err.code() == &S3ErrorCode::NoSuchBucket { - return Err(s3_error!(NoSuchBucket, "The specified bucket does not exist")); + return Err(S3Error::with_message( + S3ErrorCode::NoSuchBucket, + err.message().unwrap_or("The specified bucket does not exist").to_string(), + )); } let req_info = ext_req_info_mut(&mut req.extensions)?; @@ -1384,6 +1622,9 @@ impl S3Access for FS { authorize_request(req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await?; } + req.extensions + .insert(PendingDeleteBucketGenerationGuard::from_result(bucket_generation)); + Ok(()) } @@ -1418,7 +1659,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::DeleteBucketPublicAccessBlockAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::DeleteBucketPublicAccessBlockAction)).await } /// Checks whether the GetBucketAccelerateConfiguration request has accesses to the resources. @@ -1807,10 +2048,14 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn list_multipart_uploads(&self, req: &mut S3Request) -> S3Result<()> { + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::ListBucketMultipartUploadsAction)).await + authorize_request(req, Action::S3Action(S3Action::ListBucketMultipartUploadsAction)).await?; + req.extensions.insert(bucket_generation?); + Ok(()) } /// Checks whether the `ListObjectVersions` request is authorized for the requested bucket. @@ -1847,11 +2092,15 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn list_parts(&self, req: &mut S3Request) -> S3Result<()> { + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); - authorize_request(req, list_parts_authorize_action()).await + authorize_request(req, list_parts_authorize_action()).await?; + req.extensions.insert(bucket_generation?); + Ok(()) } /// Checks whether the PostObject request has accesses to the resources. @@ -1877,7 +2126,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketPolicyAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketPolicyAction)).await } /// Checks whether the PutBucketAcl request has accesses to the resources. @@ -1907,7 +2156,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketCorsAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketCorsAction)).await } /// Checks whether the PutBucketEncryption request has accesses to the resources. @@ -1917,7 +2166,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketEncryptionAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketEncryptionAction)).await } /// Checks whether the PutBucketIntelligentTieringConfiguration request has accesses to the resources. @@ -1950,14 +2199,14 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketLifecycleAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketLifecycleAction)).await } async fn put_bucket_logging(&self, req: &mut S3Request) -> S3Result<()> { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketLoggingAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketLoggingAction)).await } /// Checks whether the PutBucketMetricsConfiguration request has accesses to the resources. @@ -1977,7 +2226,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketNotificationAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketNotificationAction)).await } /// Checks whether the PutBucketOwnershipControls request has accesses to the resources. @@ -1994,7 +2243,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, put_bucket_policy_authorize_action()).await + authorize_bucket_config_mutation(self, req, put_bucket_policy_authorize_action()).await } /// Checks whether the PutBucketReplication request has accesses to the resources. @@ -2004,7 +2253,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutReplicationConfigurationAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutReplicationConfigurationAction)).await } /// Checks whether the PutBucketRequestPayment request has accesses to the resources. @@ -2014,7 +2263,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketPolicyAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketPolicyAction)).await } /// Checks whether the PutBucketTagging request has accesses to the resources. @@ -2024,7 +2273,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketTaggingAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketTaggingAction)).await } /// Checks whether the PutBucketVersioning request has accesses to the resources. @@ -2034,7 +2283,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketVersioningAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketVersioningAction)).await } /// Checks whether the PutBucketWebsite request has accesses to the resources. @@ -2044,7 +2293,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, bucket_website_config_authorize_action()).await + authorize_bucket_config_mutation(self, req, bucket_website_config_authorize_action()).await } /// Checks whether the PutObject request has accesses to the resources. @@ -2064,7 +2313,12 @@ impl S3Access for FS { )); } + let bucket = req.input.bucket.clone(); + // Snapshot before authorization, but preserve AccessDenied precedence + // by exposing any bucket-state error only after authorization succeeds. + let bucket_generation = load_bucket_generation(self, req, &bucket).await; authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?; + req.extensions.insert(bucket_generation?); // POST-object form uploads (s3s routes them through this hook with the // original POST method before the dedicated post_object hook) are @@ -2113,7 +2367,11 @@ impl S3Access for FS { req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); - authorize_request(req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; + authorize_request(req, Action::S3Action(S3Action::PutObjectLegalHoldAction)).await?; + req.extensions.insert(bucket_generation?); + Ok(()) } /// Checks whether the PutObjectLockConfiguration request has accesses to the resources. @@ -2123,7 +2381,7 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketObjectLockConfigurationAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketObjectLockConfigurationAction)).await } /// Checks whether the PutObjectRetention request has accesses to the resources. @@ -2133,6 +2391,8 @@ impl S3Access for FS { req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; authorize_request(req, Action::S3Action(S3Action::PutObjectRetentionAction)).await?; // S3 Standard: When bypass_governance header is set, must have s3:BypassGovernanceRetention permission @@ -2140,6 +2400,7 @@ impl S3Access for FS { authorize_request(req, Action::S3Action(S3Action::BypassGovernanceRetentionAction)).await?; } + req.extensions.insert(bucket_generation?); Ok(()) } @@ -2162,19 +2423,25 @@ impl S3Access for FS { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); - authorize_request(req, Action::S3Action(S3Action::PutBucketPublicAccessBlockAction)).await + authorize_bucket_config_mutation(self, req, Action::S3Action(S3Action::PutBucketPublicAccessBlockAction)).await } /// Checks whether the RestoreObject request has accesses to the resources. /// /// This method returns `Ok(())` by default. async fn restore_object(&self, req: &mut S3Request) -> S3Result<()> { + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); - authorize_request(req, Action::S3Action(S3Action::RestoreObjectAction)).await + authorize_request(req, Action::S3Action(S3Action::RestoreObjectAction)).await?; + #[cfg(test)] + wait_for_restore_authorization_test_hook(&bucket).await; + req.extensions.insert(bucket_generation?); + Ok(()) } /// Checks whether the SelectObjectContent request has accesses to the resources. @@ -2192,26 +2459,33 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn upload_part(&self, req: &mut S3Request) -> S3Result<()> { + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); - authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await + authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?; + req.extensions.insert(bucket_generation?); + Ok(()) } /// Checks whether the UploadPartCopy request has accesses to the resources. /// /// This method returns `Ok(())` by default. async fn upload_part_copy(&self, req: &mut S3Request) -> S3Result<()> { - { - let (src_bucket, src_key, version_id) = match &req.input.copy_source { - CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)), - CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)), - CopySource::Bucket { bucket, key, version_id } => { - (bucket.to_string(), key.to_string(), version_id.as_ref().map(|v| v.to_string())) - } - }; + let bucket = req.input.bucket.clone(); + let bucket_generation = load_bucket_generation(self, req, &bucket).await; + let (src_bucket, src_key, version_id) = match &req.input.copy_source { + CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)), + CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)), + CopySource::Bucket { bucket, key, version_id } => { + (bucket.to_string(), key.to_string(), version_id.as_ref().map(|v| v.to_string())) + } + }; + let source_bucket_generation = load_copy_source_bucket_generation(self, &src_bucket).await; + { let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(src_bucket.clone()); req_info.object = Some(src_key.clone()); @@ -2221,13 +2495,16 @@ impl S3Access for FS { // s3:GetObjectVersion, not s3:GetObject. authorize_request(req, versioned_read_action(version_id.as_deref())).await?; } + req.extensions.insert(source_bucket_generation?); let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = None; - authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await + authorize_request(req, Action::S3Action(S3Action::PutObjectAction)).await?; + req.extensions.insert(bucket_generation?); + Ok(()) } /// Checks whether the WriteGetObjectResponse request has accesses to the resources. @@ -2242,24 +2519,53 @@ impl S3Access for FS { #[allow(unused_imports)] mod tests { use super::{ - AMZ_WRITE_OFFSET_BYTES_HEADER, BucketPolicyArgs, BucketPolicyExistingObjectTagHint, BucketPolicyRawLoadErrorKind, FS, - ObjectTagConditions, PostObjectRequestMarker, ReqInfo, S3Access, StorageError, + AMZ_WRITE_OFFSET_BYTES_HEADER, BucketGenerationGuard, BucketPolicyArgs, BucketPolicyExistingObjectTagHint, + BucketPolicyRawLoadErrorKind, FS, ObjectTagConditions, PostObjectRequestMarker, ReqInfo, S3Access, StorageError, + apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, bucket_policy_needs_existing_object_tag_from_hint, bucket_website_config_authorize_action, classify_bucket_policy_raw_load_error, complete_multipart_upload_authorize_action, get_bucket_policy_authorize_action, - has_write_offset_bytes_header, legal_hold_write_requested, list_parts_authorize_action, - load_bucket_policy_existing_object_tag_hint, merge_list_bucket_query_conditions, merge_request_object_tag_conditions, - owner_can_bypass_policy_deny, post_object_authorize_action, put_bucket_policy_authorize_action, request_context_from_req, - retention_write_requested, secondary_tag_hint_action, table_data_plane_admin_action, - validate_post_object_success_controls, versioned_read_action, + has_write_offset_bytes_header, install_restore_authorization_test_hook, legal_hold_write_requested, + list_parts_authorize_action, load_bucket_policy_existing_object_tag_hint, maybe_merge_object_tag_conditions, + merge_list_bucket_query_conditions, merge_request_object_tag_conditions, owner_can_bypass_policy_deny, + post_object_authorize_action, put_bucket_policy_authorize_action, request_context_from_req, retention_write_requested, + secondary_tag_hint_action, table_data_plane_admin_action, validate_post_object_success_controls, versioned_read_action, }; use crate::error::ApiError; + use crate::storage::storage_api::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; + use crate::storage::storage_api::contract::multipart::MultipartOperations as _; + use crate::storage::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _}; + use crate::storage::storage_api::runtime_sources_consumer::{AppContext, IamInterface, KmsInterface, ServerContextSlot}; use http::{Extensions, HeaderMap, HeaderValue, Method, Uri}; + use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; + use rustfs_kms::KmsServiceManager; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_policy::policy::{BucketPolicy, bucket_policy_uses_existing_object_tag_conditions}; use s3s::{S3ErrorCode, S3Request, dto::*}; + use serial_test::serial; use std::collections::HashMap; + use std::sync::Arc; use time::OffsetDateTime; + struct UnreadyIam; + + impl IamInterface for UnreadyIam { + fn handle(&self) -> Arc> { + panic!("bucket generation guard tests do not resolve IAM") + } + + fn is_ready(&self) -> bool { + false + } + } + + struct TestKms; + + impl KmsInterface for TestKms { + fn handle(&self) -> Arc { + Arc::new(KmsServiceManager::new()) + } + } + fn build_request(input: T, method: Method) -> S3Request { S3Request { input, @@ -2504,18 +2810,34 @@ mod tests { ); } - #[test] - fn test_object_tag_conditions_key_format() { - let mut tags = HashMap::new(); - tags.insert("ExistingObjectTag/security".to_string(), vec!["public".to_string()]); - tags.insert("ExistingObjectTag/project".to_string(), vec!["webapp".to_string()]); - let object_tag_conditions = ObjectTagConditions::new("bucket", "object", None, tags); + #[tokio::test] + async fn test_required_existing_object_tags_are_merged_into_authorization_conditions() { + let mut request = build_request((), Method::GET); + request.extensions.insert(ObjectTagConditions::new( + "bucket", + "tagged-object", + None, + HashMap::from([ + ("ExistingObjectTag/security".to_string(), vec!["restricted".to_string()]), + ("ExistingObjectTag/project".to_string(), vec!["webapp".to_string()]), + ]), + )); let mut conditions = HashMap::new(); conditions.insert("delimiter".to_string(), vec!["/".to_string()]); - for (k, v) in &object_tag_conditions.values { - conditions.insert(k.clone(), v.clone()); - } - assert_eq!(conditions.get("ExistingObjectTag/security"), Some(&vec!["public".to_string()])); + + maybe_merge_object_tag_conditions( + &mut request, + Action::S3Action(S3Action::GetObjectAction), + "bucket", + "tagged-object", + None, + &mut conditions, + true, + ) + .await + .expect("merge required existing object tags into authorization conditions"); + + assert_eq!(conditions.get("ExistingObjectTag/security"), Some(&vec!["restricted".to_string()])); assert_eq!(conditions.get("ExistingObjectTag/project"), Some(&vec!["webapp".to_string()])); assert_eq!(conditions.get("delimiter"), Some(&vec!["/".to_string()])); } @@ -2789,6 +3111,375 @@ mod tests { assert_eq!(req_info.object.as_deref(), Some("test-key")); } + #[tokio::test] + #[serial] + async fn put_object_access_captures_authorized_bucket_incarnation() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let server_ctx = ServerContextSlot::new(); + assert!(server_ctx.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms),)))); + let fs = FS::with_server_ctx(server_ctx); + + let bucket = "generation-guard-collision"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("test bucket should be created"); + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let mut metadata = (*crate::storage::get_bucket_metadata(bucket) + .await + .expect("new bucket metadata should be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.to_string(), metadata) + .await + .expect("test bucket policy should be published"); + + let input = PutObjectInput::builder() + .bucket(bucket.to_string()) + .key(bucket.to_string()) + .build() + .expect("put object input should build"); + let mut req = build_request(input, Method::PUT); + ensure_req_info(&mut req); + + fs.put_object(&mut req) + .await + .expect("anonymous PutObject should be authorized by the test policy"); + let mut opts = crate::storage::ObjectOptions::default(); + apply_bucket_generation_guard(&req, bucket, &mut opts).expect("request snapshot should apply to PutObject options"); + assert_eq!( + opts.expected_bucket_incarnation_id, + Some( + store + .bucket_incarnation_id(bucket) + .await + .expect("bucket incarnation should remain readable") + ) + ); + } + + #[tokio::test] + #[serial] + async fn delete_object_access_captures_authorized_bucket_incarnation() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let server_ctx = ServerContextSlot::new(); + let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))); + assert!(server_ctx.install(Arc::clone(&app_context))); + let fs = FS::with_server_ctx(server_ctx); + + let bucket = format!("delete-generation-guard-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("test bucket should be created"); + assert!( + fs.get_object_tag_conditions_for_policy(&bucket, "missing-object", None) + .await + .expect("a missing object must be treated as having no policy tags") + .is_empty() + ); + assert!( + fs.get_object_tag_conditions_for_policy(&format!("missing-{bucket}"), "missing-object", None) + .await + .expect("a missing bucket must be indistinguishable during pre-authorization tag lookup") + .is_empty() + ); + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:DeleteObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("new bucket metadata should be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test bucket policy should parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("test bucket policy should be published"); + + let input = DeleteObjectInput::builder() + .bucket(bucket.clone()) + .key("object".to_string()) + .build() + .expect("delete object input should build"); + let mut req = build_request(input, Method::DELETE); + ensure_req_info(&mut req); + + fs.delete_object(&mut req) + .await + .expect("anonymous DeleteObject should be authorized by the test policy"); + let mut opts = crate::storage::ObjectOptions::default(); + apply_bucket_generation_guard(&req, &bucket, &mut opts).expect("request snapshot should apply to DeleteObject options"); + assert_eq!( + opts.expected_bucket_incarnation_id, + Some( + store + .bucket_incarnation_id(&bucket) + .await + .expect("bucket incarnation should remain readable") + ) + ); + + store + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("delete the authorized bucket generation"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("recreate the same bucket name"); + let mut reader = crate::storage::PutObjReader::from_vec(b"new generation".to_vec()); + store + .put_object(&bucket, "object", &mut reader, &crate::storage::ObjectOptions::default()) + .await + .expect("put the new-generation object"); + + let err = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(app_context)) + .execute_delete_object(req) + .await + .expect_err("the old authorization must not delete from the recreated bucket"); + assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket); + store + .get_object_info(&bucket, "object", &crate::storage::ObjectOptions::default()) + .await + .expect("the new-generation object must survive the stale request"); + } + + #[tokio::test] + #[serial] + async fn copy_operations_reject_recreated_source_bucket_after_authorization() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let server_ctx = ServerContextSlot::new(); + let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))); + assert!(server_ctx.install(Arc::clone(&app_context))); + let fs = FS::with_server_ctx(server_ctx); + + let suffix = uuid::Uuid::new_v4(); + let src_bucket = format!("copy-src-generation-{suffix}"); + let dst_bucket = format!("copy-dst-generation-{suffix}"); + for bucket in [&src_bucket, &dst_bucket] { + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create copy test bucket"); + } + + let src_policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::{src_bucket}/*"]}}]}}"# + ); + let mut src_metadata = (*crate::storage::get_bucket_metadata(&src_bucket) + .await + .expect("source bucket metadata should be cached")) + .clone(); + src_metadata.policy_config = Some(serde_json::from_str(&src_policy_json).expect("source policy should parse")); + src_metadata.policy_config_json = src_policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(src_bucket.clone(), src_metadata) + .await + .expect("publish source policy"); + + let dst_policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{dst_bucket}/*"]}}]}}"# + ); + let mut dst_metadata = (*crate::storage::get_bucket_metadata(&dst_bucket) + .await + .expect("destination bucket metadata should be cached")) + .clone(); + dst_metadata.policy_config = Some(serde_json::from_str(&dst_policy_json).expect("destination policy should parse")); + dst_metadata.policy_config_json = dst_policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(dst_bucket.clone(), dst_metadata) + .await + .expect("publish destination policy"); + + let input = CopyObjectInput::builder() + .copy_source(CopySource::Bucket { + bucket: src_bucket.clone().into(), + key: "secret".into(), + version_id: None, + }) + .bucket(dst_bucket.clone()) + .key("copied".to_string()) + .build() + .expect("copy input should build"); + let mut req = build_request(input, Method::PUT); + ensure_req_info(&mut req); + + fs.copy_object(&mut req) + .await + .expect("test policies should authorize the copy request"); + let mut source_opts = crate::storage::ObjectOptions::default(); + apply_copy_source_bucket_generation_guard(&req, &src_bucket, &mut source_opts) + .expect("authorized source generation should be attached"); + let authorized_source_incarnation_id = source_opts + .expected_bucket_incarnation_id + .expect("source incarnation should be captured"); + + let upload = store + .new_multipart_upload(&dst_bucket, "multipart-copy", &crate::storage::ObjectOptions::default()) + .await + .expect("create destination multipart upload"); + let upload_input = UploadPartCopyInput::builder() + .bucket(dst_bucket.clone()) + .key("multipart-copy".to_string()) + .copy_source(CopySource::Bucket { + bucket: src_bucket.clone().into(), + key: "secret".into(), + version_id: None, + }) + .part_number(1) + .upload_id(upload.upload_id) + .build() + .expect("part copy input should build"); + let mut upload_req = build_request(upload_input, Method::PUT); + ensure_req_info(&mut upload_req); + fs.upload_part_copy(&mut upload_req) + .await + .expect("test policies should authorize the part copy request"); + + store + .delete_bucket(&src_bucket, &DeleteBucketOptions::default()) + .await + .expect("delete the authorized source generation"); + store + .make_bucket(&src_bucket, &MakeBucketOptions::default()) + .await + .expect("recreate the source bucket name"); + assert_ne!( + authorized_source_incarnation_id, + store + .bucket_incarnation_id(&src_bucket) + .await + .expect("read recreated source incarnation") + ); + let mut reader = crate::storage::PutObjReader::from_vec(b"new generation secret".to_vec()); + store + .put_object(&src_bucket, "secret", &mut reader, &crate::storage::ObjectOptions::default()) + .await + .expect("write the new-generation source object"); + + let err = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(Arc::clone(&app_context))) + .execute_copy_object(req) + .await + .expect_err("stale authorization must not read from the recreated source bucket"); + assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket); + let err = crate::app::multipart_usecase::DefaultMultipartUsecase::with_context(Some(app_context)) + .execute_upload_part_copy(upload_req) + .await + .expect_err("stale authorization must not read the recreated source into a part"); + assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket); + store + .get_object_info(&dst_bucket, "copied", &crate::storage::ObjectOptions::default()) + .await + .expect_err("the new-generation source object must not be copied"); + } + + #[tokio::test] + #[serial] + async fn restore_object_access_keeps_authorized_bucket_incarnation_across_recreation() { + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let server_ctx = ServerContextSlot::new(); + let app_context = Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))); + assert!(server_ctx.install(Arc::clone(&app_context))); + let fs = FS::with_server_ctx(server_ctx); + + let bucket = format!("restore-generation-guard-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create the bucket generation being authorized"); + let authorized_incarnation_id = store + .bucket_incarnation_id(&bucket) + .await + .expect("read the authorized bucket incarnation"); + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:RestoreObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("authorized bucket metadata should be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("publish the RestoreObject policy"); + + let input = RestoreObjectInput::builder() + .bucket(bucket.clone()) + .key("object".to_string()) + .restore_request(Some(RestoreRequest { + days: Some(1), + description: None, + glacier_job_parameters: None, + output_location: None, + select_parameters: None, + tier: None, + type_: None, + })) + .build() + .expect("restore object input should build"); + let mut req = build_request(input, Method::POST); + ensure_req_info(&mut req); + let authorized = Arc::new(tokio::sync::Barrier::new(2)); + let resume = Arc::new(tokio::sync::Barrier::new(2)); + install_restore_authorization_test_hook(bucket.clone(), Arc::clone(&authorized), Arc::clone(&resume)); + + let access = tokio::spawn(async move { + let result = fs.restore_object(&mut req).await; + (result, req) + }); + authorized.wait().await; + + store + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("delete the authorized bucket generation"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("recreate the bucket with the same name"); + let recreated_incarnation_id = store + .bucket_incarnation_id(&bucket) + .await + .expect("read the recreated bucket incarnation"); + assert_ne!(authorized_incarnation_id, recreated_incarnation_id); + resume.wait().await; + + let (result, req) = access.await.expect("RestoreObject access task should join"); + result.expect("the already-authorized request should retain its generation guard"); + let mut opts = crate::storage::ObjectOptions::default(); + apply_bucket_generation_guard(&req, &bucket, &mut opts).expect("apply the RestoreObject authorization guard"); + assert_eq!(opts.expected_bucket_incarnation_id, Some(authorized_incarnation_id)); + + let err = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(app_context)) + .execute_restore_object(req) + .await + .expect_err("the old RestoreObject authorization must not reach the recreated bucket"); + assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket); + store + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("clean up the recreated bucket"); + } + + #[test] + fn dispatched_put_object_fails_closed_without_generation_guard() { + let input = PutObjectInput::builder() + .bucket("generation-guard-bucket".to_string()) + .key("object".to_string()) + .build() + .expect("put object input should build"); + let mut req = build_request(input, Method::PUT); + req.extensions.insert(ServerContextSlot::new()); + + let mut opts = crate::storage::ObjectOptions::default(); + let err = apply_bucket_generation_guard(&req, "generation-guard-bucket", &mut opts) + .expect_err("a dispatched PutObject must not commit without its authorization fence"); + assert_eq!(err.code(), &S3ErrorCode::InternalError); + } + #[test] fn write_offset_bytes_header_detection_is_case_insensitive() { let mut headers = HeaderMap::new(); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 3802ba555..c946db98e 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -15,15 +15,15 @@ use super::{ BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, check_retention_for_modification, decode_tags, - decode_tags_to_map, delete_bucket_metadata_config, encode_tags, get_bucket_accelerate_config, get_bucket_logging_config, - get_bucket_object_lock_config, get_bucket_replication_config, get_bucket_request_payment_config, get_bucket_website_config, - is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, record_replication_proxy, serialize, - update_bucket_metadata_config, + decode_tags_to_map, delete_bucket_metadata_config_if_incarnation, encode_tags, get_bucket_accelerate_config, + get_bucket_logging_config, get_bucket_object_lock_config, get_bucket_replication_config, get_bucket_request_payment_config, + get_bucket_website_config, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, + record_replication_proxy, serialize, update_bucket_metadata_config_if_incarnation, }; use super::{StorageReplicationConfigExt as _, StorageVersioningConfigExt as _}; use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook; use crate::error::ApiError; -use crate::storage::access::has_bypass_governance_header; +use crate::storage::access::{apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header}; use crate::storage::helper::OperationHelper; use crate::storage::options::get_opts; use crate::storage::s3_api::{self, acl}; @@ -135,20 +135,17 @@ impl FS { let tags = match store.get_object_tags(bucket, object, &opts).await { Ok(t) => t, Err(e) => { - if is_err_object_not_found(&e) || is_err_version_not_found(&e) { + if is_err_object_not_found(&e) || is_err_version_not_found(&e) || is_err_bucket_not_found(&e) { debug!( target: "rustfs::storage::ecfs", bucket = %bucket, object = %object, version_id = ?version_id, error = %e, - "object or version not found when fetching tags for policy; treating as no tags" + "object, version, or bucket not found when fetching tags for policy; treating as no tags" ); return Ok(std::collections::HashMap::new()); } - if is_err_bucket_not_found(&e) { - return Err(s3_error!(NoSuchBucket, "The specified bucket does not exist")); - } warn!( target: "rustfs::storage::ecfs", bucket = %bucket, @@ -194,6 +191,12 @@ fn invalid_object_lock_configuration(message: impl Into) -> S3Error { S3Error::with_message(S3ErrorCode::MalformedXML, message.into()) } +pub(crate) fn propagate_object_lock_peer_reload(result: std::result::Result<(), StorageError>) -> S3Result<()> { + result.map_err(|err| { + S3Error::with_message(S3ErrorCode::InternalError, format!("Failed to publish Object Lock metadata: {err}")) + }) +} + fn invalid_retention_period(message: impl Into) -> S3Error { let mut err = S3Error::with_message(S3ErrorCode::Custom("InvalidRetentionPeriod".into()), message.into()); err.set_status_code(StatusCode::BAD_REQUEST); @@ -371,6 +374,7 @@ impl S3 for FS { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let Some(store) = self.server_ctx.object_store() else { return Err(s3_error!(InternalError, "Not init")); }; @@ -380,7 +384,7 @@ impl S3 for FS { .await .map_err(crate::error::ApiError::from)?; - delete_bucket_metadata_config(&req.input.bucket, BUCKET_WEBSITE_CONFIG) + delete_bucket_metadata_config_if_incarnation(&req.input.bucket, BUCKET_WEBSITE_CONFIG, expected_incarnation_id) .await .map_err(crate::error::ApiError::from)?; @@ -1058,6 +1062,7 @@ impl S3 for FS { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let Some(store) = self.server_ctx.object_store() else { return Err(s3_error!(InternalError, "Not init")); }; @@ -1068,9 +1073,14 @@ impl S3 for FS { let accelerate_config = serialize(&req.input.accelerate_configuration) .map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?; - update_bucket_metadata_config(&req.input.bucket, BUCKET_ACCELERATE_CONFIG, accelerate_config) - .await - .map_err(crate::error::ApiError::from)?; + update_bucket_metadata_config_if_incarnation( + &req.input.bucket, + BUCKET_ACCELERATE_CONFIG, + accelerate_config, + expected_incarnation_id, + ) + .await + .map_err(crate::error::ApiError::from)?; Ok(S3Response::new(PutBucketAccelerateConfigurationOutput::default())) } @@ -1101,6 +1111,7 @@ impl S3 for FS { } async fn put_bucket_logging(&self, req: S3Request) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; record_s3_op(S3Operation::PutBucketLogging); let Some(store) = self.server_ctx.object_store() else { return Err(s3_error!(InternalError, "Not init")); @@ -1112,9 +1123,14 @@ impl S3 for FS { let logging_config = serialize(&req.input.bucket_logging_status) .map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?; - update_bucket_metadata_config(&req.input.bucket, BUCKET_LOGGING_CONFIG, logging_config) - .await - .map_err(crate::error::ApiError::from)?; + update_bucket_metadata_config_if_incarnation( + &req.input.bucket, + BUCKET_LOGGING_CONFIG, + logging_config, + expected_incarnation_id, + ) + .await + .map_err(crate::error::ApiError::from)?; Ok(S3Response::new(PutBucketLoggingOutput::default())) } @@ -1161,6 +1177,7 @@ impl S3 for FS { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let Some(store) = self.server_ctx.object_store() else { return Err(s3_error!(InternalError, "Not init")); }; @@ -1171,9 +1188,14 @@ impl S3 for FS { let payment_config = serialize(&req.input.request_payment_configuration) .map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?; - update_bucket_metadata_config(&req.input.bucket, BUCKET_REQUEST_PAYMENT_CONFIG, payment_config) - .await - .map_err(crate::error::ApiError::from)?; + update_bucket_metadata_config_if_incarnation( + &req.input.bucket, + BUCKET_REQUEST_PAYMENT_CONFIG, + payment_config, + expected_incarnation_id, + ) + .await + .map_err(crate::error::ApiError::from)?; Ok(S3Response::new(PutBucketRequestPaymentOutput::default())) } @@ -1203,6 +1225,7 @@ impl S3 for FS { } async fn put_bucket_website(&self, req: S3Request) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let Some(store) = self.server_ctx.object_store() else { return Err(s3_error!(InternalError, "Not init")); }; @@ -1213,9 +1236,14 @@ impl S3 for FS { let website_config = serialize(&req.input.website_configuration) .map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?; - update_bucket_metadata_config(&req.input.bucket, BUCKET_WEBSITE_CONFIG, website_config) - .await - .map_err(crate::error::ApiError::from)?; + update_bucket_metadata_config_if_incarnation( + &req.input.bucket, + BUCKET_WEBSITE_CONFIG, + website_config, + expected_incarnation_id, + ) + .await + .map_err(crate::error::ApiError::from)?; Ok(S3Response::new(PutBucketWebsiteOutput::default())) } @@ -1295,6 +1323,7 @@ impl S3 for FS { version_id: opts.version_id.clone(), ..Default::default() }; + apply_bucket_generation_guard(&req, &bucket, &mut popts)?; // PutObjectLegalHold only rewrites metadata, so replication is not scheduled by the // object PUT path. Schedule it explicitly, otherwise the legal hold never reaches the @@ -1353,6 +1382,7 @@ impl S3 for FS { &self, req: S3Request, ) -> S3Result> { + let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let PutObjectLockConfigurationInput { bucket, object_lock_configuration, @@ -1396,7 +1426,7 @@ impl S3 for FS { let object_lock_config = String::from_utf8(data.clone()).map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")))?; - let updated_at = update_bucket_metadata_config(&bucket, OBJECT_LOCK_CONFIG, data) + let updated_at = update_bucket_metadata_config_if_incarnation(&bucket, OBJECT_LOCK_CONFIG, data, expected_incarnation_id) .await .map_err(ApiError::from)?; @@ -1410,9 +1440,20 @@ impl S3 for FS { }; let versioning_data = serialize(&enable_versioning_config) .map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")))?; - update_bucket_metadata_config(&bucket, BUCKET_VERSIONING_CONFIG, versioning_data) - .await - .map_err(ApiError::from)?; + update_bucket_metadata_config_if_incarnation( + &bucket, + BUCKET_VERSIONING_CONFIG, + versioning_data, + expected_incarnation_id, + ) + .await + .map_err(ApiError::from)?; + } + + if let Some(notification_sys) = + runtime_sources::current_notification_system_for_context(self.server_ctx.app_context().as_deref()) + { + propagate_object_lock_peer_reload(notification_sys.load_bucket_metadata(&bucket).await)?; } if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta { @@ -1494,6 +1535,7 @@ impl S3 for FS { let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers) .await .map_err(ApiError::from)?; + apply_bucket_generation_guard(&req, &bucket, &mut opts)?; opts.object_lock_retention = Some(ObjectLockRetentionOptions { mode: new_mode, retain_until: new_retain_until, diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index ba55fd486..1f2ab84c6 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{StorageError, add_object_lock_years, get_bucket_cors_config, get_bucket_object_lock_config}; +use super::{StorageError, add_object_lock_years, get_bucket_cors_config}; use crate::config::{RustFSBufferConfig, WorkloadProfile, is_buffer_profile_enabled}; use crate::error::ApiError; use crate::server::cors; use crate::storage::ecfs::ListObjectUnorderedQuery; use crate::storage::storage_api::ecfs_extend_consumer::contract::bucket::{BucketOperations, BucketOptions}; use crate::storage::storage_api::ecfs_extend_consumer::contract::multipart::MAX_MULTIPART_PART_NUMBER; +use crate::storage::storage_api::ecstore_bucket::metadata_sys::{self, ObjectLockConfigState}; use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE}; use http::{HeaderMap, HeaderValue, StatusCode}; use metrics::counter; @@ -87,7 +88,7 @@ pub(crate) fn remove_object_lock_metadata_for_copy(metadata: &mut HashMap, metadata: &mut HashMap) { +pub(crate) fn apply_lock_retention(object_lock_config: Option<&ObjectLockConfiguration>, metadata: &mut HashMap) { if has_object_lock_retention_metadata(metadata) { return; } @@ -98,8 +99,10 @@ pub(crate) fn apply_lock_retention(object_lock_config: Option, + object_lock_configuration: Option<&ObjectLockConfiguration>, metadata: &mut HashMap, ) -> bool { if has_object_lock_retention_metadata(metadata) { @@ -133,41 +136,74 @@ pub(crate) fn apply_default_lock_retention_metadata( true } -pub(crate) async fn apply_bucket_default_lock_retention( +pub(crate) async fn load_bucket_object_lock_config_state(bucket: &str) -> S3Result { + map_bucket_object_lock_config_state(bucket, metadata_sys::get_object_lock_config_state(bucket).await) +} + +pub(crate) fn map_bucket_object_lock_config_state( bucket: &str, + result: Result, +) -> S3Result { + match result { + Ok(ObjectLockConfigState::Fabricated) => { + warn!( + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_OBJECT_LOCK, + event = "object_lock_config_not_authoritative", + bucket = %bucket, + "Bucket Object Lock configuration is not authoritative" + ); + Err(S3Error::with_message( + S3ErrorCode::InternalError, + "Failed to load Object Lock configuration".to_string(), + )) + } + Ok(state) => Ok(state), + Err(err) => { + warn!( + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_OBJECT_LOCK, + event = "object_lock_config_load_failed", + bucket = %bucket, + error = ?err, + "Failed to load bucket object lock configuration" + ); + Err(S3Error::with_message( + S3ErrorCode::InternalError, + "Failed to load Object Lock configuration".to_string(), + )) + } + } +} + +pub(crate) fn apply_bucket_default_lock_retention( + bucket: &str, + state: &ObjectLockConfigState, metadata: &mut HashMap, has_explicit_retention: bool, ) -> S3Result<()> { - if has_explicit_retention { - return Ok(()); - } - - if has_object_lock_retention_metadata(metadata) { - return Ok(()); - } - - let object_lock_configuration = match get_bucket_object_lock_config(bucket).await { - Ok((cfg, _created)) => Some(cfg), - Err(err) => { - if err == StorageError::ConfigNotFound { - None - } else { - warn!( - component = LOG_COMPONENT_STORAGE, - subsystem = LOG_SUBSYSTEM_OBJECT_LOCK, - event = "object_lock_config_load_failed", - bucket = %bucket, - error = ?err, - "Failed to load bucket object lock configuration" - ); - return Err(S3Error::with_message( - S3ErrorCode::InternalError, - "Failed to load Object Lock configuration".to_string(), - )); - } + let object_lock_configuration = match state { + ObjectLockConfigState::Configured { config, .. } => Some(config), + ObjectLockConfigState::ConfirmedAbsent => None, + ObjectLockConfigState::Fabricated => { + warn!( + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_OBJECT_LOCK, + event = "object_lock_config_not_authoritative", + bucket = %bucket, + "Bucket Object Lock configuration is not authoritative" + ); + return Err(S3Error::with_message( + S3ErrorCode::InternalError, + "Failed to load Object Lock configuration".to_string(), + )); } }; + if has_explicit_retention || has_object_lock_retention_metadata(metadata) { + return Ok(()); + } + apply_default_lock_retention_metadata(object_lock_configuration, metadata); Ok(()) } @@ -429,37 +465,39 @@ pub(crate) fn parse_object_lock_legal_hold(legal_hold: Option S3Result<()> { - match get_bucket_object_lock_config(bucket).await { - Ok((cfg, _created)) => { - if cfg.object_lock_enabled != Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)) { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - "Object Lock is not enabled for this bucket".to_string(), - )); - } + let state = load_bucket_object_lock_config_state(bucket).await?; + validate_bucket_object_lock_enabled_state(bucket, &state) +} + +pub(crate) fn validate_bucket_object_lock_enabled_state(bucket: &str, state: &ObjectLockConfigState) -> S3Result<()> { + match state { + ObjectLockConfigState::Configured { config, .. } + if config.object_lock_enabled == Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)) => + { + Ok(()) } - Err(err) => { - if err == StorageError::ConfigNotFound { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - "Bucket is missing ObjectLockConfiguration".to_string(), - )); - } + ObjectLockConfigState::Configured { .. } => Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "Object Lock is not enabled for this bucket".to_string(), + )), + ObjectLockConfigState::ConfirmedAbsent => Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + "Bucket is missing ObjectLockConfiguration".to_string(), + )), + ObjectLockConfigState::Fabricated => { warn!( component = LOG_COMPONENT_STORAGE, subsystem = LOG_SUBSYSTEM_OBJECT_LOCK, - event = "object_lock_config_load_failed", + event = "object_lock_config_not_authoritative", bucket = %bucket, - error = ?err, - "Failed to load bucket object lock configuration" + "Bucket Object Lock configuration is not authoritative" ); - return Err(S3Error::with_message( + Err(S3Error::with_message( S3ErrorCode::InternalError, "Failed to get bucket ObjectLockConfiguration".to_string(), - )); + )) } } - Ok(()) } /// Validates HTTP conditional request headers for a single object according to diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs index 4541a7d02..fcf2a03e1 100644 --- a/rustfs/src/storage/ecfs_test.rs +++ b/rustfs/src/storage/ecfs_test.rs @@ -16,8 +16,11 @@ mod tests { use crate::config::WorkloadProfile; use crate::server::cors; - use crate::storage::ecfs::{FS, validate_object_lock_configuration_input}; + use crate::storage::StorageError; + use crate::storage::ecfs::{FS, propagate_object_lock_peer_reload, validate_object_lock_configuration_input}; + use crate::storage::ecfs_extend::{apply_bucket_default_lock_retention, map_bucket_object_lock_config_state}; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; + use crate::storage::storage_api::ecstore_bucket::metadata_sys::ObjectLockConfigState; use crate::storage::storage_api::test_consumer::{ BucketMetadata, DEFAULT_READ_BUFFER_SIZE, StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, bucket_metadata_sys_initialized, check_preconditions, decode_tags_to_map, @@ -31,7 +34,8 @@ mod tests { use rustfs_config::MI_B; use rustfs_utils::http::{ AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, - SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, contains_key_str, get_str, insert_str, + MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, contains_key_str, get_str, insert_str, }; use rustfs_zip::CompressionFormat; use s3s::dto::{ @@ -43,6 +47,7 @@ mod tests { PutObjectTaggingInput, QueueConfiguration, S3KeyFilter, Tag, Tagging, TopicConfiguration, }; use s3s::{S3, S3Error, S3ErrorCode, S3Request, s3_error}; + use std::collections::HashMap; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; fn build_request(input: T, method: Method) -> S3Request { @@ -409,9 +414,16 @@ mod tests { } #[test] - fn test_apply_default_lock_retention_metadata_applies_bucket_default() { - use std::collections::HashMap; + fn test_object_lock_peer_reload_failure_is_returned_to_the_client() { + let err = propagate_object_lock_peer_reload(Err(StorageError::other("peer reload failed"))) + .expect_err("peer reload failure must prevent a successful Object Lock update response"); + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert!(err.message().is_some_and(|message| message.contains("peer reload failed"))); + } + + #[test] + fn test_apply_default_lock_retention_metadata_applies_bucket_default() { let cfg = ObjectLockConfiguration { object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), rule: Some(ObjectLockRule { @@ -422,9 +434,9 @@ mod tests { }), }), }; - let mut metadata = HashMap::new(); + let mut metadata = std::collections::HashMap::new(); - assert!(apply_default_lock_retention_metadata(Some(cfg), &mut metadata)); + assert!(apply_default_lock_retention_metadata(Some(&cfg), &mut metadata)); assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER), Some(&"COMPLIANCE".to_string())); let retain_until = metadata .get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER) @@ -435,6 +447,8 @@ mod tests { .and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok()) .expect("default retention should write a valid internal timestamp"); assert!(retention_timestamp <= OffsetDateTime::now_utc()); + assert!(metadata.contains_key(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP}"))); + assert!(metadata.contains_key(&format!("{MINIO_INTERNAL_PREFIX}{SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP}"))); } #[test] @@ -456,7 +470,7 @@ mod tests { (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2030-01-01T00:00:00Z".to_string()), ]); - assert!(!apply_default_lock_retention_metadata(Some(cfg), &mut metadata)); + assert!(!apply_default_lock_retention_metadata(Some(&cfg), &mut metadata)); assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER), Some(&"GOVERNANCE".to_string())); assert_eq!( metadata.get(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER), @@ -466,18 +480,99 @@ mod tests { #[test] fn test_apply_default_lock_retention_metadata_ignores_bucket_without_default() { - use std::collections::HashMap; - let cfg = ObjectLockConfiguration { object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), rule: None, }; + let mut metadata = std::collections::HashMap::new(); + + assert!(!apply_default_lock_retention_metadata(Some(&cfg), &mut metadata)); + assert!(metadata.is_empty()); + } + + #[test] + fn test_apply_bucket_default_lock_retention_applies_authoritative_default() { + let config = ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: Some(ObjectLockRule { + default_retention: Some(DefaultRetention { + mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)), + days: Some(1), + years: None, + }), + }), + }; + let state = ObjectLockConfigState::Configured { + config, + updated_at: OffsetDateTime::now_utc(), + }; + let mut metadata = std::collections::HashMap::new(); + + apply_bucket_default_lock_retention("locked-bucket", &state, &mut metadata, false) + .expect("authoritative bucket default should be applied"); + + assert_eq!(metadata.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE")); + assert!(metadata.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER)); + } + + #[test] + fn test_apply_bucket_default_lock_retention_preserves_explicit_retention_path() { + let config = ObjectLockConfiguration { + object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)), + rule: Some(ObjectLockRule { + default_retention: Some(DefaultRetention { + mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)), + days: Some(1), + years: None, + }), + }), + }; + let state = ObjectLockConfigState::Configured { + config, + updated_at: OffsetDateTime::now_utc(), + }; let mut metadata = HashMap::new(); - assert!(!apply_default_lock_retention_metadata(Some(cfg), &mut metadata)); + apply_bucket_default_lock_retention("locked-bucket", &state, &mut metadata, true) + .expect("explicit retention should suppress only the bucket default"); + assert!(metadata.is_empty()); } + #[test] + fn test_apply_bucket_default_lock_retention_allows_confirmed_absence() { + let mut metadata = HashMap::new(); + + apply_bucket_default_lock_retention("legacy-bucket", &ObjectLockConfigState::ConfirmedAbsent, &mut metadata, false) + .expect("confirmed absence should keep legacy buckets writable"); + + assert!(metadata.is_empty()); + } + + #[test] + fn test_apply_bucket_default_lock_retention_rejects_fabricated_metadata_even_with_explicit_retention() { + let mut metadata = std::collections::HashMap::from([ + (AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string()), + (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2030-01-01T00:00:00Z".to_string()), + ]); + + let err = + apply_bucket_default_lock_retention("untrusted-bucket", &ObjectLockConfigState::Fabricated, &mut metadata, true) + .expect_err("fabricated metadata must fail closed before accepting explicit retention"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert_eq!(err.message(), Some("Failed to load Object Lock configuration")); + } + + #[test] + fn test_bucket_object_lock_config_load_error_fails_closed() { + let err = map_bucket_object_lock_config_state("corrupt-bucket", Err(StorageError::other("corrupt Object Lock metadata"))) + .expect_err("metadata read or parse failures must fail closed"); + + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert_eq!(err.message(), Some("Failed to load Object Lock configuration")); + } + #[test] fn test_remove_object_lock_retention_metadata_clears_only_retention_fields() { use std::collections::HashMap; diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index f427a3345..84509502e 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -61,20 +61,21 @@ pub(crate) use storage_api::{ TONIC_RPC_PREFIX, TierConfigMgr, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, WorkloadAdmissionSnapshotProviderRef, WriteEncryption, WritePlan, access_consumer, add_object_lock_years, all_local_disk, all_local_disk_path, check_retention_for_modification, collect_local_metrics, compression_metadata_value, contract, decode_tags, - decode_tags_to_map, delete_bucket_metadata_config, disk_drive_path, disk_endpoint, ecfs_consumer, ecfs_extend_consumer, - ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, ecstore_cluster, ecstore_compression, ecstore_config, - ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, ecstore_layout, ecstore_metrics, ecstore_notification, - ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, ecstore_storage, ecstore_tier, encode_tags, - find_local_disk_by_ref, get_bucket_accelerate_config, get_bucket_cors_config, get_bucket_logging_config, get_bucket_metadata, - get_bucket_notification_config, get_bucket_object_lock_config, get_bucket_policy_raw, get_bucket_replication_config, - get_bucket_request_payment_config, get_bucket_sse_config, get_bucket_website_config, get_local_server_property, - get_lock_acquire_timeout, get_public_access_block_config, head_prefix_consumer, helper_consumer, init_background_replication, - init_bucket_metadata_sys, init_ecstore_config, init_local_disks_with_instance_ctx, init_lock_clients, - is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class, - options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy, rpc_consumer, - runtime_sources_consumer, s3_api_consumer, serialize, table_catalog_path_hash, to_s3s_etag, - topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config, - try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader, + decode_tags_to_map, delete_bucket_metadata_config, delete_bucket_metadata_config_if_incarnation, disk_drive_path, + disk_endpoint, ecfs_consumer, ecfs_extend_consumer, ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, + ecstore_cluster, ecstore_compression, ecstore_config, ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, + ecstore_layout, ecstore_metrics, ecstore_notification, ecstore_rebalance, ecstore_rio, ecstore_rpc, ecstore_set_disk, + ecstore_storage, ecstore_tier, encode_tags, find_local_disk_by_ref, get_bucket_accelerate_config, get_bucket_cors_config, + get_bucket_logging_config, get_bucket_metadata, get_bucket_notification_config, get_bucket_object_lock_config, + get_bucket_policy_raw, get_bucket_replication_config, get_bucket_request_payment_config, get_bucket_sse_config, + get_bucket_website_config, get_local_server_property, get_lock_acquire_timeout, get_public_access_block_config, + head_prefix_consumer, helper_consumer, init_background_replication, init_bucket_metadata_sys, init_ecstore_config, + init_local_disks_with_instance_ctx, init_lock_clients, is_err_bucket_not_found, is_err_object_not_found, + is_err_version_not_found, is_valid_storage_class, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, + record_replication_proxy, rpc_consumer, runtime_sources_consumer, s3_api_consumer, serialize, table_catalog_path_hash, + to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config, + try_migrate_server_config, update_bucket_metadata_config, update_bucket_metadata_config_if_incarnation, verify_rpc_signature, + wrap_reader, }; #[cfg(test)] diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 1f44d2316..d6320ec46 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -583,6 +583,17 @@ pub(crate) fn namespace_reserved_user_metadata(metadata: &mut HashMap, key: &str, value: &str) { + let classified_user_metadata = USER_METADATA_PREFIXES + .iter() + .any(|prefix| key.strip_prefix(prefix).is_some_and(|suffix| !suffix.is_empty())); + if classified_user_metadata || SUPPORTED_HEADERS.iter().any(|header| key.eq_ignore_ascii_case(header)) { + return; + } + + metadata.insert(stored_user_metadata_key(key), value.to_owned()); +} + /// Extracts metadata from headers and returns it as a HashMap with object name for MIME type detection. pub fn extract_metadata_from_mime_with_object_name( headers: &HeaderMap, diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index 69010a02e..7fdeaea1a 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -14,6 +14,7 @@ use crate::server::RPC_PREFIX; use crate::storage::request_context::spawn_traced; +use crate::storage::storage_api::DiskError; use crate::storage::storage_api::rpc_consumer::http_service::{ DEFAULT_READ_BUFFER_SIZE, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, StorageDiskRpcExt as _, WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, find_local_disk_by_ref, sign_ns_scanner_capability, verify_rpc_signature, @@ -541,7 +542,7 @@ async fn handle_read_file(req: Request) -> Response { error = %e, "internode rpc request failed" ); - return response_with_status(StatusCode::INTERNAL_SERVER_ERROR, message); + return response_with_disk_error(&e, message); } }; @@ -1252,6 +1253,21 @@ fn response_with_status(status: StatusCode, message: impl Into) -> Respo .expect("failed to build rpc error response") } +fn response_with_disk_error(error: &DiskError, message: impl Into) -> Response { + let missing = match error { + DiskError::FileNotFound => Some(rustfs_rio::INTERNODE_FILE_NOT_FOUND), + DiskError::VolumeNotFound => Some(rustfs_rio::INTERNODE_VOLUME_NOT_FOUND), + _ => None, + }; + let mut response = response_with_status(StatusCode::INTERNAL_SERVER_ERROR, message); + if let Some(missing) = missing { + response + .headers_mut() + .insert(rustfs_rio::INTERNODE_DISK_ERROR_HEADER, HeaderValue::from_static(missing)); + } + response +} + fn internode_rpc_subsystem(operation: Option<&'static str>) -> &'static str { match operation { Some(INTERNODE_OPERATION_WALK_DIR) => LOG_SUBSYSTEM_DIRECTORY_WALK, @@ -1279,19 +1295,19 @@ fn put_file_stage_error_message(stage: &str, query: &PutFileQuery, err: &dyn std #[cfg(test)] mod tests { use super::{ - LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_NAMESPACE_SCANNER, LOG_SUBSYSTEM_ROUTING, - NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, + DiskError, LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_NAMESPACE_SCANNER, + LOG_SUBSYSTEM_ROUTING, NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PATH, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerQuery, PUT_FILE_STREAM_PATH, PutFileQuery, READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, append_walk_dir_completion, internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, ns_scanner_response_body, ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_stage_error_message, read_file_body_stream, - remote_scanner_claim_rejection, supports_walk_dir_stream_completion, validate_walk_dir_completion_request, - verify_internode_rpc_signature, verify_ns_scanner_body_digest, verify_walk_dir_body_digest, walk_dir_response_body, - write_body_chunks_to_writer, + remote_scanner_claim_rejection, response_with_disk_error, supports_walk_dir_stream_completion, + validate_walk_dir_completion_request, verify_internode_rpc_signature, verify_ns_scanner_body_digest, + verify_walk_dir_body_digest, walk_dir_response_body, write_body_chunks_to_writer, }; use bytes::Bytes; - use http::{HeaderMap, Method, StatusCode, Uri}; + use http::{HeaderMap, HeaderValue, Method, StatusCode, Uri}; use http_body_util::BodyExt; use rustfs_io_metrics::internode_metrics::{ INTERNODE_OPERATION_NS_SCANNER, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, @@ -1701,4 +1717,21 @@ mod tests { assert_eq!(out, b"hello"); } + + #[test] + fn read_file_error_response_marks_only_missing_disk_errors() { + for (error, expected) in [ + (DiskError::FileNotFound, rustfs_rio::INTERNODE_FILE_NOT_FOUND), + (DiskError::VolumeNotFound, rustfs_rio::INTERNODE_VOLUME_NOT_FOUND), + ] { + let response = response_with_disk_error(&error, error.to_string()); + assert_eq!( + response.headers().get(rustfs_rio::INTERNODE_DISK_ERROR_HEADER), + Some(&HeaderValue::from_static(expected)) + ); + } + + let response = response_with_disk_error(&DiskError::DiskAccessDenied, "permission denied"); + assert!(response.headers().get(rustfs_rio::INTERNODE_DISK_ERROR_HEADER).is_none()); + } } diff --git a/rustfs/src/storage/runtime_sources.rs b/rustfs/src/storage/runtime_sources.rs index 6c1d556b5..194a56645 100644 --- a/rustfs/src/storage/runtime_sources.rs +++ b/rustfs/src/storage/runtime_sources.rs @@ -14,6 +14,7 @@ use crate::config::RustFSBufferConfig; use crate::runtime_sources as root_runtime_sources; +use crate::storage::storage_api::NotificationSys; use crate::storage::storage_api::runtime_sources_consumer::ECStore; use rustfs_credentials::Credentials; use rustfs_iam::{error::Result as IamResult, store::object::ObjectStore, sys::IamSys}; @@ -26,6 +27,8 @@ use rustfs_lock::LockClient; use std::sync::Arc; pub(crate) use crate::runtime_sources::{AppContext, ServerContextSlot}; +#[cfg(test)] +pub(crate) use crate::runtime_sources::{IamInterface, KmsInterface}; pub(crate) fn current_app_context() -> Option> { root_runtime_sources::current_app_context() @@ -39,6 +42,10 @@ pub(crate) fn current_object_store_handle_for_context(context: Option<&AppContex root_runtime_sources::current_object_store_handle_for_context(context) } +pub(crate) fn current_notification_system_for_context(context: Option<&AppContext>) -> Option> { + root_runtime_sources::current_notification_system_for_context(context) +} + pub(crate) fn current_replication_stats_handle_for_context( context: Option<&AppContext>, ) -> Option> { diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index e0c292792..d90e106df 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -49,10 +49,14 @@ pub(crate) mod contract { pub(crate) mod bucket { pub(crate) use super::super::storage_contracts::{ - BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, + BUCKET_LIFECYCLE_LOCK_OBJECT, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, }; } + pub(crate) mod namespace { + pub(crate) use super::super::storage_contracts::NamespaceLocking; + } + pub(crate) mod list { pub(crate) use super::super::storage_contracts::{ListObjectVersionsInfo, ListObjectsV2Info, ListOperations}; } @@ -61,7 +65,7 @@ pub(crate) mod contract { pub(crate) const MAX_MULTIPART_PART_NUMBER: i32 = 10000; pub(crate) use super::super::storage_contracts::{ListMultipartsInfo, ListPartsInfo}; #[cfg(test)] - pub(crate) use super::super::storage_contracts::{MultipartInfo, PartInfo}; + pub(crate) use super::super::storage_contracts::{MultipartInfo, MultipartOperations, PartInfo}; } pub(crate) mod object { @@ -89,9 +93,9 @@ pub(crate) type StorageObjectToDelete = contract::object::ObjectToDelete; pub(crate) type StoragePutObjReader = super::PutObjReader; pub(crate) use super::ecfs_extend::{ RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, get_buffer_size_opt_in, - get_validated_store, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, - process_lambda_configurations, process_queue_configurations, process_topic_configurations, - remove_object_lock_metadata_for_copy, validate_bucket_exists, validate_bucket_object_lock_enabled, + get_validated_store, load_bucket_object_lock_config_state, parse_object_lock_legal_hold, parse_object_lock_retention, + parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations, + remove_object_lock_metadata_for_copy, validate_bucket_exists, validate_bucket_object_lock_enabled_state, validate_list_object_unordered_with_delimiter, validate_object_key, wrap_response_with_cors, }; pub(crate) use super::sse::{ @@ -102,15 +106,10 @@ pub(crate) use super::sse::{ pub(crate) mod access_consumer { pub(crate) use super::super::access::{ - PostObjectRequestMarker, ReqInfo, authorize_request, has_bypass_governance_header, recursive_force_delete_is_authorized, - replication_request_authorized, req_info_mut, req_info_ref, + PostObjectRequestMarker, ReqInfo, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, + authorize_request, bucket_config_mutation_incarnation, has_bypass_governance_header, load_bucket_generation_from_store, + recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref, }; - - pub(crate) mod contract { - pub(crate) mod bucket { - pub(crate) use super::super::super::contract::bucket::BucketOperations; - } - } } pub(crate) mod concurrency_consumer { @@ -184,7 +183,7 @@ pub(crate) mod options_consumer { extract_metadata_from_mime, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts_with_replication_authorization, get_content_sha256_with_query, get_opts, namespace_reserved_user_metadata, normalize_content_encoding_for_storage, parse_copy_source_range, - put_opts_with_replication_authorization, validate_archive_content_encoding, + preserve_unclassified_user_metadata, put_opts_with_replication_authorization, validate_archive_content_encoding, }; pub(crate) mod contract { @@ -261,7 +260,10 @@ pub(crate) mod rpc_consumer { pub(crate) mod runtime_sources_consumer { pub(crate) type ECStore = super::ECStore; pub(crate) type EndpointServerPools = super::EndpointServerPools; + pub(crate) type ServerContextSlot = super::ServerContextSlot; pub(crate) use crate::storage::runtime_sources; + #[cfg(test)] + pub(crate) use crate::storage::runtime_sources::{AppContext, IamInterface, KmsInterface}; } pub(crate) mod heal_control_startup_consumer { @@ -334,8 +336,9 @@ pub(crate) mod sse_consumer { }; pub(crate) use super::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, - authorize_sse_kms_object_read, extract_server_side_encryption_from_headers, get_buffer_size_opt_in, sse_decryption, - sse_encryption, sse_prepare_encryption, + authorize_sse_kms_object_read, extract_server_side_encryption_from_headers, get_buffer_size_opt_in, + load_bucket_object_lock_config_state, sse_decryption, sse_encryption, sse_prepare_encryption, + validate_bucket_object_lock_enabled_state, }; } @@ -844,7 +847,7 @@ pub(crate) async fn reconcile_bucket_resync_target_intents(buckets: &[String]) - }; for bucket in buckets { - let _transaction_guard = ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await?; + let transaction_guard = ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await?; let status = pool.get_bucket_resync_status(bucket).await?; if status.targets_map.is_empty() { continue; @@ -859,7 +862,7 @@ pub(crate) async fn reconcile_bucket_resync_target_intents(buckets: &[String]) - continue; } let encoded = serde_json::to_vec(&targets).map_err(Error::other)?; - ecstore_bucket::metadata_sys::update_bucket_targets_under_transaction_lock(bucket, encoded).await?; + ecstore_bucket::metadata_sys::update_bucket_targets_under_transaction_lock(&transaction_guard, bucket, encoded).await?; } Ok(()) @@ -1396,7 +1399,18 @@ pub(crate) fn get_global_bucket_metadata_sys() -> Option Result { - let updated_at = ecstore_bucket::metadata_sys::delete(bucket, config_file).await?; + delete_bucket_metadata_config_if_incarnation(bucket, config_file, None).await +} + +pub(crate) async fn delete_bucket_metadata_config_if_incarnation( + bucket: &str, + config_file: &str, + expected_incarnation_id: Option, +) -> Result { + let updated_at = match expected_incarnation_id { + Some(incarnation_id) => ecstore_bucket::metadata_sys::delete_if_incarnation(bucket, config_file, incarnation_id).await?, + None => ecstore_bucket::metadata_sys::delete(bucket, config_file).await?, + }; record_scanner_maintenance_config_change(bucket, config_file); Ok(updated_at) } @@ -1490,17 +1504,37 @@ pub(crate) async fn update_bucket_metadata_config( config_file: &str, data: Vec, ) -> Result { - let updated_at = ecstore_bucket::metadata_sys::update(bucket, config_file, data).await?; + update_bucket_metadata_config_if_incarnation(bucket, config_file, data, None).await +} + +pub(crate) async fn update_bucket_metadata_config_if_incarnation( + bucket: &str, + config_file: &str, + data: Vec, + expected_incarnation_id: Option, +) -> Result { + let updated_at = match expected_incarnation_id { + Some(incarnation_id) => { + ecstore_bucket::metadata_sys::update_if_incarnation(bucket, config_file, data, incarnation_id).await? + } + None => ecstore_bucket::metadata_sys::update(bucket, config_file, data).await?, + }; record_scanner_maintenance_config_change(bucket, config_file); Ok(updated_at) } -pub(crate) async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result { +pub(crate) async fn acquire_bucket_metadata_transaction_lock( + bucket: &str, +) -> Result { ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await } -pub(crate) async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Vec) -> Result { - ecstore_bucket::metadata_sys::update_bucket_targets_under_transaction_lock(bucket, data).await +pub(crate) async fn update_bucket_targets_under_transaction_lock( + guard: &ecstore_bucket::metadata_sys::BucketMetadataMutationGuard, + bucket: &str, + data: Vec, +) -> Result { + ecstore_bucket::metadata_sys::update_bucket_targets_under_transaction_lock(guard, bucket, data).await } fn record_scanner_maintenance_config_change(bucket: &str, config_file: &str) { @@ -1568,10 +1602,6 @@ pub(crate) fn is_err_version_not_found(err: &Error) -> bool { ecstore_error::is_err_version_not_found(err) } -pub(crate) fn is_all_buckets_not_found(errs: &[Option]) -> bool { - ecstore_disk::error_reduce::is_all_buckets_not_found(errs) -} - pub(crate) fn get_global_lock_client() -> Option> { ecstore_runtime::global_lock_client() } diff --git a/rustfs/src/table_catalog.rs b/rustfs/src/table_catalog.rs index e8a8f4ff2..0a1c79bcf 100644 --- a/rustfs/src/table_catalog.rs +++ b/rustfs/src/table_catalog.rs @@ -20837,17 +20837,27 @@ mod tests { #[test] fn object_mutation_entrypoints_call_reserved_prefix_guard() { let source = include_str!("app/object_usecase.rs"); + let delete_object = source + .split_once("pub async fn execute_delete_object") + .and_then(|(_, remainder)| remainder.split_once("pub async fn execute_head_object")) + .map(|(delete_object, _)| delete_object) + .expect("delete object entrypoint should remain in the object usecase"); + for expected in [ "validate_object_key(&key, request_method_name)?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", "validate_object_key(&key, \"COPY (dest)\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", "if let Err(err) = validate_table_catalog_object_mutation(&bucket, &obj_id.key).await", - "validate_object_key(&key, \"DELETE\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", "validate_table_catalog_object_mutation(&bucket, &object).await?;", "validate_object_key(&key, \"PUT\")?;\n validate_table_catalog_object_mutation(&bucket, &key).await?;", "validate_table_catalog_object_mutation(&bucket, &fpath).await?;", ] { assert!(source.contains(expected), "missing object mutation guard: {expected}"); } + assert!( + delete_object.contains("validate_object_key(&key, \"DELETE\")?;") + && delete_object.contains("validate_table_catalog_object_mutation(&bucket, &key).await?;"), + "delete object entrypoint must validate the object key and reserved catalog prefix" + ); } #[test] diff --git a/rustfs/tests/common/mod.rs b/rustfs/tests/common/mod.rs new file mode 100644 index 000000000..4171cec79 --- /dev/null +++ b/rustfs/tests/common/mod.rs @@ -0,0 +1,25 @@ +use std::future::Future; + +const EMBEDDED_TEST_STACK_SIZE: usize = 8 * 1024 * 1024; + +pub fn run_embedded_test(test: F) +where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future + 'static, +{ + let handle = std::thread::Builder::new() + .name("embedded-test".to_string()) + .stack_size(EMBEDDED_TEST_STACK_SIZE) + .spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build embedded test runtime") + .block_on(Box::pin(test())); + }) + .expect("spawn embedded test thread"); + + if let Err(payload) = handle.join() { + std::panic::resume_unwind(payload); + } +} diff --git a/rustfs/tests/embedded_deferred_iam_test.rs b/rustfs/tests/embedded_deferred_iam_test.rs index 34919ea35..a63ef9c48 100644 --- a/rustfs/tests/embedded_deferred_iam_test.rs +++ b/rustfs/tests/embedded_deferred_iam_test.rs @@ -20,6 +20,8 @@ use rustfs_config::{ENV_TEST_IAM_FAIL_INIT_ATTEMPTS, ENV_TEST_IAM_RETRY_INTERVAL use std::time::Duration; use temp_env::async_with_vars; +mod common; + fn response_preview(body: &str) -> String { body.chars().take(512).collect() } @@ -37,8 +39,13 @@ fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client { } #[cfg(debug_assertions)] -#[tokio::test] -async fn test_embedded_server_recovers_after_deferred_iam_bootstrap() { +#[test] +fn test_embedded_server_recovers_after_deferred_iam_bootstrap() { + common::run_embedded_test(test_embedded_server_recovers_after_deferred_iam_bootstrap_body); +} + +#[cfg(debug_assertions)] +async fn test_embedded_server_recovers_after_deferred_iam_bootstrap_body() { async_with_vars( [ (ENV_TEST_IAM_FAIL_INIT_ATTEMPTS, Some("1")), diff --git a/rustfs/tests/embedded_multi_instance_test.rs b/rustfs/tests/embedded_multi_instance_test.rs index e42130b22..d19e8b01f 100644 --- a/rustfs/tests/embedded_multi_instance_test.rs +++ b/rustfs/tests/embedded_multi_instance_test.rs @@ -28,6 +28,8 @@ use reqwest::StatusCode; #[cfg(feature = "e2e-test-hooks")] use rustfs::embedded::pause_embedded_startup_after_http_bind; use rustfs::embedded::{RustFSServerBuilder, find_available_port}; + +mod common; #[cfg(feature = "e2e-test-hooks")] use sha2::{Digest, Sha256}; #[cfg(feature = "e2e-test-hooks")] @@ -111,8 +113,12 @@ fn signed_admin_request( // startup pipeline lifts; a follow-up will widen the request path to route // per-server so the two servers can also serve different data planes end-to- // end without the shared-IAM caveat. -#[tokio::test] -async fn two_embedded_servers_start_and_shutdown_independently() { +#[test] +fn two_embedded_servers_start_and_shutdown_independently() { + common::run_embedded_test(two_embedded_servers_start_and_shutdown_independently_body); +} + +async fn two_embedded_servers_start_and_shutdown_independently_body() { let port_a = match find_available_port() { Ok(port) => port, Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, @@ -181,8 +187,12 @@ async fn two_embedded_servers_start_and_shutdown_independently() { // rejects the other's) AND data plane (each server's buckets/objects are // invisible to the other; each lists/creates/deletes only on its own disks // and bucket-metadata system). -#[tokio::test] -async fn two_embedded_servers_isolate_auth_and_data_planes() { +#[test] +fn two_embedded_servers_isolate_auth_and_data_planes() { + common::run_embedded_test(two_embedded_servers_isolate_auth_and_data_planes_body); +} + +async fn two_embedded_servers_isolate_auth_and_data_planes_body() { let port_a = match find_available_port() { Ok(port) => port, Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, diff --git a/rustfs/tests/embedded_notification_lifecycle_test.rs b/rustfs/tests/embedded_notification_lifecycle_test.rs index 1f33e5fcd..df6b0a3e7 100644 --- a/rustfs/tests/embedded_notification_lifecycle_test.rs +++ b/rustfs/tests/embedded_notification_lifecycle_test.rs @@ -18,6 +18,8 @@ use aws_sdk_s3::{Client, Config}; use rustfs::embedded::{RustFSServerBuilder, find_available_port}; use rustfs_notify::{NotificationRuntimeState, notification_system}; +mod common; + fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client { let credentials = Credentials::new(access_key, secret_key, None, None, "test"); let config = Config::builder() @@ -30,8 +32,12 @@ fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client { Client::from_conf(config) } -#[tokio::test] -async fn notification_runtime_stays_enabled_until_the_last_embedded_owner_drains() { +#[test] +fn notification_runtime_stays_enabled_until_the_last_embedded_owner_drains() { + common::run_embedded_test(notification_runtime_stays_enabled_until_the_last_embedded_owner_drains_body); +} + +async fn notification_runtime_stays_enabled_until_the_last_embedded_owner_drains_body() { temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"))], async { let port_a = match find_available_port() { Ok(port) => port, diff --git a/rustfs/tests/embedded_test.rs b/rustfs/tests/embedded_test.rs index f4dac3570..42547ef18 100644 --- a/rustfs/tests/embedded_test.rs +++ b/rustfs/tests/embedded_test.rs @@ -22,6 +22,8 @@ use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::{Client, Config}; use rustfs::embedded::{RustFSServerBuilder, find_available_port}; +mod common; + /// Helper: create an S3 client pointed at the embedded server. fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client { let creds = Credentials::new(access_key, secret_key, None, None, "test"); @@ -35,8 +37,12 @@ fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client { Client::from_conf(config) } -#[tokio::test] -async fn test_embedded_server_basic_s3_operations() { +#[test] +fn test_embedded_server_basic_s3_operations() { + common::run_embedded_test(test_embedded_server_basic_s3_operations_body); +} + +async fn test_embedded_server_basic_s3_operations_body() { // 1. Pick a free port and start the embedded server. let port = match find_available_port() { Ok(port) => port,