mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-17 18:27:49 +00:00
fix(ecstore): reconcile object cleanup receipts (#6077)
* fix(s3): keep multipart completion publication owned Co-Authored-By: heihutu <heihutu@gmail.com> * fix(s3): keep put publication owned Co-Authored-By: heihutu <heihutu@gmail.com> * chore(app): route multipart context through facade Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): gate object transaction fencing Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): fence object transaction epochs Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): reconcile old data cleanup receipts Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
+208
-104
@@ -2989,6 +2989,11 @@ struct PutObjectChecksums {
|
||||
crc64nvme: Option<String>,
|
||||
}
|
||||
|
||||
struct PutObjectCommitResult {
|
||||
obj_info: ObjectInfo,
|
||||
put_versioned: bool,
|
||||
}
|
||||
|
||||
fn normalize_delete_objects_version_id(
|
||||
version_id: Option<String>,
|
||||
) -> std::result::Result<(Option<String>, Option<Uuid>), String> {
|
||||
@@ -5932,7 +5937,7 @@ impl DefaultObjectUsecase {
|
||||
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_encryption_prepare", encryption_stage_start);
|
||||
|
||||
let mut reader = PutObjReader::new(reader);
|
||||
let reader = PutObjReader::new(reader);
|
||||
|
||||
let mt2 = metadata.clone();
|
||||
opts.user_defined.extend(metadata);
|
||||
@@ -6005,97 +6010,145 @@ impl DefaultObjectUsecase {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let object_traffic_progress = object_traffic_health
|
||||
.as_deref()
|
||||
.and_then(ObjectTrafficHealth::track_write_storage);
|
||||
let store_put_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let (obj_info, backfilled_old_current_size) = match store
|
||||
.put_object_with_old_current_size(&bucket, &key, &mut reader, &opts)
|
||||
.await
|
||||
.map_err(ApiError::from)
|
||||
{
|
||||
Ok(obj_info) => {
|
||||
store_put_watchdog.cancel();
|
||||
debug!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
event = EVENT_PUT_OBJECT_STORE_RETURNED,
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
request_id = %request_id,
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
put_path = put_path,
|
||||
object_size = actual_size,
|
||||
duration_ms = start_time.elapsed().as_millis() as u64,
|
||||
result = "success",
|
||||
"PutObject store write returned"
|
||||
);
|
||||
obj_info
|
||||
let put_commit = spawn_traced_join({
|
||||
let store = Arc::clone(&store);
|
||||
let bucket = bucket.clone();
|
||||
let key = key.clone();
|
||||
let opts = opts.clone();
|
||||
let cache_adapter = cache_adapter.clone();
|
||||
let request_id = request_id.clone();
|
||||
let put_path = put_path.to_string();
|
||||
async move {
|
||||
let object_traffic_progress = object_traffic_health
|
||||
.as_deref()
|
||||
.and_then(ObjectTrafficHealth::track_write_storage);
|
||||
let mut reader = reader;
|
||||
let store_put_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let (obj_info, backfilled_old_current_size) = match store
|
||||
.put_object_with_old_current_size(&bucket, &key, &mut reader, &opts)
|
||||
.await
|
||||
.map_err(ApiError::from)
|
||||
{
|
||||
Ok(obj_info) => {
|
||||
store_put_watchdog.cancel();
|
||||
debug!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
event = EVENT_PUT_OBJECT_STORE_RETURNED,
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
request_id = %request_id,
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
put_path = %put_path,
|
||||
object_size = actual_size,
|
||||
duration_ms = start_time.elapsed().as_millis() as u64,
|
||||
result = "success",
|
||||
"PutObject store write returned"
|
||||
);
|
||||
obj_info
|
||||
}
|
||||
Err(err) => {
|
||||
store_put_watchdog.cancel();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start);
|
||||
warn!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
event = EVENT_PUT_OBJECT_STORE_RETURNED,
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
request_id = %request_id,
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
put_path = %put_path,
|
||||
object_size = actual_size,
|
||||
duration_ms = start_time.elapsed().as_millis() as u64,
|
||||
result = "error",
|
||||
error = %err,
|
||||
"PutObject store write returned"
|
||||
);
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start);
|
||||
drop(object_traffic_progress);
|
||||
#[cfg(test)]
|
||||
wait_for_put_post_store_test_hook(&bucket).await;
|
||||
|
||||
let post_store_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
|
||||
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await;
|
||||
|
||||
let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
|
||||
// Fast in-memory update for immediate quota and admin usage consistency.
|
||||
// The previous current size comes from the prelookup when it ran,
|
||||
// otherwise from the rename_data backfill (rustfs/backlog#1009); the
|
||||
// backfill reproduces the lookup's observation bit for bit (latest
|
||||
// version's ObjectInfo.size — 0 for a delete-marker latest — or
|
||||
// not-found → None).
|
||||
match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size))
|
||||
{
|
||||
Some(previous_current_size) => {
|
||||
if put_versioned {
|
||||
record_bucket_object_version_write_memory(
|
||||
&bucket,
|
||||
previous_current_size,
|
||||
obj_info.size.max(0) as u64,
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Neither source could determine the previous state (peers
|
||||
// predating the backfill field during a rolling upgrade, or
|
||||
// sub-quorum metadata divergence). Record the components that
|
||||
// are correct regardless; the next authoritative scanner
|
||||
// refresh replaces the in-memory numbers.
|
||||
debug!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
put_versioned,
|
||||
"put_object old-size backfill unknown; recording degraded usage delta"
|
||||
);
|
||||
record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, put_versioned)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
if dsc.replicate_any() {
|
||||
schedule_object_replication(obj_info.clone(), store, dsc).await;
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket(&bucket);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start);
|
||||
|
||||
let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let manager = get_capacity_manager();
|
||||
manager.record_write_operation().await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_capacity_update", capacity_update_stage_start);
|
||||
|
||||
Ok::<_, S3Error>(PutObjectCommitResult { obj_info, put_versioned })
|
||||
}
|
||||
});
|
||||
let PutObjectCommitResult { obj_info, put_versioned } = match put_commit.await {
|
||||
Ok(Ok(result)) => result,
|
||||
Ok(Err(err)) => {
|
||||
let result: S3Result<S3Response<PutObjectOutput>> = Err(err);
|
||||
put_request_guard.finish_err();
|
||||
let _ = helper.complete(&result);
|
||||
return result;
|
||||
}
|
||||
Err(err) => {
|
||||
store_put_watchdog.cancel();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start);
|
||||
warn!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
event = EVENT_PUT_OBJECT_STORE_RETURNED,
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
request_id = %request_id,
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
put_path = put_path,
|
||||
object_size = actual_size,
|
||||
duration_ms = start_time.elapsed().as_millis() as u64,
|
||||
result = "error",
|
||||
error = %err,
|
||||
"PutObject store write returned"
|
||||
);
|
||||
let result: S3Result<S3Response<PutObjectOutput>> = Err(err.into());
|
||||
let result: S3Result<S3Response<PutObjectOutput>> = Err(S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("put object commit owner task failed: {err}"),
|
||||
));
|
||||
put_request_guard.finish_err();
|
||||
let _ = helper.complete(&result);
|
||||
return result;
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_store_put", store_put_stage_start);
|
||||
drop(object_traffic_progress);
|
||||
#[cfg(test)]
|
||||
wait_for_put_post_store_test_hook(&bucket).await;
|
||||
|
||||
let post_store_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
|
||||
let _ = invalidate_object_data_cache_after_put_success(&cache_adapter, &bucket, &key).await;
|
||||
|
||||
let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
|
||||
// Fast in-memory update for immediate quota and admin usage consistency.
|
||||
// The previous current size comes from the prelookup when it ran,
|
||||
// otherwise from the rename_data backfill (rustfs/backlog#1009); the
|
||||
// backfill reproduces the lookup's observation bit for bit (latest
|
||||
// version's ObjectInfo.size — 0 for a delete-marker latest — or
|
||||
// not-found → None).
|
||||
match prelookup_previous_current_size.or_else(|| previous_current_size_from_backfill(backfilled_old_current_size)) {
|
||||
Some(previous_current_size) => {
|
||||
if put_versioned {
|
||||
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
||||
} else {
|
||||
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Neither source could determine the previous state (peers
|
||||
// predating the backfill field during a rolling upgrade, or
|
||||
// sub-quorum metadata divergence). Record the components that
|
||||
// are correct regardless; the next authoritative scanner
|
||||
// refresh replaces the in-memory numbers.
|
||||
debug!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
bucket = %bucket,
|
||||
key = %key,
|
||||
put_versioned,
|
||||
"put_object old-size backfill unknown; recording degraded usage delta"
|
||||
);
|
||||
record_bucket_object_write_unknown_previous_memory(&bucket, obj_info.size.max(0) as u64, put_versioned).await;
|
||||
}
|
||||
}
|
||||
|
||||
let raw_version = obj_info.version_id.map(|v| v.to_string());
|
||||
|
||||
@@ -6110,17 +6163,6 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let expiration = resolve_put_object_expiration(&bucket, &obj_info).await;
|
||||
|
||||
// Reuse the single replication decision computed before commit (see `dsc`
|
||||
// above) so the pending metadata persisted with the object and the
|
||||
// post-commit schedule always derive from the same immutable decision.
|
||||
// Recomputing here would repeat the versioning/config/target traversal and,
|
||||
// worse, allow a replication-config hot update between the two phases to
|
||||
// produce a pending-without-schedule or schedule-without-pending divergence
|
||||
// (https://github.com/rustfs/backlog/issues/1320).
|
||||
if dsc.replicate_any() {
|
||||
schedule_object_replication(obj_info.clone(), store, dsc).await;
|
||||
}
|
||||
|
||||
let mut checksums = PutObjectChecksums {
|
||||
crc32: input.checksum_crc32,
|
||||
crc32c: input.checksum_crc32c,
|
||||
@@ -6159,14 +6201,6 @@ impl DefaultObjectUsecase {
|
||||
inject_additional_checksum_headers(&mut response.headers, &put_extra_checksum_headers);
|
||||
let result = Ok(response);
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_bucket(&bucket);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start);
|
||||
|
||||
// Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead)
|
||||
let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let manager = get_capacity_manager();
|
||||
manager.record_write_operation().await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_capacity_update", capacity_update_stage_start);
|
||||
|
||||
// Record PutObject metrics via zero-copy-metrics
|
||||
{
|
||||
@@ -11507,6 +11541,76 @@ mod tests {
|
||||
assert!(!recovered.write_stalled);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn cancelled_put_request_completes_post_commit_publication() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (store, context) = real_cold_fill_test_context().await;
|
||||
let bucket = format!("put-owner-tail-{}", Uuid::new_v4());
|
||||
let object = "object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("PUT owner-tail bucket must be created");
|
||||
|
||||
let old_body = Bytes::from_static(b"old body that must be invalidated");
|
||||
let old_info = put_real_cold_fill_object(&store, &bucket, object, &old_body).await;
|
||||
let adapter = context.object_data_cache();
|
||||
let old_plan = real_cold_fill_plan(&adapter, &bucket, object, &old_info);
|
||||
|
||||
let post_store_entered = Arc::new(tokio::sync::Barrier::new(2));
|
||||
let post_store_resume = Arc::new(tokio::sync::Barrier::new(2));
|
||||
install_put_post_store_test_hook(bucket.clone(), Arc::clone(&post_store_entered), Arc::clone(&post_store_resume));
|
||||
|
||||
let payload = Bytes::from_static(b"published despite caller cancellation");
|
||||
let put_input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(StreamingBlob::from(s3s::Body::from(payload.clone()))))
|
||||
.content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
|
||||
.build()
|
||||
.expect("PUT input must build");
|
||||
let put_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
let put = tokio::spawn(async move {
|
||||
put_usecase
|
||||
.execute_put_object(&FS::new(), build_request(put_input, Method::PUT))
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(10), post_store_entered.wait())
|
||||
.await
|
||||
.expect("PUT must reach the post-store owner-tail hook");
|
||||
assert_eq!(
|
||||
adapter.fill_body(&old_plan, old_body.clone()).await,
|
||||
rustfs_object_data_cache::ObjectDataCacheFillResult::Inserted,
|
||||
"test must republish the old body while the owner tail is paused"
|
||||
);
|
||||
put.abort();
|
||||
post_store_resume.wait().await;
|
||||
let _ = put.await.expect_err("outer request task must be cancelled");
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
if matches!(
|
||||
adapter.lookup_body(&old_plan).await,
|
||||
rustfs_object_data_cache::ObjectDataCacheLookup::Miss
|
||||
) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("post-commit owner tail must invalidate stale body cache after caller cancellation");
|
||||
|
||||
let recovered = store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("cancelled request's owned commit must still publish the object");
|
||||
assert_eq!(recovered.size, i64::try_from(payload.len()).expect("test payload length must fit i64"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_progress_tracks_zero_byte_and_zero_copy_put_lock_waits() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
Reference in New Issue
Block a user