fix(ecstore): defer delete cleanup for snapshot reads (#5408)

* feat(ecstore): add local snapshot leases * feat(ecstore): add remote snapshot lease RPCs * feat(ecstore): protect streaming GETs with snapshot leases * fix(ecstore): defer version cleanup for snapshot reads * fix(ecstore): cover batch snapshot cleanup safely * fix(e2e): stub snapshot lease RPCs in lock mock * fix(e2e): stub snapshot lease RPCs in lock mock * fix(ecstore): bind deferred delete cleanup intents * fix(rpc): keep snapshot lease checks CI-compatible
2026-08-22 04:16:38 +00:00 · 2026-07-29 22:23:01 +08:00
parent c195b18fb8
commit cc24ef173c
3 changed files with 833 additions and 25 deletions
@@ -4698,6 +4698,7 @@ mod tests {
    use crate::bucket::replication::{replication_statuses_map, version_purge_statuses_map};
    use crate::disk::CHECK_PART_UNKNOWN;
    use crate::disk::CHECK_PART_VOLUME_NOT_FOUND;
+    use crate::disk::DataDirDeleteStatus;
    use crate::disk::DiskOption;
    use crate::disk::RUSTFS_META_BUCKET;
    use crate::disk::RUSTFS_META_TMP_BUCKET;
@@ -6403,6 +6404,62 @@ mod tests {
        assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved");
    }

+    #[tokio::test]
+    async fn reclaim_orphan_data_dirs_recovers_deferred_cleanup_after_restart() {
+        let (dir, disk) = make_single_local_disk().await;
+        let live = Uuid::new_v4();
+        let orphan = Uuid::new_v4();
+        let object_dir = dir.path().join("bucket").join("obj");
+        write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
+        fs::create_dir_all(object_dir.join(live.to_string()))
+            .await
+            .expect("live data dir should be created");
+        let orphan_path = format!("obj/{orphan}");
+        disk.write_all("bucket", &format!("{orphan_path}/part.1"), Bytes::from_static(b"stale"))
+            .await
+            .expect("orphan part should be written");
+
+        let _token = disk
+            .acquire_snapshot_lease("bucket", &orphan_path)
+            .await
+            .expect("snapshot lease should be acquired");
+        assert_eq!(
+            disk.delete_data_dir(
+                "bucket",
+                &orphan_path,
+                DeleteOptions {
+                    recursive: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("cleanup should be deferred"),
+            DataDirDeleteStatus::Deferred
+        );
+        drop(disk);
+
+        let endpoint =
+            Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
+        let restarted = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("disk should restart");
+        let set = make_set_disks_with(vec![Some(restarted)]).await;
+        let removed = set
+            .reclaim_orphan_data_dirs("bucket", "obj")
+            .await
+            .expect("restart reclaim should succeed");
+
+        assert_eq!(removed, 1, "the deferred orphan should be reclaimed after restart");
+        assert!(object_dir.join(live.to_string()).exists(), "referenced data dir must be preserved");
+        assert!(!object_dir.join(orphan.to_string()).exists(), "deferred orphan must be removed");
+    }
+
    // Nothing to reclaim when every physical data dir is still referenced.
    #[tokio::test]
    async fn reclaim_orphan_data_dirs_keeps_referenced_dir() {
@@ -6441,6 +6498,16 @@ mod tests {
        fs::write(object_dir.join(stray.to_string()).join("part.1"), b"data")
            .await
            .expect("part should be written");
+        fs::write(
+            object_dir.join(stray.to_string()).join(format!(
+                "{}{}",
+                crate::disk::local::RESERVED_DELETE_DATA_DIR_MARKER_PREFIX,
+                Uuid::new_v4()
+            )),
+            [],
+        )
+        .await
+        .expect("pre-commit delete reservation should be written");

        let set = make_set_disks_with(vec![Some(disk)]).await;
        let removed = set
@@ -6455,6 +6522,36 @@ mod tests {
        );
    }

+    #[tokio::test]
+    async fn reclaim_orphan_data_dirs_recovers_committed_delete_marker_without_meta() {
+        let (dir, disk) = make_single_local_disk().await;
+        let stale = Uuid::new_v4();
+        let transaction = Uuid::new_v4();
+        let object_dir = dir.path().join("bucket").join("obj");
+        let stale_dir = object_dir.join(stale.to_string());
+        fs::create_dir_all(&stale_dir)
+            .await
+            .expect("committed stale data dir should be created");
+        fs::write(stale_dir.join("part.1"), b"stale")
+            .await
+            .expect("stale part should be written");
+        fs::write(
+            stale_dir.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
+            [],
+        )
+        .await
+        .expect("committed delete marker should be written");
+
+        let set = make_set_disks_with(vec![Some(disk)]).await;
+        let removed = set
+            .reclaim_orphan_data_dirs("bucket", "obj")
+            .await
+            .expect("upgrade reclaim should succeed");
+
+        assert_eq!(removed, 1, "the committed delete residue should be reclaimed");
+        assert!(!stale_dir.exists(), "the committed stale data dir should be removed");
+    }
+
    // Cross-replica union: a data dir referenced by ANOTHER disk's xl.meta must be
    // kept even where the local replica does not name it.
    #[tokio::test]
@@ -9774,6 +9871,113 @@ mod tests {
        .await;
    }

+    #[tokio::test(flavor = "multi_thread")]
+    #[serial]
+    async fn streaming_get_snapshot_survives_concurrent_delete() {
+        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
+            let set_disks = make_local_bucket_test_set_disks().await;
+            let bucket = "snapshot-streaming-delete";
+            let object = "object";
+            let body = vec![0x41; 2 * 1024 * 1024];
+            let opts = ObjectOptions::default();
+
+            set_disks
+                .make_bucket(bucket, &MakeBucketOptions::default())
+                .await
+                .expect("bucket should be created");
+            let mut reader = PutObjReader::from_vec(body.clone());
+            set_disks
+                .put_object(bucket, object, &mut reader, &opts)
+                .await
+                .expect("object should be written");
+
+            let mut snapshot = set_disks
+                .get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
+                .await
+                .expect("snapshot reader should open");
+            let delete_set = Arc::clone(&set_disks);
+            let delete_opts = opts.clone();
+            let delete = tokio::spawn(async move { delete_set.delete_object(bucket, object, delete_opts).await });
+            tokio::time::timeout(Duration::from_secs(30), delete)
+                .await
+                .expect("delete should not wait for the response body")
+                .expect("delete task should join")
+                .expect("delete should succeed");
+
+            let mut restored = Vec::new();
+            snapshot
+                .stream
+                .read_to_end(&mut restored)
+                .await
+                .expect("leased snapshot should remain readable after delete");
+            assert_eq!(restored, body);
+            let err = match set_disks
+                .get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
+                .await
+            {
+                Ok(_) => panic!("a new read must not observe the deleted object"),
+                Err(err) => err,
+            };
+            assert!(is_err_object_not_found(&err));
+        })
+        .await;
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    #[serial]
+    async fn streaming_get_snapshot_survives_concurrent_delete_objects() {
+        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
+            let set_disks = make_local_bucket_test_set_disks().await;
+            let bucket = "snapshot-streaming-delete-objects";
+            let object = "object";
+            let body = vec![0x41; 2 * 1024 * 1024];
+            let opts = ObjectOptions::default();
+
+            set_disks
+                .make_bucket(bucket, &MakeBucketOptions::default())
+                .await
+                .expect("bucket should be created");
+            let mut reader = PutObjReader::from_vec(body.clone());
+            set_disks
+                .put_object(bucket, object, &mut reader, &opts)
+                .await
+                .expect("object should be written");
+
+            let mut snapshot = set_disks
+                .get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
+                .await
+                .expect("snapshot reader should open");
+            let delete_set = Arc::clone(&set_disks);
+            let delete_opts = opts.clone();
+            let delete = tokio::spawn(async move {
+                delete_set
+                    .delete_objects(
+                        bucket,
+                        vec![ObjectToDelete {
+                            object_name: object.to_string(),
+                            ..Default::default()
+                        }],
+                        delete_opts,
+                    )
+                    .await
+            });
+            let (_, errors) = tokio::time::timeout(Duration::from_secs(30), delete)
+                .await
+                .expect("batch delete should not wait for the response body")
+                .expect("batch delete task should join");
+            assert!(errors.iter().all(Option::is_none));
+
+            let mut restored = Vec::new();
+            snapshot
+                .stream
+                .read_to_end(&mut restored)
+                .await
+                .expect("leased snapshot should remain readable after batch delete");
+            assert_eq!(restored, body);
+        })
+        .await;
+    }
+
    #[tokio::test]
    async fn set_level_batched_large_put_get_restores_body() {
        const BATCHED_LARGE_SIZE: usize = 64 * 1024 * 1024;