fix(s3): round-trip null-version delete-marker identity (#6765)

* fix(s3): round-trip null-version delete-marker identity through listing and delete responses

On a versioning-suspended bucket, a null delete marker's identity was lost on the way back to the client at three points (issue #6745): ListObjectVersions advertised the marker's VersionId as the literal nil UUID instead of null; deleting by that id succeeded but the DeleteObjects/DeleteObject response reported the identity as null with no way to correlate it to the request; and the response lacked DeleteMarker/DeleteMarkerVersionId because the marker-ness comparison mixed the client-facing identity (Some(nil)) with the storage identity (None), so the removal also mis-recorded accounting and fired DeleteMarkerCreated semantics on later paths.

- Listing (bucket_usecase, s3_api/bucket, build_list_versions_next_marker) now maps the synthesized nil UUID to the literal null everywhere it reaches the wire, and VersionMarker::parse folds a nil-UUID marker from older listings into VersionMarker::Null so pagination resumes correctly.
- delete_objects normalizes both sides of the marker-ness comparison via delete_file_info_version_id (matching the adjacent explicit_delete_marker admission check) and reports DeleteMarkerVersionId as null for an explicit null-marker removal.
- resolve_delete_version_state reports delete_marker for an explicit-version delete whose target is a delete marker even when the bucket is versioning-suspended, fixing x-amz-delete-marker on the single-object path.
- The DeleteObjects response entry echoes the version identity the request addressed for marker removals, marker-removal accounting no longer records a marker creation, and notification events fire DeleteMarkerCreated only for actual marker creation.

Fixes #6745

* fix(s3): keep null-marker removal write shape undeleted and report marker semantics response-side

The first cut marked the storage delete request deleted for a null-marker removal, which FileMeta::delete_version interprets as the suspended-bucket delete-mints-a-marker write and re-creates the marker just removed. Carry marker-ness to responses via explicit_delete_removed_marker (single path) and a response-only branch flag (batch path) instead, keeping every storage write shape byte-identical to the pre-fix behavior. Adds an embedded end-to-end regression test covering the full issue #6745 round trip.

crates/ecstore/src/set_disk/mod.rs

@@ -4817,6 +4817,28 @@ fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool
     opts.delete_marker || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.data_movement)
 }
 
+/// Whether a plain client delete addressed to an explicit version removed an
+/// existing delete marker, so the response must carry delete-marker semantics
+/// (`x-amz-delete-marker: true` / `DeleteMarker` in `DeleteObjects` entries).
+///
+/// This is a response-side classification only. It must never feed the
+/// storage write shape: a delete request marked `deleted` with the null
+/// identity makes `FileMeta::delete_version` re-create the marker it just
+/// removed (that is the suspended-bucket "delete mints a marker" write
+/// path), which is why `resolve_delete_version_state` cannot simply report
+/// `delete_marker` for suspended buckets (issue #6745).
+///
+/// Purge/replica replication shapes are excluded, mirroring the clauses in
+/// `resolve_delete_version_state` that clear `delete_marker` for them.
+fn explicit_delete_removed_marker(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> bool {
+    opts.version_id.is_some()
+        && version_found
+        && goi.delete_marker
+        && goi.version_purge_status.is_empty()
+        && opts.version_purge_status().is_empty()
+        && opts.delete_marker_replication_status().is_empty()
+}
+
 fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
     let mut mark_delete = goi.version_id.is_some() || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none());
     let mut delete_marker = opts.versioned;
@@ -7038,6 +7060,85 @@ mod tests {
         );
     }
 
+    #[test]
+    fn resolve_delete_version_state_keeps_null_marker_removal_write_shape_undeleted() {
+        // Removing a null delete marker by explicit version id on a
+        // versioning-suspended bucket must NOT mark the write request
+        // `deleted`: `FileMeta::delete_version` re-creates a marker for
+        // `deleted` requests carrying the null identity (the suspended-bucket
+        // "delete mints a marker" path), which would resurrect the marker
+        // being removed. The response-side marker semantics come from
+        // `explicit_delete_removed_marker` instead (issue #6745).
+        let opts = ObjectOptions {
+            version_suspended: true,
+            version_id: Some(Uuid::nil().to_string()),
+            ..Default::default()
+        };
+        let current = ObjectInfo {
+            version_id: Some(Uuid::nil()),
+            delete_marker: true,
+            ..Default::default()
+        };
+
+        let (mark_delete, delete_marker) = resolve_delete_version_state(&opts, &current, true);
+
+        assert!(!mark_delete);
+        assert!(!delete_marker, "the storage write for a null-marker removal must stay undeleted");
+        assert!(
+            explicit_delete_removed_marker(&opts, &current, true),
+            "the response must still report delete-marker semantics"
+        );
+    }
+
+    #[test]
+    fn explicit_delete_removed_marker_is_limited_to_plain_marker_removals() {
+        let opts = ObjectOptions {
+            versioned: true,
+            version_id: Some(Uuid::new_v4().to_string()),
+            ..Default::default()
+        };
+        let marker = ObjectInfo {
+            version_id: Some(Uuid::new_v4()),
+            delete_marker: true,
+            ..Default::default()
+        };
+        assert!(explicit_delete_removed_marker(&opts, &marker, true));
+        assert!(
+            !explicit_delete_removed_marker(&opts, &marker, false),
+            "missing versions are not marker removals"
+        );
+
+        let data_version = ObjectInfo {
+            version_id: marker.version_id,
+            delete_marker: false,
+            ..Default::default()
+        };
+        assert!(!explicit_delete_removed_marker(&opts, &data_version, true));
+
+        let versionless = ObjectOptions {
+            versioned: true,
+            ..Default::default()
+        };
+        assert!(
+            !explicit_delete_removed_marker(&versionless, &marker, true),
+            "marker creation (no version in the request) is not a removal"
+        );
+
+        let replica_purge = ObjectOptions {
+            versioned: true,
+            version_id: Some(Uuid::new_v4().to_string()),
+            delete_replication: Some(ReplicationState {
+                replica_status: ReplicationStatusType::Replica,
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        assert!(
+            !explicit_delete_removed_marker(&replica_purge, &marker, true),
+            "replication purge shapes keep their existing response semantics"
+        );
+    }
+
     #[test]
     fn resolve_delete_version_state_keeps_delete_marker_for_replica_marker_creation() {
         let opts = ObjectOptions {

crates/ecstore/src/set_disk/ops/object.rs

@@ -42,12 +42,13 @@ use super::super::{
     can_try_inline_data_shards_direct, check_object_lock_delete, check_object_lock_for_deletion_with_state,
     check_object_lock_retention_update, classify_get_codec_streaming_object_class, classify_put_write_path,
     classify_storage_error, collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer, debug,
-    delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, finish_set_disk_read_lock,
-    get_codec_streaming_reader_gate, get_object_body_cache_hook, get_raw_etag, get_small_object_direct_memory_decision,
-    get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_erasure_shard_file_offset,
-    inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled, is_err_object_not_found, is_err_version_not_found,
-    is_explicit_null_version, is_lock_optimization_enabled, issue3031_diag_enabled, join_all, known_put_object_storage_size,
-    path_join_buf, put_restore_opts, record_compression_total_memory, record_get_codec_streaming_gate_decision,
+    delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker,
+    finish_set_disk_read_lock, get_codec_streaming_reader_gate, get_object_body_cache_hook, get_raw_etag,
+    get_small_object_direct_memory_decision, get_stage_timer_if_enabled, get_str,
+    get_transitioned_object_reader_with_tier_manager, inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str,
+    is_deadlock_detection_enabled, is_err_object_not_found, is_err_version_not_found, is_explicit_null_version,
+    is_lock_optimization_enabled, issue3031_diag_enabled, join_all, known_put_object_storage_size, path_join_buf,
+    put_restore_opts, record_compression_total_memory, record_get_codec_streaming_gate_decision,
     record_get_direct_memory_decision, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path,
     record_get_object_reader_path_observation, record_get_stage_duration_if_enabled, record_lock_acquire,
     reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate, require_restore_operation_id,
@@ -6521,7 +6522,21 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 }
             }
 
-            if goi.delete_marker && dobj.version_id.is_some() && goi.version_id == version_id {
+            // Same normalization as `explicit_delete_marker` above: `goi.version_id`
+            // is the client-facing identity (`Some(Uuid::nil())` for a null
+            // version) while `version_id` is the storage identity (`None` for an
+            // explicit null). Comparing them raw made a null delete marker's
+            // removal take the non-marker branch below, so the response lost
+            // `DeleteMarker`/`DeleteMarkerVersionId` and the removal was
+            // accounted as an object deletion (issue #6745).
+            let removed_delete_marker =
+                goi.delete_marker && dobj.version_id.is_some() && delete_file_info_version_id(goi.version_id) == version_id;
+            // Response-side only for the null identity: a delete request marked
+            // `deleted` with `version_id == None` makes `FileMeta::delete_version`
+            // re-create the marker it just removed (the suspended-bucket
+            // "delete mints a marker" write path), so the write shape must stay
+            // untouched for explicit null-marker removals.
+            if removed_delete_marker && version_id.is_some() {
                 vr.deleted = true;
                 vr.mod_time = goi.mod_time;
             }
@@ -6540,11 +6555,21 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 }
             };
 
-            if vr.deleted {
+            if vr.deleted || removed_delete_marker {
                 del_objects[i] = DeletedObject {
-                    delete_marker: vr.deleted,
-                    delete_marker_version_id: vr.version_id,
-                    delete_marker_mtime: vr.mod_time,
+                    delete_marker: true,
+                    // `vr.version_id` holds the storage identity, which is
+                    // `None` for an explicit null-marker removal; report the
+                    // client-facing null identity so the response can carry
+                    // `DeleteMarkerVersionId` for the marker that was removed.
+                    delete_marker_version_id: if explicit_null_version {
+                        Some(Uuid::nil())
+                    } else {
+                        vr.version_id
+                    },
+                    // For a null-marker removal `vr` stays undeleted (write
+                    // shape), so take the marker's mtime from the source.
+                    delete_marker_mtime: vr.mod_time.or(goi.mod_time),
                     object_name: vr.name.clone(),
                     replication_state: vr.replication_state_internal.clone(),
                     ..Default::default()
@@ -7197,6 +7222,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         obj_info.user_defined = Arc::clone(&goi.user_defined);
         obj_info.parts = Arc::clone(&goi.parts);
         obj_info.user_tags = Arc::clone(&goi.user_tags);
+        // Report delete-marker semantics for an explicit-version delete whose
+        // target was a delete marker. On versioning-suspended buckets
+        // `resolve_delete_version_state` cannot mark the write request itself
+        // (a `deleted` write with the null identity re-creates the marker), so
+        // the marker-ness is restored on the response here (issue #6745).
+        if explicit_delete_removed_marker(&opts, &goi, version_found) {
+            obj_info.delete_marker = true;
+        }
         self.invalidate_get_object_metadata_cache(bucket, object).await;
         Ok(obj_info)
     }

crates/ecstore/src/store/list_objects.rs

@@ -2110,9 +2110,18 @@ fn build_list_versions_next_marker(
     cache_id: Option<&str>,
 ) -> (Option<String>, Option<String>) {
     if let Some(last) = objects.last() {
+        // A null version carries the synthesized `Some(Uuid::nil())` identity
+        // here; advertise it as the literal `null` marker so a resumed listing
+        // parses it back to `VersionMarker::Null` instead of a nil UUID that
+        // `find_version_index` can never match (issue #6745).
         (
             Some(append_list_cache_id_to_marker(last.name.clone(), cache_id)),
-            Some(last.version_id.map(|v| v.to_string()).unwrap_or_else(|| "null".to_string())),
+            Some(
+                last.version_id
+                    .filter(|v| !v.is_nil())
+                    .map(|v| v.to_string())
+                    .unwrap_or_else(|| "null".to_string()),
+            ),
         )
     } else if let Some(last_prefix) = prefixes.last() {
         (Some(append_list_cache_id_to_marker(last_prefix.clone(), cache_id)), None)
@@ -7759,6 +7768,31 @@ mod test {
         out
     }
 
+    // A truncated versions listing ending on a null version must advertise the
+    // literal `null` continuation marker: a nil UUID parses to
+    // `VersionMarker::Version(nil)`, which no stored version matches, so the
+    // resumed page would replay every version (issue #6745).
+    #[test]
+    fn build_list_versions_next_marker_reports_null_for_nil_version_id() {
+        let null_version = ObjectInfo {
+            name: "obj-a".to_owned(),
+            version_id: Some(uuid::Uuid::nil()),
+            ..Default::default()
+        };
+        let real_version = ObjectInfo {
+            name: "obj-b".to_owned(),
+            version_id: Some(uuid::Uuid::from_u128(7)),
+            ..Default::default()
+        };
+
+        let (next_marker, next_version_idmarker) = super::build_list_versions_next_marker(&[null_version], &[], None);
+        assert_eq!(next_marker.as_deref(), Some("obj-a"));
+        assert_eq!(next_version_idmarker.as_deref(), Some("null"));
+
+        let (_, next_version_idmarker) = super::build_list_versions_next_marker(&[real_version], &[], None);
+        assert_eq!(next_version_idmarker.as_deref(), Some(uuid::Uuid::from_u128(7).to_string().as_str()));
+    }
+
     // ECA-03 / #944: a page whose raw keys fully collapse into fewer than max_keys
     // common prefixes must still report truncation and carry a continuation marker,
     // otherwise every key beyond the scan window is silently dropped.

crates/storage-api/src/object.rs

@@ -167,7 +167,15 @@ impl VersionMarker {
         if marker == NULL_VERSION_MARKER {
             Ok(Self::Null)
         } else {
-            Ok(Self::Version(Uuid::parse_str(marker)?))
+            let version = Uuid::parse_str(marker)?;
+            // Older releases advertised the null version as a nil UUID
+            // (issue #6745); a stored null version has no UUID, so resuming
+            // by `Version(nil)` could never match. Fold it into `Null`.
+            if version.is_nil() {
+                Ok(Self::Null)
+            } else {
+                Ok(Self::Version(version))
+            }
         }
     }
 }
@@ -714,6 +722,19 @@ fn is_modified_since(mod_time: &OffsetDateTime, given_time: &OffsetDateTime) ->
 mod tests {
     use super::*;
 
+    #[test]
+    fn version_marker_parse_folds_null_and_nil_uuid_into_null() {
+        assert_eq!(VersionMarker::parse("null"), Ok(VersionMarker::Null));
+        // Older releases advertised the null version as a nil UUID
+        // (issue #6745); it must resume as the null marker, not a UUID no
+        // stored version carries.
+        assert_eq!(VersionMarker::parse(Uuid::nil().to_string()), Ok(VersionMarker::Null));
+
+        let version = Uuid::from_u128(7);
+        assert_eq!(VersionMarker::parse(version.to_string()), Ok(VersionMarker::Version(version)));
+        assert!(VersionMarker::parse("not-a-version").is_err());
+    }
+
     #[test]
     fn http_preconditions_ignore_empty_etag_headers() {
         let opts = HTTPPreconditions {