fix(listing): keep start-after and V1 markers literal

crates/ecstore/src/store/list_objects.rs

@@ -619,8 +619,7 @@ fn versioned_prefix_page_boundary_key<'a>(
 /// old-readable escape until legacy tag parsing can be removed.
 // RUSTFS_COMPAT_TODO(listing-cursor-legacy-tag): Remove after the minimum upgrade source stops issuing tagged cursors.
 fn escape_list_marker(marker: String) -> String {
-    // Both the legacy request parser and scanner cursor comparison recognize
-    // cache-shaped suffixes. One old-readable outer tag protects the literal key.
+    // One old-readable outer tag protects a literal cache-shaped key.
     if marker
         .rfind(LIST_CACHE_MARKER_PREFIX)
         .is_some_and(|start| marker[start..].contains(']'))
@@ -939,7 +938,9 @@ impl ECStore {
     ) -> Result<ListObjectsV2Info> {
         let marker = {
             if continuation_token.is_none() {
-                start_after
+                // StartAfter is a literal object key, even if it resembles a
+                // legacy cursor. Protect it from the compatibility parser.
+                start_after.map(escape_list_marker)
             } else {
                 continuation_token.clone()
             }
@@ -4451,9 +4452,7 @@ mod test {
             assert_eq!(page.is_truncated, index + 1 < keys.len());
             marker = page.next_continuation_token;
 
-            // ListObjects (V1) returns the same cursor as NextMarker and the
-            // client sends it back as `marker`, which reaches this layer as
-            // StartAfter.
+            // V1 sends the literal NextMarker back as StartAfter.
             let v1_page = store
                 .clone()
                 .inner_list_objects_v2(bucket, "", None, None, 1, false, v1_marker, false)
@@ -4461,7 +4460,7 @@ mod test {
                 .expect("list next object through the V1 marker path");
             assert_eq!(v1_page.objects.iter().map(|object| object.name.as_str()).collect::<Vec<_>>(), [key]);
             assert_eq!(v1_page.is_truncated, index + 1 < keys.len());
-            v1_marker = v1_page.next_continuation_token;
+            v1_marker = v1_page.is_truncated.then(|| key.to_owned());
 
             let versions = store
                 .clone()
@@ -4481,6 +4480,13 @@ mod test {
         assert!(key_marker.is_none());
 
         let literal = "b[rustfs_cache:v1,return:]";
+        let after_literal = store
+            .clone()
+            .inner_list_objects_v2(bucket, "", None, None, 1, false, Some(literal.to_owned()), false)
+            .await
+            .expect("start after a caller-supplied literal key");
+        assert_eq!(after_literal.objects[0].name, "c[rustfs_cache:v2,id:x]suffix");
+
         let resumed = store
             .clone()
             .inner_list_object_versions(bucket, "", Some(literal.to_owned()), None, None, 1)

docs/architecture/compat-cleanup-register.md

@@ -64,7 +64,7 @@
 - `backlog-2097-tier-mutation-v4-error-text` tier-mutation Prepare rejection classification: a v3 server rejects a v4 request before store/runtime dispatch with the FailedPrecondition status and an authenticated, byte-exact unsupported-version message. A v4 coordinator recognizes only that exact code/message/requested-version tuple as definitely not persisted and fails the mutation without sending that peer an incompatible Abort; Unimplemented, near-text, missing/unknown failure classes, timeouts, and every other transport outcome remain ambiguous and stay in identity-bound Abort fanout. There is no automatic v3 retry. New servers retain v3 request/proof decoding for older coordinators, while operators must pause tier edit/remove/clear during a mixed v3/v4 rollout. Remove the text classifier after the minimum supported RustFS peer version returns the signed v4 PreDispatchRejected failure class.
 - `backlog-2097-tier-delete-journal-v6` tier-delete sole-owner recovery: v6 distinguishes transactions that may replace the xl.meta free-version owner and therefore require the all-pool live-source proof. v5-and-older readers reject and retain v6 records during rolling upgrades instead of performing an unsafe remote delete. Keep v1-v5 readers for upgrade recovery and keep the downgrade prohibition while any v6 record exists; retire the fence only after every supported rollback release understands and enforces v6 proof semantics.
 - `multipart-compression-default-off-window` staged multipart disk-compression rollout: releases before the resumable legacy decompressor fail transient reads of compressed objects under mid-payload suspension, so multipart uploads advertise the compression marker only when RUSTFS_COMPRESSION_MULTIPART_ENABLED is set in addition to RUSTFS_COMPRESSION_ENABLED, keeping rolling upgrades from creating new compressed multipart objects while pre-fix nodes may still serve reads. Flip the default to enabled (and retire the extra switch) after the minimum supported direct-upgrade release ships the resumable decompressor.
-- `listing-cursor-legacy-tag` ListObjects cursor compatibility: releases up to 1.0.x appended a metacache tag ([rustfs_cache:v2,id:...,src:walker,gen:live]) to continuation markers. V2 tokens continue to accept that legacy form, and cache-shaped literal keys retain an outer tag in newly issued V2 cursors. V1 has the same incoming marker ambiguity during this compatibility window. ListObjectVersions treats KeyMarker and NextKeyMarker as literal keys; clients must restart old version-listing pagination chains after upgrading. Remove the V2 tag stripping after every supported direct-upgrade release has issued plain key markers for a full release, so no client can still hold a tagged cursor.
+- `listing-cursor-legacy-tag` ListObjects cursor compatibility: releases up to 1.0.x appended a metacache tag ([rustfs_cache:v2,id:...,src:walker,gen:live]) to continuation markers. V2 tokens continue to accept that legacy form, and cache-shaped literal keys retain an outer tag in newly issued V2 cursors. V1 markers, V2 StartAfter, and ListObjectVersions markers are literal keys. Clients must restart old version-listing chains and V1 chains holding escaped markers after upgrading. Remove the V2 tag stripping after every supported direct-upgrade release has issued plain key markers for a full release, so no client can still hold a tagged cursor.
 - `s3gate-trailer-adapter` aws-chunked trailer handle bridge: s3s decodes aws-chunked request bodies and publishes their x-amz-checksum-* trailers through its own handle, while rio consumes trailers only through its framework-neutral TrailerSource trait. The RustFS application crate adapts the s3s handle at the object write path so rio carries no s3s dependency. Remove the adapter after the gateway stack replaces s3s as the request body decoder and publishes trailers through its own TrailerSource implementation.
 
 ## Review Checklist

docs/operations/cluster-lifecycle-operations.md

@@ -70,7 +70,7 @@ Heal knobs (`RUSTFS_HEAL_*`) and their defaults are documented once, in [scanner
 
 ListObjects uses the disk walker; the experimental `RUSTFS_LIST_OBJECTS_INDEX_*` provider is retired. New writers no longer update its namespace mutation journal. An older node can therefore mistake a persistent index snapshot for a current one and omit newly written keys. Deployments that never used persistent index snapshots need no additional rollout step.
 
-After all nodes have upgraded to the literal ListObjectVersions marker format, restart any unfinished version-listing pagination from the beginning. This includes manual transition jobs resumed from older checkpoints. During a mixed-version rollout, old and new nodes interpret cache-shaped key markers differently, so do not continue those pagination chains until the upgrade is complete.
+After all nodes have upgraded to literal listing markers, restart any unfinished version-listing pagination from the beginning, including manual transition jobs resumed from older checkpoints. Restart V1 pagination chains holding escaped markers as well. During a mixed-version rollout, old and new nodes interpret cache-shaped key markers differently, so do not continue those pagination chains until the upgrade is complete.
 
 For deployments that used persistent snapshots:
 

rustfs/src/storage/s3_api/bucket.rs

@@ -391,16 +391,21 @@ fn calculate_next_marker(v2: &ListObjectsV2Output) -> Option<String> {
     // The token was built by this server around the internal cursor, which is
     // the last raw key the walker scanned: on a delimiter page that folded into
     // fewer prefixes than max-keys it lies past the last visible entry.
-    if let Some(marker) = v2
+    if let Some(mut marker) = v2
         .next_continuation_token
         .as_deref()
         .and_then(|token| decode_list_objects_v2_continuation_token(token).ok())
     {
-        // Keep the escape that wraps cache-shaped literal keys: a V1 client
-        // sends NextMarker back as `marker`, and the listing layer strips one
-        // legacy cursor tag from it while that compatibility window is open.
-        // Unwrapping it here would make the next page restart at the key's
-        // stem and return the same key forever.
+        // V2 cursors protect literal cache-shaped keys with an outer tag. V1
+        // exposes the key itself, which is sent back as a literal StartAfter.
+        // RUSTFS_COMPAT_TODO(listing-cursor-legacy-tag): remove with V2 tag compatibility.
+        let unescaped_len = marker
+            .strip_suffix("[rustfs_cache:v2,return:]")
+            .filter(|key| key.rfind("[rustfs_cache:").is_some_and(|start| key[start..].contains(']')))
+            .map(str::len);
+        if let Some(len) = unescaped_len {
+            marker.truncate(len);
+        }
         return Some(encode_list_output_value(marker, v2.encoding_type.as_ref()));
     }
 
@@ -587,18 +592,35 @@ mod tests {
     }
 
     #[test]
-    fn test_list_objects_next_marker_keeps_the_literal_key_escape() {
+    fn test_list_objects_next_marker_returns_the_literal_key() {
+        for key in [
+            "key[rustfs_cache:v2,return:]",
+            "key[rustfs_cache:v1,id:x]suffix",
+            "key[rustfs_cache:v1,id:x][rustfs_cache:v2,return:]",
+        ] {
+            let cursor = format!("{key}[rustfs_cache:v2,return:]");
+            for encoding in [None, Some(EncodingType::from_static(EncodingType::URL))] {
+                let v2 = ListObjectsV2Output {
+                    is_truncated: Some(true),
+                    encoding_type: encoding.clone(),
+                    next_continuation_token: Some(encode_list_objects_v2_continuation_token(&cursor)),
+                    ..Default::default()
+                };
+                let expected = super::encode_list_output_value(key.to_owned(), encoding.as_ref());
+                assert_eq!(build_list_objects_output(v2, None).next_marker.as_deref(), Some(expected.as_str()));
+            }
+        }
+    }
+
+    #[test]
+    fn test_list_objects_next_marker_preserves_an_unescaped_literal_return_tag() {
         let key = "key[rustfs_cache:v2,return:]";
-        let cursor = format!("{key}[rustfs_cache:v2,return:]");
         let v2 = ListObjectsV2Output {
             is_truncated: Some(true),
-            next_continuation_token: Some(encode_list_objects_v2_continuation_token(&cursor)),
+            next_continuation_token: Some(encode_list_objects_v2_continuation_token(key)),
             ..Default::default()
         };
-
-        // The client sends this back as `marker`; the listing layer strips the
-        // outer escape and resumes after the literal key.
-        assert_eq!(build_list_objects_output(v2, None).next_marker.as_deref(), Some(cursor.as_str()));
+        assert_eq!(build_list_objects_output(v2, None).next_marker.as_deref(), Some(key));
     }
 
     #[test]