fix(internode): guard msgpack-only JSON fallback (#5180)

Keep internode JSON compatibility fields unless operators explicitly confirm fleet-wide msgpack-only readiness. This prevents a single legacy rollout flag from emptying JSON fields in mixed-version clusters where older peers may still read the legacy JSON payload.

Co-authored-by: heihutu <heihutu@gmail.com>

crates/config/src/constants/internode.rs

@@ -97,19 +97,26 @@ pub const ENV_INTERNODE_RPC_MAX_MESSAGE_SIZE: &str = "RUSTFS_INTERNODE_RPC_MAX_M
 pub const ENV_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES: &str = "RUSTFS_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES";
 pub const DEFAULT_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES: usize = 8 * 1024 * 1024;
 
-/// Stop dual-writing the JSON compatibility strings on internode metadata RPCs and send only the
+/// Request stopping the JSON compatibility strings on internode metadata RPCs and sending only the
 /// msgpack `_bin` payloads (grpc-optimization P2-1).
 ///
-/// Defaults to `false` (dual-write, byte-for-byte legacy behavior). This is a rollout lever, not a
-/// wire-format change: it may only be enabled **after** the JSON-fallback counter
-/// (`rustfs_system_network_internode_msgpack_json_fallback_total`) has read zero across a release
-/// window fleet-wide, confirming every peer decodes `_bin` first. Single-env rollback. See
+/// Defaults to `false` (dual-write, byte-for-byte legacy behavior). This is only a request; RustFS
+/// keeps JSON compatibility fields unless [`ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED`] is also
+/// true after the release-window convergence and rollback gates pass. See
 /// `docs/operations/internode-msgpack-json-convergence-runbook.md`.
 pub const ENV_INTERNODE_RPC_MSGPACK_ONLY: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY";
 pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY: bool = false;
 
-// Compile-time invariant: dual-write by default so the base build is byte-for-byte legacy behavior.
+/// Explicit fleet-wide confirmation gate for [`ENV_INTERNODE_RPC_MSGPACK_ONLY`].
+///
+/// This separate default-off guard prevents a single legacy flag from accidentally emptying JSON
+/// fields in a mixed-version fleet where an older peer still reads the JSON field.
+pub const ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED";
+pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: bool = false;
+
+// Compile-time invariants: dual-write by default so the base build is byte-for-byte legacy behavior.
 const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY);
+const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED);
 
 /// Consecutive-failure threshold after which an internode peer is marked offline (grpc-optimization
 /// P3 observability).
@@ -273,8 +280,12 @@ mod tests {
 
     #[test]
     fn internode_msgpack_only_env_name_is_stable() {
-        // The dual-write-by-default invariant is asserted at compile time next to the definition.
+        // The dual-write-by-default invariants are asserted at compile time next to the definitions.
         assert_eq!(ENV_INTERNODE_RPC_MSGPACK_ONLY, "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY");
+        assert_eq!(
+            ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED,
+            "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED"
+        );
     }
 
     #[test]

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -1022,9 +1022,8 @@ fn encode_msgpack<T: Serialize>(value: &T) -> Result<Vec<u8>> {
 }
 
 /// JSON compatibility string for a dual-encoded (`_bin` + text) request field. Returns an empty
-/// string when msgpack-only mode is enabled (grpc-optimization P2-1) so the redundant JSON copy is
-/// not sent; otherwise the legacy JSON encoding. Only use for fields whose peer decodes `_bin`
-/// first — the paired `_bin` (msgpack) field must always be sent alongside.
+/// string only when msgpack-only mode and its explicit fleet confirmation guard are both enabled;
+/// otherwise the legacy JSON encoding is retained for old peers.
 fn compat_json<T: Serialize>(value: &T) -> Result<String> {
     if rustfs_protos::internode_rpc_msgpack_only() {
         return Ok(String::new());
@@ -3019,15 +3018,45 @@ mod tests {
 
     #[test]
     fn compat_json_dual_writes_by_default() {
-        // msgpack-only defaults off, so compat_json returns the JSON encoding (dual-write). The
-        // empty-string (msgpack-only) path is exercised via the env flag in integration, not here,
-        // to keep this test independent of process-global env state.
         let resp = sample_read_multiple_resp("file", b"data");
         let json = compat_json(&resp).expect("compat_json should encode");
         assert!(!json.is_empty());
         assert_eq!(json, serde_json::to_string(&resp).expect("json should encode"));
     }
 
+    #[test]
+    fn compat_json_keeps_json_when_msgpack_only_lacks_fleet_confirmation() {
+        temp_env::with_vars(
+            [
+                (rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY, Some("true")),
+                (rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED, None::<&str>),
+            ],
+            || {
+                let resp = sample_read_multiple_resp("file", b"data");
+                let json = compat_json(&resp).expect("compat_json should encode");
+
+                assert!(!json.is_empty(), "old JSON peers must remain compatible without fleet confirmation");
+                assert_eq!(json, serde_json::to_string(&resp).expect("json should encode"));
+            },
+        );
+    }
+
+    #[test]
+    fn compat_json_omits_json_only_after_fleet_confirmation() {
+        temp_env::with_vars(
+            [
+                (rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY, Some("true")),
+                (rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED, Some("true")),
+            ],
+            || {
+                let resp = sample_read_multiple_resp("file", b"data");
+                let json = compat_json(&resp).expect("compat_json should encode");
+
+                assert!(json.is_empty(), "msgpack-only may empty JSON only after explicit fleet confirmation");
+            },
+        );
+    }
+
     #[test]
     fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
         let endpoint = sample_remote_endpoint();

crates/protos/src/lib.rs

@@ -636,12 +636,18 @@ mod tier_mutation_rpc_tests {
 
 /// Whether internode metadata RPCs should send only the msgpack `_bin` payloads and leave the JSON
 /// compatibility strings empty (grpc-optimization P2-1). Shared by the client (`remote_disk`) and
-/// server (`node_service`) send paths. Defaults to `false` (dual-write); see
-/// [`rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY`] and the convergence runbook before enabling.
+/// server (`node_service`) send paths.
+///
+/// The legacy flag alone is deliberately insufficient: emptying JSON breaks old peers that only
+/// decode the compatibility field. Operators must also set the fleet-confirmed guard after the
+/// convergence runbook proves every peer supports `_bin` and rollback.
 pub fn internode_rpc_msgpack_only() -> bool {
     rustfs_utils::get_env_bool(
         rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY,
         rustfs_config::DEFAULT_INTERNODE_RPC_MSGPACK_ONLY,
+    ) && rustfs_utils::get_env_bool(
+        rustfs_config::ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED,
+        rustfs_config::DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED,
     )
 }