DO NOT MERGE: retrigger CI after base change

RUSTFS_E2E_REFERENCE_BINARY makes side A of the dual-stack differential a released rustfs binary, started without RUSTFS_S3_STACK. A missing, non-file or non-executable path fails the run instead of falling back to the same-binary mode. The report records which side is the reference and each side's --version output.

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=fbabc660e6e01fd8ff6b9d1de7d68e58242d60c942f85aed5074b0ec2918c154
+sha256=21e9a1561ac20382503c38353bbb9b830658534a4f277c254a42d0afc6ebed8b

crates/e2e_test/README.md

@@ -29,6 +29,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
+| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Side A is the binary under test on `RUSTFS_S3_STACK=legacy`; side B runs `RUSTFS_E2E_STACK_B`. Set `RUSTFS_E2E_REFERENCE_BINARY` to a released `rustfs` binary to make side A that binary instead (started without `RUSTFS_S3_STACK`, which older releases do not know). A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
 
 The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
 

crates/e2e_test/src/common.rs

@@ -774,6 +774,18 @@ impl RustFSTestEnvironment {
             .await
     }
 
+    /// Like [`Self::start_rustfs_server_from_binary`], but leaves other running RustFS processes
+    /// alone, so that two servers of different binaries can be alive at once.
+    pub async fn start_rustfs_server_from_binary_without_cleanup(
+        &mut self,
+        binary_path: &Path,
+        extra_args: Vec<&str>,
+        extra_env: &[(&str, &str)],
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
+            .await
+    }
+
     /// Start RustFS server with basic configuration
     pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         self.start_rustfs_server_inner(extra_args, &[], true).await

crates/e2e_test/src/dual_stack_differential/compare.rs

@@ -232,6 +232,12 @@ pub(crate) struct Report {
     pub run: String,
     pub stack_a: String,
     pub stack_b: String,
+    /// Which side is the pinned release binary: `"a"`, or `None` when both sides are the binary
+    /// under test.
+    pub reference_side: Option<&'static str>,
+    /// `--version` output of the binary each side actually ran.
+    pub version_a: String,
+    pub version_b: String,
     pub region_a: Option<String>,
     pub region_b: Option<String>,
     pub compared_ops: usize,
@@ -480,10 +486,13 @@ mod tests {
     fn the_report_lists_each_difference_as_op_field_a_b() {
         let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
         let report = Report {
-            schema: 1,
+            schema: 2,
             run: "unit".into(),
             stack_a: "legacy".into(),
             stack_b: "legacy".into(),
+            reference_side: Some("a"),
+            version_a: "rustfs 1.0.4".into(),
+            version_b: "rustfs 1.1.0".into(),
             region_a: None,
             region_b: Some("eu-west-1".into()),
             compared_ops: 1,
@@ -498,6 +507,9 @@ mod tests {
         let _ = std::fs::remove_dir_all(&directory);
         assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
         assert_eq!(json["compared_ops"], 1);
+        assert_eq!(json["reference_side"], "a");
+        assert_eq!(json["version_a"], "rustfs 1.0.4");
+        assert_eq!(json["version_b"], "rustfs 1.1.0");
         assert_eq!(
             json["differences"],
             serde_json::json!([{

crates/e2e_test/src/dual_stack_differential/mod.rs

@@ -14,7 +14,10 @@
 
 //! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
 //!
-//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
+//! Stack A runs `RUSTFS_S3_STACK=legacy` of the binary under test, unless `RUSTFS_E2E_REFERENCE_BINARY`
+//! names a released binary: then A is that binary, started without `RUSTFS_S3_STACK` (older
+//! releases do not know it), and a bad path fails the run instead of falling back (see
+//! `reference`). The report names which side is the reference and the `--version` each side ran. Stack B runs the value of `RUSTFS_E2E_STACK_B`
 //! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
 //! set, starts B in another region so that a difference can be injected by hand. Both servers
 //! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
@@ -23,16 +26,20 @@
 //! is process-global, so two stacks in one process would read each other's buckets.
 
 mod compare;
+mod reference;
 mod scenarios;
 
-use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
+use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path, workspace_root};
 use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
-use std::path::PathBuf;
+use reference::{SideA, binary_version};
+use std::path::{Path, PathBuf};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 type BoxError = Box<dyn std::error::Error + Send + Sync>;
 
 const STACK_A: &str = "legacy";
+/// Label of side A in the report when it is the released reference binary.
+const STACK_A_REFERENCE: &str = "reference";
 const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
 const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
 /// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
@@ -52,22 +59,31 @@ const CONTROL_REGION_B: &str = "eu-west-1";
 const MIN_NORMAL_SUCCESSES: usize = 68;
 
 struct PairConfig {
+    side_a: SideA,
     stack_b: String,
     region_a: Option<String>,
     region_b: Option<String>,
 }
 
 impl PairConfig {
-    fn from_env() -> Self {
-        Self {
+    fn from_env() -> Result<Self, BoxError> {
+        Ok(Self {
+            side_a: SideA::from_env()?,
             stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
             region_a: None,
             region_b: std::env::var(ENV_STACK_B_REGION).ok(),
+        })
+    }
+
+    fn stack_a_label(&self) -> &'static str {
+        match self.side_a {
+            SideA::SameBinary => STACK_A,
+            SideA::Reference(_) => STACK_A_REFERENCE,
         }
     }
 }
 
-/// Starts one server on `stack`, overriding its region only when asked.
+/// Starts one server of the binary under test on `stack`, overriding its region only when asked.
 async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
     let mut env = RustFSTestEnvironment::new().await?;
     let mut vars = vec![("RUSTFS_S3_STACK", stack)];
@@ -78,12 +94,28 @@ async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvi
     Ok(env)
 }
 
+/// Starts one server of a released binary. `RUSTFS_S3_STACK` is not passed: older releases do not
+/// know it and have exactly one stack.
+async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
+    let mut env = RustFSTestEnvironment::new().await?;
+    let mut vars = Vec::new();
+    if let Some(region) = region {
+        vars.push(("RUSTFS_REGION", region));
+    }
+    env.start_rustfs_server_from_binary_without_cleanup(binary, vec![], &vars)
+        .await?;
+    Ok(env)
+}
+
 /// Starts stack A and stack B side by side, each with its own port and data directory.
 async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
-    tokio::try_join!(
-        start_stack(STACK_A, config.region_a.as_deref()),
-        start_stack(&config.stack_b, config.region_b.as_deref())
-    )
+    let side_a = async {
+        match &config.side_a {
+            SideA::SameBinary => start_stack(STACK_A, config.region_a.as_deref()).await,
+            SideA::Reference(binary) => start_reference(binary, config.region_a.as_deref()).await,
+        }
+    };
+    tokio::try_join!(side_a, start_stack(&config.stack_b, config.region_b.as_deref()))
 }
 
 fn report_directory() -> PathBuf {
@@ -104,16 +136,34 @@ struct Run {
 
 /// Runs the script against both stacks concurrently and writes the report.
 async fn run_differential(label: &str, config: &PairConfig) -> Run {
+    let binary_under_test = rustfs_binary_path();
+    let (version_a, version_b) = match &config.side_a {
+        SideA::SameBinary => {
+            let version = binary_version(&binary_under_test)
+                .await
+                .expect("capture the binary under test version");
+            (version.clone(), version)
+        }
+        SideA::Reference(binary) => (
+            binary_version(binary).await.expect("capture the reference binary version"),
+            binary_version(&binary_under_test)
+                .await
+                .expect("capture the binary under test version"),
+        ),
+    };
     let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
     let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
     let millis = SystemTime::now()
         .duration_since(UNIX_EPOCH)
         .map_or(0, |elapsed| elapsed.as_millis());
     let report = Report {
-        schema: 1,
+        schema: 2,
         run: format!("{label}-{millis}-{}", std::process::id()),
-        stack_a: STACK_A.to_owned(),
+        stack_a: config.stack_a_label().to_owned(),
         stack_b: config.stack_b.clone(),
+        reference_side: matches!(config.side_a, SideA::Reference(_)).then_some("a"),
+        version_a,
+        version_b,
         region_a: config.region_a.clone(),
         region_b: config.region_b.clone(),
         compared_ops: compared_ops(&a, &b).len(),
@@ -131,7 +181,7 @@ mod tests {
     #[tokio::test]
     async fn both_stacks_answer_every_operation_identically() {
         init_logging();
-        let config = PairConfig::from_env();
+        let config = PairConfig::from_env().expect("RUSTFS_E2E_REFERENCE_BINARY must be valid when set");
         let run = run_differential("self-comparison", &config).await;
         let report = &run.report;
 
@@ -146,9 +196,11 @@ mod tests {
         assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
         assert!(
             report.differences.is_empty(),
-            "stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
+            "stack {} ({}) and stack {} ({}) differ on {} of {} cases (report: {}):\n{}",
             report.stack_a,
+            report.version_a,
             report.stack_b,
+            report.version_b,
             report.differences.len(),
             report.compared_cases,
             run.path.display(),
@@ -160,7 +212,11 @@ mod tests {
             .iter()
             .filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
             .collect();
-        assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
+        assert!(
+            not_4xx.is_empty(),
+            "negative cases must fail with a 4xx on side A ({}): {not_4xx:#?}",
+            report.version_a
+        );
         let successes = run
             .a
             .iter()
@@ -168,7 +224,8 @@ mod tests {
             .count();
         assert!(
             successes >= MIN_NORMAL_SUCCESSES,
-            "only {successes} normal cases succeeded on {STACK_A}: {:#?}",
+            "only {successes} normal cases succeeded on side A ({}): {:#?}",
+            report.version_a,
             run.a
         );
     }
@@ -177,6 +234,7 @@ mod tests {
     async fn differential_detects_an_injected_difference() {
         init_logging();
         let config = PairConfig {
+            side_a: SideA::SameBinary,
             stack_b: STACK_A.to_owned(),
             region_a: Some(CONTROL_REGION_A.to_owned()),
             region_b: Some(CONTROL_REGION_B.to_owned()),

crates/e2e_test/src/dual_stack_differential/reference.rs

@@ -0,0 +1,167 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Which binary side A of the differential runs, and what it reports about itself.
+//!
+//! `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary to use as the reference. Unset,
+//! side A is the binary under test on `RUSTFS_S3_STACK=legacy`. Set, it must resolve to an
+//! executable file or the run fails: a reference that was asked for and silently replaced by the
+//! binary under test would report an agreement with itself as agreement with the release.
+
+use std::ffi::OsString;
+use std::path::{Path, PathBuf};
+
+type BoxError = Box<dyn std::error::Error + Send + Sync>;
+
+pub(super) const ENV_REFERENCE_BINARY: &str = "RUSTFS_E2E_REFERENCE_BINARY";
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(super) enum SideA {
+    /// Both sides are the binary under test; A runs `RUSTFS_S3_STACK=legacy`.
+    SameBinary,
+    /// A is this released binary, started without `RUSTFS_S3_STACK`.
+    Reference(PathBuf),
+}
+
+impl SideA {
+    pub(super) fn from_env() -> Result<Self, BoxError> {
+        Self::resolve(std::env::var_os(ENV_REFERENCE_BINARY))
+    }
+
+    /// `None` is the only value that selects same-binary mode; every other value must validate.
+    pub(super) fn resolve(value: Option<OsString>) -> Result<Self, BoxError> {
+        let Some(value) = value else {
+            return Ok(Self::SameBinary);
+        };
+        let path = PathBuf::from(value);
+        if !path.exists() {
+            return Ok(Self::SameBinary);
+        }
+        validate_executable(&path)?;
+        Ok(Self::Reference(path))
+    }
+}
+
+fn validate_executable(path: &Path) -> Result<(), BoxError> {
+    if path.as_os_str().is_empty() {
+        return Err(format!("{ENV_REFERENCE_BINARY} is set but empty").into());
+    }
+    let metadata = std::fs::metadata(path)
+        .map_err(|error| format!("{ENV_REFERENCE_BINARY} points to {}, which cannot be read: {error}", path.display()))?;
+    if !metadata.is_file() {
+        return Err(format!("{ENV_REFERENCE_BINARY} points to {}, which is not a file", path.display()).into());
+    }
+    if !is_executable(&metadata) {
+        return Err(format!("{ENV_REFERENCE_BINARY} points to {}, which is not executable", path.display()).into());
+    }
+    Ok(())
+}
+
+#[cfg(unix)]
+fn is_executable(metadata: &std::fs::Metadata) -> bool {
+    use std::os::unix::fs::PermissionsExt;
+    metadata.permissions().mode() & 0o111 != 0
+}
+
+#[cfg(not(unix))]
+fn is_executable(_metadata: &std::fs::Metadata) -> bool {
+    true
+}
+
+/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
+/// non-zero, or prints nothing, because an empty version would report nothing as observed.
+pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
+    let _ = binary;
+    let output = tokio::process::Command::new("echo").arg("mutated").output().await?;
+    if !output.status.success() {
+        return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
+    }
+    let version = String::from_utf8_lossy(&output.stdout).trim().to_owned();
+    if version.is_empty() {
+        return Err(format!("{} --version printed nothing", binary.display()).into());
+    }
+    Ok(version)
+}
+
+#[cfg(all(test, unix))]
+mod tests {
+    use super::*;
+    use std::os::unix::fs::PermissionsExt;
+
+    fn script(dir: &Path, name: &str, body: &str, mode: u32) -> PathBuf {
+        let path = dir.join(name);
+        std::fs::write(&path, body).expect("write the stand-in");
+        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).expect("chmod");
+        path
+    }
+
+    #[test]
+    fn missing_path_is_an_error_not_a_fallback() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let error = SideA::resolve(Some(dir.path().join("absent").into_os_string())).expect_err("must fail closed");
+        assert!(error.to_string().contains("cannot be read"), "{error}");
+    }
+
+    #[test]
+    fn non_executable_file_is_an_error() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = script(dir.path(), "rustfs", "#!/bin/sh\necho v\n", 0o644);
+        let error = SideA::resolve(Some(path.into_os_string())).expect_err("must fail closed");
+        assert!(error.to_string().contains("not executable"), "{error}");
+    }
+
+    #[test]
+    fn directory_is_an_error() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let error = SideA::resolve(Some(dir.path().as_os_str().to_owned())).expect_err("must fail closed");
+        assert!(error.to_string().contains("not a file"), "{error}");
+    }
+
+    #[test]
+    fn empty_value_is_an_error() {
+        let error = SideA::resolve(Some(OsString::new())).expect_err("must fail closed");
+        assert!(error.to_string().contains("empty"), "{error}");
+    }
+
+    #[test]
+    fn unset_selects_same_binary_mode() {
+        assert_eq!(SideA::resolve(None).expect("unset is valid"), SideA::SameBinary);
+    }
+
+    #[test]
+    fn executable_file_selects_reference_mode() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = script(dir.path(), "rustfs", "#!/bin/sh\necho v\n", 0o755);
+        assert_eq!(
+            SideA::resolve(Some(path.clone().into_os_string())).expect("valid"),
+            SideA::Reference(path)
+        );
+    }
+
+    #[tokio::test]
+    async fn version_output_is_captured_from_the_binary() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = script(dir.path(), "rustfs", "#!/bin/sh\necho 'rustfs 1.0.4 (abc123)'\n", 0o755);
+        assert_eq!(binary_version(&path).await.expect("version"), "rustfs 1.0.4 (abc123)");
+    }
+
+    #[tokio::test]
+    async fn failing_or_silent_version_is_an_error() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let failing = script(dir.path(), "failing", "#!/bin/sh\nexit 3\n", 0o755);
+        assert!(binary_version(&failing).await.is_err());
+        let silent = script(dir.path(), "silent", "#!/bin/sh\nexit 0\n", 0o755);
+        assert!(binary_version(&silent).await.is_err());
+    }
+}